org.refcodes.filesystem.alt.s3.impls.AbstractS3Client Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of refcodes-filesystem-alt-s3 Show documentation
Artifact for an alternate Amazon AWS S3 implementation of the refcodes file system.
The newest version!
// /////////////////////////////////////////////////////////////////////////////
// REFCODES.ORG
// =============================================================================
// This code is copyright (c) by Siegfried Steiner, Munich, Germany and licensed
// under the following (see "http://en.wikipedia.org/wiki/Multi-licensing")
// licenses:
// =============================================================================
// GNU General Public License, v3.0 ("http://www.gnu.org/licenses/gpl-3.0.html")
// together with the GPL linking exception applied; as being applied by the GNU
// Classpath ("http://www.gnu.org/software/classpath/license.html")
// =============================================================================
// Apache License, v2.0 ("http://www.apache.org/licenses/LICENSE-2.0")
// =============================================================================
// Please contact the copyright holding author(s) of the software artifacts in
// question for licensing issues not being covered by the above listed licenses,
// also regarding commercial licensing models or regarding the compatibility
// with other open source licenses.
// /////////////////////////////////////////////////////////////////////////////

package org.refcodes.filesystem.alt.s3.impls;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.refcodes.exception.ExceptionUtility;
import org.refcodes.logger.RuntimeLogger;
import org.refcodes.logger.impls.RuntimeLoggerFactorySingleton;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.Region;
import com.amazonaws.services.s3.model.S3ObjectSummary;

//@formatter:off
/**
 * Abstract class to be used for any S3 using service. An endpoint can be
 * specified (i.e. the location where Amazon SimpleDB will store the data):
 *  
 * Possible endpoints for SimpleDB can be retrieved from here:
 * 

 * {@link http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region}
 * 
 *	
 * 		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *		
 *			
 *		
 *	Region Endpoint Location Constraint Protocol
US Standard * s3.amazonaws.com (none required) HTTP and HTTPS
US West (Oregon) s3-us-west-2.amazonaws.com us-west-2 HTTP and HTTPS
US West (Northern California) s3-us-west-1.amazonaws.com us-west-1 HTTP and HTTPS
EU (Ireland) s3-eu-west-1.amazonaws.com EU HTTP and HTTPS
Asia Pacific (Singapore) s3-ap-southeast-1.amazonaws.com ap-southeast-1 HTTP and HTTPS
Asia Pacific (Tokyo) s3-ap-northeast-1.amazonaws.com ap-northeast-1 HTTP and HTTPS
South America (Sao Paulo) s3-sa-east-1.amazonaws.com sa-east-1 HTTP and HTTPS
 */
//@formatter:on
public abstract class AbstractS3Client {

	private static RuntimeLogger LOGGER = RuntimeLoggerFactorySingleton.createRuntimeLogger();

	// /////////////////////////////////////////////////////////////////////////
	// CONSTANTS:
	// /////////////////////////////////////////////////////////////////////////

	private static final int THREAD_POOL_SIZE = 20;

	// /////////////////////////////////////////////////////////////////////////
	// VARIABLES:
	// /////////////////////////////////////////////////////////////////////////

	private static final String DEFAULT_REGION = "s3-eu-west-1.amazonaws.com";

	private AmazonS3Client _amazonS3Client = null;
	private String _amazonS3BucketName = null;

	// /////////////////////////////////////////////////////////////////////////
	// CONSTRUCTORS:
	// /////////////////////////////////////////////////////////////////////////

	/**
	 * Constructs the S3 support by directly providing all needed information to
	 * setup the instance.
	 * 
	 * @param aBucketName The name of the bucket to use.
	 * @param aAccessKey The access key to use.
	 * @param aSecretKey The secret key to use.
	 */
	public AbstractS3Client( String aBucketName, String aAccessKey, String aSecretKey ) {
		this( aBucketName, aAccessKey, aSecretKey, DEFAULT_REGION );
	}

	//@formatter:off
	/**
	 * Constructs the S3 support by directly providing all needed information to
	 * setup the instance. An endpoint can be specified, i.e. the
	 * region where Amazon will store the data. 
	 * 
	 * Possible endpoints for SimpleDB can be retrieved  as stated above.
	 * 
	 * {@link http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region}
	 * 
	 * @param aBucketName The name of the bucket to use.
	 * @param aAccessKey The access key to use.
	 * @param aSecretKey The secret key to use.
	 * @param aEndPoint The endpoint (Amazon region) to use.
	 */
	//@formatter:on
	public AbstractS3Client( String aBucketName, String aAccessKey, String aSecretKey, String aEndPoint ) {
		if ( aEndPoint == null ) {
			aEndPoint = DEFAULT_REGION;
		}
		_amazonS3Client = new AmazonS3Client( new BasicAWSCredentials( aAccessKey, aSecretKey ) );
		_amazonS3Client.setEndpoint( aEndPoint );
		_amazonS3BucketName = aBucketName;
	}

	// /////////////////////////////////////////////////////////////////////////
	// HELPER:
	// /////////////////////////////////////////////////////////////////////////

	/**
	 * Retrieves the bucket name to be used.
	 * 
	 * @return The bucket name.
	 */
	protected String getAmazonS3BucketName() {
		return _amazonS3BucketName;
	}

	/**
	 * Sets the bucket name to be used.
	 * 
	 * @param aAmazonS3BucketName The bucket name to be used.
	 * 
	 */
	protected void setAmazonS3BucketName( String aAmazonS3BucketName ) {
		_amazonS3BucketName = aAmazonS3BucketName;
	}

	/**
	 * Retrieves the amazon S3 client to be used.
	 * 
	 * @return The S3 client to be used.
	 */
	protected AmazonS3Client getAmazonS3Client() {
		return _amazonS3Client;
	}

	/**
	 * Creates an S3 bucket.
	 * 
	 * @param aAmazonS3 The {@link AmazonS3} client.
	 * @param aBucketId The ID of the bucket to be created.
	 */
	public static void createBucket( AmazonS3 aAmazonS3, String aBucketId ) {
		aAmazonS3.createBucket( aBucketId, Region.EU_Ireland );
	}

	/**
	 * Creates an S3 bucket.
	 * 
	 * @param aAmazonS3 The {@link AmazonS3} client.
	 * @param aBucketId The ID of the bucket to be created.
	 * @param aRegion The region in the Amazon landscape where to create the
	 *        bucket.
	 */
	protected static void createBucket( AmazonS3 aAmazonS3, String aBucketId, String aRegion ) {
		aAmazonS3.createBucket( aBucketId, aRegion );
	}

	/**
	 * Clears (removes all content from) an S3 bucket.
	 * 
	 * @param aAmazonS3 The {@link AmazonS3} client.
	 * 
	 * @param aBucketId The ID of the bucket to be cleared.
	 */
	protected static void clearBucket( final AmazonS3 aAmazonS3, final String aBucketId ) {
		ObjectListing theObjectListing = aAmazonS3.listObjects( aBucketId );
		while ( theObjectListing.getObjectSummaries() != null && !theObjectListing.getObjectSummaries().isEmpty() ) {
			List theObjectSummaries = theObjectListing.getObjectSummaries();
			deleteS3Objects( aAmazonS3, aBucketId, theObjectSummaries );
			theObjectListing = aAmazonS3.listNextBatchOfObjects( theObjectListing );
		}
	}
	
	/**
	 * Deletes an S3 bucket.
	 * 
	 * @param aAmazonS3 The {@link AmazonS3} client.
	 * 
	 * @param aBucketId The ID of the bucket to be deleted.
	 */
	public static void deleteBucket( final AmazonS3 aAmazonS3, final String aBucketId ) {
		clearBucket( aAmazonS3, aBucketId );
		aAmazonS3.deleteBucket( aBucketId );
	}

	/**
	 * Creates an {@link AmazonS3} "client".
	 * 
	 * @param aAccessKey The according access key for accessing amazon AWS.
	 * @param aSecretKey The secret access key for accessing amazon AWS.
	 * 
	 * @return The client represented by an {@link AmazonS3} instance.
	 */
	protected static AmazonS3 createAmazonS3( String aAccessKey, String aSecretKey ) {
		return new AmazonS3Client( new BasicAWSCredentials( aAccessKey, aSecretKey ) );
	}

	/**
	 * Deletes the content described by the given {@link S3ObjectSummary} list
	 * from an S3 bucket.
	 * 
	 * @param aAmazonS3 The {@link AmazonS3} client.
	 * 
	 * @param aBucketId The ID of the bucket from which the objects are to be
	 *        deleted.
	 * 
	 * @param aS3SummaryObjects The {@link S3ObjectSummary} list describing the
	 *        objects to be deleted.
	 */
	protected static void deleteS3Objects( final AmazonS3 aAmazonS3, final String aBucketId, List aS3SummaryObjects ) {
		ExecutorService theExecutorService = Executors.newFixedThreadPool( THREAD_POOL_SIZE );
		final CountDownLatch theCountDownLatch = new CountDownLatch( aS3SummaryObjects.size() );
		for ( final S3ObjectSummary eSummary : aS3SummaryObjects ) {
			theExecutorService.execute( new Runnable() {

				@Override
				public void run() {
					try {
						aAmazonS3.deleteObject( aBucketId, eSummary.getKey() );
					}
					catch ( Exception e ) {
						LOGGER.warn( ExceptionUtility.toMessage( e ), e );
					}
					finally {
						theCountDownLatch.countDown();
					}
				}
			} );
		}
		try {
			theCountDownLatch.await();
		}
		catch ( InterruptedException ignored ) {}
		theExecutorService.shutdown();
	}

	/**
	 * Retrieves an {@link AmazonS3Client} from a configuration file containing
	 * the access- and the secret key.
	 * 
	 * @param aConfigFile The configuration file used to configure the
	 *        {@link AmazonS3Client}.
	 * 
	 * @return An {@link AmazonS3Client}.
	 * 
	 * @throws IOException In case there were problems reading the configuration
	 *         file.
	 */
	protected static AmazonS3Client getAmazonS3Client( File aConfigFile ) throws IOException {
		Properties theProperties = new Properties();
		theProperties.load( new FileInputStream( aConfigFile ) );
		AmazonS3Client theAmazonS3Client = new AmazonS3Client( new PropertiesCredentials( aConfigFile ) );
		return theAmazonS3Client;
	}
}
Region	Endpoint	Location Constraint	Protocol
US Standard *	s3.amazonaws.com	(none required)	HTTP and HTTPS
US West (Oregon)	s3-us-west-2.amazonaws.com	us-west-2	HTTP and HTTPS
US West (Northern California)	s3-us-west-1.amazonaws.com	us-west-1	HTTP and HTTPS
EU (Ireland)	s3-eu-west-1.amazonaws.com	EU	HTTP and HTTPS
Asia Pacific (Singapore)	s3-ap-southeast-1.amazonaws.com	ap-southeast-1	HTTP and HTTPS
Asia Pacific (Tokyo)	s3-ap-northeast-1.amazonaws.com	ap-northeast-1	HTTP and HTTPS
South America (Sao Paulo)	s3-sa-east-1.amazonaws.com	sa-east-1	HTTP and HTTPS