org.refcodes.filesystem.alt.s3.impls.S3FileSystemImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of refcodes-filesystem-alt-s3 Show documentation
Artifact for an alternate Amazon AWS S3 implementation of the refcodes file system.
The newest version!
// /////////////////////////////////////////////////////////////////////////////
// REFCODES.ORG
// =============================================================================
// This code is copyright (c) by Siegfried Steiner, Munich, Germany and licensed
// under the following (see "http://en.wikipedia.org/wiki/Multi-licensing")
// licenses:
// =============================================================================
// GNU General Public License, v3.0 ("http://www.gnu.org/licenses/gpl-3.0.html")
// together with the GPL linking exception applied; as being applied by the GNU
// Classpath ("http://www.gnu.org/software/classpath/license.html")
// =============================================================================
// Apache License, v2.0 ("http://www.apache.org/licenses/LICENSE-2.0")
// =============================================================================
// Please contact the copyright holding author(s) of the software artifacts in
// question for licensing issues not being covered by the above listed licenses,
// also regarding commercial licensing models or regarding the compatibility
// with other open source licenses.
// /////////////////////////////////////////////////////////////////////////////

package org.refcodes.filesystem.alt.s3.impls;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.refcodes.data.ControlFlowConsts;
import org.refcodes.data.DateConsts;
import org.refcodes.exception.ExceptionUtility;
import org.refcodes.filesystem.ConcurrentAccessException;
import org.refcodes.filesystem.FileAlreadyExistsException;
import org.refcodes.filesystem.FileHandle;
import org.refcodes.filesystem.FileHandle.MutableFileHandle;
import org.refcodes.filesystem.FileSystem;
import org.refcodes.filesystem.FileSystemUtility;
import org.refcodes.filesystem.IllegalFileHandleException;
import org.refcodes.filesystem.IllegalKeyException;
import org.refcodes.filesystem.IllegalNameException;
import org.refcodes.filesystem.IllegalPathException;
import org.refcodes.filesystem.NoCreateAccessException;
import org.refcodes.filesystem.NoDeleteAccessException;
import org.refcodes.filesystem.NoListAccessException;
import org.refcodes.filesystem.NoReadAccessException;
import org.refcodes.filesystem.NoWriteAccessException;
import org.refcodes.filesystem.UnknownFileException;
import org.refcodes.filesystem.UnknownFileSystemException;
import org.refcodes.filesystem.UnknownKeyException;
import org.refcodes.filesystem.UnknownPathException;
import org.refcodes.filesystem.impls.FileHandleImpl;
import org.refcodes.logger.RuntimeLogger;
import org.refcodes.logger.impls.RuntimeLoggerFactorySingleton;

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;

/**
 * TODO Update the file handle size after writing. Provide a size argument for
 * stream based operations.
 */
public class S3FileSystemImpl extends AbstractS3Client implements FileSystem {

	private static RuntimeLogger LOGGER = RuntimeLoggerFactorySingleton.createRuntimeLogger();

	private static String METADATA_CREATED_DATE = "CreatedDate";
	private HashSet _threadCache = new HashSet();

	// /////////////////////////////////////////////////////////////////////////
	// CONSTRUCTORS:
	// /////////////////////////////////////////////////////////////////////////
	public S3FileSystemImpl( String aBucketName, String aAccessKey, String aSecretKey ) {
		super( aBucketName, aAccessKey, aSecretKey );
	}

	public S3FileSystemImpl( String aBucketName, String aAccessKey, String aSecretKey, String aEndPoint ) {
		super( aBucketName, aAccessKey, aSecretKey, aEndPoint );
	}

	// /////////////////////////////////////////////////////////////////////////
	// METHODS:
	// /////////////////////////////////////////////////////////////////////////
	@Override
	public boolean hasFile( String aKey ) throws IllegalKeyException, NoListAccessException, UnknownFileSystemException, IOException {
		if ( StringUtils.isEmpty( aKey ) ) {
			throw new IllegalKeyException( aKey, "The provided key is invalid." );
		}
		try {
			getAmazonS3Client().getObjectMetadata( getAmazonS3BucketName(), aKey );
		}
		catch ( AmazonClientException e ) {
			// LOGGER.warn( "Exception while testing key \"" + aKey + "\"" +
			// ExceptionUtility.toMessage( e ), e );
			return false;
		}
		return true;
	}

	@Override
	public boolean hasFile( String aPath, String aName ) throws IllegalPathException, IllegalNameException, NoListAccessException, UnknownFileSystemException, IOException {
		if ( StringUtils.isEmpty( aPath ) ) {
			throw new IllegalPathException( aPath, "The provided path is invalid." );
		}
		if ( StringUtils.isEmpty( aName ) ) {
			throw new IllegalNameException( aName, "The provided name is invalid." );
		}
		boolean result = false;
		try {
			result = hasFile( FileSystemUtility.toKey( aPath, aName ) );
		}
		catch ( IllegalKeyException e ) {
			throw new IOException( ExceptionUtility.toMessage( e ), e );
		}
		return result;
	}

	@Override
	public boolean hasFile( FileHandle aFileHandle ) throws NoListAccessException, UnknownFileSystemException, IOException, IllegalFileHandleException {
		boolean result = false;
		try {
			result = hasFile( aFileHandle.toKey() );
		}
		catch ( IllegalKeyException e ) {
			throw new IOException( ExceptionUtility.toMessage( e ), e );
		}
		return result;
	}

	@Override
	public FileHandle createFile( String aKey ) throws FileAlreadyExistsException, NoCreateAccessException, IllegalKeyException, UnknownFileSystemException, IOException, NoListAccessException {
		if ( StringUtils.isEmpty( aKey ) ) {
			throw new IllegalKeyException( aKey, "The provided key is invalid." );
		}
		if ( hasFile( aKey ) ) {
			throw new FileAlreadyExistsException( aKey, "A file handle with the given key \"" + aKey + "\" already exists." );
		}
		String path = FileSystemUtility.getPath( aKey );
		String name = FileSystemUtility.getName( aKey );
		FileHandle fileHandle = new FileHandleImpl( path, name, 0L, new Date(), new Date() );
		return fileHandle;
	}

	@Override
	public FileHandle createFile( String aPath, String aName ) throws FileAlreadyExistsException, NoCreateAccessException, IllegalNameException, IllegalPathException, UnknownFileSystemException, IOException, NoListAccessException {
		if ( StringUtils.isEmpty( aPath ) ) {
			throw new IllegalPathException( aPath, "The provided path is invalid." );
		}
		if ( StringUtils.isEmpty( aName ) ) {
			throw new IllegalNameException( aName, "The provided name is invalid." );
		}
		FileHandle fileHandle = new FileHandleImpl( aPath, aName, 0L, new Date(), new Date() );
		if ( hasFile( aPath, aName ) ) {
			throw new FileAlreadyExistsException( fileHandle.toKey(), "A file handle with the given path \"" + aPath + "\" and name \"" + aName + "\" (key = \"" + FileSystemUtility.toKey( aPath, aName ) + "\") already exists." );
		}
		return fileHandle;
	}

	@Override
	public FileHandle getFileHandle( String aKey ) throws NoListAccessException, IllegalKeyException, UnknownFileSystemException, IOException, UnknownKeyException {
		if ( StringUtils.isEmpty( aKey ) ) {
			throw new IllegalKeyException( aKey, "The provided key is invalid." );
		}
		if ( !hasFile( aKey ) ) {
			throw new UnknownKeyException( aKey, "A file with the given key \"" + aKey + "\"does not exist!" );
		}
		ObjectMetadata metadata = getAmazonS3Client().getObjectMetadata( getAmazonS3BucketName(), aKey );
		Date createdDate = getCreatedDate( metadata );
		String path = FileSystemUtility.getPath( aKey );
		String name = FileSystemUtility.getName( aKey );
		FileHandle fileHandle = new FileHandleImpl( path, name, metadata.getContentLength(), createdDate, metadata.getLastModified() );
		return fileHandle;
	}

	@Override
	public FileHandle getFileHandle( String aPath, String aName ) throws NoListAccessException, IllegalNameException, IllegalPathException, UnknownFileSystemException, IOException, UnknownKeyException {
		if ( StringUtils.isEmpty( aPath ) ) {
			throw new IllegalPathException( aPath, "The provided path is invalid." );
		}
		if ( StringUtils.isEmpty( aName ) ) {
			throw new IllegalNameException( aName, "The provided name is invalid." );
		}
		if ( !hasFile( aPath, aName ) ) {
			throw new UnknownKeyException( FileSystemUtility.toKey( aPath, aName ), "A filke with the given path \"" + aPath + "\" and name \"" + aName + "\" (= key \"" + FileSystemUtility.toKey( aPath, aName ) + "\") does not exists." );
		}
		String key = FileSystemUtility.toKey( aPath, aName );
		ObjectMetadata metadata = getAmazonS3Client().getObjectMetadata( getAmazonS3BucketName(), key );
		Date createdDate = getCreatedDate( metadata );
		FileHandle fileHandle = new FileHandleImpl( aPath, aName, metadata.getContentLength(), createdDate, metadata.getLastModified() );
		return fileHandle;
	}

	@Override
	public void fromFile( FileHandle aFromFileHandle, OutputStream aOutputStream ) throws ConcurrentAccessException, UnknownFileException, NoReadAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( !hasFile( aFromFileHandle ) ) {
			throw new UnknownFileException( aFromFileHandle, "The given \"from\" file handle was not found." );
		}
		S3Object s3Object = getAmazonS3Client().getObject( getAmazonS3BucketName(), aFromFileHandle.toKey() );
		if ( s3Object == null ) {
			throw new UnknownFileException( aFromFileHandle, "The given \"from\" file handle was not found." );
		}
		try {
			IOUtils.copy( s3Object.getObjectContent(), aOutputStream );
		}
		finally {
			IOUtils.closeQuietly( s3Object.getObjectContent() );
		}
	}

	/**
	 * -------------------------------------------------------------------------
	 * ATTENTION: As of Amazon's S3 shortcomings, Amazon requires a content
	 * length being provided: "Content length must be specified before data can
	 * be uploaded to Amazon S3. If the caller doesn't provide it, the library
	 * will have to buffer the contents of the input stream in order to
	 * calculate it because Amazon S3 explicitly requires that the content
	 * length be sent in the request headers before any of the data is sent."
	 * -------------------------------------------------------------------------
	 * As the advantage of a stream is *not* to require knowledge of the length
	 * beforehand, please use the {@link #toFile(FileHandle, FileHandle)} or
	 * {@link #toFile(FileHandle, byte[])} methods wherever possible.
	 * -------------------------------------------------------------------------
	 * 
	 * {@inheritDoc}
	 */
	@Override
	public void toFile( FileHandle aToFileHandle, InputStream aInputStream ) throws ConcurrentAccessException, UnknownFileException, NoWriteAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		try {
			ObjectMetadata metadata = new ObjectMetadata();
			if ( aToFileHandle.getCreatedDate() != null ) {
				String createdDateString = DateConsts.NORM_DATE_FORMAT.toString( aToFileHandle.getCreatedDate() );
				metadata.getUserMetadata().put( METADATA_CREATED_DATE, createdDateString );
			}
			getAmazonS3Client().putObject( getAmazonS3BucketName(), aToFileHandle.toKey(), aInputStream, metadata );
		}
		catch ( AmazonClientException ase ) {
			throw new IOException( ExceptionUtility.toMessage( ase ), ase );
		}
	}

	@Override
	public InputStream fromFile( FileHandle aFromFileHandle ) throws ConcurrentAccessException, UnknownFileException, UnknownFileException, NoReadAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( !hasFile( aFromFileHandle ) ) {
			throw new UnknownFileException( aFromFileHandle, "The given \"from\" file handle was not found." );
		}
		S3Object s3Object = getAmazonS3Client().getObject( getAmazonS3BucketName(), aFromFileHandle.toKey() );
		if ( s3Object == null ) {
			throw new UnknownFileException( aFromFileHandle, "The given \"from\" file handle was not found." );
		}
		return s3Object.getObjectContent();
	}

	/**
	 * -------------------------------------------------------------------------
	 * ATTENTION: As of Amazon's S3 shortcomings, Amazon requires a content
	 * length being provided: "Content length must be specified before data can
	 * be uploaded to Amazon S3. If the caller doesn't provide it, the library
	 * will have to buffer the contents of the input stream in order to
	 * calculate it because Amazon S3 explicitly requires that the content
	 * length be sent in the request headers before any of the data is sent."
	 * -------------------------------------------------------------------------
	 * As the advantage of a stream is *not* to require knowledge of the length
	 * beforehand, please use the {@link #toFile(FileHandle, FileHandle)} or
	 * {@link #toFile(FileHandle, byte[])} methods wherever possible.
	 * -------------------------------------------------------------------------
	 * 
	 * {@inheritDoc}
	 */
	@Override
	public OutputStream toFile( final FileHandle aToFileHandle ) throws ConcurrentAccessException, UnknownFileException, NoWriteAccessException, UnknownFileSystemException, IOException, IllegalFileHandleException {
		final PipedInputStream thePipedInputStream = new PipedInputStream();
		PipedOutputStream thePipedOutputStream = new PipedOutputStream( thePipedInputStream );
		// ---------------------------------------------------------------------
		// Start a new thread for the request, since the request is blocking the
		// current thread because the amazon SDK waits for the given
		// InputStream to be closed .
		// ---------------------------------------------------------------------
		Thread t = new Thread() {

			@Override
			public void run() {
				try {
					ObjectMetadata metadata = new ObjectMetadata();
					if ( aToFileHandle.getCreatedDate() != null ) {
						String createdDateString = DateConsts.NORM_DATE_FORMAT.toString( aToFileHandle.getCreatedDate() );
						metadata.getUserMetadata().put( METADATA_CREATED_DATE, createdDateString );
					}
					getAmazonS3Client().putObject( getAmazonS3BucketName(), aToFileHandle.toKey(), thePipedInputStream, metadata );
				}
				finally {
					// Remove this thread instance from the thread cache.
					_threadCache.remove( this );
				}
			}
		};
		// Prevent the thread from dying upon a system's exit:
		t.setDaemon( true );
		// Add the thread to the thread cache.
		_threadCache.add( t );
		t.start();
		return thePipedOutputStream;
	}

	@Override
	public void fromFile( FileHandle aFileHandle, File aToFile ) throws ConcurrentAccessException, UnknownFileException, NoReadAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( !hasFile( aFileHandle ) ) {
			throw new UnknownFileException( aFileHandle, "The given file handle was not found." );
		}
		S3Object s3Object = getAmazonS3Client().getObject( getAmazonS3BucketName(), aFileHandle.toKey() );
		if ( s3Object == null ) {
			throw new UnknownFileException( aFileHandle, "The given file handle was not found." );
		}
		InputStream theInputStream = s3Object.getObjectContent();
		OutputStream theOutputStream = new FileOutputStream( aToFile );
		IOUtils.copy( theInputStream, theOutputStream );
	}

	@Override
	public void toFile( FileHandle aFileHandle, File aFile ) throws ConcurrentAccessException, UnknownFileException, NoWriteAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		getAmazonS3Client().putObject( getAmazonS3BucketName(), aFileHandle.toKey(), aFile );
	}

	@Override
	public void toFile( FileHandle aFileHandle, byte[] aBuffer ) throws ConcurrentAccessException, UnknownFileException, NoWriteAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		ObjectMetadata theMetadata = new ObjectMetadata();
		if ( aFileHandle.getCreatedDate() != null ) {
			String createdDateString = DateConsts.NORM_DATE_FORMAT.toString( aFileHandle.getCreatedDate() );
			theMetadata.getUserMetadata().put( METADATA_CREATED_DATE, createdDateString );
		}
		ByteArrayInputStream theInputStream = new ByteArrayInputStream( aBuffer );
		theMetadata.setContentLength( aBuffer.length );
		getAmazonS3Client().putObject( getAmazonS3BucketName(), aFileHandle.toKey(), theInputStream, theMetadata );
	}

	@Override
	public FileHandle renameFile( FileHandle aFileHandle, String aNewName ) throws UnknownFileException, ConcurrentAccessException, FileAlreadyExistsException, NoCreateAccessException, NoDeleteAccessException, IllegalNameException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( StringUtils.isEmpty( aNewName ) ) {
			throw new IllegalNameException( aNewName, "The provided new name \"" + aFileHandle + "\" is invalid." );
		}
		FileHandle fileHandle = null;
		try {
			fileHandle = moveFile( aFileHandle, FileSystemUtility.toKey( aFileHandle.getPath(), aNewName ) );
		}
		catch ( IllegalKeyException e ) {
			throw new IllegalNameException( aNewName, "The provided new name \"" + aNewName + "\" is invalid." );
		}
		return fileHandle;
	}

	@Override
	public FileHandle moveFile( FileHandle aFileHandle, String aNewKey ) throws UnknownFileException, ConcurrentAccessException, FileAlreadyExistsException, NoCreateAccessException, NoDeleteAccessException, IllegalKeyException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( StringUtils.isEmpty( aNewKey ) ) {
			throw new IllegalKeyException( aNewKey, "The provided new key \"" + aNewKey + "\" is invalid." );
		}
		if ( !hasFile( aFileHandle ) ) {
			throw new UnknownFileException( aFileHandle, "The given file handle handle was not found." );
		}
		if ( hasFile( aNewKey ) ) {
			throw new FileAlreadyExistsException( aNewKey, "A file handle with the given new key \"" + aNewKey + "\" already exists." );
		}
		getAmazonS3Client().copyObject( getAmazonS3BucketName(), aFileHandle.toKey(), getAmazonS3BucketName(), aNewKey );
		deleteFile( aFileHandle );
		FileHandle theNewFileHandle = null;

		try {
			theNewFileHandle = getFileHandle( aNewKey );
		}
		catch ( UnknownKeyException e ) { /* ignore, handled below */}

		// -----------------------------------------------------------------
		// The file was deleted at its new key just after we moved it. We don't
		// throw an exception as the deletion of the new key could as well have
		// happened after returning from this method.
		// -----------------------------------------------------------------
		if ( theNewFileHandle == null ) {
			LOGGER.warn( "The file handle which moved from \"" + aFileHandle.toKey() + "\" to  \"" + aFileHandle.getName() + "\" does not exist any more at it's target location! This should never happen (we probably have encountered a thread race condition)." );
			MutableFileHandle theMutableNewFileHandle = aFileHandle.toMutableFileHandle();
			theMutableNewFileHandle.setPath( FileSystemUtility.getPath( aNewKey ) );
			theMutableNewFileHandle.setName( FileSystemUtility.getName( aNewKey ) );
			return theMutableNewFileHandle;
		}
		return theNewFileHandle;
	}

	@Override
	public void deleteFile( FileHandle aFileHandle ) throws ConcurrentAccessException, UnknownFileException, NoDeleteAccessException, UnknownFileSystemException, IOException, NoListAccessException, IllegalFileHandleException {
		if ( !hasFile( aFileHandle ) ) {
			throw new UnknownFileException( aFileHandle, "The given file handle was not found." );
		}
		getAmazonS3Client().deleteObject( getAmazonS3BucketName(), aFileHandle.toKey() );
	}

	@Override
	public boolean hasFiles( String aPath, boolean isRecursively ) throws NoListAccessException, IllegalPathException, UnknownFileSystemException, IOException {
		if ( StringUtils.isEmpty( aPath ) ) {
			throw new IllegalPathException( aPath, "The provided path \"" + aPath + "\" is invalid." );
		}
		try {
			List fileHandles = getFileHandles( aPath, isRecursively );
			if ( fileHandles.isEmpty() ) {
				return false;
			}
			else {
				return true;
			}
		}
		catch ( UnknownPathException e ) {
			return false;
		}
	}

	@Override
	public List getFileHandles( String aPath, boolean isRecursively ) throws NoListAccessException, UnknownPathException, IllegalPathException, UnknownFileSystemException, IOException {
		if ( StringUtils.isEmpty( aPath ) ) {
			throw new IllegalPathException( aPath, "The provided path \"" + aPath + "\" is invalid." );
		}
		if ( !aPath.endsWith( "" + FileSystem.PATH_DELIMETER ) ) {
			aPath += FileSystem.PATH_DELIMETER;
		}
		List result = new ArrayList();
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
		listObjectsRequest.setBucketName( getAmazonS3BucketName() );
		listObjectsRequest.setDelimiter( "" + FileSystem.PATH_DELIMETER );
		listObjectsRequest.setPrefix( aPath );
		ObjectListing objectListing = getAmazonS3Client().listObjects( listObjectsRequest );
		while ( objectListing.getObjectSummaries() != null && !objectListing.getObjectSummaries().isEmpty() ) {
			List batchObjects = objectListing.getObjectSummaries();
			for ( S3ObjectSummary summary : batchObjects ) {
				if ( !aPath.equals( summary.getKey() ) ) {
					ObjectMetadata metadata = getAmazonS3Client().getObjectMetadata( getAmazonS3BucketName(), summary.getKey() );
					Date createdDate = getCreatedDate( metadata );
					FileHandle fileHandle = new FileHandleImpl( FileSystemUtility.getPath( summary.getKey() ), FileSystemUtility.getName( summary.getKey() ), metadata.getContentLength(), createdDate, summary.getLastModified() );
					result.add( fileHandle );
				}
			}
			if ( isRecursively ) {
				for ( String prefix : objectListing.getCommonPrefixes() ) {
					result.addAll( getFileHandles( prefix, isRecursively ) );
				}
			}
			objectListing = getAmazonS3Client().listNextBatchOfObjects( objectListing );
		}
		return result;
	}

	@Override
	public void destroy() {
		LOGGER.debug( "Destroying the S3 file system..." );
		while ( !_threadCache.isEmpty() ) {
			LOGGER.info( "Waiting for destroying the S3 file system because some threads aren't finished yet." );
			try {
				Thread.sleep( ControlFlowConsts.NORM_CODE_LOOP_SLEEP_TIME_IN_MS );
			}
			catch ( InterruptedException ignored ) {}
		}
		LOGGER.debug( "Destroyed the S3 file system." );
	}

	// /////////////////////////////////////////////////////////////////////////
	// HELPER:
	// /////////////////////////////////////////////////////////////////////////

	/**
	 * Retrieves the created date from the provided meta data.
	 * 
	 * @param aMetadata The meta data from which to retrieve the created date.
	 * 
	 * @return TRhe created date.
	 */
	private Date getCreatedDate( ObjectMetadata aMetadata ) {
		String theCreatedDateString = aMetadata.getUserMetadata().get( METADATA_CREATED_DATE );
		Date theCreatedDate = null;
		if ( theCreatedDateString != null ) {
			try {
				theCreatedDate = DateConsts.NORM_DATE_FORMAT.parse( theCreatedDateString );
			}
			catch ( ParseException e ) {
				theCreatedDate = null;
			}
		}
		return theCreatedDate;
	}
}