org.cyclopsgroup.cym2.awss3.S3Wagon Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of awss3-maven-wagon Show documentation
Show all versions of awss3-maven-wagon Show documentation
Maven Wagon to support S3 repository
package org.cyclopsgroup.cym2.awss3;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.activation.MimetypesFileTypeMap;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.maven.wagon.ConnectionException;
import org.apache.maven.wagon.InputData;
import org.apache.maven.wagon.OutputData;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.StreamWagon;
import org.apache.maven.wagon.TransferFailedException;
import org.apache.maven.wagon.authentication.AuthenticationException;
import org.apache.maven.wagon.authorization.AuthorizationException;
import org.apache.maven.wagon.proxy.ProxyInfo;
import org.apache.maven.wagon.resource.Resource;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
/**
* Amazon S3 wagon provider implementation
*
* @author Jiaqi Guo
*/
public class S3Wagon
extends StreamWagon
{
private static Map loadMimeTypes()
throws IOException
{
Map map = new HashMap();
InputStream in =
S3Wagon.class.getClassLoader().getResourceAsStream( "mime.types" );
try
{
LineNumberReader reader =
new LineNumberReader( new InputStreamReader( in ) );
for ( String line = reader.readLine(); line != null; line =
reader.readLine() )
{
if ( StringUtils.isBlank( line ) || line.startsWith( "#" ) )
{
continue;
}
line = line.trim();
String[] pieces = line.split( "\\s+" );
if ( pieces.length <= 1 )
{
continue;
}
String mimeType = pieces[0];
for ( int i = 1; i < pieces.length; i++ )
{
map.put( pieces[i], mimeType );
}
}
return map;
}
finally
{
IOUtils.closeQuietly( in );
}
}
private String bucketName;
private String keyPrefix;
private final Map mimeTypes;
private AmazonS3 s3;
private final MimetypesFileTypeMap typeMap = new MimetypesFileTypeMap();
/**
* Default constructor reads mime type mapping from generated properties
* file for later use
*
* @throws IOException Allows IO errors
*/
public S3Wagon()
throws IOException
{
this.mimeTypes = Collections.unmodifiableMap( loadMimeTypes() );
}
/**
* @inheritDoc
*/
@Override
public void closeConnection()
throws ConnectionException
{
}
private void doPutFromStream( InputStream in, File inFile,
String destination, long contentLength,
long lastModified )
{
Resource resource = new Resource( destination );
firePutInitiated( resource, inFile );
String dest = StringUtils.removeStart( destination, "./" );
dest = StringUtils.removeStart( dest, "/" );
String key = keyPrefix + dest;
fireTransferDebug( "{keyPreix = " + keyPrefix + ", dest=" + destination
+ "} -> " + dest );
// Prepare for meta data
ObjectMetadata meta = new ObjectMetadata();
// Content length is important. Many S3 client relies on it
if ( contentLength != -1 )
{
meta.setContentLength( contentLength );
}
// Last modified data is used by CloudFront
meta.setLastModified( new Date( lastModified ) );
// Find mime type based on file extension
int lastDot = destination.lastIndexOf( '.' );
String mimeType = null;
if ( lastDot != -1 )
{
String ext =
destination.substring( lastDot + 1, destination.length() );
mimeType = mimeTypes.get( ext );
}
if ( mimeType == null )
{
mimeType = typeMap.getContentType( destination );
}
else
{
fireTransferDebug( "Mime type of " + dest + " is " + mimeType
+ " according to build-in types" );
}
if ( mimeType != null )
{
meta.setContentType( mimeType );
}
try
{
fireTransferDebug( "Uploading file " + inFile + " to key " + key
+ " in S3 bucket " + bucketName );
firePutStarted( resource, inFile );
s3.putObject( bucketName, key, in, meta );
firePutCompleted( resource, inFile );
}
finally
{
IOUtils.closeQuietly( in );
}
}
/**
* @inheritDoc
*/
@Override
public void fillInputData( InputData in )
throws TransferFailedException, ResourceDoesNotExistException,
AuthorizationException
{
fireTransferDebug( "Filling input data" );
String key = keyPrefix + in.getResource().getName();
S3Object object;
try
{
object = s3.getObject( bucketName, key );
}
catch ( AmazonServiceException e )
{
if ( e.getStatusCode() == 404 )
{
throw new ResourceDoesNotExistException( "Key " + key
+ " does not exist in S3 bucket " + bucketName );
}
else if ( e.getStatusCode() == 403 )
{
// 403 is thrown when key does not exist and configuration
// doesn't allow user to list keys
throw new ResourceDoesNotExistException(
"403 implies that key "
+ key
+ " does not exist in bucket "
+ bucketName, e );
}
throw new TransferFailedException( "Can't get object " + key
+ " from S4 bucket " + bucketName, e );
}
in.getResource().setContentLength( object.getObjectMetadata().getContentLength() );
in.getResource().setLastModified( object.getObjectMetadata().getLastModified().getTime() );
in.setInputStream( object.getObjectContent() );
}
/**
* @inheritDoc
*/
@Override
public void fillOutputData( OutputData out )
throws TransferFailedException
{
throw new UnsupportedOperationException( "This call is not supported" );
}
/**
* @inheritDoc
*/
public void get( String resourceName, File destination )
throws ResourceDoesNotExistException, TransferFailedException
{
Resource resource = new Resource( resourceName );
fireGetInitiated( resource, destination );
String key = keyPrefix + resourceName;
try
{
fireGetStarted( resource, destination );
// This is a bit more efficient than copying stream
s3.getObject( new GetObjectRequest( bucketName, key ), destination );
fireGetCompleted( resource, destination );
}
catch ( AmazonServiceException e )
{
if ( e.getStatusCode() == 404 )
{
throw new ResourceDoesNotExistException( "Key " + key
+ " does not exist in bucket " + bucketName, e );
}
else if ( e.getStatusCode() == 403 )
{
throw new ResourceDoesNotExistException(
"403 implies that key "
+ key
+ " does not exist in bucket "
+ bucketName, e );
}
throw new TransferFailedException( "Getting metadata of key " + key
+ " failed", e );
}
}
/**
* @inheritDoc
*/
@SuppressWarnings( "rawtypes" )
public List getFileList( String destinationDirectory )
throws TransferFailedException, ResourceDoesNotExistException
{
String path = keyPrefix + destinationDirectory;
if ( !path.endsWith( "/" ) )
{
path += "/";
}
fireSessionDebug( "Listing objects with prefix " + path
+ " under bucket " + bucketName );
// Since S3 does not have concept of directory, result contains all
// contents with given prefix
ObjectListing result =
s3.listObjects( new ListObjectsRequest().withBucketName( bucketName ).withPrefix( path ).withDelimiter( "/" ) );
if ( result.getObjectSummaries().isEmpty() )
{
throw new ResourceDoesNotExistException(
"No keys exist with prefix "
+ path );
}
Set results = new HashSet();
for ( S3ObjectSummary summary : result.getObjectSummaries() )
{
String name = StringUtils.removeStart( summary.getKey(), path );
if ( name.indexOf( '/' ) == -1 )
{
results.add( name );
}
else
{
results.add( name.substring( 0, name.indexOf( '/' ) ) );
}
}
fireSessionDebug( "Returning result " + results );
return new ArrayList( results );
}
/**
* @inheritDoc
*/
public boolean getIfNewer( String resourceName, File destination,
long timestamp )
throws ResourceDoesNotExistException, TransferFailedException
{
ObjectMetadata meta = getRequiredMetadata( resourceName );
if ( meta == null )
{
return false;
}
if ( meta.getLastModified() != null
&& meta.getLastModified().getTime() > timestamp )
{
fireSessionDebug( "Remote timestamp " + meta.getLastModified()
+ " is greater than local timestamp " + timestamp
+ ", ignore get" );
return false;
}
get( resourceName, destination );
return true;
}
/**
* @inheritDoc
*/
@Override
public boolean getIfNewerToStream( String resourceName, OutputStream out,
long timestamp )
throws ResourceDoesNotExistException, TransferFailedException
{
ObjectMetadata meta = getRequiredMetadata( resourceName );
if ( meta == null )
{
return false;
}
if ( meta.getLastModified() != null
&& meta.getLastModified().getTime() > timestamp )
{
fireSessionDebug( "Remote timestamp " + meta.getLastModified()
+ " is greater than local timestamp " + timestamp
+ ", ignore get" );
return false;
}
Resource resource = new Resource( resourceName );
fireGetInitiated( resource, null );
InputStream in =
s3.getObject( bucketName, keyPrefix ).getObjectContent();
try
{
fireGetStarted( resource, null );
IOUtils.copy( in, out );
out.flush();
out.close();
fireGetCompleted( resource, null );
return true;
}
catch ( IOException e )
{
throw new TransferFailedException( "Stream copy failed", e );
}
finally
{
IOUtils.closeQuietly( in );
}
}
private ObjectMetadata getRequiredMetadata( String resourceName )
throws ResourceDoesNotExistException, TransferFailedException
{
String key = keyPrefix + resourceName;
try
{
return s3.getObjectMetadata( bucketName, key );
}
catch ( AmazonServiceException e )
{
if ( e.getStatusCode() == 404 )
{
throw new ResourceDoesNotExistException( "Key " + key
+ " does not exist in bucket " + bucketName, e );
}
throw new TransferFailedException( "Getting metadata of key " + key
+ "failed", e );
}
}
/**
* @inheritDoc
*/
@Override
protected void openConnectionInternal()
throws ConnectionException, AuthenticationException
{
// Retrieve credentials from authentication information is always
// required since it has access key and secret
// key
if ( authenticationInfo == null )
{
throw new AuthenticationException(
"S3 access requires authentication information. Repository is "
+ getRepository() );
}
if ( StringUtils.isEmpty( authenticationInfo.getUserName() ) )
{
throw new AuthenticationException(
"Tag must set to valid AWS access key ID in server configuration, either in pom.xml or settings.xml. Repository is "
+ getRepository() );
}
if ( StringUtils.isEmpty( authenticationInfo.getPassword() ) )
{
throw new AuthenticationException(
"Tag must set to valid AWS secret key in server configuration, either in pom.xml or settings.xml. Repository is "
+ getRepository() );
}
AWSCredentials credentials =
new BasicAWSCredentials( authenticationInfo.getUserName(),
authenticationInfo.getPassword() );
// Pass timeout configuration to AWS client config
ClientConfiguration config = new ClientConfiguration();
config.setConnectionTimeout( getTimeout() );
config.setSocketTimeout( getTimeout() );
fireSessionDebug( "Connect timeout and socket timeout is set to "
+ getTimeout() + " ms" );
// Possible proxy
ProxyInfo proxy = getProxyInfo();
fireSessionDebug( "Setting up AWS S3 client with source "
+ ToStringBuilder.reflectionToString( getRepository() )
+ ", authentication information and proxy "
+ ToStringBuilder.reflectionToString( proxy ) );
if ( proxy != null )
{
config.setProxyDomain( proxy.getNtlmDomain() );
config.setProxyHost( proxy.getHost() );
config.setProxyPassword( proxy.getPassword() );
config.setProxyPort( proxy.getPort() );
config.setProxyUsername( proxy.getUserName() );
config.setProxyWorkstation( proxy.getNtlmHost() );
}
fireSessionDebug( "AWS Client config is "
+ ToStringBuilder.reflectionToString( config ) );
// Create client
s3 = new AmazonS3Client( credentials, config );
bucketName = getRepository().getHost();
fireSessionDebug( "Bucket name is " + bucketName );
// Figure out path defined in pom.xml
String prefix = StringUtils.trimToEmpty( getRepository().getBasedir() );
if ( !prefix.endsWith( "/" ) )
{
prefix = prefix + "/";
}
prefix = StringUtils.removeStart( prefix, "/" );
keyPrefix = prefix;
fireSessionDebug( "Key prefix " + keyPrefix );
}
/**
* @inheritDoc
*/
public void put( File source, String destination )
throws TransferFailedException, ResourceDoesNotExistException
{
try
{
doPutFromStream( new FileInputStream( source ), source,
destination, source.length(),
source.lastModified() );
}
catch ( FileNotFoundException e )
{
throw new ResourceDoesNotExistException( "Source file " + source
+ " does not exist", e );
}
}
/**
* @inheritDoc
*/
public void putDirectory( File sourceDirectory, String destinationDirectory )
throws TransferFailedException, ResourceDoesNotExistException,
AuthorizationException
{
if ( destinationDirectory.equals( "." ) )
{
destinationDirectory = "";
}
fireTransferDebug( "Putting " + sourceDirectory + " to "
+ destinationDirectory + " which is noop" );
for ( File file : sourceDirectory.listFiles() )
{
String dest =
StringUtils.isBlank( destinationDirectory ) ? file.getName()
: ( destinationDirectory + "/" + file.getName() );
fireTransferDebug( "Putting child element " + file + " to " + dest );
if ( file.isDirectory() )
{
putDirectory( file, dest );
}
else
{
put( file, dest );
}
}
}
/**
* @inheritDoc
*/
@Override
public void putFromStream( InputStream in, String destination )
throws TransferFailedException, ResourceDoesNotExistException
{
doPutFromStream( in, null, destination, -1, System.currentTimeMillis() );
}
/**
* @inheritDoc
*/
@Override
public void putFromStream( InputStream in, String destination,
long contentLength, long lastModified )
throws TransferFailedException, ResourceDoesNotExistException
{
doPutFromStream( in, null, destination, contentLength, lastModified );
}
/**
* @inheritDoc
*/
@Override
public boolean resourceExists( String resourceName )
throws TransferFailedException, AuthorizationException
{
String key = keyPrefix + resourceName;
try
{
s3.getObjectMetadata( bucketName, key );
return true;
}
catch ( AmazonServiceException e )
{
if ( e.getStatusCode() == 404 )
{
return false;
}
throw new TransferFailedException( "Can't verify if resource key "
+ key + " exist or not", e );
}
}
/**
* @inheritDoc
*/
@Override
public boolean supportsDirectoryCopy()
{
return true;
}
}