com.amazonaws.mobileconnectors.s3.transfermanager.TransferManager Maven / Gradle / Ivy
Show all versions of aws-java-sdk-s3 Show documentation
/*
* Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazonaws.mobileconnectors.s3.transfermanager;
import static com.amazonaws.services.s3.internal.ServiceUtils.APPEND_MODE;
import static com.amazonaws.services.s3.internal.ServiceUtils.OVERWRITE_MODE;
import com.amazonaws.AbortedException;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.AmazonWebServiceRequest;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressListenerChain;
import com.amazonaws.event.ProgressListenerChain.ProgressEventFilter;
import com.amazonaws.mobileconnectors.s3.transfermanager.Transfer.TransferState;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.CopyCallable;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.CopyImpl;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.CopyMonitor;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.DownloadImpl;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.DownloadMonitor;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.MultipleFileDownloadImpl;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.MultipleFileTransferMonitor;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.MultipleFileUploadImpl;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.S3ProgressListener;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.S3ProgressListenerChain;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.TransferManagerUtils;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.TransferProgressUpdatingListener;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.TransferStateChangeListener;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.UploadCallable;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.UploadImpl;
import com.amazonaws.mobileconnectors.s3.transfermanager.internal.UploadMonitor;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.internal.ServiceUtils;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.MultipartUpload;
import com.amazonaws.services.s3.model.MultipartUploadListing;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.util.Mimetypes;
import com.amazonaws.util.VersionInfoUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Stack;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
/**
* High level utility for managing transfers to Amazon S3.
*
* TransferManager provides a simple API for uploading content to
* Amazon S3, and makes extensive use of Amazon S3 multipart uploads to achieve
* enhanced throughput, performance and reliability.
*
* When possible, TransferManager attempts to use multiple threads
* to upload multiple parts of a single upload at once. When dealing with large
* content sizes and high bandwidth, this can have a significant increase on
* throughput.
*
* TransferManager is responsible for managing resources such as
* connections and threads; share a single instance of
* TransferManager whenever possible. TransferManager,
* like all the client classes in the AWS SDK for Java, is thread safe. Call
* TransferManager.shutdownNow() to release the resources once the
* transfer is complete.
*
* Using TransferManager to upload options to Amazon S3 is easy:
*
*
* DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
* TransferManager tx = new TransferManager(
* credentialProviderChain.getCredentials());
* Upload myUpload = tx.upload(myBucket, myFile.getName(), myFile);
*
* // You can poll your transfer's status to check its progress
* if (myUpload.isDone() == false) {
* System.out.println("Transfer: " + myUpload.getDescription());
* System.out.println(" - State: " + myUpload.getState());
* System.out.println(" - Progress: "
* + myUpload.getProgress().getBytesTransferred());
* }
*
* // Transfers also allow you to set a <code>ProgressListener</code> to receive
* // asynchronous notifications about your transfer's progress.
* myUpload.addProgressListener(myProgressListener);
*
* // Or you can block the current thread and wait for your transfer to
* // to complete. If the transfer fails, this method will throw an
* // AmazonClientException or AmazonServiceException detailing the reason.
* myUpload.waitForCompletion();
*
* // After the upload is complete, call shutdownNow to release the resources.
* tx.shutdownNow();
*
*
* Transfers can be paused and resumed at a later time. It can also survive JVM
* crash, provided the information that is required to resume the transfer is
* given as input to the resume operation. For more information on pause and
* resume,
*
* @see Upload#pause()
* @see Download#pause()
* @see TransferManager#resumeUpload(PersistableUpload)
* @see TransferManager#resumeDownload(PersistableDownload)
* @deprecated The Transfer Manager is now deprecated in favor of the
* {@link com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility}
*/
@Deprecated
public class TransferManager {
/** The low level client we use to make the actual calls to Amazon S3. */
private final AmazonS3 s3;
/** Configuration for how TransferManager processes requests. */
private TransferManagerConfiguration configuration;
/** The thread pool in which transfers are uploaded or downloaded. */
private final ExecutorService threadPool;
/**
* Thread used for periodicially checking transfers and updating thier
* state.
*/
private final ScheduledExecutorService timedThreadPool = new ScheduledThreadPoolExecutor(1,
daemonThreadFactory);
private static final Log log = LogFactory.getLog(TransferManager.class);
/**
* Constructs a new TransferManager and Amazon S3 client using
* the credentials from DefaultAWSCredentialsProviderChain
*
* TransferManager and client objects may pool connections and
* threads. Reuse TransferManager and client objects and share
* them throughout applications.
*
* TransferManager and all AWS client objects are thread safe.
*/
public TransferManager() {
this(new AmazonS3Client(new DefaultAWSCredentialsProviderChain()));
}
/**
* Constructs a new TransferManager and Amazon S3 client using
* the specified AWS security credentials provider.
*
* TransferManager and client objects may pool connections and
* threads. Reuse TransferManager and client objects and share
* them throughout applications.
*
* TransferManager and all AWS client objects are thread safe.
*
* @param credentialsProvider The AWS security credentials provider to use
* when making authenticated requests.
*/
public TransferManager(AWSCredentialsProvider credentialsProvider) {
this(new AmazonS3Client(credentialsProvider));
}
/**
* Constructs a new TransferManager and Amazon S3 client using
* the specified AWS security credentials.
*
* TransferManager and client objects may pool connections and
* threads. Reuse TransferManager and client objects and share
* them throughout applications.
*
* TransferManager and all AWS client objects are thread safe.
*
* @param credentials The AWS security credentials to use when making
* authenticated requests.
*/
public TransferManager(AWSCredentials credentials) {
this(new AmazonS3Client(credentials));
}
/**
* Constructs a new TransferManager, specifying the client to
* use when making requests to Amazon S3.
*
* TransferManager and client objects may pool connections and
* threads. Reuse TransferManager and client objects and share
* them throughout applications.
*
* TransferManager and all AWS client objects are thread safe.
*
*
* @param s3 The client to use when making requests to Amazon S3.
*/
public TransferManager(AmazonS3 s3) {
this(s3, TransferManagerUtils.createDefaultExecutorService());
}
/**
* Constructs a new TransferManager specifying the client and
* thread pool to use when making requests to Amazon S3.
*
* TransferManager and client objects may pool connections and
* threads. Reuse TransferManager and client objects and share
* them throughout applications.
*
* TransferManager and all AWS client objects are thread safe.
*
* @param s3 The client to use when making requests to Amazon S3.
* @param threadPool The thread pool in which to execute requests.
*/
public TransferManager(AmazonS3 s3, ExecutorService threadPool) {
this.s3 = s3;
this.threadPool = threadPool;
this.configuration = new TransferManagerConfiguration();
}
/**
* Sets the configuration which specifies how this
* TransferManager processes requests.
*
* @param configuration The new configuration specifying how this
* TransferManager processes requests.
*/
public void setConfiguration(TransferManagerConfiguration configuration) {
this.configuration = configuration;
}
/**
* Returns the configuration which specifies how this
* TransferManager processes requests.
*
* @return The configuration settings for this TransferManager.
*/
public TransferManagerConfiguration getConfiguration() {
return configuration;
}
/**
* Returns the underlying Amazon S3 client used to make requests to Amazon
* S3.
*
* @return The underlying Amazon S3 client used to make requests to Amazon
* S3.
*/
public AmazonS3 getAmazonS3Client() {
return s3;
}
/**
*
* Schedules a new transfer to upload data to Amazon S3. This method is
* non-blocking and returns immediately (i.e. before the upload has
* finished).
*
*
* When uploading options from a stream, callers must supply the size
* of options in the stream through the content length field in the
* ObjectMetadata parameter. If no content length is specified
* for the input stream, then TransferManager will attempt to buffer all the
* stream contents in memory and upload the options as a traditional, single
* part upload. Because the entire stream contents must be buffered in
* memory, this can be very expensive, and should be avoided whenever
* possible.
*
*
* Use the returned Upload object to query the progress of the
* transfer, add listeners for progress events, and wait for the upload to
* complete.
*
*
* If resources are available, the upload will begin immediately. Otherwise,
* the upload is scheduled and started as soon as resources become
* available.
*
*
* @param bucketName The name of the bucket to upload the new object to.
* @param key The key in the specified bucket by which to store the new
* object.
* @param input The input stream containing the options to upload to Amazon
* S3.
* @param objectMetadata Additional information about the object being
* uploaded, including the size of the options, content type,
* additional custom user metadata, etc.
* @return A new Upload object to use to check the state of the
* upload, listen for progress notifications, and otherwise manage
* the upload.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Upload upload(final String bucketName, final String key, final InputStream input,
ObjectMetadata objectMetadata)
throws AmazonServiceException, AmazonClientException {
return upload(new PutObjectRequest(bucketName, key, input, objectMetadata));
}
/**
* Schedules a new transfer to upload data to Amazon S3. This method is
* non-blocking and returns immediately (i.e. before the upload has
* finished).
*
* The returned Upload object allows you to query the progress of the
* transfer, add listeners for progress events, and wait for the upload to
* complete.
*
* If resources are available, the upload will begin immediately, otherwise
* it will be scheduled and started as soon as resources become available.
*
* @param bucketName The name of the bucket to upload the new object to.
* @param key The key in the specified bucket by which to store the new
* object.
* @param file The file to upload.
* @return A new Upload object which can be used to check state of the
* upload, listen for progress notifications, and otherwise manage
* the upload.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Upload upload(final String bucketName, final String key, final File file)
throws AmazonServiceException, AmazonClientException {
return upload(new PutObjectRequest(bucketName, key, file));
}
/**
*
* Schedules a new transfer to upload data to Amazon S3. This method is
* non-blocking and returns immediately (i.e. before the upload has
* finished).
*
*
* Use the returned Upload object to query the progress of the
* transfer, add listeners for progress events, and wait for the upload to
* complete.
*
*
* If resources are available, the upload will begin immediately. Otherwise,
* the upload is scheduled and started as soon as resources become
* available.
*
*
* @param putObjectRequest The request containing all the parameters for the
* upload.
* @return A new Upload object to use to check the state of the
* upload, listen for progress notifications, and otherwise manage
* the upload.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Upload upload(final PutObjectRequest putObjectRequest)
throws AmazonServiceException, AmazonClientException {
return doUpload(putObjectRequest, null, null, null);
}
/**
*
* Schedules a new transfer to upload data to Amazon S3. This method is
* non-blocking and returns immediately (i.e. before the upload has
* finished).
*
*
* Use the returned Upload object to query the progress of the
* transfer, add listeners for progress events, and wait for the upload to
* complete.
*
*
* If resources are available, the upload will begin immediately. Otherwise,
* the upload is scheduled and started as soon as resources become
* available.
*
*
* @param putObjectRequest The request containing all the parameters for the
* upload.
* @param progressListener An optional callback listener to receive the
* progress of the upload.
* @return A new Upload object to use to check the state of the
* upload, listen for progress notifications, and otherwise manage
* the upload.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Upload upload(final PutObjectRequest putObjectRequest,
final S3ProgressListener progressListener)
throws AmazonServiceException, AmazonClientException {
return doUpload(putObjectRequest, null, progressListener, null);
}
/**
*
* Schedules a new transfer to upload data to Amazon S3. This method is
* non-blocking and returns immediately (i.e. before the upload has
* finished).
*
*
* Use the returned Upload object to query the progress of the
* transfer, add listeners for progress events, and wait for the upload to
* complete.
*
*
* If resources are available, the upload will begin immediately. Otherwise,
* the upload is scheduled and started as soon as resources become
* available.
*
*
* @param putObjectRequest The request containing all the parameters for the
* upload.
* @param stateListener The transfer state change listener to monitor the
* upload.
* @param progressListener An optional callback listener to receive the
* progress of the upload.
* @return A new Upload object to use to check the state of the
* upload, listen for progress notifications, and otherwise manage
* the upload.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
private Upload doUpload(final PutObjectRequest putObjectRequest,
final TransferStateChangeListener stateListener,
final S3ProgressListener progressListener,
final PersistableUpload persistableUpload) throws AmazonServiceException,
AmazonClientException {
appendSingleObjectUserAgent(putObjectRequest);
final String multipartUploadId = persistableUpload != null ? persistableUpload
.getMultipartUploadId() : null;
if (putObjectRequest.getMetadata() == null) {
putObjectRequest.setMetadata(new ObjectMetadata());
}
final ObjectMetadata metadata = putObjectRequest.getMetadata();
final File file = TransferManagerUtils.getRequestFile(putObjectRequest);
if (file != null) {
// Always set the content length, even if it's already set
metadata.setContentLength(file.length());
// Only set the content type if it hasn't already been set
if (metadata.getContentType() == null) {
metadata.setContentType(Mimetypes.getInstance().getMimetype(file));
}
} else {
if (multipartUploadId != null) {
throw new IllegalArgumentException(
"Unable to resume the upload. No file specified.");
}
}
final String description = "Uploading to " + putObjectRequest.getBucketName()
+ "/" + putObjectRequest.getKey();
final TransferProgress transferProgress = new TransferProgress();
transferProgress.setTotalBytesToTransfer(TransferManagerUtils
.getContentLength(putObjectRequest));
final S3ProgressListenerChain listenerChain = new S3ProgressListenerChain(
new TransferProgressUpdatingListener(transferProgress),
putObjectRequest.getGeneralProgressListener(), progressListener);
putObjectRequest.setGeneralProgressListener(listenerChain);
final UploadImpl upload = new UploadImpl(description, transferProgress,
listenerChain, stateListener);
final UploadCallable uploadCallable = new UploadCallable(this, threadPool,
upload, putObjectRequest, listenerChain, multipartUploadId, transferProgress);
final UploadMonitor watcher = new UploadMonitor(this, upload, threadPool,
uploadCallable, putObjectRequest, listenerChain);
watcher.setTimedThreadPool(timedThreadPool);
upload.setMonitor(watcher);
return upload;
}
/**
* Schedules a new transfer to download data from Amazon S3 and save it to
* the specified file. This method is non-blocking and returns immediately
* (i.e. before the data has been fully downloaded).
*
* Use the returned Download object to query the progress of the transfer,
* add listeners for progress events, and wait for the download to complete.
*
* @param bucket The name of the bucket containing the object to download.
* @param key The key under which the object to download is stored.
* @param file The file to download the object's data to.
* @return A new Download object to use to check the state of
* the download, listen for progress notifications, and otherwise
* manage the download.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Download download(String bucket, String key, File file) {
return download(new GetObjectRequest(bucket, key), file);
}
/**
* Schedules a new transfer to download data from Amazon S3 and save it to
* the specified file. This method is non-blocking and returns immediately
* (i.e. before the data has been fully downloaded).
*
* Use the returned Download object to query the progress of the transfer,
* add listeners for progress events, and wait for the download to complete.
*
* @param getObjectRequest The request containing all the parameters for the
* download.
* @param file The file to download the object data to.
* @return A new Download object to use to check the state of
* the download, listen for progress notifications, and otherwise
* manage the download.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Download download(final GetObjectRequest getObjectRequest, final File file) {
return doDownload(getObjectRequest, file, null, null, OVERWRITE_MODE);
}
/**
* Schedules a new transfer to download data from Amazon S3 and save it to
* the specified file. This method is non-blocking and returns immediately
* (i.e. before the data has been fully downloaded).
*
* Use the returned Download object to query the progress of the transfer,
* add listeners for progress events, and wait for the download to complete.
*
* @param getObjectRequest The request containing all the parameters for the
* download.
* @param file The file to download the object data to.
* @param progressListener An optional callback listener to get the progress
* of the download.
* @return A new Download object to use to check the state of
* the download, listen for progress notifications, and otherwise
* manage the download.
* @throws AmazonClientException If any errors are encountered in the client
* while making the request or handling the response.
* @throws AmazonServiceException If any errors occurred in Amazon S3 while
* processing the request.
*/
public Download download(final GetObjectRequest getObjectRequest,
final File file, final S3ProgressListener progressListener) {
return doDownload(getObjectRequest, file, null, progressListener,
OVERWRITE_MODE);
}
/**
* Same as public interface, but adds a state listener so that callers can
* be notified of state changes to the download.
*
* @see TransferManager#download(GetObjectRequest, File)
*/
private Download doDownload(final GetObjectRequest getObjectRequest,
final File file, final TransferStateChangeListener stateListener,
final S3ProgressListener s3progressListener,
final boolean resumeExistingDownload) {
appendSingleObjectUserAgent(getObjectRequest);
final String description = "Downloading from " + getObjectRequest.getBucketName() + "/"
+ getObjectRequest.getKey();
final TransferProgress transferProgress = new TransferProgress();
// S3 progress listener to capture the persistable transfer when
// available
final S3ProgressListenerChain listenerChain = new S3ProgressListenerChain(
new TransferProgressUpdatingListener(transferProgress), // The
// listener
// for
// updating
// transfer
// progress
getObjectRequest.getGeneralProgressListener(),
s3progressListener); // Listeners included in the original
// request
// The listener chain used by the low-level GetObject request.
// This listener chain ignores any COMPLETE event, so that we could
// delay firing the signal until the high-level download fully finishes.
final ProgressListenerChain listeners = new ProgressListenerChain(
new ProgressEventFilter() {
@Override
public ProgressEvent filter(ProgressEvent progressEvent) {
if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
// Block COMPLETE events from the low-level
// GetObject operation,
// but we still want to keep the BytesTransferred
progressEvent.setEventCode(0);
}
return progressEvent;
}
},
listenerChain);
getObjectRequest.setGeneralProgressListener(listeners);
final GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(
getObjectRequest.getBucketName(), getObjectRequest.getKey());
if (getObjectRequest.getSSECustomerKey() != null) {
getObjectMetadataRequest.setSSECustomerKey(getObjectRequest.getSSECustomerKey());
}
final ObjectMetadata objectMetadata = s3.getObjectMetadata(getObjectMetadataRequest);
// We still pass the unfiltered listener chain into DownloadImpl
final DownloadImpl download = new DownloadImpl(description,
transferProgress, listenerChain, null, stateListener,
getObjectRequest, file);
long startingByte = 0;
long lastByte = objectMetadata.getContentLength() - 1;
if (getObjectRequest.getRange() != null
&& getObjectRequest.getRange().length == 2) {
startingByte = getObjectRequest.getRange()[0];
lastByte = getObjectRequest.getRange()[1];
}
long totalBytesToDownload = lastByte - startingByte + 1;
transferProgress.setTotalBytesToTransfer(totalBytesToDownload);
if (resumeExistingDownload) {
if (file.exists()) {
final long numberOfBytesRead = file.length();
startingByte = startingByte + numberOfBytesRead;
getObjectRequest.setRange(startingByte, lastByte);
transferProgress.updateProgress(Math.min(numberOfBytesRead,
totalBytesToDownload));
totalBytesToDownload = lastByte - startingByte + 1;
}
}
if (totalBytesToDownload < 0) {
throw new IllegalArgumentException(
"Unable to determine the range for download operation.");
}
final CountDownLatch latch = new CountDownLatch(1);
final Future future = submitDownloadTask(getObjectRequest, file,
resumeExistingDownload, latch, download);
download.setMonitor(new DownloadMonitor(download, future));
latch.countDown();
return download;
}
private Future submitDownloadTask(
final GetObjectRequest getObjectRequest, final File file,
final boolean resumeExistingDownload,
final CountDownLatch latch,
final DownloadImpl download) {
final Future future = threadPool.submit(new Callable