com.qcloud.cos.transfer.CopyCallable Maven / Gradle / Ivy

Go to download
package com.qcloud.cos.transfer;

import static com.qcloud.cos.event.SDKProgressPublisher.publishProgress;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.qcloud.cos.COS;
import com.qcloud.cos.event.ProgressEventType;
import com.qcloud.cos.event.ProgressListenerChain;
import com.qcloud.cos.internal.CopyImpl;
import com.qcloud.cos.model.AbortMultipartUploadRequest;
import com.qcloud.cos.model.CopyObjectRequest;
import com.qcloud.cos.model.CopyObjectResult;
import com.qcloud.cos.model.CopyPartRequest;
import com.qcloud.cos.model.CopyResult;
import com.qcloud.cos.model.InitiateMultipartUploadRequest;
import com.qcloud.cos.model.ObjectMetadata;
import com.qcloud.cos.model.PartETag;
import com.qcloud.cos.region.Region;
import com.qcloud.cos.transfer.Transfer.TransferState;

public class CopyCallable implements Callable {

    /**
     * A reference to the COS client using which copy or copy part
     * requests are initiated.
     */
    private final COS cos;
    /** Thread pool used during multi-part copy is performed. */
    private final ExecutorService threadPool;
    /** A reference to the original copy request received. */
    private final CopyObjectRequest copyObjectRequest;
    /** Upload id to be used when sending copy part requests. */
    private String multipartUploadId;
    /** Metadata of the object in the source bucket to be copied. */
    private final ObjectMetadata metadata;
    private final CopyImpl copy;

    private static final Log log = LogFactory.getLog(CopyCallable.class);
    /**
     * TransferManager configuration that provides details on when
     * to use multi-part copy, part size etc.,
     */
    private final TransferManagerConfiguration configuration;
    /**
     * A list of future objects to be returned when multi-part copy is
     * initiated.
     */
    private final List> futures = new ArrayList>();

    private final ProgressListenerChain listenerChain;

    public CopyCallable(TransferManager transferManager,
            ExecutorService threadPool, CopyImpl copy,
            CopyObjectRequest copyObjectRequest, ObjectMetadata metadata,
            ProgressListenerChain progressListenerChain) {
        this.cos = transferManager.getCOSClient();
        this.configuration = transferManager.getConfiguration();
        this.threadPool = threadPool;
        this.copyObjectRequest = copyObjectRequest;
        this.metadata = metadata;
        this.listenerChain = progressListenerChain;
        this.copy = copy;
    }

    List> getFutures() {
        return futures;
    }

    String getMultipartUploadId() {
        return multipartUploadId;
    }

    /**
     * Returns true if this CopyCallable is processing a multi-part copy.
     *
     * @return True if this CopyCallable is processing a multi-part copy.
     */
    public boolean isMultipartCopy() {
        Region sourceRegion = copyObjectRequest.getSourceBucketRegion();
        Region destRegion = cos.getClientConfig().getRegion();
        // 如果没有设置source region, 表示使用的和clientconfig里面同一region, 这里不适用分块copy，使用put object copy即可
        if (sourceRegion == null) {
            return false;
        } else {
            // 如果设置了source region, 且和目的region相同, 则也是用put object copy.
            if (sourceRegion.equals(destRegion)) {
                return false;
            }
        }
        return (metadata.getContentLength() > configuration
                .getMultipartCopyThreshold());
    }

    public CopyResult call() throws Exception {
        copy.setState(TransferState.InProgress);
        if (isMultipartCopy()) {
            publishProgress(listenerChain, ProgressEventType.TRANSFER_STARTED_EVENT);
            copyInParts();
            return null;
        } else {
            return copyInOneChunk();
        }
    }

    /**
     * Performs the copy of the COS object from source bucket to
     * destination bucket. The COS object is copied to destination in one
     * single request.
     *
     * @returns CopyResult response information from the server.
     */
    private CopyResult copyInOneChunk() {
        CopyObjectResult copyObjectResult = cos.copyObject(copyObjectRequest);

        CopyResult copyResult = new CopyResult();
        copyResult.setSourceBucketName(copyObjectRequest.getSourceBucketName());
        copyResult.setSourceKey(copyObjectRequest.getSourceKey());
        copyResult.setDestinationBucketName(copyObjectRequest
                .getDestinationBucketName());
        copyResult.setDestinationKey(copyObjectRequest.getDestinationKey());
        copyResult.setETag(copyObjectResult.getETag());
        copyResult.setVersionId(copyObjectResult.getVersionId());
        return copyResult;
    }

    /**
     * Performs the copy of an COS object from source bucket to
     * destination bucket as multiple copy part requests. The information about
     * the part to be copied is specified in the request as a byte range
     * (first-last)
     *
     * @throws Exception
     *             Any Exception that occurs while carrying out the request.
     */
    private void copyInParts() throws Exception {
        multipartUploadId = initiateMultipartUpload(copyObjectRequest);

        long optimalPartSize = getOptimalPartSize(metadata.getContentLength());

        try {
            CopyPartRequestFactory requestFactory = new CopyPartRequestFactory(
                    copyObjectRequest, multipartUploadId, optimalPartSize,
                    metadata.getContentLength());
            copyPartsInParallel(requestFactory);
        } catch (Exception e) {
            publishProgress(listenerChain, ProgressEventType.TRANSFER_FAILED_EVENT);
            abortMultipartCopy();
            throw new RuntimeException("Unable to perform multipart copy", e);
        }
    }

    /**
     * Computes and returns the optimal part size for the copy operation.
     */
    private long getOptimalPartSize(long contentLengthOfSource) {

        long optimalPartSize = TransferManagerUtils
                .calculateOptimalPartSizeForCopy(copyObjectRequest,
                        configuration, contentLengthOfSource);
        log.debug("Calculated optimal part size: " + optimalPartSize);
        return optimalPartSize;
    }

    /**
     * Submits a callable for each part to be copied to our thread pool and
     * records its corresponding Future.
     */
    private void copyPartsInParallel(CopyPartRequestFactory requestFactory) {
        while (requestFactory.hasMoreRequests()) {
            if (threadPool.isShutdown())
                throw new CancellationException(
                        "TransferManager has been shutdown");
            CopyPartRequest request = requestFactory.getNextCopyPartRequest();
            futures.add(threadPool.submit(new CopyPartCallable(cos, request)));
        }
    }

    /**
     * Initiates a multipart upload and returns the upload id
     */
    private String initiateMultipartUpload(CopyObjectRequest origReq) {

        InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest(
                origReq.getDestinationBucketName(),
                origReq.getDestinationKey()).withCannedACL(
                origReq.getCannedAccessControlList())
                .withAccessControlList(origReq.getAccessControlList())
                .withStorageClass(origReq.getStorageClass())
                .withGeneralProgressListener(origReq.getGeneralProgressListener())
           ;

        ObjectMetadata newObjectMetadata = origReq.getNewObjectMetadata();
        if (newObjectMetadata == null){
            newObjectMetadata = new ObjectMetadata();
        }
        if (newObjectMetadata.getContentType() == null){
            newObjectMetadata.setContentType(metadata.getContentType());
        }

        req.setObjectMetadata(newObjectMetadata);

        String uploadId = cos.initiateMultipartUpload(req).getUploadId();
        log.debug("Initiated new multipart upload: " + uploadId);

        return uploadId;
    }


    private void abortMultipartCopy() {
        try {
            AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(
                    copyObjectRequest.getDestinationBucketName(),
                    copyObjectRequest.getDestinationKey(), multipartUploadId);
            cos.abortMultipartUpload(abortRequest);
        } catch (Exception e) {
            log.info(
                    "Unable to abort multipart upload, you may need to manually remove uploaded parts: "
                            + e.getMessage(), e);
        }
    }
}