com.qcloud.cos.transfer.CopyCallable Maven / Gradle / Ivy
/*
* Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
* According to cos feature, we modify some class，comment, field name, etc.
*/
package com.qcloud.cos.transfer;
import static com.qcloud.cos.event.SDKProgressPublisher.publishProgress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.qcloud.cos.COS;
import com.qcloud.cos.event.ProgressEventType;
import com.qcloud.cos.event.ProgressListenerChain;
import com.qcloud.cos.internal.CopyImpl;
import com.qcloud.cos.model.AbortMultipartUploadRequest;
import com.qcloud.cos.model.CopyObjectRequest;
import com.qcloud.cos.model.CopyObjectResult;
import com.qcloud.cos.model.CopyPartRequest;
import com.qcloud.cos.model.CopyResult;
import com.qcloud.cos.model.InitiateMultipartUploadRequest;
import com.qcloud.cos.model.ObjectMetadata;
import com.qcloud.cos.model.PartETag;
import com.qcloud.cos.region.Region;
import com.qcloud.cos.transfer.Transfer.TransferState;
import com.qcloud.cos.Headers;
public class CopyCallable implements Callable {
/**
* A reference to the COS client using which copy or copy part requests are initiated.
*/
private final COS cos;
/** Thread pool used during multi-part copy is performed. */
private final ExecutorService threadPool;
/** A reference to the original copy request received. */
private final CopyObjectRequest copyObjectRequest;
/** Upload id to be used when sending copy part requests. */
private String multipartUploadId;
/** Metadata of the object in the source bucket to be copied. */
private final ObjectMetadata metadata;
private final CopyImpl copy;
private static final Log log = LogFactory.getLog(CopyCallable.class);
/**
* TransferManager configuration that provides details on when to use multi-part
* copy, part size etc.,
*/
private final TransferManagerConfiguration configuration;
/**
* A list of future objects to be returned when multi-part copy is initiated.
*/
private final List> futures = new ArrayList>();
private final ProgressListenerChain listenerChain;
public CopyCallable(TransferManager transferManager, ExecutorService threadPool, CopyImpl copy,
CopyObjectRequest copyObjectRequest, ObjectMetadata metadata,
ProgressListenerChain progressListenerChain) {
this.cos = transferManager.getCOSClient();
this.configuration = transferManager.getConfiguration();
this.threadPool = threadPool;
this.copyObjectRequest = copyObjectRequest;
this.metadata = metadata;
this.listenerChain = progressListenerChain;
this.copy = copy;
}
List> getFutures() {
return futures;
}
String getMultipartUploadId() {
return multipartUploadId;
}
/**
* Returns true if this CopyCallable is processing a multi-part copy.
*
* @return True if this CopyCallable is processing a multi-part copy.
*/
public boolean isMultipartCopy() {
Region sourceRegion = copyObjectRequest.getSourceBucketRegion();
Region destRegion = cos.getClientConfig().getRegion();
String sourceStorageClass = (String)metadata.getRawMetadataValue(Headers.STORAGE_CLASS);
String destStorageClass = copyObjectRequest.getStorageClass();
if(sourceStorageClass == null) {
sourceStorageClass = "Standard";
}
// 如果源和目的对象的存储类型相同
if(sourceStorageClass.equalsIgnoreCase(destStorageClass)) {
// 如果没有设置source region, 表示使用的和clientconfig里面同一region,
// 或者设置了相同的region，使用put object copy即可
if(sourceRegion == null || sourceRegion.equals(destRegion)) {
return false;
}
}
return (metadata.getContentLength() > configuration.getMultipartCopyThreshold());
}
public CopyResult call() throws Exception {
copy.setState(TransferState.InProgress);
if (isMultipartCopy()) {
publishProgress(listenerChain, ProgressEventType.TRANSFER_STARTED_EVENT);
copyInParts();
return null;
} else {
return copyInOneChunk();
}
}
/**
* Performs the copy of the COS object from source bucket to destination bucket. The COS object
* is copied to destination in one single request.
*
* @returns CopyResult response information from the server.
*/
private CopyResult copyInOneChunk() {
CopyObjectResult copyObjectResult = cos.copyObject(copyObjectRequest);
if (copyObjectResult == null) {
return null;
}
CopyResult copyResult = new CopyResult();
copyResult.setSourceBucketName(copyObjectRequest.getSourceBucketName());
copyResult.setSourceKey(copyObjectRequest.getSourceKey());
copyResult.setDestinationBucketName(copyObjectRequest.getDestinationBucketName());
copyResult.setDestinationKey(copyObjectRequest.getDestinationKey());
copyResult.setETag(copyObjectResult.getETag());
copyResult.setVersionId(copyObjectResult.getVersionId());
copyResult.setRequestId(copyObjectResult.getRequestId());
copyResult.setDateStr(copyObjectResult.getDateStr());
copyResult.setCrc64Ecma(copyObjectResult.getCrc64Ecma());
return copyResult;
}
/**
* Performs the copy of an COS object from source bucket to destination bucket as multiple copy
* part requests. The information about the part to be copied is specified in the request as a
* byte range (first-last)
*
* @throws Exception Any Exception that occurs while carrying out the request.
*/
private void copyInParts() throws Exception {
multipartUploadId = initiateMultipartUpload(copyObjectRequest);
long optimalPartSize = getOptimalPartSize(metadata.getContentLength());
try {
CopyPartRequestFactory requestFactory = new CopyPartRequestFactory(copyObjectRequest,
multipartUploadId, optimalPartSize, metadata.getContentLength());
copyPartsInParallel(requestFactory);
} catch (Exception e) {
publishProgress(listenerChain, ProgressEventType.TRANSFER_FAILED_EVENT);
abortMultipartCopy();
throw new RuntimeException("Unable to perform multipart copy", e);
}
}
/**
* Computes and returns the optimal part size for the copy operation.
*/
private long getOptimalPartSize(long contentLengthOfSource) {
long optimalPartSize = TransferManagerUtils.calculateOptimalPartSizeForCopy(
copyObjectRequest, configuration, contentLengthOfSource);
log.debug("Calculated optimal part size: " + optimalPartSize);
return optimalPartSize;
}
/**
* Submits a callable for each part to be copied to our thread pool and records its
* corresponding Future.
*/
private void copyPartsInParallel(CopyPartRequestFactory requestFactory) {
while (requestFactory.hasMoreRequests()) {
if (threadPool.isShutdown())
throw new CancellationException("TransferManager has been shutdown");
CopyPartRequest request = requestFactory.getNextCopyPartRequest();
futures.add(threadPool.submit(new CopyPartCallable(cos, request)));
}
}
/**
* Initiates a multipart upload and returns the upload id
*/
private String initiateMultipartUpload(CopyObjectRequest origReq) {
InitiateMultipartUploadRequest req =
new InitiateMultipartUploadRequest(origReq.getDestinationBucketName(),
origReq.getDestinationKey())
.withCannedACL(origReq.getCannedAccessControlList())
.withAccessControlList(origReq.getAccessControlList())
.withStorageClass(origReq.getStorageClass())
.withGeneralProgressListener(origReq.getGeneralProgressListener());
ObjectMetadata newObjectMetadata = origReq.getNewObjectMetadata();
if (newObjectMetadata == null) {
newObjectMetadata = new ObjectMetadata();
}
if (newObjectMetadata.getContentType() == null) {
newObjectMetadata.setContentType(metadata.getContentType());
}
req.setObjectMetadata(newObjectMetadata);
TransferManagerUtils.populateEndpointAddr(origReq, req);
String uploadId = cos.initiateMultipartUpload(req).getUploadId();
log.debug("Initiated new multipart upload: " + uploadId);
return uploadId;
}
private void abortMultipartCopy() {
try {
AbortMultipartUploadRequest abortRequest =
new AbortMultipartUploadRequest(copyObjectRequest.getDestinationBucketName(),
copyObjectRequest.getDestinationKey(), multipartUploadId);
TransferManagerUtils.populateEndpointAddr(copyObjectRequest, abortRequest);
cos.abortMultipartUpload(abortRequest);
} catch (Exception e) {
log.info(
"Unable to abort multipart upload, you may need to manually remove uploaded parts: "
+ e.getMessage(),
e);
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy