All Downloads are FREE. Search and download functionalities are using the official Maven repository.

software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper Maven / Gradle / Ivy

Go to download

The AWS Java SDK for Amazon S3 module holds the client classes that are used for communicating with Amazon Simple Storage Service

There is a newer version: 2.30.1
Show newest version
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *  http://aws.amazon.com/apache2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package software.amazon.awssdk.services.s3.internal.multipart;


import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.stream.IntStream;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.internal.crt.UploadPartCopyRequestIterable;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CopyPartResult;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest;
import software.amazon.awssdk.services.s3.model.UploadPartCopyResponse;
import software.amazon.awssdk.utils.CompletableFutureUtils;
import software.amazon.awssdk.utils.Logger;

/**
 * An internal helper class that automatically uses multipart copy based on the size of the source object
 */
@SdkInternalApi
public final class CopyObjectHelper {
    private static final Logger log = Logger.loggerFor(S3AsyncClient.class);

    private final S3AsyncClient s3AsyncClient;
    private final long partSizeInBytes;
    private final GenericMultipartHelper genericMultipartHelper;
    private final long uploadThreshold;

    public CopyObjectHelper(S3AsyncClient s3AsyncClient, long partSizeInBytes, long uploadThreshold) {
        this.s3AsyncClient = s3AsyncClient;
        this.partSizeInBytes = partSizeInBytes;
        this.genericMultipartHelper = new GenericMultipartHelper<>(s3AsyncClient,
                                                                   SdkPojoConversionUtils::toAbortMultipartUploadRequest,
                                                                   SdkPojoConversionUtils::toCopyObjectResponse);
        this.uploadThreshold = uploadThreshold;
    }

    public CompletableFuture copyObject(CopyObjectRequest copyObjectRequest) {

        CompletableFuture returnFuture = new CompletableFuture<>();

        try {
            CompletableFuture headFuture =
                s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest));

            // Ensure cancellations are forwarded to the head future
            CompletableFutureUtils.forwardExceptionTo(returnFuture, headFuture);

            headFuture.whenComplete((headObjectResponse, throwable) -> {
                if (throwable != null) {
                    genericMultipartHelper.handleException(returnFuture, () -> "Failed to retrieve metadata from the source "
                                                                               + "object", throwable);
                } else {
                    doCopyObject(copyObjectRequest, returnFuture, headObjectResponse);
                }
            });
        } catch (Throwable throwable) {
            returnFuture.completeExceptionally(throwable);
        }

        return returnFuture;
    }

    private void doCopyObject(CopyObjectRequest copyObjectRequest, CompletableFuture returnFuture,
                              HeadObjectResponse headObjectResponse) {
        Long contentLength = headObjectResponse.contentLength();

        if (contentLength <= partSizeInBytes || contentLength <= uploadThreshold) {
            log.debug(() -> "Starting the copy as a single copy part request");
            copyInOneChunk(copyObjectRequest, returnFuture);
        } else {
            log.debug(() -> "Starting the copy as multipart copy request");
            copyInParts(copyObjectRequest, contentLength, returnFuture);
        }
    }

    private void copyInParts(CopyObjectRequest copyObjectRequest,
                             Long contentLength,
                             CompletableFuture returnFuture) {

        CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest);
        CompletableFuture createMultipartUploadFuture =
            s3AsyncClient.createMultipartUpload(request);

        // Ensure cancellations are forwarded to the createMultipartUploadFuture future
        CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);

        createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> {
            if (throwable != null) {
                genericMultipartHelper.handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable);
            } else {
                log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId());
                doCopyInParts(copyObjectRequest, contentLength, returnFuture, createMultipartUploadResponse.uploadId());
            }
        });
    }

    private void doCopyInParts(CopyObjectRequest copyObjectRequest,
                               Long contentLength,
                               CompletableFuture returnFuture,
                               String uploadId) {

        long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes);

        int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize);
        if (optimalPartSize > partSizeInBytes) {
            log.debug(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts "
                                         + "allowed", partSizeInBytes, optimalPartSize));
        }

        log.debug(() -> String.format("Starting multipart copy with partCount: %s, optimalPartSize: %s",
                                      partCount, optimalPartSize));

        // The list of completed parts must be sorted
        AtomicReferenceArray completedParts = new AtomicReferenceArray<>(partCount);

        List> futures = sendUploadPartCopyRequests(copyObjectRequest,
                                                                                    contentLength,
                                                                                    uploadId,
                                                                                    completedParts,
                                                                                    optimalPartSize);
        CompletableFutureUtils.allOfExceptionForwarded(futures.toArray(new CompletableFuture[0]))
                              .thenCompose(ignore -> completeMultipartUpload(copyObjectRequest, uploadId, completedParts))
                              .handle(genericMultipartHelper.handleExceptionOrResponse(copyObjectRequest, returnFuture,
                                                                                       uploadId))
                              .exceptionally(throwable -> {
                                  genericMultipartHelper.handleException(returnFuture, () -> "Unexpected exception occurred",
                                                                         throwable);
                                  return null;
                              });
    }

    private CompletableFuture completeMultipartUpload(
        CopyObjectRequest copyObjectRequest, String uploadId, AtomicReferenceArray completedParts) {
        log.debug(() -> String.format("Sending completeMultipartUploadRequest, uploadId: %s",
                                      uploadId));
        CompletedPart[] parts =
            IntStream.range(0, completedParts.length())
                     .mapToObj(completedParts::get)
                     .toArray(CompletedPart[]::new);
        CompleteMultipartUploadRequest completeMultipartUploadRequest =
            CompleteMultipartUploadRequest.builder()
                                          .bucket(copyObjectRequest.destinationBucket())
                                          .key(copyObjectRequest.destinationKey())
                                          .uploadId(uploadId)
                                          .multipartUpload(CompletedMultipartUpload.builder()
                                                                                   .parts(parts)
                                                                                   .build())
                                          .sseCustomerAlgorithm(copyObjectRequest.sseCustomerAlgorithm())
                                          .sseCustomerKey(copyObjectRequest.sseCustomerKey())
                                          .sseCustomerKeyMD5(copyObjectRequest.sseCustomerKeyMD5())
                                          .build();
        return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest);
    }

    private List> sendUploadPartCopyRequests(CopyObjectRequest copyObjectRequest,
                                                                              long contentLength,
                                                                              String uploadId,
                                                                              AtomicReferenceArray completedParts,
                                                                              long optimalPartSize) {
        List> futures = new ArrayList<>();

        UploadPartCopyRequestIterable uploadPartCopyRequests = new UploadPartCopyRequestIterable(uploadId,
                                                                                                 optimalPartSize,
                                                                                                 copyObjectRequest,
                                                                                                 contentLength);

        uploadPartCopyRequests.forEach(uploadPartCopyRequest ->
                                           sendIndividualUploadPartCopy(uploadId, completedParts, futures,
                                                                        uploadPartCopyRequest));

        return futures;
    }

    private void sendIndividualUploadPartCopy(String uploadId,
                                              AtomicReferenceArray completedParts,
                                              List> futures,
                                              UploadPartCopyRequest uploadPartCopyRequest) {
        Integer partNumber = uploadPartCopyRequest.partNumber();
        log.debug(() -> "Sending uploadPartCopyRequest with range: " + uploadPartCopyRequest.copySourceRange() + " uploadId: "
                        + uploadId);

        CompletableFuture uploadPartCopyFuture =
            s3AsyncClient.uploadPartCopy(uploadPartCopyRequest);

        CompletableFuture convertFuture =
            uploadPartCopyFuture.thenApply(uploadPartCopyResponse ->
                                               convertUploadPartCopyResponse(completedParts, partNumber, uploadPartCopyResponse));
        futures.add(convertFuture);

        CompletableFutureUtils.forwardExceptionTo(convertFuture, uploadPartCopyFuture);
    }

    private static CompletedPart convertUploadPartCopyResponse(AtomicReferenceArray completedParts,
                                                               Integer partNumber,
                                                               UploadPartCopyResponse uploadPartCopyResponse) {
        CopyPartResult copyPartResult = uploadPartCopyResponse.copyPartResult();
        CompletedPart completedPart =
            SdkPojoConversionUtils.toCompletedPart(copyPartResult,
                                                   partNumber);

        completedParts.set(partNumber - 1, completedPart);
        return completedPart;
    }

    private void copyInOneChunk(CopyObjectRequest copyObjectRequest,
                                CompletableFuture returnFuture) {
        CompletableFuture copyObjectFuture =
            s3AsyncClient.copyObject(copyObjectRequest);
        CompletableFutureUtils.forwardExceptionTo(returnFuture, copyObjectFuture);
        CompletableFutureUtils.forwardResultTo(copyObjectFuture, returnFuture);
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy