com.aliyun.uploader.BaseOssUploader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of toolkit-maven-plugin
There is a newer version: 1.1.9
package com.aliyun.uploader;

import com.aliyun.oss.ClientConfiguration;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.event.ProgressEvent;
import com.aliyun.oss.event.ProgressEventType;
import com.aliyun.oss.event.ProgressListener;
import com.aliyun.oss.model.AbortMultipartUploadRequest;
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
import com.aliyun.oss.model.InitiateMultipartUploadRequest;
import com.aliyun.oss.model.InitiateMultipartUploadResult;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.PutObjectRequest;
import com.aliyun.Context;
import com.aliyun.oss.model.UploadPartRequest;
import com.aliyun.oss.model.UploadPartResult;
import com.aliyuncs.DefaultAcsClient;
import com.aliyuncs.edas.model.v20170801.QueryRegionConfigRequest;
import com.aliyuncs.edas.model.v20170801.QueryRegionConfigResponse;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

public abstract class BaseOssUploader implements Uploader {
    private static final String OSS_ENDPOINT_PTN = "http://oss-%s.aliyuncs.com";
    private static final String OSS_VPC_ENDPOINT_PTN = "http://oss-%s-internal.aliyuncs.com";
    private static final String OSS_DOWNLOAD_URL_PTN = "https://%s.oss-%s-internal.aliyuncs.com/%s";
    private static final long MULTIPART_THRESHOLD = 20 * 1024 * 1024L;
    private static final int MULTIPART_UPLOAD_COUNT = 5;

    protected String doUpload(UploadContext context) throws Exception {
        String ossEndpoint = String.format(OSS_ENDPOINT_PTN, context.getRegionId());
        String downloadUrl = String.format(OSS_DOWNLOAD_URL_PTN, context.getBucket(), context.getRegionId(),
                URLEncoder.encode(context.getKey(), "UTF-8").replaceAll("%2F", "/"));

        if (context.getEdasEndpoint().endsWith("aliyuncs.com")) {
            if (context.isUseVpcEndpoint()) {
                ossEndpoint = String.format(OSS_VPC_ENDPOINT_PTN, context.getRegionId());
            }
            if (context.isUseHttps() && ossEndpoint.startsWith("http://")) {
                ossEndpoint = ossEndpoint.replaceFirst("http://", "https://");
            }
        } else {
            DefaultAcsClient acsClient = Context.getAcsClient();
            QueryRegionConfigRequest request = new QueryRegionConfigRequest();
            QueryRegionConfigResponse response = acsClient.getAcsResponse(request);
            if (response.getCode() != 200) {
                throw new Exception(String.format(
                        "Unable to get oss endpoint by QueryRegionConfigRequest, code: %d, msg: %s",
                        response.getCode(), response.getMessage()));
            }
            QueryRegionConfigResponse.RegionConfig regionConfig = response.getRegionConfig();
            if ("oss".equalsIgnoreCase(regionConfig.getFileServerType())
                    && regionConfig.getFileServerConfig() != null) {
                String internalHost = regionConfig.getFileServerConfig().getInternalUrl();
                String publicHost = regionConfig.getFileServerConfig().getPublicUrl();
                if (publicHost != null && internalHost != null) {
                    ossEndpoint = "http://" + publicHost;
                    downloadUrl = "http://" + context.getBucket() + "." + internalHost + "/"
                            + URLEncoder.encode(context.getKey(), "UTF-8").replaceAll("%2F", "/");
                    Context.getLogger().info("Use oss endpoint: " + ossEndpoint);
                    Context.getLogger().info("Use oss downloadUrl: " + downloadUrl);
                } else {
                    Context.getLogger().info("InternalUrl in region config: " + internalHost);
                    Context.getLogger().info("PublicUrl in region config: " + publicHost);
                }
            } else {
                throw new Exception(String.format(
                        "Fs type is %s, Fs server config is %s for current endpoint: %s",
                        regionConfig.getFileServerType(),
                        regionConfig.getFileServerConfig(),
                        context.getEdasEndpoint()));
            }
        }
        Context.getLogger().info("OSS Endpoint: " + ossEndpoint);

        ClientConfiguration configuration = new ClientConfiguration();
        configuration.setSupportCname(false);
        OSSClient ossClient = (context.getToken() == null) ?
                new OSSClient(ossEndpoint, context.getAk(), context.getSk(), configuration)
                : new OSSClient(ossEndpoint, context.getAk(), context.getSk(), context.getToken(), configuration);
        upload(context.getFile(), ossClient, context.getBucket(), context.getKey(), context.isUseMultiPartUpload());
        downloadUrl = generateDownloadUrl(ossClient, downloadUrl);

        ossClient.shutdown();

        return downloadUrl;
    }

    protected String generateDownloadUrl(OSSClient ossClient, String defaultUrl) {
        return defaultUrl;
    }

    private void upload(File file, OSSClient ossClient, String bucket, String key, boolean useMultiPartUpload) throws Exception {
        if (!useMultiPartUpload || file.length() < MULTIPART_THRESHOLD) {
            ossClient.putObject(new PutObjectRequest(bucket, key, new FileInputStream(file))
                .withProgressListener(new OssUploadProgressListener(file.length())));
        } else {
            Context.getLogger().info("using multi part uploading " + file.getName());
            multipartUpload(bucket, key, file, ossClient);
        }
    }

    private void multipartUpload(String bucketName, String objectName, File file, OSSClient ossClient) throws Exception {
        String uploadId = getMultipartUploadId(bucketName, objectName, ossClient);
        try {
            multiUploadInner(uploadId, bucketName, objectName, file, ossClient);
        } catch (Exception e) {
            // 分片上传失败后，取消分片上传
            try {
                AbortMultipartUploadRequest abortMultipartUploadRequest =
                    new AbortMultipartUploadRequest(bucketName, objectName, uploadId);
                ossClient.abortMultipartUpload(abortMultipartUploadRequest);
            } catch (Exception ex) {
                Context.getLogger().debug(ex.getMessage());
            }
            throw e;
        }
    }

    private String getMultipartUploadId(String bucketName, String objectName, OSSClient ossClient) {
        InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName);
        InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
        return result.getUploadId();
    }

    private void multiUploadInner(String uploadId, String bucketName, String objectName, File file, final OSSClient ossClient)
        throws InterruptedException {
        final long fileLength = file.length();
        long uploadSize = fileLength / MULTIPART_UPLOAD_COUNT;
        List tags = new ArrayList<>();
        ExecutorService threadPool = Executors.newFixedThreadPool(MULTIPART_UPLOAD_COUNT);
        ProgressListener listener = new ProgressListener() {
            private long written = 0L;

            @Override
            public void progressChanged(ProgressEvent progressEvent) {
                if (progressEvent.getEventType() == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT) {
                    long byteCount = progressEvent.getBytes();
                    written += byteCount;
                    double percent = (double)written / fileLength;
                    int totalSymbolCount = 50;
                    long poundCount = Math.round(percent * totalSymbolCount);
                    StringBuilder sb = new StringBuilder("[INFO] [");
                    for (int i = 0; i < totalSymbolCount; i++) {
                        if (i < poundCount) {
                            sb.append("#");
                        } else {
                            sb.append("=");
                        }
                    }
                    double mbs = (double)written / (1024 * 1024);
                    sb.append("] ").append(String.format("%.2f", mbs)).append(" MBs uploaded");
                    synchronized (this) {
                        System.out.print("\r");
                        System.out.print(sb.toString());
                    }
                }
            }
        };
        for (int i = 0; i < MULTIPART_UPLOAD_COUNT; i++) {
            int finali = i;
            threadPool.execute(() -> {
                try {
                    long startPos = finali * uploadSize;
                    long curPartSize = (finali + 1 == MULTIPART_UPLOAD_COUNT) ? (fileLength - startPos) : uploadSize;
                    InputStream instream = new FileInputStream(file);
                    instream.skip(startPos);
                    UploadPartRequest uploadPartRequest = new UploadPartRequest();
                    uploadPartRequest.setBucketName(bucketName);
                    uploadPartRequest.setKey(objectName);
                    uploadPartRequest.setUploadId(uploadId);
                    uploadPartRequest.setInputStream(instream);
                    uploadPartRequest.setPartSize(curPartSize);
                    uploadPartRequest.setPartNumber(finali + 1);
                    uploadPartRequest.setProgressListener(listener);
                    UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
                    tags.add(uploadPartResult.getPartETag());
                } catch (IOException e) {
                    Context.getLogger().error(e);
                    throw new RuntimeException(e);
                }
            });
        }
        threadPool.shutdown();
        threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
        if (tags.size() != MULTIPART_UPLOAD_COUNT) {
            throw new IllegalStateException("wrong tags count " + tags.size());
        }
        tags.sort(Comparator.comparingInt(PartETag::getPartNumber));
        CompleteMultipartUploadRequest completeMultipartUploadRequest =
            new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, tags);
        ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        System.out.print("\n");
    }
}