com.alibaba.schedulerx.worker.util.FileDownloader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of schedulerx2-worker Show documentation
There is a newer version: 1.12.2
package com.alibaba.schedulerx.worker.util;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;

import com.alibaba.schedulerx.common.domain.JSONResult;
import com.alibaba.schedulerx.common.util.ConfigUtil;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.worker.domain.WorkerConstants;

import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;

/**
 * @author yanxun on 2018/11/29.
 */
public class FileDownloader {
    private static final String DOWNLOADED_JAR_DIR = ConfigUtil.getWorkerConfig().getString(
            WorkerConstants.WORKER_JAR_FILE_DIR, WorkerConstants.WORKER_JAR_FILE_DIR_DEFAULT);
    //private static final String OSS_ENDPOINT = ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_OSS_ENDPOINT, WorkerConstants.WORKER_OSS_ENDPOINT_DEFAULT);
    //
    /**
     * download file from oss and return downloaded file store path,
     * store path is /, this method
     * will also create /_metadata file recording file metadata, now just
     * record file last modified time used to check whether we need to download new file in the future.
     * @param url resource url
     * @return file store path
     * @throws Exception
     */
    //@Deprecated
    //public static String ossDownload(String url) throws Exception{
    //    OSSClient ossClient = null;
    //    try {
    //        ossClient = new OSSClient(OSS_ENDPOINT,
    //            WorkerConstants.WORKER_OSS_ACCESS_KEY_DEFAULT, WorkerConstants.WORKER_OSS_SECRETE_KEY_DEFAULT);
    //        String bucketName = url.substring("http://".length(), url.indexOf("."));
    //        String fileName = url.substring(url.lastIndexOf("/") + 1);
    //        String storedDir = DOWNLOADED_JAR_DIR;
    //        File dir = new File(storedDir);
    //        if (!dir.exists()) {
    //            dir.mkdirs();
    //        }
    //        String storedJarPath = storedDir + fileName;
    //        String storedJarMetaPath = storedJarPath + "_metadata";
    //        SimplifiedObjectMeta objectMeta = ossClient.getSimplifiedObjectMeta(bucketName, fileName);
    //        long newModified = objectMeta.getLastModified().getTime();
    //        if (!needDownload(newModified, storedJarMetaPath)) {
    //            return storedJarPath;
    //        }
    //        Files.deleteIfExists(Paths.get(storedJarMetaPath));
    //        Files.createFile(Paths.get(storedJarMetaPath));
    //        ossClient.getObject(new GetObjectRequest(bucketName, fileName), new File(storedJarPath));
    //        try (BufferedWriter writer = new BufferedWriter(new FileWriter(storedJarMetaPath))) {
    //            writer.write(String.valueOf(newModified));
    //        }
    //        return storedJarPath;
    //    } finally {
    //        if (ossClient != null) {
    //            ossClient.shutdown();
    //        }
    //    }
    //}

    /**
     * compare local file modified time with current remote file modified time, check whether we need download remote file
     * @param modified remote file modified time
     * @param storedJarMetaPath metadata contain old modified time
     * @return true if local file modified smaller than  remote file modified time, otherwise false
     * @throws Exception
     */
    //private static boolean needDownload(long modified, String storedJarMetaPath) throws Exception{
    //    File path = new File(storedJarMetaPath);
    //    if (!path.exists()) {
    //        return true;
    //    }
    //    try (BufferedReader reader = new BufferedReader(new FileReader(storedJarMetaPath))) {
    //        long oldModified = Long.valueOf(reader.readLine());
    //        return modified > oldModified;
    //    }
    //}

    /**
     * for backup
     * @param url
     * @return
     * @throws Exception
     */
    public static String httpDownload(String url) throws Exception {
        InputStream inStream = null;
        FileOutputStream outStream = null;
        String outFilePath = null;
        try {
            String fileName = url.substring(url.lastIndexOf("/") + 1);
            File dir = new File(DOWNLOADED_JAR_DIR);
            if (!dir.exists()) {
                dir.mkdirs();
            }
            outFilePath = DOWNLOADED_JAR_DIR + fileName;
            if (new File(outFilePath).exists()) {
                return outFilePath;
            }
            String consoleDomain = ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME);
            String acquireDownloadSignatureUrl = "http://" + consoleDomain + "/job/getDownloadSignature.json"
                + "?objName=" + fileName;
            HttpResponse jsonResponse = Unirest.get(acquireDownloadSignatureUrl).asJson();
            JSONResult jsonResult = JsonUtil.fromJson(jsonResponse.getBody().toString(), JSONResult.class);
            //noinspection unchecked
            String signedUrl = ((Map)jsonResult.getData()).get("signedUrl");
            URLConnection conn = new URL(signedUrl).openConnection();
            inStream = conn.getInputStream();
            outStream = new FileOutputStream(outFilePath);
            byte[] buffer = new byte[1024];
            int len = 0;
            while ((len = inStream.read(buffer)) != -1) {
                outStream.write(buffer, 0, len);
            }
        } finally {
            if (inStream != null) {
                inStream.close();
            }
            if (outStream != null) {
                outStream.close();
            }
        }
        return outFilePath;
    }
}