com.alibaba.schedulerx.worker.util.FileDownloader Maven / Gradle / Ivy
package com.alibaba.schedulerx.worker.util;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;
import com.alibaba.schedulerx.common.domain.JSONResult;
import com.alibaba.schedulerx.common.util.ConfigUtil;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.worker.domain.WorkerConstants;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;
/**
* @author yanxun on 2018/11/29.
*/
public class FileDownloader {
private static final String DOWNLOADED_JAR_DIR = ConfigUtil.getWorkerConfig().getString(
WorkerConstants.WORKER_JAR_FILE_DIR, WorkerConstants.WORKER_JAR_FILE_DIR_DEFAULT);
//private static final String OSS_ENDPOINT = ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_OSS_ENDPOINT, WorkerConstants.WORKER_OSS_ENDPOINT_DEFAULT);
//
/**
* download file from oss and return downloaded file store path,
* store path is /, this method
* will also create /_metadata file recording file metadata, now just
* record file last modified time used to check whether we need to download new file in the future.
* @param url resource url
* @return file store path
* @throws Exception
*/
//@Deprecated
//public static String ossDownload(String url) throws Exception{
// OSSClient ossClient = null;
// try {
// ossClient = new OSSClient(OSS_ENDPOINT,
// WorkerConstants.WORKER_OSS_ACCESS_KEY_DEFAULT, WorkerConstants.WORKER_OSS_SECRETE_KEY_DEFAULT);
// String bucketName = url.substring("http://".length(), url.indexOf("."));
// String fileName = url.substring(url.lastIndexOf("/") + 1);
// String storedDir = DOWNLOADED_JAR_DIR;
// File dir = new File(storedDir);
// if (!dir.exists()) {
// dir.mkdirs();
// }
// String storedJarPath = storedDir + fileName;
// String storedJarMetaPath = storedJarPath + "_metadata";
// SimplifiedObjectMeta objectMeta = ossClient.getSimplifiedObjectMeta(bucketName, fileName);
// long newModified = objectMeta.getLastModified().getTime();
// if (!needDownload(newModified, storedJarMetaPath)) {
// return storedJarPath;
// }
// Files.deleteIfExists(Paths.get(storedJarMetaPath));
// Files.createFile(Paths.get(storedJarMetaPath));
// ossClient.getObject(new GetObjectRequest(bucketName, fileName), new File(storedJarPath));
// try (BufferedWriter writer = new BufferedWriter(new FileWriter(storedJarMetaPath))) {
// writer.write(String.valueOf(newModified));
// }
// return storedJarPath;
// } finally {
// if (ossClient != null) {
// ossClient.shutdown();
// }
// }
//}
/**
* compare local file modified time with current remote file modified time, check whether we need download remote file
* @param modified remote file modified time
* @param storedJarMetaPath metadata contain old modified time
* @return true if local file modified smaller than remote file modified time, otherwise false
* @throws Exception
*/
//private static boolean needDownload(long modified, String storedJarMetaPath) throws Exception{
// File path = new File(storedJarMetaPath);
// if (!path.exists()) {
// return true;
// }
// try (BufferedReader reader = new BufferedReader(new FileReader(storedJarMetaPath))) {
// long oldModified = Long.valueOf(reader.readLine());
// return modified > oldModified;
// }
//}
/**
* for backup
* @param url
* @return
* @throws Exception
*/
public static String httpDownload(String url) throws Exception {
InputStream inStream = null;
FileOutputStream outStream = null;
String outFilePath = null;
try {
String fileName = url.substring(url.lastIndexOf("/") + 1);
File dir = new File(DOWNLOADED_JAR_DIR);
if (!dir.exists()) {
dir.mkdirs();
}
outFilePath = DOWNLOADED_JAR_DIR + fileName;
if (new File(outFilePath).exists()) {
return outFilePath;
}
String consoleDomain = ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME);
String acquireDownloadSignatureUrl = "http://" + consoleDomain + "/job/getDownloadSignature.json"
+ "?objName=" + fileName;
HttpResponse jsonResponse = Unirest.get(acquireDownloadSignatureUrl).asJson();
JSONResult jsonResult = JsonUtil.fromJson(jsonResponse.getBody().toString(), JSONResult.class);
//noinspection unchecked
String signedUrl = ((Map)jsonResult.getData()).get("signedUrl");
URLConnection conn = new URL(signedUrl).openConnection();
inStream = conn.getInputStream();
outStream = new FileOutputStream(outFilePath);
byte[] buffer = new byte[1024];
int len = 0;
while ((len = inStream.read(buffer)) != -1) {
outStream.write(buffer, 0, len);
}
} finally {
if (inStream != null) {
inStream.close();
}
if (outStream != null) {
outStream.close();
}
}
return outFilePath;
}
}