com.qiniu.entry.QSuitsEntry Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of qsuits Show documentation
Show all versions of qsuits Show documentation
qiniu-suits is a efficient tools for qiniu api implemented by java8.
package com.qiniu.entry;
import com.aliyun.oss.ClientConfiguration;
import com.baidubce.services.bos.BosClientConfiguration;
import com.google.gson.JsonObject;
import com.obs.services.ObsConfiguration;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.region.Region;
import com.qiniu.common.Constants;
import com.qiniu.common.SuitsException;
import com.qiniu.common.Zone;
import com.qiniu.convert.MapToString;
import com.qiniu.datasource.*;
import com.qiniu.interfaces.*;
import com.qiniu.process.filtration.*;
import com.qiniu.process.other.*;
import com.qiniu.process.qai.*;
import com.qiniu.process.qdora.*;
import com.qiniu.process.qos.*;
import com.qiniu.sdk.UpYunConfig;
import com.qiniu.storage.Configuration;
import com.qiniu.util.*;
import java.io.IOException;
import java.util.*;
public class QSuitsEntry {
private IEntryParam entryParam;
private CommonParams commonParams;
private int connectTimeout;
private int readTimeout;
private int requestTimeout;
private Configuration qiniuConfig;
private ClientConfig tenClientConfig;
private ClientConfiguration aliClientConfig;
private UpYunConfig upYunConfig;
private com.amazonaws.ClientConfiguration s3ClientConfig;
private ObsConfiguration obsConfiguration;
private BosClientConfiguration bosClientConfiguration;
private String source;
private String qiniuAccessKey;
private String qiniuSecretKey;
private String regionName;
private String bucket;
private Map indexMap;
private int unitLen;
private int threads;
private boolean saveTotal;
private String process;
private int retryTimes;
private String savePath;
private String saveFormat;
private String saveSeparator;
private List rmFields;
public QSuitsEntry(IEntryParam entryParam) throws Exception {
this.entryParam = entryParam;
this.commonParams = new CommonParams(entryParam);
setMembers();
}
public QSuitsEntry(IEntryParam entryParam, CommonParams commonParams) {
this.entryParam = entryParam;
this.commonParams = commonParams;
setMembers();
}
public void updateEntry(IEntryParam entryParam) throws Exception {
this.entryParam = entryParam;
this.commonParams = new CommonParams(entryParam);
setMembers();
}
public void updateEntry(CommonParams commonParams) {
this.commonParams = commonParams;
setMembers();
}
public void setQiniuConfig(Configuration configuration) throws IOException {
if (configuration == null) throw new IOException("the configuration can not be null when you set it.");
this.qiniuConfig = configuration;
}
public void setTenClientConfig(ClientConfig clientConfig) throws IOException {
if (clientConfig == null) throw new IOException("the clientConfig can not be null when you set it.");
this.tenClientConfig = clientConfig;
}
public void setAliClientConfig(ClientConfiguration clientConfig) throws IOException {
if (clientConfig == null) throw new IOException("the clientConfiguration can not be null when you set it.");
this.aliClientConfig = clientConfig;
}
public void setUpYunConfig(UpYunConfig upYunConfig) throws IOException {
if (upYunConfig == null) throw new IOException("the clientConfiguration can not be null when you set it.");
this.upYunConfig = upYunConfig;
}
private void setMembers() {
this.connectTimeout = commonParams.getConnectTimeout();
this.readTimeout = commonParams.getReadTimeout();
this.requestTimeout = commonParams.getRequestTimeout();
this.source = commonParams.getSource();
this.qiniuAccessKey = commonParams.getQiniuAccessKey();
this.qiniuSecretKey = commonParams.getQiniuSecretKey();
this.bucket = commonParams.getBucket();
this.regionName = commonParams.getRegionName();
this.indexMap = commonParams.getIndexMap();
this.unitLen = commonParams.getUnitLen();
this.threads = commonParams.getThreads();
this.saveTotal = commonParams.getSaveTotal();
this.rmFields = commonParams.getRmFields();
this.process = commonParams.getProcess();
this.retryTimes = commonParams.getRetryTimes();
this.savePath = commonParams.getSavePath();
this.saveFormat = commonParams.getSaveFormat();
this.saveSeparator = commonParams.getSaveSeparator();
}
public IEntryParam getEntryParam() {
return entryParam;
}
public CommonParams getCommonParams() {
return commonParams;
}
public Configuration getQiniuConfig() {
return qiniuConfig == null ? getDefaultQiniuConfig() : qiniuConfig;
}
private Configuration getDefaultQiniuConfig() {
Zone zone = CloudApiUtils.getQiniuRegion(regionName);
Configuration configuration = new Configuration(zone);
if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
if (readTimeout> Constants.READ_TIMEOUT) configuration.readTimeout = readTimeout;
if (requestTimeout > Constants.WRITE_TIMEOUT) configuration.writeTimeout = requestTimeout;
return configuration;
}
public ClientConfig getTenClientConfig() throws IOException {
return tenClientConfig == null ? getDefaultTenClientConfig() : tenClientConfig;
}
private ClientConfig getDefaultTenClientConfig() throws SuitsException {
if (regionName == null || "".equals(regionName)) regionName = CloudApiUtils.getTenCosRegion(
commonParams.getTencentSecretId(), commonParams.getTencentSecretKey(), bucket);
ClientConfig clientConfig = new ClientConfig(new Region(regionName));
if (1000 * connectTimeout > clientConfig.getConnectionTimeout())
clientConfig.setConnectionTimeout(1000 * connectTimeout);
if (1000 * readTimeout > clientConfig.getSocketTimeout())
clientConfig.setSocketTimeout(1000 * readTimeout);
if (1000 * requestTimeout > clientConfig.getConnectionRequestTimeout())
clientConfig.setConnectionRequestTimeout(1000 * requestTimeout);
return clientConfig;
}
public ClientConfiguration getAliClientConfig() {
return aliClientConfig == null ? getDefaultAliClientConfig() : aliClientConfig;
}
private ClientConfiguration getDefaultAliClientConfig() {
ClientConfiguration clientConfig = new ClientConfiguration();
if (1000 * connectTimeout > clientConfig.getConnectionTimeout())
clientConfig.setConnectionTimeout(1000 * connectTimeout);
if (1000 * readTimeout > clientConfig.getSocketTimeout())
clientConfig.setSocketTimeout(1000 * readTimeout);
if (1000 * requestTimeout > clientConfig.getConnectionRequestTimeout())
clientConfig.setConnectionRequestTimeout(1000 * requestTimeout);
return clientConfig;
}
public UpYunConfig getUpYunConfig() {
return upYunConfig == null ? getDefaultUpYunConfig() : upYunConfig;
}
private UpYunConfig getDefaultUpYunConfig() {
UpYunConfig upYunConfig = new UpYunConfig();
if (1000 * connectTimeout > upYunConfig.connectTimeout)
upYunConfig.connectTimeout = 1000 * connectTimeout;
if (1000 * readTimeout > upYunConfig.readTimeout)
upYunConfig.readTimeout = 1000 * readTimeout;
return upYunConfig;
}
public com.amazonaws.ClientConfiguration getS3ClientConfig() {
return s3ClientConfig == null ? getDefaultS3ClientConfig() : s3ClientConfig;
}
private com.amazonaws.ClientConfiguration getDefaultS3ClientConfig() {
com.amazonaws.ClientConfiguration clientConfig = new com.amazonaws.ClientConfiguration();
if (1000 * connectTimeout > clientConfig.getConnectionTimeout())
clientConfig.setConnectionTimeout(1000 * connectTimeout);
if (1000 * readTimeout > clientConfig.getSocketTimeout())
clientConfig.setSocketTimeout(1000 * readTimeout);
if (1000 * requestTimeout > clientConfig.getRequestTimeout())
clientConfig.setRequestTimeout(1000 * requestTimeout);
return clientConfig;
}
public ObsConfiguration getObsConfiguration() {
return obsConfiguration == null ? getDefaultObsConfiguration() : obsConfiguration;
}
private ObsConfiguration getDefaultObsConfiguration() {
ObsConfiguration obsConfiguration = new ObsConfiguration();
if (1000 * connectTimeout > obsConfiguration.getConnectionTimeout())
obsConfiguration.setConnectionTimeout(1000 * connectTimeout);
if (1000 * readTimeout > obsConfiguration.getSocketTimeout())
obsConfiguration.setSocketTimeout(1000 * readTimeout);
// if (1000 * requestTimeout > obsConfiguration.getConnectionRequestTimeout())
// obsConfiguration.setConnectionRequestTimeout(1000 * requestTimeout);
return obsConfiguration;
}
private BosClientConfiguration getBosClientConfiguration() {
return bosClientConfiguration == null ? getDefaultBosClientConfiguration() : bosClientConfiguration;
}
private BosClientConfiguration getDefaultBosClientConfiguration() {
BosClientConfiguration bosClientConfiguration = new BosClientConfiguration();
if (1000 * connectTimeout > bosClientConfiguration.getConnectionTimeoutInMillis())
bosClientConfiguration.setConnectionTimeoutInMillis(1000 * connectTimeout);
if (1000 * readTimeout > bosClientConfiguration.getSocketTimeoutInMillis())
bosClientConfiguration.setSocketTimeoutInMillis(1000 * readTimeout);
return bosClientConfiguration;
}
public IDataSource getDataSource() throws IOException {
if ("qiniu".equals(source)) {
return getQiniuQosContainer();
} else if ("tencent".equals(source)) {
return getTenCosContainer();
} else if ("aliyun".equals(source)) {
return getAliOssContainer();
} else if ("upyun".equals(source)) {
return getUpYosContainer();
} else if ("s3".equals(source)) {
return getAwsS3Container();
} else if ("huawei".equals(source)) {
return getHuaweiObsContainer();
} else if ("baidu".equals(source)) {
return getBaiduBosContainer();
} else if ("local".equals(source)) {
return getLocalFileContainer();
} else {
return null;
}
}
public InputSource getInputSource() {
String parse = commonParams.getParse();
String separator = commonParams.getSeparator();
String addKeyPrefix = commonParams.getAddKeyPrefix();
String rmKeyPrefix = commonParams.getRmKeyPrefix();
return new InputSource(parse, separator, addKeyPrefix, rmKeyPrefix, indexMap);
}
public LocalFileContainer getLocalFileContainer() throws IOException {
String filePath = commonParams.getPath();
String parse = commonParams.getParse();
String separator = commonParams.getSeparator();
String addKeyPrefix = commonParams.getAddKeyPrefix();
String rmKeyPrefix = commonParams.getRmKeyPrefix();
Map linesMap = commonParams.getLinesMap();
LocalFileContainer localFileContainer = new LocalFileContainer(filePath, parse, separator, addKeyPrefix,
rmKeyPrefix, linesMap, indexMap, null, unitLen, threads);
localFileContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
localFileContainer.setRetryTimes(retryTimes);
return localFileContainer;
}
public QiniuQosContainer getQiniuQosContainer() throws IOException {
if (qiniuConfig == null) qiniuConfig = getDefaultQiniuConfig();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
QiniuQosContainer qiniuQosContainer = new QiniuQosContainer(qiniuAccessKey, qiniuSecretKey, qiniuConfig, bucket,
prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen, threads);
qiniuQosContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
qiniuQosContainer.setRetryTimes(retryTimes);
return qiniuQosContainer;
}
public TenCosContainer getTenCosContainer() throws IOException {
String secretId = commonParams.getTencentSecretId();
String secretKey = commonParams.getTencentSecretKey();
if (tenClientConfig == null) tenClientConfig = getDefaultTenClientConfig();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
TenCosContainer tenCosContainer = new TenCosContainer(secretId, secretKey, tenClientConfig, bucket,
prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen, threads);
tenCosContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
tenCosContainer.setRetryTimes(retryTimes);
return tenCosContainer;
}
public AliOssContainer getAliOssContainer() throws IOException {
String accessId = commonParams.getAliyunAccessId();
String accessSecret = commonParams.getAliyunAccessSecret();
String endPoint;
if (regionName == null || "".equals(regionName)) regionName = CloudApiUtils.getAliOssRegion(accessId, accessSecret, bucket);
if (regionName.matches("https?://.+")) {
endPoint = regionName;
} else {
if (!regionName.startsWith("oss-")) regionName = "oss-" + regionName;
endPoint = "http://" + regionName + ".aliyuncs.com";
}
if (aliClientConfig == null) aliClientConfig = getDefaultAliClientConfig();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
AliOssContainer aliOssContainer = new AliOssContainer(accessId, accessSecret, aliClientConfig, endPoint, bucket,
prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen, threads);
aliOssContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
aliOssContainer.setRetryTimes(retryTimes);
return aliOssContainer;
}
public UpYosContainer getUpYosContainer() throws IOException {
String username = commonParams.getUpyunUsername();
String password = commonParams.getUpyunPassword();
if (upYunConfig == null) upYunConfig = getDefaultUpYunConfig();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
// boolean prefixLeft = commonParams.getPrefixLeft();
// boolean prefixRight = commonParams.getPrefixRight();
UpYosContainer upYosContainer = new UpYosContainer(username, password, upYunConfig, bucket, prefixesMap, antiPrefixes,
// prefixLeft, prefixRight,
indexMap, null, unitLen, threads);
upYosContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
upYosContainer.setRetryTimes(retryTimes);
return upYosContainer;
}
public AwsS3Container getAwsS3Container() throws IOException {
String s3AccessId = commonParams.getS3AccessId();
String s3SecretKey = commonParams.getS3SecretKey();
if (s3ClientConfig == null) s3ClientConfig = getDefaultS3ClientConfig();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
String endpoint = entryParam.getValue("endpoint", "").trim();
if (endpoint.isEmpty() && (regionName == null || "".equals(regionName)))
regionName = CloudApiUtils.getS3Region(s3AccessId, s3SecretKey, bucket);
AwsS3Container awsS3Container = new AwsS3Container(s3AccessId, s3SecretKey, s3ClientConfig, endpoint, regionName, bucket,
prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen, threads);
awsS3Container.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
awsS3Container.setRetryTimes(retryTimes);
return awsS3Container;
}
public HuaweiObsContainer getHuaweiObsContainer() throws IOException {
String accessId = commonParams.getHuaweiAccessId();
String secretKey = commonParams.getHuaweiSecretKey();
if (obsConfiguration == null) obsConfiguration = getDefaultObsConfiguration();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
String endPoint;
if (regionName == null || "".equals(regionName)) regionName = CloudApiUtils.getHuaweiObsRegion(accessId, secretKey, bucket);
if (regionName.matches("https?://.+")) {
endPoint = regionName;
} else {
if (!regionName.startsWith("obs.")) regionName = "obs." + regionName;
endPoint = "http://" + regionName + ".myhuaweicloud.com";
}
HuaweiObsContainer huaweiObsContainer = new HuaweiObsContainer(accessId, secretKey, new ObsConfiguration(), endPoint,
bucket, prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen,
threads);
huaweiObsContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
huaweiObsContainer.setRetryTimes(retryTimes);
return huaweiObsContainer;
}
public BaiduBosContainer getBaiduBosContainer() throws IOException {
String accessId = commonParams.getBaiduAccessId();
String secretKey = commonParams.getBaiduSecretKey();
if (bosClientConfiguration == null) bosClientConfiguration = getDefaultBosClientConfiguration();
Map> prefixesMap = commonParams.getPrefixesMap();
List antiPrefixes = commonParams.getAntiPrefixes();
boolean prefixLeft = commonParams.getPrefixLeft();
boolean prefixRight = commonParams.getPrefixRight();
String endPoint;
if (regionName == null || "".equals(regionName)) regionName = CloudApiUtils.getHuaweiObsRegion(accessId, secretKey, bucket);
if (regionName.matches("https?://.+")) {
endPoint = regionName;
} else {
endPoint = "http://" + regionName + ".bcebos.com";
}
BaiduBosContainer baiduBosContainer = new BaiduBosContainer(accessId, secretKey, bosClientConfiguration, endPoint,
bucket, prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, null, unitLen, threads);
baiduBosContainer.setSaveOptions(saveTotal, savePath, saveFormat, saveSeparator, rmFields);
baiduBosContainer.setRetryTimes(retryTimes);
return baiduBosContainer;
}
public ILineProcess> getProcessor() throws Exception {
ILineProcess> nextProcessor = process == null ? null : whichNextProcessor(false);
BaseFilter> baseFilter = commonParams.getBaseFilter();
SeniorFilter> seniorFilter = commonParams.getSeniorFilter();
ILineProcess> processor;
if (baseFilter != null || seniorFilter != null) {
List fields = ConvertingUtils.getOrderedFields(new ArrayList<>(indexMap.values()), rmFields);
if (nextProcessor == null) {
processor = new FilterProcess>(baseFilter, seniorFilter, savePath, saveFormat,
saveSeparator, rmFields) {
@Override
protected ITypeConvert, String> newTypeConverter() throws IOException {
return new MapToString(saveFormat, saveSeparator, fields);
}
};
} else {
processor = new FilterProcess>(baseFilter, seniorFilter){};
processor.setNextProcessor(nextProcessor);
}
} else {
if ("filter".equals(process)) {
throw new Exception("please set the correct filter conditions.");
} else {
processor = nextProcessor;
}
}
return processor;
}
public ILineProcess> whichNextProcessor(boolean single) throws Exception {
ILineProcess> processor = null;
ILineProcess> privateProcessor = null;
boolean useQuery = true;
Map indexes = new HashMap<>(indexMap);
if (ProcessUtils.canPrivateToNext(process)) {
privateProcessor = getPrivateTypeProcessor(single);
if (privateProcessor != null) {
indexes.put("url", "url");
single = true;
useQuery = false;
}
}
switch (process) {
case "status": processor = getChangeStatus(single); break;
case "type": processor = getChangeType(single); break;
case "lifecycle": processor = getChangeLifecycle(single); break;
case "copy": processor = getCopyFile(indexes, single); break;
case "move":
case "rename": processor = getMoveFile(indexes, single); break;
case "delete": processor = getDeleteFile(single); break;
case "asyncfetch": processor = getAsyncFetch(indexes, single); break;
case "avinfo": processor = getQueryAvinfo(indexes, single); break;
case "pfopcmd": processor = getPfopCommand(indexes, single); break;
case "pfop": processor = getPfop(indexes, single); break;
case "pfopresult": processor = getPfopResult(indexes, single); break;
case "qhash": processor = getQueryHash(indexes, single); break;
case "stat": processor = getStatFile(single); break;
case "privateurl": processor = getPrivateUrl(indexes, single); break;
case "mirror": processor = getMirrorFile(single); break;
case "exportts": processor = getExportTs(indexes, single); break;
case "tenprivate": processor = getTencentPrivateUrl(single); break;
case "s3private": case "awsprivate": processor = getAwsS3PrivateUrl(single); break;
case "aliprivate": processor = getAliyunPrivateUrl(single); break;
case "huaweiprivate": processor = getHuaweiPrivateUrl(single); break;
case "baiduprivate": processor = getBaiduPrivateUrl(single); break;
case "download": processor = getDownloadFile(indexes, single, useQuery); break;
case "imagecensor": processor = getImageCensor(indexes, single, useQuery); break;
case "videocensor": processor = getVideoCensor(indexes, single); break;
case "censorresult": processor = getCensorResult(indexes, single); break;
case "filter": case "": break;
default: throw new IOException("unsupported process: " + process);
}
if (processor != null) {
if (ProcessUtils.canBatch(processor.getProcessName())) processor.setBatchSize(commonParams.getBatchSize());
// 为了保证程序出现因网络等原因产生的非预期异常时正常运行需要设置重试次数
processor.setRetryTimes(retryTimes);
if (privateProcessor != null) {
privateProcessor.setNextProcessor(processor);
return privateProcessor;
}
}
return processor;
}
private ILineProcess> getPrivateTypeProcessor(boolean single) throws IOException {
ILineProcess> processor = null;
String privateType = commonParams.getPrivateType();
if ("qiniu".equals(privateType)) {
processor = getPrivateUrl(indexMap, single);
} else if ("tencent".equals(privateType)) {
processor = getTencentPrivateUrl(single);
} else if ("aliyun".equals(privateType)) {
processor = getAliyunPrivateUrl(single);
} else if ("s3".equals(privateType) || "aws".equals(privateType)) {
processor = getAwsS3PrivateUrl(single);
} else if ("huawei".equals(privateType)) {
processor = getHuaweiPrivateUrl(single);
} else if ("baidu".equals(privateType)) {
processor = getBaiduPrivateUrl(single);
} else if (privateType != null && !"".equals(privateType)) {
throw new IOException("unsupported private process: " + privateType + " for asyncfetch's url.");
}
return processor;
}
private ILineProcess> getChangeStatus(boolean single) throws IOException {
String status = ParamsUtils.checked(entryParam.getValue("status").trim(), "status", "[01]");
return single ? new ChangeStatus(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(status))
: new ChangeStatus(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(status), savePath);
}
private ILineProcess> getChangeType(boolean single) throws IOException {
String type = ParamsUtils.checked(entryParam.getValue("type").trim(), "type", "[01]");
return single ? new ChangeType(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(type))
: new ChangeType(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(type), savePath);
}
private ILineProcess> getChangeLifecycle(boolean single) throws IOException {
String days = ParamsUtils.checked(entryParam.getValue("days").trim(), "days", "\\d+");
return single ? new ChangeLifecycle(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(days))
: new ChangeLifecycle(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.valueOf(days), savePath);
}
private ILineProcess> getCopyFile(Map indexMap, boolean single) throws IOException {
String toBucket = entryParam.getValue("to-bucket").trim();
String toKeyIndex = indexMap.containsValue("toKey") ? "toKey" : null;
String addPrefix = entryParam.getValue("add-prefix", null);
String rmPrefix = entryParam.getValue("rm-prefix", null);
return single ? new CopyFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, toBucket, toKeyIndex, addPrefix,
rmPrefix)
: new CopyFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, toBucket, toKeyIndex, addPrefix,
rmPrefix, savePath);
}
private ILineProcess> getMoveFile(Map indexMap, boolean single) throws IOException {
String toBucket = entryParam.getValue("to-bucket", "").trim();
if ("move".equals(process)) {
if (toBucket.isEmpty()) throw new IOException("no incorrect to-bucket, please set it.");
} else {
toBucket = null;
}
String toKeyIndex = indexMap.containsValue("toKey") ? "toKey" : null;
String addPrefix = entryParam.getValue("add-prefix", null);
String rmPrefix = entryParam.getValue("rm-prefix", null);
String force = entryParam.getValue("prefix-force", "false").trim();
force = ParamsUtils.checked(force, "prefix-force", "(true|false)");
return single ? new MoveFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, toBucket, toKeyIndex, addPrefix,
rmPrefix, Boolean.valueOf(force))
: new MoveFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, toBucket, toKeyIndex, addPrefix,
rmPrefix, Boolean.valueOf(force), savePath);
}
private ILineProcess> getDeleteFile(boolean single) throws IOException {
return single ? new DeleteFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket)
: new DeleteFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, savePath);
}
private ILineProcess> getAsyncFetch(Map indexMap, boolean single) throws IOException {
String ak = entryParam.getValue("qiniu-ak", qiniuAccessKey).trim();
String sk = entryParam.getValue("qiniu-sk", qiniuSecretKey).trim();
String toBucket = entryParam.getValue("to-bucket").trim();
if (toBucket.equals(bucket) && "qiniu".equals(source))
throw new IOException("the to-bucket can not be same as bucket if source is qiniu.");
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String addPrefix = entryParam.getValue("add-prefix", null);
String rmPrefix = entryParam.getValue("rm-prefix", null);
String host = entryParam.getValue("host", "").trim();
String md5Index = indexMap.containsValue("md5") ? "md5" : null;
String callbackUrl = entryParam.getValue("callback-url", "").trim();
String checkUrl = entryParam.getValue("check-url", "true").trim();
if ("true".equals(checkUrl) && !"".equals(callbackUrl)) RequestUtils.checkCallbackUrl(callbackUrl);
String callbackBody = entryParam.getValue("callback-body", "").trim();
String callbackBodyType = entryParam.getValue("callback-body-type", "").trim();
String callbackHost = entryParam.getValue("callback-host", "").trim();
String type = entryParam.getValue("file-type", "0").trim();
String ignore = entryParam.getValue("ignore-same-key", "false").trim();
ignore = ParamsUtils.checked(ignore, "ignore-same-key", "(true|false)");
String region = entryParam.getValue("qiniu-region", regionName).trim();
Zone zone = CloudApiUtils.getQiniuRegion(region);
Configuration configuration = new Configuration(zone);
if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
if (readTimeout> Constants.READ_TIMEOUT) configuration.readTimeout = readTimeout;
if (requestTimeout > Constants.WRITE_TIMEOUT) configuration.writeTimeout = requestTimeout;
ILineProcess> processor = single ? new AsyncFetch(ak, sk, configuration, toBucket, protocol,
domain, urlIndex, addPrefix, rmPrefix)
: new AsyncFetch(ak, sk, configuration, toBucket, protocol, domain, urlIndex, addPrefix, rmPrefix, savePath);
if (!host.isEmpty() || md5Index != null || !callbackUrl.isEmpty() || !callbackBody.isEmpty() ||
!callbackBodyType.isEmpty() || !callbackHost.isEmpty() || "1".equals(type) || "true".equals(ignore)) {
((AsyncFetch) processor).setFetchArgs(host, md5Index, callbackUrl, callbackBody,
callbackBodyType, callbackHost, Integer.valueOf(type), Boolean.valueOf(ignore));
}
return processor;
}
private ILineProcess> getQueryAvinfo(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
return single ? new QueryAvinfo(getQiniuConfig(), protocol, domain, urlIndex)
: new QueryAvinfo(getQiniuConfig(), protocol, domain, urlIndex, savePath);
}
private ILineProcess> getPfopCommand(Map indexMap, boolean single) throws IOException {
String avinfoIndex = indexMap.containsValue("avinfo") ? "avinfo" : null;
String duration = entryParam.getValue("duration", "false").trim();
duration = ParamsUtils.checked(duration, "duration", "(true|false)");
String size = entryParam.getValue("size", "false").trim();
size = ParamsUtils.checked(size, "size", "(true|false)");
String configJson = entryParam.getValue("pfop-config", "").trim();
List pfopConfigs = commonParams.getPfopConfigs();
return single ? new PfopCommand(getQiniuConfig(), avinfoIndex, Boolean.valueOf(duration), Boolean.valueOf(size),
configJson, pfopConfigs)
: new PfopCommand(getQiniuConfig(), avinfoIndex, Boolean.valueOf(duration), Boolean.valueOf(size), configJson,
pfopConfigs, savePath);
}
private ILineProcess> getPfop(Map indexMap, boolean single) throws IOException {
String pipeline = entryParam.getValue("pipeline", "").trim();
String forcePublic = entryParam.getValue("force-public", "false").trim();
if (pipeline.isEmpty() && !"true".equals(forcePublic)) {
throw new IOException("please set pipeline, if you don't want to use" +
" private pipeline, please set the force-public as true.");
}
String configJson = entryParam.getValue("pfop-config", "").trim();
List pfopConfigs = commonParams.getPfopConfigs();
String fopsIndex = indexMap.containsValue("fops") ? "fops" : null;
return single ? new QiniuPfop(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, pipeline, configJson,
pfopConfigs, fopsIndex)
: new QiniuPfop(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, pipeline, configJson, pfopConfigs,
fopsIndex, savePath);
}
private ILineProcess> getPfopResult(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String pIdIndex = indexMap.containsValue("id") ? "id" : null;
return single ? new QueryPfopResult(getQiniuConfig(), protocol, pIdIndex)
: new QueryPfopResult(getQiniuConfig(), protocol, pIdIndex, savePath);
}
private ILineProcess> getQueryHash(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String algorithm = entryParam.getValue("algorithm", "md5").trim();
algorithm = ParamsUtils.checked(algorithm, "algorithm", "(md5|sha1)");
return single ? new QueryHash(getQiniuConfig(), protocol, domain, urlIndex, algorithm)
: new QueryHash(getQiniuConfig(), protocol, domain, urlIndex, algorithm, savePath);
}
private ILineProcess> getStatFile(boolean single) throws IOException {
return single ? new StatFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, saveFormat, saveSeparator, rmFields)
: new StatFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, savePath, saveFormat, saveSeparator, rmFields);
}
private ILineProcess> getPrivateUrl(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String queries = entryParam.getValue("queries", "").trim();
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
return single ? new PrivateUrl(qiniuAccessKey, qiniuSecretKey, protocol, domain, urlIndex, queries, Long.valueOf(expires))
: new PrivateUrl(qiniuAccessKey, qiniuSecretKey, protocol, domain, urlIndex, queries, Long.valueOf(expires), savePath);
}
private ILineProcess> getMirrorFile(boolean single) throws IOException {
return single ? new MirrorFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket)
: new MirrorFile(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, savePath);
}
private ILineProcess> getExportTs(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
return single ? new ExportTS(getQiniuConfig(), protocol, domain, urlIndex)
: new ExportTS(getQiniuConfig(), protocol, domain, urlIndex, savePath);
}
private Map getQueriesMap() {
String queries = entryParam.getValue("queries", "").trim();
if (queries.startsWith("\\?")) queries = queries.substring(1);
String[] items = queries.split("&");
Map queriesMap = new HashMap<>();
int index;
String key;
String value;
for (String item : items) {
index = item.indexOf("=");
if (index < 0) {
key = item;
value = "";
} else {
key = item.substring(0, index);
value = item.substring(index + 1);
}
queriesMap.put(key, value);
}
return queriesMap;
}
private com.qiniu.process.tencent.PrivateUrl getTencentPrivateUrl(boolean single) throws IOException {
String secretId = entryParam.getValue("ten-id", commonParams.getTencentSecretId());
String secretKey = entryParam.getValue("ten-secret", commonParams.getTencentSecretKey());
String tenBucket = bucket == null || bucket.isEmpty() ? entryParam.getValue("bucket") : bucket;
String region = regionName == null || regionName.isEmpty() ? entryParam.getValue("region", regionName) : regionName;
if (region == null || "".equals(region)) region = CloudApiUtils.getTenCosRegion(secretId, secretKey, tenBucket);
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
return single ? new com.qiniu.process.tencent.PrivateUrl(secretId, secretKey, tenBucket, region,
1000 * Long.valueOf(expires), getQueriesMap()) : new com.qiniu.process.tencent.PrivateUrl(secretId,
secretKey, tenBucket, region, 1000 * Long.valueOf(expires), getQueriesMap(), savePath);
}
private com.qiniu.process.aliyun.PrivateUrl getAliyunPrivateUrl(boolean single) throws IOException {
String accessId = entryParam.getValue("ali-id", commonParams.getAliyunAccessId());
String accessSecret = entryParam.getValue("ali-secret", commonParams.getAliyunAccessSecret());
String aliBucket = bucket == null || bucket.isEmpty() ? entryParam.getValue("bucket", bucket) : bucket;
String endPoint = regionName == null || regionName.isEmpty() ? entryParam.getValue("region", regionName) : regionName;
if (endPoint == null || "".equals(endPoint)) endPoint = CloudApiUtils.getAliOssRegion(accessId, accessSecret, aliBucket);
if (!endPoint.matches("https?://.+")) {
if (endPoint.startsWith("oss-")) endPoint = "http://" + endPoint + ".aliyuncs.com";
else endPoint = "http://oss-" + endPoint + ".aliyuncs.com";
}
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
return single ? new com.qiniu.process.aliyun.PrivateUrl(accessId, accessSecret, aliBucket, endPoint,
1000 * Long.valueOf(expires), getQueriesMap()) : new com.qiniu.process.aliyun.PrivateUrl(accessId,
accessSecret, aliBucket, endPoint, 1000 * Long.valueOf(expires), getQueriesMap(), savePath);
}
private com.qiniu.process.aws.PrivateUrl getAwsS3PrivateUrl(boolean single) throws IOException {
String accessId = entryParam.getValue("s3-id", commonParams.getS3AccessId());
String secretKey = entryParam.getValue("s3-secret", commonParams.getS3SecretKey());
String s3Bucket = bucket == null || bucket.isEmpty() ? entryParam.getValue("bucket", bucket) : bucket;
String region = regionName == null || regionName.isEmpty() ? entryParam.getValue("region", regionName) : regionName;
String endpoint = entryParam.getValue("endpoint", "").trim();
if (endpoint.isEmpty() && (region == null || "".equals(region)))
region = CloudApiUtils.getS3Region(accessId, secretKey, s3Bucket);
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
return single ? new com.qiniu.process.aws.PrivateUrl(accessId, secretKey, s3Bucket, endpoint, region,
1000 * Long.valueOf(expires), getQueriesMap()) : new com.qiniu.process.aws.PrivateUrl(accessId,
secretKey, s3Bucket, endpoint, region, 1000 * Long.valueOf(expires), getQueriesMap(), savePath);
}
private com.qiniu.process.huawei.PrivateUrl getHuaweiPrivateUrl(boolean single) throws IOException {
String accessId = entryParam.getValue("hua-id", commonParams.getS3AccessId());
String secretKey = entryParam.getValue("hua-secret", commonParams.getS3SecretKey());
String huaweiBucket = bucket == null || bucket.isEmpty() ? entryParam.getValue("bucket", bucket) : bucket;
String endPoint = regionName == null || regionName.isEmpty() ? entryParam.getValue("region", regionName) : regionName;
if (endPoint == null || "".equals(endPoint)) endPoint = CloudApiUtils.getHuaweiObsRegion(accessId, secretKey, huaweiBucket);
if (!endPoint.matches("https?://.+")) {
if (endPoint.startsWith("obs.")) endPoint = "http://" + endPoint + ".myhuaweicloud.com";
else endPoint = "http://obs." + endPoint + ".myhuaweicloud.com";
}
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
// 华为 sdk 的过期时间按秒设置
return single ? new com.qiniu.process.huawei.PrivateUrl(accessId, secretKey, huaweiBucket, endPoint, Long.valueOf(expires),
getQueriesMap()) : new com.qiniu.process.huawei.PrivateUrl(accessId, secretKey, huaweiBucket, endPoint,
Long.valueOf(expires), getQueriesMap(), savePath);
}
private com.qiniu.process.baidu.PrivateUrl getBaiduPrivateUrl(boolean single) throws IOException {
String accessId = entryParam.getValue("bai-id", commonParams.getS3AccessId());
String secretKey = entryParam.getValue("bai-secret", commonParams.getS3SecretKey());
String baiduBucket = bucket == null || bucket.isEmpty() ? entryParam.getValue("bucket", bucket) : bucket;
String endPoint = regionName == null || regionName.isEmpty() ? entryParam.getValue("region", regionName) : regionName;
if (endPoint == null || "".equals(endPoint)) endPoint = CloudApiUtils.getBaiduBosRegion(accessId, secretKey, baiduBucket);
if (!endPoint.matches("https?://.+")) {
endPoint = "http://" + endPoint + ".bcebos.com";
}
String expires = entryParam.getValue("expires", "3600").trim();
expires = ParamsUtils.checked(expires, "expires", "[1-9]\\d*");
// 华为 sdk 的过期时间按秒设置
return single ? new com.qiniu.process.baidu.PrivateUrl(accessId, secretKey, baiduBucket, endPoint, Integer.valueOf(expires),
getQueriesMap()) : new com.qiniu.process.baidu.PrivateUrl(accessId, secretKey, baiduBucket, endPoint,
Integer.valueOf(expires), getQueriesMap(), savePath);
}
private ILineProcess> getDownloadFile(Map indexMap, boolean single, boolean useQuery)
throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String queries = useQuery ? entryParam.getValue("queries", "").trim() : null;
String host = entryParam.getValue("host", "").trim();
String bytes = entryParam.getValue("bytes", "").trim();
if (bytes.equals("0")) throw new IOException("range bytes can not be 0.");
int[] range = new int[0];
if (!"".equals(bytes)) {
String[] ranges = bytes.split("-");
if (ranges.length > 2) throw new IOException("range bytes should be like \"0-1024\".");
try {
if (ranges.length > 1) {
range = new int[2];
range[0] = Integer.valueOf(ranges[0]);
String byteSize = ranges[1];
if (byteSize != null && !"".equals(byteSize)) range[1] = Integer.valueOf(byteSize);
} else {
range = new int[]{Integer.valueOf(ranges[0])};
}
} catch (Exception e) {
throw new IOException("incorrect range bytes value, " + e.toString());
}
}
String preDown = entryParam.getValue("pre-down", "false").trim();
preDown = ParamsUtils.checked(preDown, "pre-down", "(true|false)");
String addPrefix = entryParam.getValue("add-prefix", null);
String rmPrefix = entryParam.getValue("rm-prefix", null);
String timeOut = entryParam.getValue("download-timeout", null);
Configuration configuration = null;
if (timeOut != null) {
configuration = new Configuration();
configuration.connectTimeout = getQiniuConfig().connectTimeout;
configuration.readTimeout = Integer.valueOf(timeOut);
}
return single ? new DownloadFile(configuration, protocol, domain, urlIndex, host, range, queries, "true".equals(preDown)
? null : savePath, addPrefix, rmPrefix)
: new DownloadFile(configuration, protocol, domain, urlIndex, host, range, queries, Boolean.valueOf(preDown),
addPrefix, rmPrefix, savePath);
}
private ILineProcess> getImageCensor(Map indexMap, boolean single, boolean useQuery)
throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String queries = useQuery ? entryParam.getValue("queries", "").trim() : null;
String[] scenes = entryParam.getValue("scenes").trim().split(",");
return single ? new ImageCensor(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), protocol, domain, urlIndex, queries, scenes) :
new ImageCensor(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), protocol, domain, urlIndex, queries, scenes, savePath);
}
private ILineProcess> getVideoCensor(Map indexMap, boolean single) throws IOException {
String protocol = entryParam.getValue("protocol", "http").trim();
protocol = ParamsUtils.checked(protocol, "protocol", "https?");
String domain = entryParam.getValue("domain", "").trim();
String urlIndex = indexMap.containsValue("url") ? "url" : null;
String[] scenes = entryParam.getValue("scenes").trim().split(",");
String interval = entryParam.getValue("interval", "0").trim();
String saverBucket = entryParam.getValue("save-bucket", "").trim();
String saverPrefix = entryParam.getValue("saver-prefix", "").trim();
String hookUrl = entryParam.getValue("callback-url", "").trim();
String checkUrl = entryParam.getValue("check-url", "true").trim();
if ("true".equals(checkUrl) && !"".equals(hookUrl)) RequestUtils.checkCallbackUrl(hookUrl);
return single ? new VideoCensor(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), protocol, domain, urlIndex, scenes,
Integer.valueOf(interval), saverBucket, saverPrefix, hookUrl) : new VideoCensor(qiniuAccessKey, qiniuSecretKey,
getQiniuConfig(), protocol, domain, urlIndex, scenes, Integer.valueOf(interval), saverBucket, saverPrefix, hookUrl,
savePath);
}
private ILineProcess> getCensorResult(Map indexMap, boolean single) throws IOException {
String jobIdIndex = indexMap.containsValue("id") ? "id" : null;
return single ? new QueryCensorResult(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), jobIdIndex)
: new QueryCensorResult(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), jobIdIndex, savePath);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy