Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.alibaba.ververica.connectors.odps.util.OdpsMetadataProvider Maven / Gradle / Ivy
package com.alibaba.ververica.connectors.odps.util;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsException;
import com.aliyun.odps.Partition;
import com.aliyun.odps.PartitionSpec;
import com.aliyun.odps.Project;
import com.aliyun.odps.Table;
import com.aliyun.odps.tunnel.TableTunnel;
import com.aliyun.odps.tunnel.TunnelException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Provider for ODPS metadata.
*/
public class OdpsMetadataProvider {
private static final Logger LOGGER = LoggerFactory.getLogger(OdpsMetadataProvider.class);
private static final int MAX_RETRY_TIMES = 5;
private static final Map projectCache = new HashMap<>();
private static final Map tableCache = new HashMap<>();
private static final Map> allPartitionCache = new HashMap<>();
private static final Map partitionCache = new HashMap<>();
private static final Map downloadSessionMap = new HashMap<>();
public static Project getProject(Odps odps, String project) throws OdpsException {
if (projectCache.containsKey(project)) {
return projectCache.get(project);
}
long start = System.currentTimeMillis();
Project p = odps.projects().get(project);
p.reload();
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(String.format("get project %s cost %s ms", project, (end - start)));
}
projectCache.put(project, p);
return p;
}
public static Table getTable(Odps odps, String project, String table) throws OdpsException {
String cacheKey = project + "." + table;
if (tableCache.containsKey(cacheKey)) {
return tableCache.get(cacheKey);
}
long start = System.currentTimeMillis();
Table t = odps.tables().get(project, table);
t.reload();
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(String.format("get table %s cost %s ms", cacheKey, (end - start)));
}
tableCache.put(cacheKey, t);
return t;
}
public static List getAllPartitions(
Odps odps,
String project,
String table) throws OdpsException {
String cacheKey = project + "." + table;
if (allPartitionCache.containsKey(cacheKey)) {
return allPartitionCache.get(cacheKey);
}
Table t = getTable(odps, project, table);
long start = System.currentTimeMillis();
List allPartitions = t.getPartitions();
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(String.format("get %s all partitions cost %s ms", cacheKey, (end - start)));
}
allPartitionCache.put(cacheKey, allPartitions);
return allPartitions;
}
public static Partition getPartition(Odps odps,
String project,
String table,
PartitionSpec partitionSpec) throws OdpsException {
String cacheKey = project + "." + table + ":" + partitionSpec.toString();
if (partitionCache.containsKey(cacheKey)) {
return partitionCache.get(cacheKey);
}
Table t = getTable(odps, project, table);
checkArgument(t.isPartitioned(), "The table is not a partitioned table!");
long start = System.currentTimeMillis();
Partition p = t.getPartition(partitionSpec);
p.reload();
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(String.format("get partition %s cost %s ms", cacheKey, (end - start)));
}
partitionCache.put(cacheKey, p);
return p;
}
public static TableTunnel.DownloadSession createDownloadSession(
TableTunnel tunnel,
String project,
String table) {
String cacheKey = project + "." + table;
TableTunnel.DownloadSession downloadSession = downloadSessionMap.get(cacheKey);
if (downloadSession == null) {
long start = System.currentTimeMillis();
downloadSession = retryableCreateDownLoadSession(tunnel, project, table, null);
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(
String.format("create %s download session cost %s ms", cacheKey, (end - start)));
}
downloadSessionMap.put(cacheKey, downloadSession);
}
return downloadSession;
}
public static TableTunnel.DownloadSession createDownloadSession(
TableTunnel tunnel,
String project,
String table,
PartitionSpec partitionSpec) {
checkNotNull(partitionSpec);
String cacheKey = project + "." + table + ":" + partitionSpec.toString();
TableTunnel.DownloadSession downloadSession = downloadSessionMap.get(cacheKey);
if (downloadSession == null) {
long start = System.currentTimeMillis();
downloadSession = retryableCreateDownLoadSession(tunnel, project, table, partitionSpec);
downloadSessionMap.put(cacheKey, downloadSession);
if (LOGGER.isDebugEnabled()) {
long end = System.currentTimeMillis();
LOGGER.debug(
String.format("create %s download session cost %s ms", cacheKey, (end - start)));
}
}
return downloadSession;
}
private static TableTunnel.DownloadSession retryableCreateDownLoadSession(
TableTunnel tunnel,
String project,
String table,
PartitionSpec partitionSpec) {
int tryTimes = 0;
while (true) {
try {
TableTunnel.DownloadSession downloadSession;
if (partitionSpec == null) {
downloadSession = tunnel.createDownloadSession(project, table);
} else {
downloadSession = tunnel.createDownloadSession(project, table, partitionSpec);
}
return downloadSession;
} catch (TunnelException e) {
String downloadObjectStr;
if (partitionSpec == null) {
downloadObjectStr = String.format("odps table [%s].[%s]!", project, table);
} else {
downloadObjectStr = String.format("partition [%s] of odps table [%s].[%s]!",
partitionSpec.toString(), project, table);
}
if (tryTimes++ >= MAX_RETRY_TIMES) {
logAndPropagateException(e,
"Give up to create download session of %s after try %d times",
downloadObjectStr, MAX_RETRY_TIMES);
} else {
LOGGER.warn("Retry to create download session of {} after try {} times",
downloadObjectStr, tryTimes, e);
try {
TimeUnit.SECONDS.sleep(tryTimes);
} catch (InterruptedException ie) {
logAndPropagateException(ie,
"Stop to create download session of %s because of interruption",
downloadObjectStr);
}
}
}
}
}
private static void logAndPropagateException(Throwable t, String format, Object... arguments) {
String warnMsg = String.format(format, arguments);
LOGGER.warn(warnMsg, t);
throw new RuntimeException(warnMsg, t);
}
private OdpsMetadataProvider() {
}
}
