Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.ververica.connectors.odps.util;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.runtime.typeutils.RowDataTypeInfo;
import org.apache.flink.table.types.logical.LogicalType;
import com.alibaba.ververica.connectors.odps.OdpsConf;
import com.alibaba.ververica.connectors.odps.schema.ODPSColumn;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsException;
import com.aliyun.odps.Partition;
import com.aliyun.odps.PartitionSpec;
import com.aliyun.odps.Table;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.tunnel.TableTunnel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.alibaba.ververica.connectors.odps.OdpsOptions.ACCESS_ID;
import static com.alibaba.ververica.connectors.odps.OdpsOptions.ACCESS_KEY;
import static com.alibaba.ververica.connectors.odps.OdpsOptions.END_POINT;
import static com.alibaba.ververica.connectors.odps.OdpsOptions.PROJECT_NAME;
import static com.alibaba.ververica.connectors.odps.OdpsOptions.TUNNEL_END_POINT;
/**
* Utility for Odps.
*/
public class OdpsUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(OdpsUtils.class);
public static final String DONE_FLAG = ".done";
/**
* Creates odps instance using odps configuration.
*
* @param odpsConf odps configuration
* @return odps instance
*/
public static Odps initOdps(OdpsConf odpsConf) {
Account account = new AliyunAccount(odpsConf.getAccessId(), odpsConf.getAccessKey());
Odps odps = new Odps(account);
odps.setEndpoint(odpsConf.getEndpoint());
if (odpsConf.getProject() != null) {
odps.setDefaultProject(odpsConf.getProject());
}
return odps;
}
public static OdpsConf createOdpsConf(ReadableConfig options) {
String accessId = options.get(ACCESS_ID);
String accessKey = options.get(ACCESS_KEY);
String endpoint = options.get(END_POINT);
String project = options.get(PROJECT_NAME);
String tunnelEndpoint = options.get(TUNNEL_END_POINT);
return new OdpsConf(accessId, accessKey, endpoint, project, tunnelEndpoint);
}
public static TableTunnel createTableTunnel(OdpsConf odpsConf) {
Odps odps = initOdps(odpsConf);
TableTunnel tunnel = new TableTunnel(odps);
// set tunnel endpoint if user specify the value
if (odpsConf.getTunnelEndpoint() != null) {
tunnel.setEndpoint(odpsConf.getTunnelEndpoint());
}
return tunnel;
}
/**
* Checks whether a {@link Partition} is .done flag partition.
*
* @param partition Partition to check
* @return true if the partition is .done flag partition
*/
static boolean isDoneFlagPartition(Partition partition) {
PartitionSpec spec = partition.getPartitionSpec();
for (String key : spec.keys()) {
if (spec.get(key).endsWith(DONE_FLAG)) {
return true;
}
}
return false;
}
/**
* Converts a {@link PartitionSpec} to string.
*
* @param spec PartitionSpec to convert
* @return string converted from PartitionSpec. e.g. pt=2014,ds=03
*/
public static String partitionSpecToString(PartitionSpec spec) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (String key : spec.keys()) {
if (first) {
first = false;
} else {
sb.append(',');
}
sb.append(key).append("=").append(spec.get(key));
}
return sb.toString();
}
/**
* Fetches size of an odps table.
*
* @param odpsConf odps configuration
* @param table odps table name
* @return total size of an odps table
*/
public static long getTableSize(OdpsConf odpsConf, String table) {
Odps odps = initOdps(odpsConf);
try {
Table t = OdpsMetadataProvider.getTable(odps, odpsConf.getProject(), table);
return t.getSize();
} catch (OdpsException e) {
LOGGER.error("Fail to get table size of table {} !", table, e);
throw new TableException("Fail to get table size of table !", e);
}
}
/**
* Fetches size of an odps table partition.
*
* @param odpsConf odps configuration
* @param table odps table name
* @param partition a specified odps table partition, e.g pt='1',ds='2'
* @return total size of an odps table partition
*/
public static long getPartitionSize(OdpsConf odpsConf, String table, String partition) {
Odps odps = initOdps(odpsConf);
try {
PartitionSpec spec = new PartitionSpec(partition);
Partition p = OdpsMetadataProvider.getPartition(odps, odpsConf.getProject(), table, spec);
return p.getSize();
} catch (OdpsException e) {
LOGGER.error("Fail to get partition size of table {}, partition {} !", table, partition, e);
throw new TableException("Fail to get partition size !", e);
}
}
/**
* Fetches last modify time of an odps table.
*
* @param odpsConf odps configuration
* @param table odps table name
* @return last modify time of an odps table
*/
public static long getTableLastModifyTime(OdpsConf odpsConf, String table) {
Odps odps = initOdps(odpsConf);
try {
Table t = OdpsMetadataProvider.getTable(odps, odpsConf.getProject(), table);
return t.getLastDataModifiedTime().getTime();
} catch (OdpsException e) {
LOGGER.error("Fail to get last modify time of table {} !", table, e);
throw new TableException("Fail to get last modify time of table !", e);
}
}
/**
* Fetches last modify time of an odps table partition.
*
* @param odpsConf odps configuration
* @param table odps table name
* @param partition a specified odps table partition, e.g pt='1',ds='2'
* @return last modify time of an odps table partition
*/
public static long getPartitionLastModifyTime(OdpsConf odpsConf, String table, String partition) {
Odps odps = initOdps(odpsConf);
PartitionSpec spec = new PartitionSpec(partition);
try {
Partition p = OdpsMetadataProvider.getPartition(odps, odpsConf.getProject(), table, spec);
return p.getLastDataModifiedTime().getTime();
} catch (OdpsException e) {
LOGGER.error("Fail to get partition last modify time of table {}, partition {} !", table, partition, e);
throw new TableException("Fail to get partition last modify time of table !", e);
}
}
public static RowDataTypeInfo deriveRowType(TableSchema tableSchema, ODPSColumn[] columns) {
int columnsNum = columns.length;
String[] fieldNames = new String[columnsNum];
LogicalType[] fieldTypes = new LogicalType[columnsNum];
for (int idx = 0; idx < columnsNum; idx++) {
ODPSColumn column = columns[idx];
fieldNames[idx] = column.getName();
fieldTypes[idx] = tableSchema.getFieldDataType(fieldNames[idx]).get().getLogicalType();
}
return new RowDataTypeInfo(fieldTypes, fieldNames);
}
/**
* Deprecates default constructor.
*/
private OdpsUtils() {
}
}
