com.aliyun.datahub.client.example.examples.ConnectorExample Maven / Gradle / Ivy
The newest version!
package com.aliyun.datahub.client.example.examples;
import com.aliyun.datahub.client.DatahubClient;
import com.aliyun.datahub.client.DatahubClientBuilder;
import com.aliyun.datahub.client.auth.AliyunAccount;
import com.aliyun.datahub.client.common.DatahubConfig;
import com.aliyun.datahub.client.exception.AuthorizationFailureException;
import com.aliyun.datahub.client.exception.DatahubClientException;
import com.aliyun.datahub.client.exception.InvalidParameterException;
import com.aliyun.datahub.client.exception.ResourceNotFoundException;
import com.aliyun.datahub.client.http.HttpConfig;
import com.aliyun.datahub.client.model.*;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class ConnectorExample {
private static DatahubClient datahubClient;
private static String shardId;
private static String connectorId;
public static void init() {
HttpConfig httpConfig = new HttpConfig();
httpConfig.setCompressType(CompressType.DEFLATE);
// 创建DataHubClient实例
datahubClient = DatahubClientBuilder.newBuilder()
.setDatahubConfig(
new DatahubConfig(Constant.endpoint,
new AliyunAccount(Constant.accessId, Constant.accessKey)))
.setHttpConfig(httpConfig)
.build();
shardId = "0";
// config your connectorID
connectorId = "";
}
public static void createConnector() {
List columnFields = Arrays.asList("field1", "field2");
SinkOdpsConfig config = new SinkOdpsConfig() {{
setEndpoint(Constant.odps_endpoint);
setProject(Constant.odps_project);
setTable(Constant.odps_table);
setAccessId(Constant.odps_accessId);
setAccessKey(Constant.odps_accessKey);
setPartitionMode(PartitionMode.SYSTEM_TIME);
setTimeRange(60);
}};
//设置分区格式
SinkOdpsConfig.PartitionConfig partitionConfig = new SinkOdpsConfig.PartitionConfig() {{
addConfig("ds", "%Y%m%d");
addConfig("hh", "%H");
addConfig("mm", "%M");
}};
config.setPartitionConfig(partitionConfig);
try {
//创建Connector
CreateConnectorResult result = datahubClient.createConnector(Constant.projectName, Constant.topicName, ConnectorType.SINK_ODPS, columnFields, config);
System.out.println("create connector successful");
System.out.println(result.getConnectorId());
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void deleteConnector() {
try {
datahubClient.deleteConnector(Constant.projectName, Constant.topicName, connectorId);
System.out.println("delete connector successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void updateConnector() {
SinkOdpsConfig config = (SinkOdpsConfig) datahubClient.getConnector(Constant.projectName, Constant.topicName, connectorId).getConfig();
//修改配置
config.setTimeRange(100);
config.setAccessId(Constant.accessId);
config.setAccessKey(Constant.accessKey);
try {
datahubClient.updateConnector(Constant.projectName, Constant.topicName, ConnectorType.SINK_ODPS, config);
System.out.println("update connector successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void updateConnectorState() {
try {
datahubClient.updateConnectorState(Constant.projectName, Constant.topicName, connectorId, ConnectorState.STOPPED);
datahubClient.updateConnectorState(Constant.projectName, Constant.topicName, connectorId, ConnectorState.RUNNING);
System.out.println("update connector state successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void updateConnectorOffset() {
ConnectorOffset offset = new ConnectorOffset() {{
setSequence(10);
setTimestamp(1000);
}};
try {
//更新Connector点位需要先停止Connector
datahubClient.updateConnectorState(Constant.projectName, Constant.topicName, connectorId, ConnectorState.STOPPED);
datahubClient.updateConnectorOffset(Constant.projectName, Constant.topicName, connectorId, shardId, offset);
datahubClient.updateConnectorState(Constant.projectName, Constant.topicName, connectorId, ConnectorState.RUNNING);
System.out.println("update connector offset successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void listConnector() {
try {
ListConnectorResult listConnectorResult = datahubClient.listConnector(Constant.projectName, Constant.topicName);
for (String cName : listConnectorResult.getConnectorNames()) {
System.out.println(cName);
}
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void getConnector() {
try {
GetConnectorResult getConnectorResult = datahubClient.getConnector(Constant.projectName, Constant.topicName, connectorId);
System.out.println(getConnectorResult.getState() + "\t" + getConnectorResult.getSubId());
for (String fieldName : getConnectorResult.getColumnFields()) {
System.out.println(fieldName);
}
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void getConnectorShardStatusByShard() {
try {
ConnectorShardStatusEntry connectorShardStatusEntry = datahubClient.getConnectorShardStatus(Constant.projectName, Constant.topicName, connectorId, shardId);
System.out.println(connectorShardStatusEntry.getState() + "\t"
+ connectorShardStatusEntry.getCurrSequence() + "\t"
+ connectorShardStatusEntry.getDiscardCount() + "\t"
+ connectorShardStatusEntry.getUpdateTime());
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void getConnectorShardStatus() {
try {
GetConnectorShardStatusResult getConnectorShardStatusResult = datahubClient.getConnectorShardStatus(Constant.projectName, Constant.topicName, connectorId);
for (Map.Entry entry : getConnectorShardStatusResult.getStatusEntryMap().entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue().getState() + "\t"
+ entry.getValue().getCurrSequence() + "\t"
+ entry.getValue().getDiscardCount() + "\t"
+ entry.getValue().getUpdateTime());
}
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void reloadConnector() {
try {
datahubClient.reloadConnector(Constant.projectName, Constant.topicName, connectorId);
System.out.println("reload connector successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void reloadConnectorByShard() {
try {
datahubClient.reloadConnector(Constant.projectName, Constant.topicName, connectorId, shardId);
System.out.println("reload connector successful");
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void getDoneTime() {
try {
GetConnectorDoneTimeResult getConnectorDoneTimeResult = datahubClient.getConnectorDoneTime(Constant.projectName, Constant.topicName, connectorId);
System.out.println(getConnectorDoneTimeResult.getDoneTime());
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void appendConnectorField() {
String newField = "newfield";
try {
//要求topic的schema和MaxCompute的table中都存在列newfield，并且表结构完全一致
datahubClient.appendConnectorField(Constant.projectName, Constant.topicName, connectorId, newField);
} catch (InvalidParameterException e) {
// invalid parameter, please check your parameter
e.printStackTrace();
throw e;
} catch (AuthorizationFailureException e) {
// AK error, please check your accessId and accessKey
e.printStackTrace();
throw e;
} catch (ResourceNotFoundException e) {
// project or topic or connector not found
e.printStackTrace();
throw e;
} catch (DatahubClientException e) {
// other error
e.printStackTrace();
throw e;
}
}
public static void main(String[] args) {
init();
createConnector();
// getConnector();
//
// updateConnector();
//
// updateConnectorState();
//
// updateConnectorOffset();
//
// listConnector();
//
// getConnector();
//
// getConnectorShardStatus();
//
// getConnectorShardStatusByShard();
//
// reloadConnector();
//
// reloadConnectorByShard();
//
// appendConnectorField();
//
// deleteConnector();
}
}