com.taobao.drc.clusterclient.BaseClusterContext Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of consumer-core Show documentation
Show all versions of consumer-core Show documentation
The java consumer core component for Data Transmission Service
package com.taobao.drc.clusterclient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
/**
* BaseClusterContext:作为Context基类，应尽量通用，便于扩展（主要属性值存储在properties中）
*
* @author yangyang
* @since 2017/7/4
*/
public class BaseClusterContext {
private static Logger logger = LoggerFactory.getLogger(BaseClusterContext.class);
protected final Properties properties;
/**************DRC RM管控地址参数相关*******************/
public static final String TAG_RM_ADDR = "rm.addr";
/***此类属性:可被发送给server,通常由用户提供*****/
public static final String TAG_USER_ASK_SELFUNIT = "askSelfUnit"; //是否只抓取单元内应用写入的数据:true:只抓取本单元内的数据,默认为false
private final String DEFAULT_USER_ASK_SELFUNIT = "false";
/*****************属性名称************************/
//位点ACK相关
public final static String TAG_ACKFORWARD = "ack.forward";
public final static String TAG_TASKTYPE = "taskType";
public final static String TAG_ACCESSKEYID = "accessKeyId";
public final static String TAG_SECRET = "secret";
public final static String TAG_TS = "ts";
public final static String TAG_GUID = "guid";
public final static String TAG_CMURL = "cmUrl";
public final static String TAG_MANAGER_HOST = "manager.host";
public final static String TAG_USER = "user";
public final static String TAG_PASSWORD = "password";
public final static String TAG_DATATYPE = "dataType";
public final static String TAG_USEPUBLICIP = "usePublicIp";
public final static String TAG_USEBINARY = "useBinary";
public final static String TAG_USEDRCNET = "useDrcNet";
public final static String TAG_CM_MAXRETRYTIMES = "cm.maxRetriedTimes";
public final static String TAG_STORE_MAXRETRYTIMES = "server.maxRetriedTimes"; //Store重试次数
public final static String TAG_THREADNAMEPREFIX = "threadNamePrefix";
public static final String TAG_CM_DEFAULT_MAX_COUNT_OF_BUFFERED_RECORDS_NOT_ACKED = "cm.defaultMaxCountOfBufferedRecordNotAcked";
public static final String TAG_CM_DEFAULT_MAX_SIZE_OF_NOTIFY_MESSAGE_QUEUE = "cm.defaultMaxSizeOfNotifyMessageQueue";
/***************系统参数配置相关*********************/
public static final String TAG_DTS_TASK_SCHEDULE_INTERVAL_MS = "dts.taskScheduleIntervalMs"; /*调度任务调度间隔*/
public static final String DEFAULT_DTS_TASK_SCHEDULE_INTERVAL_MS = "10";
final static String TAG_SERVER_MESSAGE_TYPE = "server.messageType";
static final String TAG_SERVER_MAX_RETRIES = "server.maxRetriedTimes";
final static String TAG_USE_DRC_NET = "useDrcNet";
public final static String TAG_GROUP = "group";
/*过滤条件相关*/
public static final String TAG_FILTER_TXN = "filter.txn"; /*是否需要消息类型::begin/commit, 默认为:true, 设置为false:则不需要此类信息*/
/*单元化配置相关*/
public static final String TAG_ASKSELFUNIT = "unit.askSelfUnit"; /*是否只拉取本Unit数据，默认:false*/
/*客户段Hash相关*/
public static final String TAG_CLIENT_HASH_KEY = "client.hashKey";
public static final String TAG_CLIENT_HASH_MASK = "client.hashMask";
/**************属性默认值相关************************/
//位点ACK默认值相关
public static final String DEFAULT_ACK_FORWARD_DTS = "cm";
public static final String DEFAULT_ACK_FORWARD_DRC = "local";
public static final String DEFAULT_ACK_FORWARD_DSTORE = "local";
private static final String DEFAULT_THREADNAMEPREFIX = "DTS_";
private final String DEFAULT_STORE_MAXRETRYTIMES = "100"; //设置Client重连Store次数
protected final String DEFAULT_USEDRCNET = "false"; //
protected final String DEFAULT_USEBINARY = "true";
private final String DEFAULT_USEPUBLICIP = "false";
private final String DEFAULT_FILTER_TXN = "true";
private final String DEFAULT_ASKSELFUNIT = "false";
protected final int DEFAULT_MAX_COUNT_OF_BUFFERED_RECORDS_NOT_ACKED = 8 * 1024; //最多未ACK的Record数目
public static final int DEFAULT_MAX_SIZE_OF_NOTIFY_MESSAGE_QUEUE = 4 * 1024; //Notify消息队列最大条数
private int maxConns = 10240;
private String coordinatorThreadNamePrefix = "DTS_";
private int maxConsumerNumPerCoordinator = 1024;
private int maxIOThreadNumPerCoordinator = Runtime.getRuntime().availableProcessors() * 2;
private boolean notifierFair = false;
public BaseClusterContext() {
this(new Properties());
}
public BaseClusterContext(Properties properties) {
this.properties = properties;
}
/**
* 获取Properties
*
* @return 属性
*/
public final Properties getProperties() {
return this.properties;
}
/*Store重试次数*/
public int getStoreMaxRetryTimes() {
return Integer.valueOf(properties.getProperty(TAG_STORE_MAXRETRYTIMES, DEFAULT_STORE_MAXRETRYTIMES));
}
public void setStoreMaxRetryTimes(int maxRetryTimes) {
properties.setProperty(TAG_STORE_MAXRETRYTIMES, String.valueOf(maxRetryTimes));
}
public int getTaskScheduleIntervalMs() {
return Integer.valueOf(properties.getProperty(TAG_DTS_TASK_SCHEDULE_INTERVAL_MS,DEFAULT_DTS_TASK_SCHEDULE_INTERVAL_MS));
}
/**
* 获取任务类型
*/
public final String getTasktype() { return properties.getProperty(TAG_TASKTYPE); }
public String getDataType() { return properties.getProperty(TAG_DATATYPE); }
/**
* 设置ClusterClinet指定的集群地址
*
* @param clusterUrl 集群地址
*/
public void setClusterUrl(final String clusterUrl) {
properties.setProperty(TAG_MANAGER_HOST, clusterUrl);
}
/**
* 获取ClusterClinet指定的集群地址
*
* @return 集群地址
*/
public final String getClusterUrl() {
return properties.getProperty(TAG_MANAGER_HOST);
}
/**
* 设置ClusterClient的消费GUID
*
* @param guid 消费GUID
*/
public void setAppGuid(final String guid) {
properties.setProperty(TAG_GUID, guid);
}
/**
* 获取ClusterClient的消费的GUID
*
* @return 消费GUID
*/
public final String getAppGuid() {
return properties.getProperty(TAG_GUID);
}
/**
* 设置ClusterClient属于的应用组名
*
* @param group 用户组名
*/
public void setAppGroup(final String group) {
properties.setProperty(TAG_GROUP, group);
}
/**
* 获取ClusterClient所属的应用组名
*
* @return 用户组名
*/
public final String getAppGroup() {
return properties.getProperty(TAG_GROUP);
}
/**
* 设置用户所属组用户名
*
* @param username 用户名
*/
public void setAppGroupUserName(final String username) {
properties.setProperty(TAG_USER,username);
}
/**
* 获取用户组用户名
*
* @return 用户名
*/
public final String getAppGroupUserName() {
return properties.getProperty(TAG_USER);
}
/**
* 设置用户组密码
*
* @param password 密码
*/
public void setAppGroupPassword(final String password) {
properties.setProperty(TAG_PASSWORD, password);
}
/**
* 获取用户组密码
*
* @return 密码
*/
public final String getAppGroupPassword() {
return properties.getProperty(TAG_PASSWORD);
}
public boolean isUseBinary() {
return Boolean.valueOf(properties.getProperty(TAG_USEBINARY, DEFAULT_USEBINARY));
}
public void setUseBinary(boolean useBinary) {
this.properties.put(TAG_USEBINARY, String.valueOf(useBinary));
}
/**USER_ASK_SELFUNIT 属性设置**/
public void askSelfUnit(){ this.properties.put(TAG_USER_ASK_SELFUNIT, String.valueOf(true)); }
public void setAskSelfUnit(boolean booleanVal){ this.properties.put(TAG_USER_ASK_SELFUNIT, String.valueOf(booleanVal)); }
public boolean getAskSelfUnit() { return Boolean.valueOf(properties.getProperty(TAG_USER_ASK_SELFUNIT, DEFAULT_USER_ASK_SELFUNIT)); }
/**
* 设置drcnet开关
*
* @param useDrcNet 如果为true，则开启使用drcnet模式进行数据传输
*/
public void setUseDrcNet(final boolean useDrcNet) {
properties.setProperty(TAG_USE_DRC_NET, String.valueOf(useDrcNet));
}
/**
* 获取是否开启drcnet
*
* @return drcnet
*/
public final boolean getUseDrcNet() {
return Boolean.valueOf(properties.getProperty(TAG_USE_DRC_NET, DEFAULT_USEDRCNET));
}
public boolean isUsePublicIp() {
return Boolean.valueOf(properties.getProperty(TAG_USEPUBLICIP, DEFAULT_USEPUBLICIP));
}
public void setUsePublicIp(boolean usePublicIp) {
properties.setProperty(TAG_USEPUBLICIP, String.valueOf(usePublicIp));
}
/**
* 设置该ClusterClient支持partition的最大连接数
*
* @param maxConns 允许的最大连接数
*/
public void setMaxConns(int maxConns) {
this.maxConns = maxConns;
}
/**
* 获取该ClusterClient支持partition的最大连接数
*
* @return 返回当前设置的最大连接数
*/
public final int getMaxConns() {
return this.maxConns;
}
/**
* 设置clusterClient内部线程名字的前缀
* @param prefix 线程名前缀
*/
public void setThreadNamePrefix(String prefix) {
properties.setProperty(TAG_THREADNAMEPREFIX,prefix);
}
/**
* 返回线程命名的前缀
* @return 线程名前缀
*/
public String getThreadNamePrefix() {
return (String) properties.getOrDefault(TAG_THREADNAMEPREFIX,DEFAULT_THREADNAMEPREFIX);
}
/**
* 获取Coordinator线程池使用的线程名前缀。
*
* @return 线程名前缀
*/
public String getCoordinatorThreadNamePrefix() {
return coordinatorThreadNamePrefix;
}
/**
* 设置Coordinator线程池使用的线程名前缀
* *
*
* @param coordinatorThreadNamePrefix 线程名前缀
*/
public void setCoordinatorThreadNamePrefix(String coordinatorThreadNamePrefix) {
this.coordinatorThreadNamePrefix = coordinatorThreadNamePrefix;
}
/**
* 获取一个Coordinator上允许管理的最大的consumer的数量
*
* @return 一个Coordinator上允许管理的最大的consumer的数量
*/
public int getMaxConsumerNumPerCoordinator() {
return maxConsumerNumPerCoordinator;
}
/**
* 设置一个Coordinator上允许管理的最大的consumer的数量
*
* @param maxConsumerNumPerCoordinator 一个Coordinator上允许管理的最大的consumer的数量
*/
public void setMaxConsumerNumPerCoordinator(int maxConsumerNumPerCoordinator) {
this.maxConsumerNumPerCoordinator = maxConsumerNumPerCoordinator;
}
/**
* 获取一个Coordinator上最大的用于IO访问，包括启动、停止client，获取partition的信息等操作的线程数
*
* @return 一个Coordinator上IO线程数的最大值
*/
public int getMaxIOThreadNumPerCoordinator() {
return maxIOThreadNumPerCoordinator;
}
/**
* 设置一个Coordinator上最大的用于IO操作，包括启动、停止client，获取partition的信息等操作的线程数
*
* @param maxIOThreadNumPerCoordinator 一个Coordinator上IO操作线程数的最大值
*/
public void setMaxIOThreadNumPerCoordinator(int maxIOThreadNumPerCoordinator) {
this.maxIOThreadNumPerCoordinator = maxIOThreadNumPerCoordinator;
}
public boolean isNotifierFair() {
return notifierFair;
}
public void setNotifierFair(boolean notifierFair) {
this.notifierFair = notifierFair;
}
/**
* 获取最大允许缓存的记录数量
*
* @return 当前设置的最大允许缓存的记录数量
*/
public int getMaxCountOfBufferedRecordsNotAcked() {
return Integer.valueOf(
properties.getProperty(TAG_CM_DEFAULT_MAX_COUNT_OF_BUFFERED_RECORDS_NOT_ACKED,
String.valueOf(DEFAULT_MAX_COUNT_OF_BUFFERED_RECORDS_NOT_ACKED))
);
}
/**
* 设置最大允许缓存的记录数量
*
* @param maxCountOfBufferedRecordsNotAcked 要设置的最大允许缓存的记录数量
*/
public void setMaxCountOfBufferedRecordsNotAcked(int maxCountOfBufferedRecordsNotAcked) {
getProperties().put(TAG_CM_DEFAULT_MAX_COUNT_OF_BUFFERED_RECORDS_NOT_ACKED,
String.valueOf(maxCountOfBufferedRecordsNotAcked));
}
/**
* 获取notify线程中消息缓存队列设置的大小
*
* @return 返回当前notify线程中的消息缓存队列大小
*/
public int getMaxSizeOfNotifyMessageQueue() {
return Integer.valueOf(
properties.getProperty(TAG_CM_DEFAULT_MAX_SIZE_OF_NOTIFY_MESSAGE_QUEUE,
String.valueOf(DEFAULT_MAX_SIZE_OF_NOTIFY_MESSAGE_QUEUE))
);
}
/**
* 设置notify线程中消息缓存队列的大小
*
* @param maxSizeOfNotifyMessageQueue 要设置的notify线程中的消息缓存队列大小
*/
public void setMaxSizeOfNotifyMessageQueue(int maxSizeOfNotifyMessageQueue) {
getProperties().put(TAG_CM_DEFAULT_MAX_SIZE_OF_NOTIFY_MESSAGE_QUEUE, String.valueOf(maxSizeOfNotifyMessageQueue));
}
/*accessKeyId 与 secret 相关*/
public String getAccessKey() {
return properties.getProperty(TAG_ACCESSKEYID);
}
public String getSecret() {
return properties.getProperty(TAG_SECRET);
}
}