com.huaweicloud.dws.client.TableConfig Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client;

import com.huaweicloud.dws.client.model.ConflictStrategy;
import com.huaweicloud.dws.client.model.Constants;
import com.huaweicloud.dws.client.model.CopyMode;
import com.huaweicloud.dws.client.model.CreateTempTableMode;
import com.huaweicloud.dws.client.model.WriteMode;
import lombok.Getter;

import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @ProjectName: dws-connector
 * @ClassName: TableConfig
 * @Description:
 * @Date: 2023/2/15 19:09
 * @Version: 1.0
 */
@Getter
public class TableConfig implements Serializable {

    /**
     * 运行参数
     */
    protected ConflictStrategy conflictStrategy = ConflictStrategy.INSERT_OR_UPDATE;

    /**
     * 写入模式
     */
    protected WriteMode writeMode = WriteMode.AUTO;

    protected CopyMode copyMode = CopyMode.CSV;
    protected String delimiter = copyMode.getDelimiter();
    protected String eof = copyMode.getEof();

    /**
     * 在 WriteMode.AUTO 为自动模式下，数据量最低多少使用copy方式入库
     */
    protected int copyWriteBatchSize = 6000;
    protected long autoFlushMaxIntervalMs = 3000L;
    protected int autoFlushBatchSize = 5000;

    protected boolean updateAll = true;
    protected Set compareField;

    /**
     * 事务自动提交
     */
    protected boolean autoCommit = true;

    /**
     * 缓存溢出系数
     */
    protected int batchOutWeighRatio = 1;

    /**
     * 内核优化参数
     */
    protected boolean enableHstoreUpsertAutocommit;

    protected List uniqueKeys;

    protected boolean caseSensitive;

    protected CreateTempTableMode createTempTableMode = CreateTempTableMode.AS;
    protected String tempType = "";

    /**
     * 如果数据库是时间类型，数据为数字类型将当时间戳入库
     */
    protected boolean numberAsEpochMsForDatetime;

    /**
     * 如果数据库是时间类型，数据为字符串，将用该格式转换为时间类型
     */
    protected String stringToDatetimeFormat;

    protected boolean securityMode;

    /**
     * 是否开启binlog
     */
    protected boolean binlog = Constants.BINLOG;

    /**
     * 是否是cdc模式（会影响一定性能，内核侧会反查）
     */
    protected boolean cdcMode = Constants.CDC_MODE;

    /**
     * 复制槽名
     */
    protected String binlogSlotName;

    /**
     * 读取binlog数据出错后重试的次数
     */
    protected int binlogMaxRetryTimes = Constants.BINLOG_MAX_RETRY_TIMES;

    /**
     * 读取binlog数据出错后的重试时间间隔，单位为毫秒
     */
    protected long binlogRetryInterval = Constants.BINLOG_RETRY_INTERVAL;

    /**
     * 增量读取binlog的数据行数
     */
    protected int binlogBatchReadSize = Constants.BINLOG_BATCH_READ_SIZE;

    /**
     * 全量读取binlog的数据行数
     */
    protected int fullSyncBinlogBatchReadSize = Constants.FULL_SYNC_BINLOG_BATCH_READ_SIZE;

    /**
     * 读取binlog数据时启动的线程数
     */
    protected int binlogParallelNum = Constants.BINLOG_PARALLEL_NUM;

    /**
     * 读取binlog超时时间,单位毫秒
     */
    protected long binlogReadTimeout = Constants.BINLOG_READ_TIMEOUT;

    /**
     * 全量同步binlog时超时时间,单位毫秒
     */
    protected long fullSyncBinlogReadTimeout = Constants.FULL_SYNC_BINLOG_READ_TIMEOUT;

    /**
     * 读取不到binlog数据时休眠的时间,单位毫秒
     */
    protected long binlogSleepTime = Constants.BINLOG_SLEEP_TIME;

    /**
     * 读取不到binlog数据时最大休眠的时间,单位毫秒
     */
    protected long binlogMaxSleepTime = Constants.BINLOG_MAX_SLEEP_TIME;

    /**
     * 是否在主线程中统一获取同步点以及提交同步点(目前仅用于长稳测试的jar包)
     */
    protected boolean waitRegisterSyncPoint = Constants.WAIT_REGISTER_SYNC_POINT;

    /**
     * 是否需要适配dws扩容重分布流程（需要和内核版本匹配）
     */
    protected boolean needRedistribution = Constants.NEED_REDISTRIBUTION;

    /**
     * 是否使用新的系统字段来处理获取增量函数（需要和内核版本匹配）
     */
    protected boolean newSystemValue = Constants.NEW_SYSTEM_VALUE;

    /**
     * binlog数据队列大小,如果是0则为无界队列
     */
    protected int binlogQueueSize = Constants.BINLOG_QUEUE_SIZE;

    /**
     * 节点变化时调用内核接口报错信息
     */
    protected String errorMessage = Constants.ERROR_MESSAGE;

    /**
     * 控制更新同步点超时时间,单位: 秒
     */
    protected int updateSyncPointTimeOut = Constants.UPDATE_SYNC_POINT_TIME_OUT;

    /**
     * tableConfig对应的表名
     */
    protected String tableName;

    /**
     * 表中是否包含binlog系统表字段
     */
    protected boolean containBinlogSysValue;

    /**
     * 检测节点变化时间间隔,单位毫秒
     */
    protected long checkNodeChangeInterval = Constants.CHECK_NODE_CHANGE_INTERVAL;

    public TableConfig withNumberAsEpochMsForDatetime(boolean numberAsEpochMsForDatetime) {
        this.numberAsEpochMsForDatetime = numberAsEpochMsForDatetime;
        return this;
    }

    public TableConfig withStringToDatetimeFormat(String stringToDatetimeFormat) {
        this.stringToDatetimeFormat = stringToDatetimeFormat;
        return this;
    }

    public TableConfig withUniqueKeys(List uniqueKeys) {
        this.uniqueKeys = uniqueKeys;
        return this;
    }

    public TableConfig withConflictStrategy(ConflictStrategy conflictStrategy) {
        this.conflictStrategy = conflictStrategy;
        return this;
    }

    public TableConfig withWriteMode(WriteMode writeMode) {
        this.writeMode = writeMode;
        return this;
    }

    public TableConfig withCopyWriteBatchSize(int copyWriteBatchSize) {
        this.copyWriteBatchSize = copyWriteBatchSize;
        return this;
    }

    public TableConfig withAutoFlushMaxIntervalMs(long autoFlushMaxIntervalMs) {
        this.autoFlushMaxIntervalMs = autoFlushMaxIntervalMs;
        return this;
    }

    public TableConfig withAutoFlushBatchSize(int autoFlushBatchSize) {
        this.autoFlushBatchSize = autoFlushBatchSize;
        return this;
    }

    public TableConfig withBatchOutWeighRatio(int batchOutWeighRatio) {
        this.batchOutWeighRatio = batchOutWeighRatio;
        return this;
    }

    public TableConfig withEnableHstoreUpsertAutocommit(boolean enableHstoreUpsertAutocommit) {
        this.enableHstoreUpsertAutocommit = enableHstoreUpsertAutocommit;
        return this;
    }

    public TableConfig withCopyMode(CopyMode mode, String delimiter, String eof) {
        this.copyMode = mode;
        this.delimiter = delimiter;
        this.eof = eof;
        return this;
    }

    public TableConfig withCopyMode(CopyMode mode) {
        this.copyMode = mode;
        this.delimiter = mode.getDelimiter();
        this.eof = mode.getEof();
        return this;
    }

    public TableConfig withCaseSensitive(boolean caseSensitive) {
        this.caseSensitive = caseSensitive;
        return this;
    }

    public TableConfig withCreateTempTableMode(CreateTempTableMode createTempTableMode) {
        this.createTempTableMode = createTempTableMode;
        return this;
    }

    public TableConfig withUpdateAll(boolean updateAll) {
        this.updateAll = updateAll;
        return this;
    }

    public TableConfig withBinlog(boolean binlog) {
        this.binlog = binlog;
        return this;
    }

    public TableConfig withBinlogSlotName(String binlogSlotName) {
        this.binlogSlotName = binlogSlotName;
        return this;
    }

    public TableConfig withBinlogMaxRetryTimes(int binlogMaxRetryTimes) {
        this.binlogMaxRetryTimes = binlogMaxRetryTimes;
        return this;
    }

    public TableConfig withBinlogRetryInterval(long binlogRetryInterval) {
        this.binlogRetryInterval = binlogRetryInterval;
        return this;
    }

    public TableConfig withTempTableType(String type) {
        this.tempType = type;
        return this;
    }

    public TableConfig withBinlogBatchReadSize(int binlogBatchReadSize) {
        this.binlogBatchReadSize = binlogBatchReadSize;
        return this;
    }

    public TableConfig withFullSyncBinlogBatchReadSize(int fullSyncBinlogBatchReadSize) {
        this.fullSyncBinlogBatchReadSize = fullSyncBinlogBatchReadSize;
        return this;
    }

    public TableConfig withBinlogParallelNum(int binlogParallelNum) {
        this.binlogParallelNum = binlogParallelNum;
        return this;
    }

    public TableConfig withBinlogReadTimeout(long binlogReadTimeout) {
        this.binlogReadTimeout = binlogReadTimeout;
        return this;
    }

    public TableConfig withBinlogSleepTime(long binlogSleepTime) {
        this.binlogSleepTime = binlogSleepTime;
        return this;
    }

    public TableConfig withCdcMode(boolean cdcMode) {
        this.cdcMode = cdcMode;
        return this;
    }

    public TableConfig withFullSyncBinlogReadTimeout(long fullSyncBinlogReadTimeout) {
        this.fullSyncBinlogReadTimeout = fullSyncBinlogReadTimeout;
        return this;
    }

    public TableConfig withBinlogMaxSleepTime(long binlogMaxSleepTime) {
        this.binlogMaxSleepTime = binlogMaxSleepTime;
        return this;
    }

    public TableConfig withWaitRegisterSyncPoint(boolean waitRegisterSyncPoint) {
        this.waitRegisterSyncPoint = waitRegisterSyncPoint;
        return this;
    }

    public TableConfig withNeedRedistribution(boolean needRedistribution) {
        this.needRedistribution = needRedistribution;
        return this;
    }

    public TableConfig withNewSystemValue(boolean newSystemValue) {
        this.newSystemValue = newSystemValue;
        return this;
    }

    public TableConfig withUpdateSyncPointTimeOut(int updateSyncPointTimeOut) {
        this.updateSyncPointTimeOut = updateSyncPointTimeOut;
        return this;
    }

    public TableConfig withTableName(String tableName) {
        this.tableName = tableName;
        return this;
    }

    public TableConfig withContainBinlogSysValue(boolean containBinlogSysValue) {
        this.containBinlogSysValue = containBinlogSysValue;
        return this;
    }

    public TableConfig withBinlogQueueSize(int binlogQueueSize) {
        this.binlogQueueSize = binlogQueueSize;
        return this;
    }

    public TableConfig withErrorMessage(String errorMessage) {
        this.errorMessage = errorMessage;
        return this;
    }

    public TableConfig withCheckNodeChangeInterval(long checkNodeChangeInterval) {
        this.checkNodeChangeInterval = checkNodeChangeInterval;
        return this;
    }

    public TableConfig withCompareField(String compareField) {
        if (compareField == null) {
            return this;
        }
        this.compareField = Arrays.stream(compareField.split(",")).collect(Collectors.toSet());
        return this;
    }

    public TableConfig withCompareField(Set compareField) {
        this.compareField = compareField;
        return this;
    }

    public TableConfig withAutoCommit(boolean autoCommit) {
        this.autoCommit = autoCommit;
        return this;
    }

    public TableConfig withSecurityMode(boolean securityMode) {
        this.securityMode = securityMode;
        return this;
    }

    public TableConfig copy() {
        return new TableConfig()
                .withAutoFlushBatchSize(this.autoFlushBatchSize)
                .withAutoFlushMaxIntervalMs(this.autoFlushMaxIntervalMs)
                .withConflictStrategy(this.conflictStrategy)
                .withCopyMode(this.copyMode, this.delimiter, this.eof)
                .withEnableHstoreUpsertAutocommit(this.enableHstoreUpsertAutocommit)
                .withBatchOutWeighRatio(this.batchOutWeighRatio)
                .withCopyWriteBatchSize(this.copyWriteBatchSize)
                .withCaseSensitive(this.caseSensitive)
                .withCreateTempTableMode(this.createTempTableMode)
                .withNumberAsEpochMsForDatetime(this.numberAsEpochMsForDatetime)
                .withStringToDatetimeFormat(this.stringToDatetimeFormat)
                .withWriteMode(this.writeMode)
                .withBinlog(this.binlog)
                .withBinlogMaxRetryTimes(this.binlogMaxRetryTimes)
                .withBinlogRetryInterval(this.binlogRetryInterval)
                .withBinlogBatchReadSize(this.binlogBatchReadSize)
                .withFullSyncBinlogBatchReadSize(this.fullSyncBinlogBatchReadSize)
                .withBinlogParallelNum(this.binlogParallelNum)
                .withBinlogReadTimeout(this.binlogReadTimeout)
                .withBinlogSleepTime(this.binlogSleepTime)
                .withCdcMode(this.cdcMode)
                .withFullSyncBinlogReadTimeout(this.fullSyncBinlogReadTimeout)
                .withBinlogMaxSleepTime(this.binlogMaxSleepTime)
                .withWaitRegisterSyncPoint(this.waitRegisterSyncPoint)
                .withNeedRedistribution(this.needRedistribution)
                .withNewSystemValue(this.newSystemValue)
                .withUpdateSyncPointTimeOut(this.updateSyncPointTimeOut)
                .withTableName(this.tableName)
                .withErrorMessage(this.errorMessage)
                .withBinlogQueueSize(this.binlogQueueSize)
                .withCompareField(this.compareField)
                .withUpdateAll(this.updateAll)
                .withAutoCommit(this.autoCommit)
                .withWriteMode(this.writeMode);
    }
}