com.huaweicloud.dws.client.DwsConfig Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client;

import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.InvalidException;
import com.huaweicloud.dws.client.function.DwsClientExceptionFunction;
import com.huaweicloud.dws.client.model.ConflictStrategy;
import com.huaweicloud.dws.client.model.Constants;
import com.huaweicloud.dws.client.model.CopyMode;
import com.huaweicloud.dws.client.model.CreateTempTableMode;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableName;
import com.huaweicloud.dws.client.model.WriteMode;
import com.huaweicloud.dws.client.util.AssertUtil;
import lombok.Getter;

import java.io.Serializable;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;

/**
 * @ProjectName: dws-connector
 * @ClassName: DwsConfig
 * @Description:
 * @Date: 2022/12/22 11:15
 * @Version: 1.0
 */
@Getter
public class DwsConfig extends TableConfig {
    public static final int DEFAULT_READ_BATCH_SIZE = 128;

    public static final int DEFAULT_READ_QUEUE = 256;

    private DwsConfig() {
        super();
    }

    /**
     * 成功后回调
     */
    private Consumer> successFunction;

    /**
     * 后台执行失败时回调
     */
    private DwsClientExceptionFunction errorFunction;

    private int maxFlushRetryTimes = 3;

    /**
     * 连接参数
     */
    private String url;
    private String driverName = "com.huawei.gauss200.jdbc.Driver";
    private String username;
    private String password;

    /**
     * 用于执行入库的线程数量
     */
    private int threadSize = 3;

    /**
     * 执行异步读表的线程数量
     */
    private int readThreadSize = 3;

    /**
     * 最多一次将readBatchSize条Get请求合并提交，默认128.
     */
    int readBatchSize = DEFAULT_READ_BATCH_SIZE;

    /**
     * get请求缓冲池大小，默认256.
     */
    int readBatchQueueSize = DEFAULT_READ_QUEUE;

    /**
     * get操作的超时时间.
     * 默认值0表示不超时，会在两处位置生效：
     * 1.get操作从用户开始执行到client准备提交到dws的等待时间，如果这个阶段超时不会重试直接抛出异常.
     * 2.get sql的执行超时，即statement query timeout，最小1s.
     */
    int readTimeoutMilliseconds = 0;
    int scanFetchSize = 2000;
    int scanTimeoutSeconds = 60;

    /**
     * 检查超时时间（秒）
     */
    private int connectionCheckTimeoutSeconds = 60;

    /**
     * 连接最大使用时间（秒）
     */
    private int connectionMaxUseTimeSeconds = 3600;

    /**
     * 连接最大空闲时间（毫秒）
     */
    private long connectionMaxIdleMs = 60000L;

    /**
     * 元数据的最大缓存时间
     */
    private int metadataCacheSeconds = 180;

    /**
     * 打印详细日志开关
     */
    private boolean logSwitch = false;

    /**
     * 用于配置需要打印入库数据的表名，方便在调试阶段排查问题
     */
    private Set logDataTables;

    /**
     * 重试时等待时间基数
     */
    private long retryBaseTime = 1000L;

    /**
     * 重试时等待 随机时间因子
     */
    private int retryRandomTime = 300;

    /**
     * 读取binlog对应的表
     */
    private String binlogTableName;


    /**
     * 是否自动关闭dws-client服务
     */
    private boolean autoCloseDwsClient = true;


    /**
     * 表及参数收集
     */
    private final Map tableConfigMap = new ConcurrentHashMap<>();

    private long timeOutMs = Constants.CONNECTION_TIME_OUT;

    // 连接池相关参数
    private String connectionPoolName = Constants.CONNECTION_POOL_NAME;

    /**
     * 连接池大小
     */
    private int connectionPoolSize = Constants.CONNECTION_POOL_SIZE;

    /**
     * 获取从连接池中获取连接的超时时间，单位毫秒
     */
    private long connectionPoolTimeout = Constants.CONNECTION_POOL_TIMEOUT;

    /**
     * 连接处理超时时间，单位毫秒
     */
    private int connectionSocketTimeout = Constants.CONNECTION_SOCKET_TIMEOUT;

    /**
     * 单个连接最大使用次数
     */
    private long connectionMaxUseCount = Constants.CONNECTION_MAX_USE_COUNT;

    public TableConfig getTableConfig(TableName tableName) {
        return Optional.ofNullable(tableConfigMap.get(tableName)).orElse(this);
    }

    public TableConfig getTableConfig(String tableName) {
        return Optional.ofNullable(tableConfigMap.get(TableName.valueOf(tableName))).orElse(this);
    }

    public static Builder builder() {
        return new Builder();
    }

    public static final class Builder implements Serializable {
        final DwsConfig config;

        public Builder() {
            config = new DwsConfig();
        }

        public Builder withCopyWriteBatchSize(int size) {
            config.copyWriteBatchSize = size;
            return this;
        }

        public Builder withConnectionMaxIdleMs(long ms) {
            config.connectionMaxIdleMs = ms;
            return this;
        }

        public Builder withMetadataCacheSeconds(int seconds) {
            config.metadataCacheSeconds = seconds;
            return this;
        }

        public Builder withConnectionMaxUseTime(int maxTime) {
            config.connectionMaxUseTimeSeconds = maxTime;
            return this;
        }

        public Builder withThreadSize(int threadSize) {
            config.threadSize = threadSize;
            return this;
        }

        public Builder withReadThreadSize(int threadSize) {
            config.readThreadSize = threadSize;
            return this;
        }

        public Builder withReadBatchSize(int batchSize) {
            config.readBatchSize = batchSize;
            return this;
        }

        public Builder withReadBatchQueueSize(int batchQueueSize) {
            config.readBatchQueueSize = batchQueueSize;
            return this;
        }

        public Builder withReadTimeoutMilliseconds(int readTimeoutMilliseconds) {
            config.readTimeoutMilliseconds = readTimeoutMilliseconds;
            return this;
        }

        public Builder withScanFetchSize(int scanFetchSize) {
            config.scanFetchSize = scanFetchSize;
            return this;
        }

        public Builder withScanTimeoutSeconds(int timeoutSeconds) {
            config.scanTimeoutSeconds = timeoutSeconds;
            return this;
        }

        public Builder withConnectionCheckTimeout(int timeout) {
            config.connectionCheckTimeoutSeconds = timeout;
            return this;
        }

        /**
         * 功能同withConflictStrategy，该名称错误
         *
         * @deprecated withConflictStrategy
         */
        @Deprecated
        public Builder withPolicyStrategy(ConflictStrategy conflictStrategy) throws InvalidException {
            AssertUtil.nonNull(conflictStrategy, new InvalidException("policyMode is null"));
            config.conflictStrategy = conflictStrategy;
            return this;
        }

        public Builder withConflictStrategy(ConflictStrategy conflictStrategy) throws InvalidException {
            AssertUtil.nonNull(conflictStrategy, new InvalidException("conflictStrategy is null"));
            config.conflictStrategy = conflictStrategy;
            return this;
        }

        public Builder withWriteMode(WriteMode writeMode) throws InvalidException {
            AssertUtil.nonNull(writeMode, new InvalidException("writeMode is null"));
            config.writeMode = writeMode;
            return this;
        }

        public Builder withAutoFlushBatchSize(int size) {
            config.autoFlushBatchSize = size;
            return this;
        }

        public Builder withAutoFlushMaxIntervalMs(long intervalMs) {
            config.autoFlushMaxIntervalMs = intervalMs;
            return this;
        }

        public Builder withMaxFlushRetryTimes(int maxRetries) {
            config.maxFlushRetryTimes = maxRetries;
            return this;
        }

        public Builder withUrl(String url) {
            config.url = url.startsWith("jdbc:postgresql") ?
                    url.replace("jdbc:postgresql", "jdbc:gaussdb") :
                    url;
            return this;
        }

        /**
         * @deprecated 固定驱动，无法再配置，只能使用gauss200
         */
        @Deprecated
        public Builder withDriverName(String driverName) {
            return this;
        }

        public Builder withUsername(String username) {
            config.username = username;
            return this;
        }

        public Builder withPassword(String password) {
            config.password = password;
            return this;
        }

        public Builder withLogSwitch(boolean logSwitch) {
            config.logSwitch = logSwitch;
            return this;
        }

        public Builder withTimeOutMs(long timeOut) {
            config.timeOutMs = timeOut;
            return this;
        }

        public Builder withTableConfig(String tableName, TableConfig tableConfig) {
            config.tableConfigMap.put(TableName.valueOf(tableName), tableConfig.withTableName(tableName));
            return this;
        }

        public TableConfig withTableConfig(String tableName) {
            return config.tableConfigMap.computeIfAbsent(TableName.valueOf(tableName), tableName1 -> config.copy());
        }

        public Builder withCopyMode(CopyMode mode, String delimiter, String eof) {
            config.copyMode = mode;
            config.delimiter = delimiter;
            config.eof = eof;
            return this;
        }

        public Builder withCopyMode(CopyMode mode) {
            config.copyMode = mode;
            config.delimiter = mode.getDelimiter();
            config.eof = mode.getEof();
            return this;
        }

        public Builder withLogDataTables(String... tables) {
            Set tableNames = new HashSet<>(tables.length);
            Arrays.stream(tables).forEach(table -> tableNames.add(TableName.valueOf(table)));
            config.logDataTables = tableNames;
            return this;
        }

        public Builder withRetryBaseTime(long retryBaseTime) {
            config.retryBaseTime = retryBaseTime;
            return this;
        }

        public Builder withRetryRandomTime(int retryRandomTime) {
            config.retryRandomTime = retryRandomTime;
            return this;
        }

        public Builder withBatchOutWeighRatio(int batchOutWeighRatio) {
            config.batchOutWeighRatio = batchOutWeighRatio;
            return this;
        }

        public Builder onFlushSuccess(Consumer> consumer) {
            config.successFunction = consumer;
            return this;
        }

        public Builder onError(DwsClientExceptionFunction consumer) {
            config.errorFunction = consumer;
            return this;
        }

        public Builder withEnableHstoreUpsertAutocommit(boolean enableHstoreUpsertAutocommit) {
            config.enableHstoreUpsertAutocommit = enableHstoreUpsertAutocommit;
            return this;
        }

        public Builder withCaseSensitive(boolean caseSensitive) {
            config.caseSensitive = caseSensitive;
            return this;
        }

        public Builder withNumberAsEpochMsForDatetime(boolean numberAsEpochMsForDatetime) {
            config.numberAsEpochMsForDatetime = numberAsEpochMsForDatetime;
            return this;
        }

        public Builder withStringToDatetimeFormat(String stringToDatetimeFormat) {
            config.stringToDatetimeFormat = stringToDatetimeFormat;
            return this;
        }

        public Builder withCompareField(String compareField) {
            if (compareField == null) {
                return this;
            }
            config.compareField = Arrays.stream(compareField.split(",")).collect(Collectors.toSet());
            return this;
        }

        public Builder withCompareField(Set compareField) {
            config.compareField = compareField;
            return this;
        }

        public Builder withAutoCommit(boolean autoCommit) {
            config.autoCommit = autoCommit;
            return this;
        }

        public Builder withBinlogTableName(String binlogTableName) {
            config.binlogTableName = binlogTableName;
            return this;
        }

        public Builder withConnectionPoolName(String connectionPoolName) {
            config.connectionPoolName = connectionPoolName;
            return this;
        }

        public Builder withConnectionPoolSize(int connectionPoolSize) {
            config.connectionPoolSize = connectionPoolSize;
            return this;
        }

        public Builder withConnectionPoolTimeout(long connectionPoolTimeout) {
            config.connectionPoolTimeout = connectionPoolTimeout;
            return this;
        }

        public Builder withConnectionSocketTimeout(int connectionSocketTimeout) {
            config.connectionSocketTimeout = connectionSocketTimeout;
            return this;
        }

        public Builder withConnectionMaxUseCount(long connectionMaxUseCount) {
            config.connectionMaxUseCount = connectionMaxUseCount;
            return this;
        }

        public Builder withSecurityMode(boolean securityMode) {
            config.securityMode = securityMode;
            return this;
        }

        public Builder withCreateTempTableMode(CreateTempTableMode createTempTableMode) {
            config.createTempTableMode = createTempTableMode;
            return this;
        }

        public Builder withAutoCloseDwsClient(boolean autoCloseDwsClient) {
            config.autoCloseDwsClient = autoCloseDwsClient;
            return this;
        }

        public DwsConfig build() {
            return config;
        }
    }
}