com.huaweicloud.dws.client.DwsConfig Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client;

import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.InvalidException;
import com.huaweicloud.dws.client.function.DwsClientExceptionFunction;
import com.huaweicloud.dws.client.model.ConflictStrategy;
import com.huaweicloud.dws.client.model.CopyMode;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableName;
import com.huaweicloud.dws.client.model.WriteMode;
import com.huaweicloud.dws.client.util.AssertUtil;
import lombok.Getter;

import java.io.Serializable;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;

/**
 * @ProjectName: dws-connector
 * @ClassName: DwsConfig
 * @Description:
 * @Date: 2022/12/22 11:15
 * @Version: 1.0
 */
@Getter
public class DwsConfig extends TableConfig {

    private DwsConfig() {
        super();
    }

    /**
     * 成功后回调
     */
    private Consumer> successFunction;

    /**
     * 后台执行失败时回调
     */
    private DwsClientExceptionFunction errorFunction;

    private int maxFlushRetryTimes = 3;

    /**
     * 连接参数
     */
    private String url;
    private String driverName = "com.huawei.gauss200.jdbc.Driver";
    private String username;
    private String password;

    /**
     * 用于执行入库的线程数量
     */
    private int threadSize = 3;

    /**
     * 检查超时时间（秒）
     */
    private int connectionCheckTimeoutSeconds = 60;

    /**
     * 连接最大使用时间（秒）
     */
    private int connectionMaxUseTimeSeconds = 3600;

    /**
     * 连接最大空闲时间（毫秒）
     */
    private long connectionMaxIdleMs = 60000L;

    /**
     * 元数据的最大缓存时间
     */
    private int metadataCacheSeconds = 180;

    /**
     * 打印详细日志开关
     */
    private boolean logSwitch = false;

    /**
     * 用于配置需要打印入库数据的表名，方便在调试阶段排查问题
     */
    private Set logDataTables;

    /**
     * 重试时等待时间基数
     */
    private long retryBaseTime = 1000L;

    /**
     *  重试时等待 随机时间因子
     */
    private int retryRandomTime = 300;

    /**
     * 表及参数收集
     */
    private final Map tableConfigMap = new ConcurrentHashMap<>();

    private long timeOutMs = 1000 * 60 * 5L;

    public TableConfig getTableConfig(TableName tableName) {
        return Optional.ofNullable(tableConfigMap.get(tableName)).orElse(this);
    }

    public static Builder builder() {
        return new Builder();
    }

    public static final class Builder implements Serializable {
        final DwsConfig config;

        public Builder() {
            config = new DwsConfig();
        }

        public Builder withCopyWriteBatchSize(int size) {
            config.copyWriteBatchSize = size;
            return this;
        }

        public Builder withConnectionMaxIdleMs(long ms) {
            config.connectionMaxIdleMs = ms;
            return this;
        }

        public Builder withMetadataCacheSeconds(int seconds) {
            config.metadataCacheSeconds = seconds;
            return this;
        }

        public Builder withConnectionMaxUseTime(int maxTime) {
            config.connectionMaxUseTimeSeconds = maxTime;
            return this;
        }

        public Builder withThreadSize(int threadSize) {
            config.threadSize = threadSize;
            return this;
        }

        public Builder withConnectionCheckTimeout(int timeout) {
            config.connectionCheckTimeoutSeconds = timeout;
            return this;
        }

        /**
         *  功能同withConflictStrategy，该名称错误
         * @deprecated withConflictStrategy
         */
        @Deprecated
        public Builder withPolicyStrategy(ConflictStrategy conflictStrategy) throws InvalidException {
            AssertUtil.nonNull(conflictStrategy, new InvalidException("policyMode is null"));
            config.conflictStrategy = conflictStrategy;
            return this;
        }

        public Builder withConflictStrategy(ConflictStrategy conflictStrategy) throws InvalidException {
            AssertUtil.nonNull(conflictStrategy, new InvalidException("conflictStrategy is null"));
            config.conflictStrategy = conflictStrategy;
            return this;
        }

        public Builder withWriteMode(WriteMode writeMode) throws InvalidException {
            AssertUtil.nonNull(writeMode, new InvalidException("writeMode is null"));
            config.writeMode = writeMode;
            return this;
        }

        public Builder withAutoFlushBatchSize(int size) {
            config.autoFlushBatchSize = size;
            return this;
        }

        public Builder withAutoFlushMaxIntervalMs(long intervalMs) {
            config.autoFlushMaxIntervalMs = intervalMs;
            return this;
        }

        public Builder withMaxFlushRetryTimes(int maxRetries) {
            config.maxFlushRetryTimes = maxRetries;
            return this;
        }

        public Builder withUrl(String url) {
            config.url = url.startsWith("jdbc:postgresql") ?
                    url.replace("jdbc:postgresql", "jdbc:gaussdb") :
                    url;
            return this;
        }

        /**
         * 固定驱动，无法再配置，只能使用gauss200
         * @deprecated
         */
        @Deprecated
        public Builder withDriverName(String driverName) {
            return this;
        }

        public Builder withUsername(String username) {
            config.username = username;
            return this;
        }

        public Builder withPassword(String password) {
            config.password = password;
            return this;
        }

        public Builder withLogSwitch(boolean logSwitch) {
            config.logSwitch = logSwitch;
            return this;
        }

        public Builder withTimeOutMs(long timeOut) {
            config.timeOutMs = timeOut;
            return this;
        }

        public Builder withTableConfig(String tableName, TableConfig tableConfig) {
            config.tableConfigMap.put(TableName.valueOf(tableName), tableConfig);
            return this;
        }

        public TableConfig withTableConfig(String tableName) {
            return config.tableConfigMap.computeIfAbsent(TableName.valueOf(tableName), tableName1 -> config.copy());
        }

        public Builder withCopyMode(CopyMode mode, String delimiter, String eof) {
            config.copyMode = mode;
            config.delimiter = delimiter;
            config.eof = eof;
            return this;
        }

        public Builder withCopyMode(CopyMode mode) {
            config.copyMode = mode;
            config.delimiter = mode.getDelimiter();
            config.eof = mode.getEof();
            return this;
        }

        public Builder withLogDataTables(String... tables) {
            Set tableNames = new HashSet<>(tables.length);
            Arrays.stream(tables).forEach(table -> tableNames.add(TableName.valueOf(table)));
            config.logDataTables = tableNames;
            return this;
        }

        public Builder withRetryBaseTime(long retryBaseTime) {
            config.retryBaseTime = retryBaseTime;
            return this;
        }

        public Builder withRetryRandomTime(int retryRandomTime) {
            config.retryRandomTime = retryRandomTime;
            return this;
        }

        public Builder withBatchOutWeighRatio(int batchOutWeighRatio) {
            config.batchOutWeighRatio = batchOutWeighRatio;
            return this;
        }

        public Builder onFlushSuccess(Consumer> consumer) {
            config.successFunction = consumer;
            return this;
        }

        public Builder onError(DwsClientExceptionFunction consumer) {
            config.errorFunction = consumer;
            return this;
        }

        public Builder withEnableHstoreUpsertAutocommit(boolean enableHstoreUpsertAutocommit) {
            config.enableHstoreUpsertAutocommit = enableHstoreUpsertAutocommit;
            return this;
        }

        public Builder withCaseSensitive(boolean caseSensitive) {
            config.caseSensitive = caseSensitive;
            return this;
        }

        public Builder withNumberAsEpochMsForDatetime(boolean numberAsEpochMsForDatetime) {
            config.numberAsEpochMsForDatetime = numberAsEpochMsForDatetime;
            return this;
        }

        public Builder withStringToDatetimeFormat(String stringToDatetimeFormat) {
            config.stringToDatetimeFormat = stringToDatetimeFormat;
            return this;
        }

        public Builder withCompareField(String compareField) {
            if (compareField == null) {
                return this;
            }
            config.compareField = Arrays.stream(compareField.split(",")).collect(Collectors.toSet());
            return this;
        }

        public Builder withCompareField(Set compareField) {
            config.compareField = compareField;
            return this;
        }

        public Builder withAutoCommit(boolean autoCommit) {
            config.autoCommit = autoCommit;
            return this;
        }


        public DwsConfig build() {
            return config;
        }
    }
}