com.huaweicloud.dws.client.executor.CopyMergeExecutor Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client.executor;

import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableSchema;
import com.huaweicloud.dws.client.util.JdbcUtil;
import com.huaweicloud.dws.client.util.LogUtil;
import com.huaweicloud.dws.client.worker.ConnectionProvider;
import lombok.extern.slf4j.Slf4j;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;

/**
 * @ProjectName: dws-connector
 * @ClassName: CopyException
 * @Description: 通过copy merge方式入库
 * @Date: 2023/1/17 16:50
 * @Version: 1.0
 */
@Slf4j
public class CopyMergeExecutor {

    private final MergeExecutor mergeExecutor = new MergeExecutor();
    private final CopyExecutor copyExecutor = new CopyExecutor();
    protected final ConnectionProvider connectionProvider;

    public CopyMergeExecutor(ConnectionProvider connectionProvider) {
        this.connectionProvider = connectionProvider;
    }

    public void execute(List records, Connection connection, DwsConfig config) throws DwsClientException {
        LogUtil.withLogSwitch(config, () -> log.info("start copy merge data to dws. executor is {}, records size = {}", this.getClass().getName(), records.size()));
        long startTime = System.currentTimeMillis();
        Record first = records.get(0);
        TableSchema schema = first.getTableSchema();
        List keys = schema.getPrimaryKeyNames();
        // 需要copy到的表
        String copyTable = schema.getTableName().getFullName();
        if (!keys.isEmpty()) {
            // 如果有主键，copy到临时表, 并且需要创建临时表
            copyTable = connectionProvider.getTempTable(schema);
        }
        // 将数据copy到数据库：临时表或者目标表
        copyExecutor.execute(records, connection, copyTable, config);
        // 如果有主键 从临时表merge 并且删除临时表
        if (!keys.isEmpty()) {
            try {
                storeTarget(records, connection, config, schema, copyTable);
            } finally {
                try {
                    JdbcUtil.executeSql(connection, "TRUNCATE " + copyTable);
                } catch (SQLException e) {
                    log.error("TRUNCATE temp table fail. {}", copyTable, e);
                }
            }

        }
        log.info("end copy merge userTime = {}", (System.currentTimeMillis() - startTime));
    }

    protected void storeTarget(List records, Connection connection, DwsConfig config, TableSchema schema, String copyTable) throws DwsClientException {
        mergeExecutor.execute(records, connection, config, schema.getTableName().getFullName(), copyTable);
    }
}