com.huaweicloud.dws.client.executor.CopyMergeExecutor Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dws-client Show documentation
There is a newer version: 2.0.0-r0
package com.huaweicloud.dws.client.executor;

import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableSchema;
import com.huaweicloud.dws.client.util.JdbcUtil;
import com.huaweicloud.dws.client.util.LogUtil;
import lombok.extern.slf4j.Slf4j;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;

/**
 * @ProjectName: dws-connector
 * @ClassName: CopyException
 * @Description: 通过copy merge方式入库
 * @Date: 2023/1/17 16:50
 * @Version: 1.0
 */
@Slf4j
public class CopyMergeExecutor {

    private final MergeExecutor mergeExecutor = new MergeExecutor();
    private final CopyExecutor copyExecutor = new CopyExecutor();

    public void execute(List records, Connection connection, DwsConfig config) throws DwsClientException {
        LogUtil.withLogSwitch(config, () -> log.info("start copy merge data to dws. executor is {}, records size = {}", this.getClass().getName(), records.size()));
        long startTime = System.currentTimeMillis();
        Record first = records.get(0);
        TableSchema schema = first.getTableSchema();
        List keys = schema.getPrimaryKeyNames();
        // 需要copy到的表
        String copyTable = schema.getTableName().getFullName();
        if (!keys.isEmpty()) {
            // 如果有主键，copy到临时表, 并且需要创建临时表
            copyTable = String.format("tmp_%s_%s", schema.getTableName().getTableName(), System.currentTimeMillis());
            String createTempTableSql = JdbcUtil.getCreateTempTableSql(schema.getTableName().getFullName(), copyTable);
            LogUtil.withLogSwitch(config, () -> log.info("this table has pks, will copy to temp table. create temp table sql {}", createTempTableSql));
            try {
                connection.prepareStatement(createTempTableSql).execute();
            } catch (SQLException e) {
                log.error("create temp table fail. sql is {}", createTempTableSql, e);
                throw DwsClientException.fromException(e);
            }
        }
        // 将数据copy到数据库：临时表或者目标表
        copyExecutor.execute(records, connection, copyTable, config);
        // 如果有主键 从临时表merge 并且删除临时表
        if (!keys.isEmpty()) {
            try {
                storeTarget(records, connection, config, schema, copyTable);
            } finally {
                try {
                    connection.prepareStatement("DROP table " + copyTable).execute();
                } catch (SQLException e) {
                    log.error("drop temp table fail.", e);
                }
            }

        }
        log.info("end copy merge userTime = {}", (System.currentTimeMillis() - startTime));
    }

    protected void storeTarget(List records, Connection connection, DwsConfig config, TableSchema schema, String copyTable) throws DwsClientException {
        mergeExecutor.execute(records, connection, config, schema.getTableName().getFullName(), copyTable);
    }
}