com.huaweicloud.dws.client.collector.ActionCollector Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client.collector;

import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.action.AbstractAction;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.DwsClientRecordException;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableName;
import com.huaweicloud.dws.client.model.TableSchema;
import com.huaweicloud.dws.client.util.AssertUtil;
import com.huaweicloud.dws.client.util.LogUtil;
import com.huaweicloud.dws.client.worker.ExecutionPool;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;

/**
 * @ProjectName: dws-connector
 * @ClassName: ActionCollector
 * @Description: 所有事件收集
 * @Date: 2023/1/10 15:47
 * @Version: 1.0
 */
@Slf4j
public class ActionCollector {

    /**
     * 为每个表设置一个事件收集器，避免不同表之间相互影响
     */
    final Map tableCollectors = new ConcurrentHashMap<>();

    /**
     * 全局配置
     */
    final DwsConfig config;

    /**
     * 资源池
     */
    final ExecutionPool pool;

    /**
     * 保存最后的执行异常
     */
    public ActionCollector(DwsConfig config, ExecutionPool pool) {
        this.config = config;
        this.pool = pool;
    }

    /**
     * 尝试缓存入库
     */
    public void tryFlush() throws DwsClientException {
        flush(false);
    }

    private synchronized void flush(boolean force) throws DwsClientException {
        LogUtil.withLogSwitch(config, () -> log.info("action collector flush. force = {}", force));
        List> tasks = new ArrayList<>();
        for (TableActionCollector value : tableCollectors.values()) {
            try {
                List> action = force ? value.flush() : value.tryFlush();
                Optional.ofNullable(action).ifPresent(tasks::addAll);
            } catch (Exception e) {
                log.error("action collector flush error. force = {}", force, e);
                throw DwsClientException.fromException(e);
            }
        }
        LogUtil.withLogSwitch(config, () -> log.info("submit task successful. task num is {}", tasks.size()));
        DwsClientException e = null;
        DwsClientRecordException recordException = null;
        for (AbstractAction action : tasks) {
            try {
                if (recordException == null) {
                    recordException = DwsClientRecordException.fromAction(action);
                    continue;
                }
                DwsClientRecordException finalRecordException = recordException;
                Optional.ofNullable(DwsClientRecordException.fromAction(action)).ifPresent(finalRecordException::merge);
            } catch (DwsClientException dwsClientException) {
                log.error("action result exception.", dwsClientException);
                e = dwsClientException;
            }
        }
        AssertUtil.isNull(recordException, recordException);
        AssertUtil.isNull(e, e);
    }

    /**
     * 强制入库
     */
    public void flush() throws DwsClientException {
        flush(true);
    }


    /**
     * ===============添加数据API===============
     */
    public void collector(Record record) throws DwsClientException {
        TableSchema schema = record.getTableSchema();
        // 获取该表对应的收集器
        TableActionCollector collector = tableCollectors.computeIfAbsent(schema.getTableName(), tableName -> new TableActionCollector(config, pool));
        collector.collector(record);

    }

    public ExecutionPool getPool() {
        return pool;
    }

    public DwsConfig getConfig() {
        return config;
    }
}