com.huaweicloud.dws.client.collector.ActionCollector Maven / Gradle / Ivy

Go to download

package com.huaweicloud.dws.client.collector;

import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.action.GetAction;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.ExceptionCode;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableName;
import com.huaweicloud.dws.client.model.TableSchema;
import com.huaweicloud.dws.client.op.Get;
import com.huaweicloud.dws.client.util.LogUtil;
import com.huaweicloud.dws.client.worker.ExecutionPool;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/**
 * @ProjectName: dws-connector
 * @ClassName: ActionCollector
 * @Description: 所有事件收集
 * @Date: 2023/1/10 15:47
 * @Version: 1.0
 */
@Slf4j
@Getter
public class ActionCollector implements ICollector {
    // 插入queue等待时间，默认10s
    private static final long OFFER_QUEUE_TIME_OUT = 10000L;
    private static final long POLL_QUEUE_TIME_OUT = 2000L;


    /**
     * 为每个表设置一个事件收集器，避免不同表之间相互影响
     */
    final Map tableCollectors = new ConcurrentHashMap<>();

    /**
     * 全局配置
     */
    protected final DwsConfig config;

    /**
     * 资源池
     */
    protected final ExecutionPool pool;

    /**
     * get请求队列
     */
    final ArrayBlockingQueue queue;

    /**
     * 保存最后的执行异常
     */
    public ActionCollector(DwsConfig config, ExecutionPool pool) {
        this.config = config;
        this.pool = pool;
        this.queue = pool.getQueue();
    }

    /**
     * 尝试缓存入库
     */
    @Override
    public void tryFlush() throws DwsClientException {
        flush(false);
    }



    private void flush(boolean force) throws DwsClientException {
        for (ICollector value : tableCollectors.values()) {
            try {
                if (force) {
                    value.flush();
                } else {
                    value.tryFlush();
                }
            } catch (Exception e) {
                log.error("action collector flush error. force = {}", force, e);
                throw DwsClientException.fromException(e);
            }
        }
    }


    /**
     * 强制入库
     */
    @Override
    public void flush() throws DwsClientException {
        flush(true);
    }


    @Override
    public ITableCollector getTableActionCollector(TableSchema schema) {
        return tableCollectors.computeIfAbsent(schema.getTableName(), table -> {
            if (config.getTableConfig(table).isSecurityMode()) {
                return new SecurityModeCollector(config, pool, schema);
            }
            return new TableActionCollector(config, pool, schema);
        });
    }

    public void submit(List recordList, int batchSize) throws DwsClientException {
        DwsClientException exception;
        recordList.clear();
        try {
            Get firstGet = queue.poll(POLL_QUEUE_TIME_OUT, TimeUnit.MILLISECONDS);
            if (firstGet != null) {
                recordList.add(firstGet);
                queue.drainTo(recordList, batchSize - 1);
                if (config.isLogSwitch()) {
                    log.info("submit gets action to dws, size: {}", recordList.size());
                }
                submit(recordList);
            }
        } catch (Exception e) {
            exception = DwsClientException.fromException(e);
            for (Get get : recordList) {
                if (!get.getFuture().isDone()) {
                    get.getFuture().completeExceptionally(exception);
                }
            }
        }
    }

    public void submit(List recordList) throws DwsClientException {
        Map> getsByTable = new HashMap<>();
        for (Get get : recordList) {
            if (config.getReadTimeoutMilliseconds() > 0) {
                long waitingTime = (System.nanoTime() - get.getStartTime()) / 1000000L;
                if (waitingTime - config.getReadTimeoutMilliseconds() > 0) {
                    get.getFuture()
                            .completeExceptionally(new DwsClientException(ExceptionCode.TIMEOUT,
                                    String.format(
                                            "Get action timeout before submit to dws, %s ms set in the config, actual cost %s ms",
                                            waitingTime, config.getReadTimeoutMilliseconds())));
                }
            }
            // 区分不同的tableSchema，用于后续批量sql拼接
            List list = getsByTable.computeIfAbsent(get.getRecord().getTableSchema(), (s) -> new ArrayList<>());
            list.add(get);
        }
        for (Map.Entry> entry : getsByTable.entrySet()) {
            GetAction getAction = new GetAction(entry.getValue(), config);
            while (!pool.submit(getAction)) {
                LogUtil.withLogSwitch(config, () -> log.warn("try submit."));
            }
        }
    }

    /**
     * ===============添加数据API===============
     */
    @Override
    public void collector(Record record) throws DwsClientException {
        // 获取该表对应的收集器
        getTableActionCollector(record.getTableSchema()).collector(record);

    }

    public void appendGet(Get get) {
        try {
            if (!queue.offer(get, OFFER_QUEUE_TIME_OUT, TimeUnit.MILLISECONDS)) {
                get.getFuture().completeExceptionally(new TimeoutException());
            }
        } catch (InterruptedException e) {
            get.getFuture().completeExceptionally(e);
        }
    }

    @Override
    public ExecutionPool getPool() {
        return pool;
    }

    @Override
    public DwsConfig getConfig() {
        return config;
    }
}