com.huaweicloud.dws.client.collector.TableActionCollector Maven / Gradle / Ivy

Go to download
package com.huaweicloud.dws.client.collector;

import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.TableConfig;
import com.huaweicloud.dws.client.action.PutAction;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.DwsClientRecordException;
import com.huaweicloud.dws.client.model.ColumnKey;
import com.huaweicloud.dws.client.model.Record;
import com.huaweicloud.dws.client.model.TableSchema;
import com.huaweicloud.dws.client.worker.ExecutionPool;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;

/**
 * @ProjectName: dws-connector
 * @ClassName: TableActionCollector
 * @Description: 表事件收集， 用于按表维处理事件
 * @Date: 2023/1/10 15:48
 * @Version: 1.0
 */
@Slf4j
public class TableActionCollector extends ActionCollector implements ITableCollector {


    /**
     * 记录当前收集器正在处理的表定义，如果表结构改变这里会被体现
     */
    protected TableSchema currentTableSchema;

    /**
     * 表中操作缓存
     */
    protected final RecordBuffer buffer;

    /**
     * 记录表级配置
     */
    protected TableConfig tableConfig;

    protected final ReentrantLock lock = new ReentrantLock();


    public TableActionCollector(DwsConfig config, ExecutionPool pool, TableSchema schema) {
        super(config, pool);
        this.buffer = new RecordBuffer(config);
        this.currentTableSchema = schema;
        if (schema != null) {
            this.tableConfig = config.getTableConfig(schema.getTableName());
        }
    }

    @Override
    public ITableCollector getTableActionCollector(TableSchema schema) {
        return this;
    }

    @Override
    public void collector(Record record) throws DwsClientException {
        pool.checkStatus();
        // 收集操作是，判断表结构有没变更，如果变更直接入库后再收集新的
        if (currentTableSchema != null && !currentTableSchema.equals(record.getTableSchema())) {
            log.info("schema change. table = {}", currentTableSchema.getTableName());
            currentTableSchema = record.getTableSchema();
            flush();
        }
        lock.lock();
        try {
            // 刷库后就会将字段清空，重新缓存
            if (currentTableSchema == null) {
                currentTableSchema = record.getTableSchema();
            }
            if (tableConfig == null) {
                this.tableConfig = config.getTableConfig(currentTableSchema.getTableName());
            }
            // 将数据入缓存
            buffer.write(record);
        } finally {
            lock.unlock();
        }
        if (buffer.flush() && buffer.getSize() >= tableConfig.getAutoFlushBatchSize() * tableConfig.getBatchOutWeighRatio()) {
            // 只有在数量超过容量规定倍数后 才强制业务线程提交刷库，否则等待定时任务刷库即可
            flush();
        }
    }


    @Override
    public void flush() throws DwsClientException {
        // 同一张表中的数据，按列宽分类对齐后，同列宽批量入库
        lock.lock();
        try {
            flushBuffer();
            // 清空缓存
        } finally {
            lock.unlock();
        }
    }

    private void flushBuffer() throws DwsClientException {
        if (buffer.getSize() <= 0) {
            return;
        }
        try {
            List records = buffer.getRecords();
            List delRecords = buffer.getDelRecords();
            if (!delRecords.isEmpty()) {
                long startTime = System.currentTimeMillis();
                log.info("will flush buffer to dws, buffer size {}, ", records.size());
                PutAction action = new PutAction(records, config);
                while (!pool.submit(action, true)) {
                    log.debug("try submit.");
                }
                log.info("submit action successful. use time = {}", System.currentTimeMillis() - startTime);
            } else {
                writeNoDelete();
            }

        } finally {
            buffer.clear();
        }
    }


    private void writeNoDelete() throws DwsClientRecordException {
        DwsClientRecordException exception = null;
        Map> cache = new HashMap<>();
        for (Record record : buffer.getRecords()) {
            cache.computeIfAbsent(new ColumnKey(record.getColumnBit(), record.getIgnoreUpdate()),
                    columnKey -> new ArrayList<>(buffer.getSize())).add(record);
        }
        for (Map.Entry> entry : cache.entrySet()) {
            try {
                long startTime = System.currentTimeMillis();
                log.info("will flush buffer to dws, buffer size {}, ", entry.getValue().size());
                PutAction action = new PutAction(entry.getValue(), config);
                while (!pool.submit(action, true)) {
                    log.debug("try submit.");
                }
                log.info("submit action successful. use time = {}", System.currentTimeMillis() - startTime);
            } catch (DwsClientRecordException e) {
                if (exception == null) {
                    exception = e;
                } else {
                    exception.merge(e);
                }
            } catch (Exception unknown) {
                DwsClientRecordException clientRecordException = new DwsClientRecordException(DwsClientException.fromException(unknown), entry.getValue());
                if (exception == null) {
                    exception = clientRecordException;
                } else {
                    exception.merge(clientRecordException);
                }
            }
        }
        if (exception != null) {
            throw exception;
        }
    }

    /**
     * 根据 缓存是否达到刷库要求决定是否需要刷库
     * @return
     */
    @Override
    public void tryFlush() throws DwsClientException {
        if (buffer.flush()) {
            flush();
        }
    }

    @Override
    public ExecutionPool getPool() {
        return pool;
    }

    @Override
    public TableSchema getTableSchema() {
        return currentTableSchema;
    }

    @Override
    public DwsConfig getConfig() {
        return config;
    }

    @Override
    public TableConfig getTableConfig() {
        return tableConfig;
    }
}