com.huaweicloud.dws.client.binlog.collector.BinlogCollector Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved.
 */

package com.huaweicloud.dws.client.binlog.collector;

import com.huaweicloud.dws.client.TableConfig;
import com.huaweicloud.dws.client.binlog.model.BinlogRecord;
import com.huaweicloud.dws.client.binlog.model.Slot;
import com.huaweicloud.dws.client.exception.DwsBinlogException;
import com.huaweicloud.dws.client.worker.DwsConnectionPool;

import lombok.extern.slf4j.Slf4j;

import java.sql.Connection;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

/**
 * @ProjectName: dws-connector
 * @Description: binlog信息收集器, 负责去DN上读取binlog信息
 * @Date: 2023/07/11 10:46
 * @Version: 1.0
 **/
@Slf4j
public class BinlogCollector implements Runnable {

    private final BlockingQueue queue;

    private final List slots;

    private final AtomicBoolean started;

    private final List columnNames;

    private final DwsConnectionPool dwsConnectionPool;

    private final AtomicLong totalRecord;

    private final TableConfig tableConfig;

    private final AtomicBoolean running = new AtomicBoolean(false);

    public BinlogCollector(BlockingQueue queue, List slots, AtomicBoolean started,
        TableConfig tableConfig, DwsConnectionPool dwsConnectionPool, List columnNames,
        AtomicLong totalRecord) {
        this.queue = queue;
        this.slots = slots;
        this.started = started;
        this.tableConfig = tableConfig;
        this.dwsConnectionPool = dwsConnectionPool;
        this.columnNames = columnNames;
        this.totalRecord = totalRecord;
        running.set(true);
    }

    @Override
    public void run() {
        if (!started.get() || !running.get()) {
            log.error("binlogReader not started or incremental sync collector not running...");
            return;
        }
        int total = 0;
        long start = System.currentTimeMillis();
        for (Slot slot : slots) {
            if (slot.isEnd()) {
                // 表示该slot上当前批次的数据已消费完,直接返回
                continue;
            }
            do {
                try (Connection connection = dwsConnectionPool.getConnection()) {
                    total += BinlogApi.getBinlogRecords(connection, slot, columnNames, tableConfig,
                            queue, running);
                } catch (Exception e) {
                    throw new DwsBinlogException(e);
                }
                if (running.get()) {
                    // 推进同步点
                    slot.setCurrentStartCsn(slot.getConsumeEndScn(tableConfig.getBinlogBatchReadSize()));
                } else {
                    return;
                }
            } while (!slot.isEnd());

        }
        if (total > 0) {
            totalRecord.addAndGet(total);
            log.info("[incremental sync]get binlog data from slot: {}, total: {}, cost time: {}", slots, total,
                (System.currentTimeMillis() - start));
        }
    }

    public void stop() {
        running.set(false);
    }
}