com.huaweicloud.dws.client.binlog.collector.BinlogFullSyncCollector Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved.
 */

package com.huaweicloud.dws.client.binlog.collector;

import com.huaweicloud.dws.client.binlog.model.BinlogRecord;
import com.huaweicloud.dws.client.binlog.model.Slot;
import com.huaweicloud.dws.client.exception.DwsBinlogException;
import com.huaweicloud.dws.client.worker.DwsConnectionPool;

import lombok.extern.slf4j.Slf4j;

import java.sql.Connection;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

/**
 * @ProjectName: dws-connector
 * @Description: 全量同步
 * @Date: 2023/07/11 10:46
 * @Version: 1.0
 **/
@Slf4j
public class BinlogFullSyncCollector implements Runnable {

    private final BlockingQueue queue;

    private final List slots;

    private final AtomicBoolean started;

    private final int fullSyncBinlogBatchReadSize;

    private final List columnNames;

    private final DwsConnectionPool dwsConnectionPool;

    private final String tableName;

    private final AtomicLong totalRecord;

    private final AtomicBoolean running = new AtomicBoolean(false);

    public BinlogFullSyncCollector(BlockingQueue queue, List slots, AtomicBoolean started,
        int fullSyncBinlogBatchReadSize, DwsConnectionPool dwsConnectionPool, List columnNames, String tableName,
        AtomicLong totalRecord) {
        this.queue = queue;
        this.slots = slots;
        this.started = started;
        this.fullSyncBinlogBatchReadSize = fullSyncBinlogBatchReadSize;
        this.dwsConnectionPool = dwsConnectionPool;
        this.columnNames = columnNames;
        this.totalRecord = totalRecord;
        this.tableName = tableName;
        running.set(true);
    }

    @Override
    public void run() {
        int total = 0;
        long start = System.currentTimeMillis();
        if (!started.get() || !running.get()) {
            log.error("binlogReader not started or full sync collector not running...");
            return;
        }
        for (Slot slot : slots) {
            if (slot.isEnd()) {
                // 表示该slot上当前批次的数据已消费完,直接返回
                continue;
            }
            try (Connection connection = dwsConnectionPool.getConnection()) {
                total += BinlogApi.fullSyncBinlog(connection, tableName, slot.getDnNodeId(), columnNames,
                        fullSyncBinlogBatchReadSize, queue, running);
            } catch (Exception e) {
                throw new DwsBinlogException(e);
            }
            // 推进同步点
            if (running.get()) {
                slot.finish();
            } else {
                return;
            }
        }
        if (total > 0) {
            totalRecord.addAndGet(total);
            log.info("[full sync]get binlog data from slot: {}, total: {}, cost time: {}", slots, total,
                (System.currentTimeMillis() - start));
        }
    }

    public void stop() {
        running.set(false);
    }
}