com.huaweicloud.dws.client.binlog.reader.BinlogReader Maven / Gradle / Ivy
/*
* Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved.
*/
package com.huaweicloud.dws.client.binlog.reader;
import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.TableConfig;
import com.huaweicloud.dws.client.binlog.collector.BinlogApi;
import com.huaweicloud.dws.client.binlog.collector.BinlogCollector;
import com.huaweicloud.dws.client.binlog.collector.BinlogFullSyncCollector;
import com.huaweicloud.dws.client.binlog.model.BinlogRecord;
import com.huaweicloud.dws.client.binlog.model.FullSyncSlot;
import com.huaweicloud.dws.client.binlog.model.Slot;
import com.huaweicloud.dws.client.exception.DwsBinlogException;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.ExceptionCode;
import com.huaweicloud.dws.client.exception.InvalidException;
import com.huaweicloud.dws.client.model.Constants;
import com.huaweicloud.dws.client.util.AssertUtil;
import com.huaweicloud.dws.client.util.LogUtil;
import com.huaweicloud.dws.client.worker.DwsConnectionPool;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* @ProjectName: dws-connector
* @Description: binlog处理类总入口，负责启动读取binlog线程以及提交同步点
* 大体流程：
* 1. 在主线程中获取同步点信息（要兼容GTM和GTM-free的场景，前者返回统一的同步点信息，后者分别返回各个DN上的同步点信息）
* 1.1 按照步长分批消费同步点间的数据（如果开始的同步点+步长 < 结束的同步点,则无需再次从内核中获取同步点信息）
* 2. 开启多线程来增量同步binlog信息或全量同步binlog信息
* 3. 等待第二步所有线程读取完毕后，回到主线程并提交同步点
* @Date: 2023/07/11 15:40
* @Version 1.0
**/
@Slf4j
public class BinlogReader {
private static final long AWAIT_TIME = 1000L;
private static final int SLEEP_RANDOM_MS = 100;
private final SecureRandom sleepRandom;
private final DwsConfig dwsConfig;
@Getter
private final TableConfig tableConfig;
private final ExecutorService binlogCollectExecutorService;
private final ExecutorService binlogGetRecordExecutorService;
private final AtomicBoolean started = new AtomicBoolean(false);
// 是否需要全量同步(只要有一个同步点的startCsn为-1则进行全量同步)
private final AtomicBoolean needFullSync = new AtomicBoolean(false);
private final AtomicLong totalRecord = new AtomicLong(0);
private final List> completableFutureList = new ArrayList<>();
// 全量读取binlog的数据行数
private final int fullSyncBinlogBatchReadSize;
private final AtomicBoolean running = new AtomicBoolean(false);
private final List binlogCollectors = new ArrayList<>();
private final List binlogFullSyncCollectors = new ArrayList<>();
private final int binlogParallelNum;
// 存放读取binlog数据的队列
private final BlockingQueue queue;
// 指定的同步点信息
private final List specifySlots;
@Getter
private Exception exception;
private final List columnNames;
// 保存当前所需消费的同步点信息
@Setter
private List currentSlots = new ArrayList<>();
@Getter
private final String tableName;
@Getter
private final String slotName;
@Getter
private final DwsConnectionPool dwsConnectionPool;
@Getter
@Setter
private List nodeIds = new CopyOnWriteArrayList<>();
@Getter
private final boolean isNeedRedistribution;
@Getter
private final String errorMessage;
private int noDataTimes = 0;
@Setter
private String binlogWorkerThreadPrefix = "binlog-worker";
@Setter
private String binlogGetRecordThreadPrefix = "binlog-get-record";
public BinlogReader(DwsConfig dwsConfig, BlockingQueue queue, List specifySlots,
List columnNames, DwsConnectionPool dwsConnectionPool) {
checkParams(dwsConfig);
this.dwsConfig = dwsConfig;
this.queue = queue;
this.specifySlots = specifySlots;
this.columnNames = columnNames;
this.tableName = dwsConfig.getBinlogTableName();
this.dwsConnectionPool = dwsConnectionPool;
// 获取表级设置
this.tableConfig = dwsConfig.getTableConfig(tableName);
this.isNeedRedistribution = tableConfig.isNeedRedistribution();
this.errorMessage = tableConfig.getErrorMessage();
this.slotName = tableConfig.getBinlogSlotName();
this.fullSyncBinlogBatchReadSize = tableConfig.getFullSyncBinlogBatchReadSize();
this.binlogParallelNum = tableConfig.getBinlogParallelNum();
// 初始化binlog-worker线程相关信息
ThreadFactory binlogCollectThreadFactory = new ThreadFactory() {
private final AtomicInteger threadNumber = new AtomicInteger(1);
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName(binlogWorkerThreadPrefix + "-" + tableName + "-" + threadNumber.getAndIncrement());
t.setDaemon(true);
return t;
}
};
// 初始化binlog-reader相关线程池
this.binlogCollectExecutorService =
new ThreadPoolExecutor(binlogParallelNum, binlogParallelNum, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(10), binlogCollectThreadFactory, new ThreadPoolExecutor.AbortPolicy());
this.binlogGetRecordExecutorService = Executors.newFixedThreadPool(1, r -> {
Thread t = new Thread(r);
t.setName(binlogGetRecordThreadPrefix + "-" + tableName);
t.setDaemon(true);
return t;
});
try {
this.sleepRandom = SecureRandom.getInstanceStrong();
} catch (NoSuchAlgorithmException e) {
throw new InvalidException(e);
}
}
public BinlogReader(DwsConfig dwsConfig, BlockingQueue queue, List columnNames,
DwsConnectionPool dwsConnectionPool) {
this(dwsConfig, queue, null, columnNames, dwsConnectionPool);
}
/**
* 读取binlog信息入口方法: 读取到的数据实时往queue中写入
*
* @throws Exception 异常信息
*/
public synchronized void getRecords() throws Exception {
if (!started.get()) {
log.error("binlogReader not started...");
throw new DwsClientException(ExceptionCode.BINLOG_READER_NOT_STARTED, "binlogReader not started...");
}
if (Objects.nonNull(exception)) {
throw new DwsBinlogException(exception.getMessage(), exception);
}
if (running.get()) {
return;
}
// 开始获取binlog,另起一个线程,保证有数据就消费
binlogGetRecordExecutorService.execute(() -> {
try {
if (running.compareAndSet(false, true)) {
while (started.get()) {
if (Objects.nonNull(exception)) {
throw DwsClientException.fromException(exception);
}
doCollect();
}
}
} catch (Exception e) {
log.error("get binlog records has error", e);
exception = e;
}
});
}
public void checkException() throws DwsBinlogException {
if (Objects.nonNull(exception)) {
throw new DwsBinlogException(exception.getMessage(), exception);
}
}
public boolean hasException() {
return Objects.nonNull(exception);
}
/**
* 开启读取binlog流程, 大致如下:
* 获取同步点 -> 读取同步点内的binlog数据 -> 提交同步点
*
* @throws Exception 异常信息
*/
public synchronized void doCollect() throws Exception {
int binlogMaxRetryTimes = tableConfig.getBinlogMaxRetryTimes();
for (int retryCount = 1; retryCount <= binlogMaxRetryTimes; retryCount++) {
try {
long start = System.currentTimeMillis();
// 第一步: 获取binlog数据的前置动作(获取slot信息、初始化BinlogCollectors)
prepareToCollect();
// 第二步: 开启线程全量或增量读取binlog信息
collectBinlog();
// 第三步: 向内核注册同步点信息
registerSlot();
// 重置标志位
reset();
if (totalRecord.get() == 0) {
long sleepTime =
Math.min(tableConfig.getBinlogSleepTime() * (++noDataTimes), Constants.BINLOG_MAX_SLEEP_TIME);
LogUtil.withLogSwitch(dwsConfig,
() -> log.warn("there is no new data in currentSlots, sleepTime: {}", sleepTime));
// 如果从GTM上拿数据的话，即使表没进行DML操作，同步点也会推进，所以在这里判断是否读取到了数据，没有读到则sleep
TimeUnit.MILLISECONDS.sleep(sleepTime);
} else {
noDataTimes = 0;
log.info("read binlog info cost time: {}, slots: {},", (System.currentTimeMillis() - start),
currentSlots);
}
return;
} catch (Exception e) {
log.error("[retry]collectBinlog has error: ", e);
stopCollectors();
if (handleRedistribution(e)) {
log.info("no need retry, begin re-consume...");
throw DwsClientException.fromException(e);
}
// 重试
if (retryCount < binlogMaxRetryTimes) {
long sleepTime =
tableConfig.getBinlogRetryInterval() * retryCount + sleepRandom.nextInt(SLEEP_RANDOM_MS);
log.info("reader binlog fail, start retry, retryCount: {}, sleepTime: {}", retryCount, sleepTime);
try {
TimeUnit.MILLISECONDS.sleep(sleepTime);
} catch (InterruptedException ex) {
log.error("sleep has error: ", ex);
throw DwsClientException.fromException(ex);
}
} else {
log.info("reach to MaxRetryTimes: " + retryCount);
throw DwsClientException.fromException(e);
}
}
}
}
/**
* 处理扩缩容数据重分布逻辑
*
* @param e 异常信息,根据该异常信息来判断是否处于扩缩容阶段
* @return 是否需要处理
*/
public boolean handleRedistribution(Exception e) {
if (Objects.isNull(e.getMessage())) {
return false;
}
boolean isRedistribution = e.getMessage().contains(errorMessage);
if (!isRedistribution) {
return false;
}
log.info("start handle redistribution...");
return true;
}
/**
* 同步binlog的前置动作:
* 1. 从内核侧获取同步点信息
* 2. 按照消费步长,来初始化当前的同步点信息
* 3. 使用同步点信息来初始化BinlogCollector
*
* @throws DwsClientException 异常信息
*/
private void prepareToCollect() throws Exception {
if (nextCycle()) {
initCurrentSlots();
}
// 使用指定的slot信息来初始化当前的slot(故障恢复场景)
mergeCurrentSlotList();
if (currentSlots.stream().anyMatch(slot -> slot.getCurrentStartCsn() == Constants.FULL_SYNC_SLOT_CSN)) {
// 设置全量同步标识, 增量同步和全量同步调用内核接口不同
needFullSync.set(true);
}
// 每次使用新获取的同步点信息来初始化BinlogCollectors
initBinlogCollectors();
}
protected void initCurrentSlots() throws SQLException {
try (Connection connection = dwsConnectionPool.getConnection()) {
if (isNeedRedistribution && BinlogApi.NODE_COUNT.get() == 0) {
BinlogApi.NODE_COUNT.set(BinlogApi.getNodeIds(connection, getTableName()).size());
}
currentSlots = BinlogApi.getSyncPoint(connection, tableName, slotName, 0, false, isNeedRedistribution);
LogUtil.withLogSwitch(dwsConfig, () -> log.info("get slot info from db currentSlots: {}", currentSlots));
}
}
/**
* 根据内核返回的点位信息和当前保存的点位信息进行比较，判断是否有新数据
* 注意：由于每次获取同步点都会推进，所以暂时无需该判断方法。
*
* @return 是否有新数据
* @throws Exception 异常信息
*/
private boolean hasNewData() throws Exception {
if (currentSlots.isEmpty()) {
return true;
}
List slotsInDB = getSlotsInDb();
// 判断是否获取到新的点位
Map slotMap = slotsInDB.stream().collect(Collectors.toMap(Slot::getDnNodeId, Function.identity()));
for (Slot slot : currentSlots) {
int dnNodeId = slot.getDnNodeId();
if (slotMap.containsKey(dnNodeId) && slotMap.get(dnNodeId).getEndCsn() > slot.getCurrentStartCsn()) {
return true;
}
}
// 没有新数据,休眠一段时间(等待同步点推进)
log.warn("there is no new data, currentSlots: {}, slotsInDB: {}", currentSlots, slotsInDB);
return false;
}
protected List getSlotsInDb() throws SQLException {
try (Connection connection = dwsConnectionPool.getConnection()) {
List slotsInDB =
BinlogApi.getSyncPoint(connection, tableName, slotName, 0, false, isNeedRedistribution);
LogUtil.withLogSwitch(dwsConfig, () -> log.info("get slot info from db slotsInDB: {}", slotsInDB));
return slotsInDB;
}
}
public List getNodesFromDB() throws SQLException {
try (Connection connection = getDwsConnectionPool().getConnection()) {
return BinlogApi.getNodeIdsWithTableName(connection, getTableName());
}
}
/**
* 判断currentStartCsn是否小于endCsn
* 1.小于表示继续消费当前批次的数据
* 2.否则表示需要重新从内核获取同步点信息
*
* @return 当前批次的数据是否完全消费完
*/
private boolean nextCycle() {
return currentSlots.stream().allMatch(Slot::isEnd);
}
/**
* 使用传入的点位信息(specifySlots)来替换当前的点位信息
*/
private void mergeCurrentSlotList() {
if (Objects.isNull(specifySlots) || specifySlots.size() == 0) {
return;
}
if (specifySlots.size() == 1 && specifySlots.get(0) instanceof FullSyncSlot) {
currentSlots.forEach(slot -> {
slot.setStartCsn(Constants.FULL_SYNC_SLOT_CSN);
slot.setCurrentStartCsn(Constants.FULL_SYNC_SLOT_CSN);
});
} else {
// 使用指定slot的currentStartCsn来替代内核返回的(增量场景)
Map slotMap =
specifySlots.stream().collect(Collectors.toMap(Slot::getDnNodeId, Function.identity()));
for (Slot slot : currentSlots) {
int dnNodeId = slot.getDnNodeId();
if (slotMap.containsKey(dnNodeId)) {
Slot specifySlot = slotMap.get(dnNodeId);
if (specifySlot.getCurrentStartCsn() >= slot.getCurrentStartCsn()) {
// 属于异常清空,直接忽略
log.warn("specify slot is large than current slot, ignore...");
continue;
}
// 使用指定同步点来更新当前消费点位,之后从specifySlots中删除
slot.setCurrentStartCsn(specifySlot.getCurrentStartCsn());
log.info("use specify slot. specifySlot: {} and remove...", specifySlot);
specifySlots.removeIf(toBeRemoveSlot -> toBeRemoveSlot.getDnNodeId() == dnNodeId);
log.info("replaced current slot. currentSlots: {}, specifySlots: {}", currentSlots, specifySlots);
}
}
}
}
/**
* 调用内核接口全量或增量读取binlog信息
*/
private void collectBinlog() {
if (needFullSync.get()) {
binlogFullSyncCollectors.forEach(binlogFullSyncCollector -> completableFutureList
.add(CompletableFuture.runAsync(binlogFullSyncCollector, binlogCollectExecutorService)));
} else {
binlogCollectors.forEach(binlogCollector -> completableFutureList
.add(CompletableFuture.runAsync(binlogCollector, binlogCollectExecutorService)));
}
}
/**
* 初始化BinlogCollectors,需要根据同步点数量以及用户传入的并发度来初始化
*/
private void initBinlogCollectors() {
binlogCollectors.clear();
binlogFullSyncCollectors.clear();
totalRecord.set(0);
int slotNum = currentSlots.size();
// 如果用户传入的并发度小于slotNum,则一个线程需要读取多个DN上的数据
// 如果用户传入的并发度大于等于slotNum,则只需要起slotNum个线程
Map> map = new HashMap<>();
for (int i = 0; i < slotNum; i++) {
int index = i % binlogParallelNum;
map.computeIfAbsent(index, r -> new ArrayList<>()).add(currentSlots.get(i));
}
map.forEach((key, slotList) -> {
boolean isEnd = slotList.stream().allMatch(Slot::isEnd);
if (isEnd) {
// 表示该dn上的数据在此批次已经消费完毕
return;
}
if (needFullSync.get()) {
BinlogFullSyncCollector binlogFullSyncCollector = new BinlogFullSyncCollector(queue, slotList, started,
fullSyncBinlogBatchReadSize, dwsConnectionPool, columnNames, tableName, totalRecord);
binlogFullSyncCollectors.add(binlogFullSyncCollector);
} else {
BinlogCollector binlogCollector = new BinlogCollector(queue, slotList, started, tableConfig,
dwsConnectionPool, columnNames, totalRecord);
binlogCollectors.add(binlogCollector);
}
});
}
/**
* 向内核注册同步点信息
*
* @throws Exception 异常信息
*/
private void registerSlot() throws Exception {
CompletableFuture completableFuture =
CompletableFuture.allOf(completableFutureList.toArray(new CompletableFuture[0])).whenComplete((v, err) -> {
completableFutureList.clear();
});
long timeout =
needFullSync.get() ? tableConfig.getFullSyncBinlogReadTimeout() : tableConfig.getBinlogReadTimeout();
completableFuture.get(timeout, TimeUnit.MILLISECONDS);
if (needFullSync.get() || nextCycle()) {
// 需要往每个dn上提交
for (Slot slot : currentSlots) {
try (Connection connection = dwsConnectionPool.getConnection()) {
BinlogApi.updateSyncPoint(connection, tableName, slotName, slot.getEndCsn(), slot.getDnNodeId(),
slot.getXmin(), false, tableConfig.getUpdateSyncPointTimeOut());
}
}
}
}
/**
* 重置标志位
*/
private void reset() {
needFullSync.set(false);
}
/**
* 启动BinlogReader
*/
public void start() {
if (!started.compareAndSet(false, true)) {
log.warn("BinlogReader has started...");
return;
}
log.info("start BinlogReader...");
// 系统停止时执行关闭
Runtime.getRuntime().addShutdownHook(new Thread(this::close));
}
/**
* binlog相关参数校验
*
* @param dwsConfig 配置参数
*/
private void checkParams(DwsConfig dwsConfig) {
String binlogTableName = dwsConfig.getBinlogTableName();
AssertUtil.nonNull(binlogTableName, new InvalidException("tableName not set"));
TableConfig binlogTableConfig = dwsConfig.getTableConfig(binlogTableName);
AssertUtil.nonNull(binlogTableConfig, new InvalidException("tableConfig not set"));
boolean binlog = binlogTableConfig.isBinlog();
AssertUtil.isTrue(binlog, new InvalidException("binlog is false"));
String binlogSlotName = binlogTableConfig.getBinlogSlotName();
if (Objects.isNull(binlogSlotName) || "".equals(binlogSlotName)) {
// 如果slotName为空则默认用tableName来设置slotName
binlogTableConfig.withBinlogSlotName(binlogTableName);
}
}
public void stop() {
started.set(false);
}
public boolean isStart() {
return started.get();
}
private void stopCollectors() {
log.info("stop collectors...");
// 清理资源
binlogFullSyncCollectors.forEach(BinlogFullSyncCollector::stop);
binlogCollectors.forEach(BinlogCollector::stop);
}
public void close() {
try {
if (!started.compareAndSet(true, false)) {
log.warn("BinlogReader has closed...");
return;
}
log.info("BinlogReader close start");
BinlogApi.NODE_COUNT.remove();
stopCollectors();
binlogCollectExecutorService.shutdownNow();
while (!binlogCollectExecutorService.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)) {
log.info("wait binlog collect executorService termination");
}
binlogGetRecordExecutorService.shutdownNow();
while (!binlogGetRecordExecutorService.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)) {
log.info("wait binlog get record executorService termination");
}
dwsConnectionPool.close();
log.info("BinlogReader close end");
} catch (Exception e) {
log.error("shutdown binlog executorService error.", e);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy