net.wicp.tams.common.flink.source.binlog.BinlogSource Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of common-flink-source-binlog Show documentation
There is a newer version: 3.6.14
Show newest version
package net.wicp.tams.common.flink.source.binlog;

import java.util.ArrayList;
import java.util.List;

import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

import lombok.extern.slf4j.Slf4j;
import net.wicp.tams.common.binlog.alone.BusiAssit;
import net.wicp.tams.common.binlog.alone.ListenerConf.CheckPoint;
import net.wicp.tams.common.binlog.alone.ListenerConf.ColHis;
import net.wicp.tams.common.binlog.alone.ListenerConf.ConnConf;
import net.wicp.tams.common.binlog.alone.ListenerConf.ConnConf.Builder;
import net.wicp.tams.common.binlog.alone.ListenerConf.Position;
import net.wicp.tams.common.binlog.alone.parser.ParseLogOnline;
import net.wicp.tams.duckula.client.Protobuf3.DuckulaEvent;

@Slf4j
public class BinlogSource extends RichSourceFunction implements ListCheckpointed {

	private static final long serialVersionUID = 1L;

	private ConnConf connConf;
	private ParseLogOnline logFetcher;
	
	private static final String listener="net.wicp.tams.common.flink.source.binlog.FlinkBinlogListener";

	public BinlogSource(ConnConf.Builder connConfBuilder) {
		connConfBuilder.setListener(listener);
		this.connConf = connConfBuilder.build();
	}

	/***
	 * 需要在配置文件中存在相关的配置
	 * 
	 * @param configKey 监听配置key
	 */
	public BinlogSource(String configKey) {
		ConnConf.Builder connConfBuilder = BusiAssit.configMap(configKey);
		connConfBuilder.setListener(listener);
		this.connConf = connConfBuilder.build();
	}

	public BinlogSource() {
		this("default");
	}

	@Override
	public void setRuntimeContext(RuntimeContext t) {
		super.setRuntimeContext(t);
	}

	@Override
	public void open(Configuration parameters) throws Exception {
		super.open(parameters);
		this.logFetcher = new ParseLogOnline(connConf.toBuilder());
	}

	@Override
	public List snapshotState(long checkpointId, long timestamp) throws Exception {
		List listState = new ArrayList<>();
		CheckPoint checkPoint = logFetcher.getCheckPointCur();// .getCheckPoint(timestamp);
		listState.add(checkPoint);
		return listState;
	}

	private List colsList = null;

	@Override
	public void restoreState(List state) throws Exception {
		Builder newBuilder = connConf.toBuilder();
		CheckPoint checkPoint = state.get(0);
		Position pos = checkPoint.getPos();
		log.info("the binlog begin from:{}", pos.getGtids());
		newBuilder.setPos(pos);
		this.connConf = newBuilder.build();
		this.colsList = checkPoint.getColsList();
	}

	@Override
	public void run(SourceContext ctx) throws Exception {
		FlinkBinlogListener binlogListener = (FlinkBinlogListener) this.logFetcher.getBinlogListener();
		binlogListener.setCtx(ctx);
		this.logFetcher.setColHis(this.colsList);

		this.logFetcher.read();
	}

	@Override
	public void cancel() {
		if (this.logFetcher != null) {
			log.info("============cancel the logFetcher");
			this.logFetcher.close();
		}
	}

}