net.ibizsys.dataflow.flink.FlinkPSDataFlowSystemEngine Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of ibiz-dataflow-flink
There is a newer version: 8.1.0.559
package net.ibizsys.dataflow.flink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.connector.source.Source;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.springframework.util.ObjectUtils;

import net.ibizsys.dataflow.core.PSDataFlowSystemEngineBase;
import net.ibizsys.dataflow.flink.dataentity.datasync.IFlinkPSDEDataSyncEngine;
import net.ibizsys.dataflow.flink.eai.IFlinkPSSysDataSyncAgentEngine;
import net.ibizsys.model.PSModelEnums.DataSyncAgentDir;
import net.ibizsys.model.PSModelEnums.DataSyncAgentType;
import net.ibizsys.model.PSModelEnums.DataSyncDir;
import net.ibizsys.model.dataentity.IPSDataEntity;
import net.ibizsys.model.dataentity.datasync.IPSDEDataSync;
import net.ibizsys.model.res.IPSSysDataSyncAgent;

public class FlinkPSDataFlowSystemEngine extends PSDataFlowSystemEngineBase implements IFlinkPSDataFlowSystemEngine{

	private StreamExecutionEnvironment streamExecutionEnvironment = null;
	
	@Override
	protected void onInit() throws Exception {
		
		if(this.getStreamExecutionEnvironment(true) == null) {
			this.prepareStreamExecutionEnvironment();
		}
		super.onInit();
		
		
		java.util.List psSysDataSyncAgentList = this.getPSSystem().getAllPSSysDataSyncAgents();
		if(!ObjectUtils.isEmpty(psSysDataSyncAgentList)) {
			for(IPSSysDataSyncAgent iPSSysDataSyncAgent : psSysDataSyncAgentList) {
				
				if(!DataSyncAgentType.KAFKA.value.equalsIgnoreCase(iPSSysDataSyncAgent.getAgentType())){
					continue;
				}
				
				if(!DataSyncAgentDir.IN.value.equalsIgnoreCase(iPSSysDataSyncAgent.getSyncDir())) {
					continue;
				}
				
				IFlinkPSSysDataSyncAgentEngine iPSSysDataSyncAgentEngine = this.getPSModelEngineHolder().getPSModelEngine(iPSSysDataSyncAgent, IFlinkPSSysDataSyncAgentEngine.class);
				
				if(iPSSysDataSyncAgentEngine instanceof ISourceProvider) {
					Source source = ((ISourceProvider)iPSSysDataSyncAgentEngine).getSource();
					 DataStreamSource s = this.getStreamExecutionEnvironment().fromSource(source, WatermarkStrategy.noWatermarks(), iPSSysDataSyncAgent.getName());
					 
					 
					 
					 boolean sinkFlag= false;
					 //枚举所有的实体数据同步
					 java.util.List psDataEntityList = this.getPSSystem().getAllPSDataEntities();
					 if(!ObjectUtils.isEmpty(psDataEntityList)) {
						 for(IPSDataEntity iPSDataEntity : psDataEntityList) {
							 java.util.List psDEDataSyncList = iPSDataEntity.getAllPSDEDataSyncs();
							 if(!ObjectUtils.isEmpty(psDEDataSyncList)) {
								 for(IPSDEDataSync iPSDEDataSync : psDEDataSyncList) {
									 if(!DataSyncDir.IN.value.equalsIgnoreCase(iPSDEDataSync.getSyncDir())) {
										 continue;
									 }
									 
									 if(!iPSDEDataSync.getInPSSysDataSyncAgentMust().getId().equalsIgnoreCase(iPSSysDataSyncAgent.getId())) {
										 continue;
									 }
									 
									 IFlinkPSDEDataSyncEngine iPSDEDataSyncEngine = this.getPSModelEngineHolder().getPSModelEngine(iPSDEDataSync, IFlinkPSDEDataSyncEngine.class);
									 if(iPSDEDataSyncEngine.isEnableSinkFunction()) {
										 s.addSink(iPSDEDataSyncEngine.getSinkFunction());			
										 sinkFlag = true;
									 }
									 else
										 throw new Exception(String.format("实体数据同步[%1$s]引擎未提供Sink功能", iPSDEDataSync.getName()));
										 
								 }
							 }
						 }
					 }
					 
					 if(!sinkFlag) {
						 //没有消费，使用打印
						 s.print();
					 }
				}
			}
		}
	}
	
	protected void prepareStreamExecutionEnvironment() throws Exception{
		this.setStreamExecutionEnvironment(StreamExecutionEnvironment.getExecutionEnvironment());
	}
	
	protected StreamExecutionEnvironment getStreamExecutionEnvironment() throws Exception {
		return this.getStreamExecutionEnvironment(false);
	}
	
	protected void setStreamExecutionEnvironment(StreamExecutionEnvironment streamExecutionEnvironment) {
		this.streamExecutionEnvironment = streamExecutionEnvironment;
	}
	
	protected StreamExecutionEnvironment getStreamExecutionEnvironment(boolean tryMode) throws Exception{
		if(this.streamExecutionEnvironment != null || tryMode) {
			return this.streamExecutionEnvironment;
		}
		throw new Exception(String.format("未指定流执行环境"));
	}
	
//	@Override
//	protected Object onExecute(Object[] args) throws Throwable {
//		return this.getStreamExecutionEnvironment().execute(this.getPSSystem().getName());
//	}
//	
//	
	
	
}