net.ibizsys.dataflow.spark.ba.SparkPSSysBDSchemeEngine Maven / Gradle / Ivy
The newest version!
package net.ibizsys.dataflow.spark.ba;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.spark.sql.DataFrameReader;
import org.apache.spark.sql.DataFrameWriter;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.streaming.DataStreamReader;
import org.apache.spark.sql.streaming.DataStreamWriter;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import net.ibizsys.dataflow.core.ba.PSSysBDSchemeEngineBase;
import net.ibizsys.model.PSModelEnums.BDType;
import net.ibizsys.model.PSModelEnums.DEType;
import net.ibizsys.model.ba.IPSSysBDScheme;
import net.ibizsys.model.ba.IPSSysBDTable;
import net.ibizsys.model.dataentity.IPSDataEntity;
import net.ibizsys.model.dataentity.defield.IPSDEFGroup;
import net.ibizsys.model.dataentity.defield.IPSDEField;
import net.ibizsys.model.dataentity.der.IPSDER1N;
import net.ibizsys.model.dataentity.der.IPSDERBase;
import net.ibizsys.model.engine.util.IAction;
public class SparkPSSysBDSchemeEngine extends PSSysBDSchemeEngineBase implements ISparkPSSysBDSchemeEngine {
private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(SparkPSSysBDSchemeEngine.class);
@Override
public boolean isEnableDataStream() {
return false;
}
@Override
public DataFrameReader getDataFrameReader(SparkSession sparkSession, Object target) {
return this.executeAction("获取数据源读取对象", new IAction() {
@Override
public DataFrameReader execute(Object[] args) throws Throwable {
return onGetDataFrameReader(sparkSession, target);
}
}, DataFrameReader.class);
}
protected DataFrameReader onGetDataFrameReader(SparkSession sparkSession, Object target) throws Throwable {
String dbName = this.getDBName();
if(!StringUtils.hasLength(dbName)) {
throw new Exception(String.format("未指定数据库名称"));
}
String tableName = null;
IPSSysBDTable iPSSysBDTable = null;
IPSDataEntity iPSDataEntity = null;
IPSDEFGroup iPSDEFGroup = null;
if(target != null) {
if(target instanceof String) {
tableName = (String)target;
}
else
if(target instanceof IPSSysBDTable) {
iPSSysBDTable = (IPSSysBDTable)target;
tableName = iPSSysBDTable.getName();
}
else
if(target instanceof IPSDataEntity) {
iPSDataEntity = (IPSDataEntity)target;
}
else
if(target instanceof IPSDEFGroup) {
iPSDEFGroup = (IPSDEFGroup)target;
iPSDataEntity = iPSDEFGroup.getParentPSModelObject(IPSDataEntity.class);
}
else
throw new Exception(String.format("无法识别的目标对象[%1$s]", target));
if(iPSDataEntity!=null) {
tableName = iPSDataEntity.getTableName();
}
}
tableName = getRealDBObjName(tableName);
if (BDType.MONGODB.value.equalsIgnoreCase(this.getDBType())) {
DataFrameReader reader = sparkSession
.read()
.format("mongodb");
reader.option("connection.uri", this.getServiceUrl());
reader.option("database", dbName);
if(StringUtils.hasLength(tableName)) {
reader.option("collection", tableName);
}
return reader;
}
if (BDType.NEO4J.value.equalsIgnoreCase(this.getDBType())) {
DataFrameReader reader = sparkSession
.read()
.format("org.neo4j.spark.DataSource");
reader.option("url", this.getServiceUrl());
if(StringUtils.hasLength(this.getClientId())) {
reader.option("authentication.basic.username", this.getClientId());
}
if(StringUtils.hasLength(this.getClientSecret())) {
reader.option("authentication.basic.password", this.getClientSecret());
}
return reader;
}
throw new Exception(String.format("无法识别的大数据库类型[%1$s]", this.getDBType()));
}
@Override
public DataFrameWriter getDataFrameWriter(SparkSession sparkSession, Dataset dataset, Object target, SaveMode saveMode) {
// 判断是否为输出
return this.executeAction("获取数据源写入对象", new IAction() {
@Override
public DataFrameWriter execute(Object[] args) throws Throwable {
return onGetDataFrameWriter(sparkSession, dataset, target, saveMode);
}
}, DataFrameWriter.class);
}
protected DataFrameWriter onGetDataFrameWriter(SparkSession sparkSession, Dataset dataset, Object target, SaveMode saveMode) throws Throwable {
String dbName = this.getDBName();
if(!StringUtils.hasLength(dbName)) {
throw new Exception(String.format("未指定数据库名称"));
}
String tableName = null;
IPSSysBDTable iPSSysBDTable = null;
IPSDataEntity iPSDataEntity = null;
IPSDEFGroup iPSDEFGroup = null;
if(target != null) {
if(target instanceof String) {
tableName = (String)target;
}
else
if(target instanceof IPSSysBDTable) {
iPSSysBDTable = (IPSSysBDTable)target;
tableName = iPSSysBDTable.getName();
}
else
if(target instanceof IPSDataEntity) {
iPSDataEntity = (IPSDataEntity)target;
}
else
if(target instanceof IPSDEFGroup) {
iPSDEFGroup = (IPSDEFGroup)target;
iPSDataEntity = iPSDEFGroup.getParentPSModelObject(IPSDataEntity.class);
}
else
throw new Exception(String.format("无法识别的目标对象[%1$s]", target));
if(iPSDataEntity!=null) {
tableName = iPSDataEntity.getTableName();
}
}
tableName = getRealDBObjName(tableName);
if (BDType.MONGODB.value.equalsIgnoreCase(this.getDBType())) {
DataFrameWriter writer = dataset
.write()
.format("mongodb");
writer.option("connection.uri", this.getServiceUrl());
writer.option("database", dbName);
if(StringUtils.hasLength(tableName)) {
writer.option("collection", tableName);
}
if(saveMode!=null) {
writer.mode(saveMode);
}
return writer;
}
if (BDType.NEO4J.value.equalsIgnoreCase(this.getDBType())) {
DataFrameWriter writer = dataset
.write()
.format("org.neo4j.spark.DataSource");
writer.option("url", this.getServiceUrl());
if(StringUtils.hasLength(this.getClientId())) {
writer.option("authentication.basic.username", this.getClientId());
}
if(StringUtils.hasLength(this.getClientSecret())) {
writer.option("authentication.basic.password", this.getClientSecret());
}
if(iPSDataEntity != null) {
//判断关系类型
if(iPSDataEntity.getDEType() == DEType.RELATED.value) {
//关系模式
writer.option("relationship", tableName);
writer.option("relationship.save.strategy", "keys");
Map psDERMap = new LinkedHashMap();
List psDERBaseList = iPSDataEntity.getMinorPSDERs();
if(!ObjectUtils.isEmpty(psDERBaseList)) {
for(IPSDERBase iPSDERBase : psDERBaseList) {
if("SOURCE".equalsIgnoreCase(iPSDERBase.getCodeName())) {
psDERMap.put("SOURCE", iPSDERBase);
continue;
}
if("TARGET".equalsIgnoreCase(iPSDERBase.getCodeName())) {
psDERMap.put("TARGET", iPSDERBase);
continue;
}
IPSDER1N iPSDER1N = (IPSDER1N)iPSDERBase;
String name = iPSDER1N.getPSPickupDEFieldMust().getName();
if("SOURCE".equalsIgnoreCase(name)) {
psDERMap.put("SOURCE", iPSDERBase);
continue;
}
if("TARGET".equalsIgnoreCase(name)) {
psDERMap.put("TARGET", iPSDERBase);
continue;
}
}
if(psDERMap.size()!=2) {
psDERMap.clear();
boolean source = true;
for(IPSDERBase iPSDERBase : psDERBaseList) {
if(source) {
source = false;
psDERMap.put("SOURCE", iPSDERBase);
}
else {
psDERMap.put("TARGET", iPSDERBase);
break;
}
}
}
if(psDERMap.size()!=2) {
throw new Exception("未指定源及目标关系");
}
IPSDER1N sourcePSDERBase = (IPSDER1N)psDERMap.get("SOURCE");
//起始节点
writer.option("relationship.source.labels", this.getRealDBObjName(sourcePSDERBase.getMajorPSDataEntityMust().getTableName()));
String sourceId = this.getRealDBObjName(sourcePSDERBase.getPSPickupDEFieldMust().getName());
String sourceId2 = this.getRealDBObjName(sourcePSDERBase.getPSPickupDEFieldMust().getRelatedPSDEFieldMust().getName());
writer.option("relationship.source.node.keys", String.format("%1$s:%2$s", sourceId, sourceId2));
writer.option("relationship.source.save.mode", "Overwrite");
IPSDER1N targetPSDERBase = (IPSDER1N)psDERMap.get("TARGET");
//目标节点
writer.option("relationship.target.labels", this.getRealDBObjName(targetPSDERBase.getMajorPSDataEntityMust().getTableName()));
String targetId = this.getRealDBObjName(targetPSDERBase.getPSPickupDEFieldMust().getName());
String targetId2 = this.getRealDBObjName(targetPSDERBase.getPSPickupDEFieldMust().getRelatedPSDEFieldMust().getName());
writer.option("relationship.target.node.keys", String.format("%1$s:%2$s", targetId, targetId2));
writer.option("relationship.target.save.mode", "Overwrite");
//关系是属性
List fieldList = new ArrayList();
for(IPSDEField iPSDEField : iPSDataEntity.getAllPSDEFields()) {
fieldList.add(this.getRealDBObjName(iPSDEField.getName()));
}
writer.option("relationship.properties", StringUtils.collectionToDelimitedString(fieldList, ","));
}
}
else {
//节点模式
writer.option("labels", tableName);
writer.option("node.keys", this.getRealDBObjName(iPSDataEntity.getKeyPSDEFieldMust().getName()));
}
if(saveMode!=null) {
writer.mode(saveMode);
}
}
return writer;
}
throw new Exception(String.format("无法识别的大数据库类型[%1$s]", this.getDBType()));
}
@Override
public DataStreamReader getDataStreamReader(SparkSession sparkSession, Object target) {
return this.executeAction("获取数据源读取对象", new IAction() {
@Override
public DataStreamReader execute(Object[] args) throws Throwable {
return onGetDataStreamReader(sparkSession, target);
}
}, DataStreamReader.class);
}
protected DataStreamReader onGetDataStreamReader(SparkSession sparkSession, Object target) throws Throwable {
throw new Exception(String.format("无法识别的大数据库类型[%1$s]", this.getDBType()));
}
@Override
public DataStreamWriter getDataStreamWriter(SparkSession sparkSession, Dataset dataset, Object target, SaveMode saveMode) {
// 判断是否为输出
return this.executeAction("获取数据源写入对象", new IAction() {
@Override
public DataStreamWriter execute(Object[] args) throws Throwable {
return onGetDataStreamWriter(sparkSession, dataset, Row.class, target, saveMode);
}
}, DataStreamWriter.class);
}
protected DataStreamWriter onGetDataStreamWriter(SparkSession sparkSession, Dataset dataset, Class cls, Object tableName, SaveMode saveMode) throws Throwable {
throw new Exception(String.format("无法识别的大数据库类型[%1$s]", this.getDBType()));
}
}