org.apache.kylin.tool.bisync.SyncModelBuilder Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kylin.tool.bisync;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.model.SelectRule;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.tool.bisync.model.ColumnDef;
import org.apache.kylin.tool.bisync.model.JoinTreeNode;
import org.apache.kylin.tool.bisync.model.MeasureDef;
import org.apache.kylin.tool.bisync.model.SyncModel;
public class SyncModelBuilder {
private final SyncContext syncContext;
public SyncModelBuilder(SyncContext syncContext) {
this.syncContext = syncContext;
}
public SyncModel buildSourceSyncModel(List dimensions, List measures) {
NDataModel dataModelDesc = syncContext.getDataflow().getModel();
IndexPlan indexPlan = syncContext.getDataflow().getIndexPlan();
// init joinTree, dimension cols, measure cols, hierarchies
Map columnDefMap = authColumns(dataModelDesc, syncContext.isAdmin(), ImmutableSet.of(),
ImmutableSet.of());
List measureDefs = dataModelDesc.getEffectiveMeasures().values().stream() //
.map(MeasureDef::new).collect(Collectors.toList());
markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, ImmutableSet.of(), dimensions,
measures, syncContext.getModelElement());
markComputedColumnVisibility(columnDefMap, measureDefs, syncContext.getKylinConfig().exposeComputedColumn());
Set hierarchies = getHierarchies(indexPlan);
JoinTreeNode joinTree = generateJoinTree(dataModelDesc.getJoinTables(), dataModelDesc.getRootFactTableName());
return getSyncModel(dataModelDesc, columnDefMap, measureDefs, hierarchies, joinTree);
}
public SyncModel buildHasPermissionSourceSyncModel(Set authTables, Set authColumns,
List dimensions, List measures) {
NDataModel dataModelDesc = syncContext.getDataflow().getModel();
IndexPlan indexPlan = syncContext.getDataflow().getIndexPlan();
Set allAuthColumns = addHasPermissionCCColumn(dataModelDesc, authColumns);
// init joinTree, dimension cols, measure cols, hierarchies
Map columnDefMap = authColumns(dataModelDesc, syncContext.isAdmin(), authTables,
allAuthColumns);
List measureDefs = dataModelDesc.getEffectiveMeasures().values().stream() //
.filter(measure -> checkMeasurePermission(allAuthColumns, measure)) //
.map(MeasureDef::new).collect(Collectors.toList());
markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, allAuthColumns, dimensions,
measures, syncContext.getModelElement());
markComputedColumnVisibility(columnDefMap, measureDefs, syncContext.getKylinConfig().exposeComputedColumn());
Set omitDbColSet = renameColumnName(allAuthColumns);
Set hierarchies = getHierarchies(indexPlan).stream()
.map(hierarchyArray -> Arrays.stream(hierarchyArray).filter(omitDbColSet::contains)
.collect(Collectors.toSet()).toArray(new String[0]))
.collect(Collectors.toSet()).stream().filter(x -> !Arrays.asList(x).isEmpty())
.collect(Collectors.toSet());
JoinTreeNode joinTree = generateJoinTree(dataModelDesc.getJoinTables(), dataModelDesc.getRootFactTableName());
return getSyncModel(dataModelDesc, columnDefMap, measureDefs, hierarchies, joinTree);
}
private SyncModel getSyncModel(NDataModel dataModelDesc, Map columnDefMap,
List measureDefs, Set hierarchies, JoinTreeNode joinTree) {
// populate CubeSyncModel
SyncModel syncModel = new SyncModel();
syncModel.setColumnDefMap(columnDefMap);
syncModel.setJoinTree(joinTree);
syncModel.setMetrics(measureDefs);
syncModel.setHierarchies(hierarchies);
syncModel.setProject(syncContext.getProjectName());
syncModel.setModelName(dataModelDesc.getAlias());
syncModel.setHost(syncContext.getHost());
syncModel.setPort(String.valueOf(syncContext.getPort()));
return syncModel;
}
private boolean checkMeasurePermission(Set columns, NDataModel.Measure measure) {
Set measureColumns = measure.getFunction().getParameters().stream()
.filter(parameterDesc -> parameterDesc.getColRef() != null)
.map(parameterDesc -> parameterDesc.getColRef().getAliasDotName()).collect(Collectors.toSet());
return columns.containsAll(measureColumns);
}
private void markComputedColumnVisibility(Map columnDefMap, List measureDefs,
boolean exposeComputedColumns) {
if (exposeComputedColumns) {
return;
}
// hide all CC cols and related measures
for (ColumnDef columnDef : columnDefMap.values()) {
if (columnDef.isComputedColumn()) {
columnDef.setHidden(true);
}
}
for (MeasureDef measureDef : measureDefs) {
for (TblColRef paramColRef : measureDef.getMeasure().getFunction().getColRefs()) {
ColumnDef columnDef = columnDefMap.get(paramColRef.getAliasDotName());
if (columnDef != null && columnDef.isComputedColumn()) {
measureDef.setHidden(true);
break;
}
}
}
}
private void markHasPermissionIndexedColumnsAndMeasures(Map columnDefMap,
List measureDefs, IndexPlan indexPlan, Set authorizedCols, List dimensions,
List measures, SyncContext.ModelElement modelElement) {
Set colsToShow = Sets.newHashSet();
Set measuresToShow = Sets.newHashSet();
switch (modelElement) {
case AGG_INDEX_COL:
ImmutableBitSet aggDimBitSet = indexPlan.getAllIndexes().stream() //
.filter(index -> !index.isTableIndex()) //
.map(IndexEntity::getDimensionBitset) //
.reduce(ImmutableBitSet.EMPTY, ImmutableBitSet::or);
Set tblColRefs = indexPlan.getEffectiveDimCols().entrySet().stream() //
.filter(entry -> aggDimBitSet.get(entry.getKey())) //
.map(Map.Entry::getValue) //
.collect(Collectors.toSet());
colsToShow = tblColRefs.stream() //
.filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
.map(TblColRef::getAliasDotName) //
.collect(Collectors.toSet());
measuresToShow = indexPlan.getEffectiveMeasures().values().stream() //
.filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
.map(MeasureDesc::getName) //
.collect(Collectors.toSet());
break;
case AGG_INDEX_AND_TABLE_INDEX_COL:
colsToShow = indexPlan.getEffectiveDimCols().values().stream() //
.filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
.map(TblColRef::getAliasDotName) //
.collect(Collectors.toSet());
measuresToShow = indexPlan.getEffectiveMeasures().values().stream()
.filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
.map(MeasureDesc::getName) //
.collect(Collectors.toSet());
break;
case ALL_COLS:
colsToShow = indexPlan.getModel().getEffectiveDimensions().values().stream()
.filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
.map(TblColRef::getAliasDotName) //
.collect(Collectors.toSet());
measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream() //
.filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
.map(MeasureDesc::getName) //
.collect(Collectors.toSet());
break;
case CUSTOM_COLS:
Set dimensionSet = Sets.newHashSet(dimensions);
colsToShow = indexPlan.getModel().getEffectiveDimensions().values().stream()
.filter(colRef -> testAuthorizedDimensions(dimensionSet, colRef)) //
.map(TblColRef::getAliasDotName) //
.collect(Collectors.toSet());
measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream() //
.map(MeasureDesc::getName) //
.filter(measures::contains) //
.collect(Collectors.toSet());
break;
default:
break;
}
Set dimensionSet = indexPlan.getModel().getEffectiveDimensions().values().stream()
.map(TblColRef::getAliasDotName).collect(Collectors.toSet());
showDimsAndMeasures(columnDefMap, measureDefs, colsToShow, measuresToShow, dimensionSet);
}
private boolean testAuthorizedCols(Set authorizedCols, TblColRef colRef) {
return syncContext.isAdmin() || authorizedCols.contains(colRef.getColumnWithTableAndSchema())
|| authorizedCols.contains(colRef.getAliasDotName());
}
private boolean testAuthorizedDimensions(Set dimensions, TblColRef colRef) {
return dimensions.contains(colRef.getColumnWithTableAndSchema())
|| dimensions.contains(colRef.getAliasDotName());
}
private boolean testAuthorizedMeasures(Set authorizedCols, NDataModel.Measure measureDef) {
return syncContext.isAdmin() || checkMeasurePermission(authorizedCols, measureDef);
}
private void showDimsAndMeasures(Map columnDefMap, List measureDefs,
Set colsToShow, Set measuresToShow, Set dimensionSet) {
for (String colToShow : colsToShow) {
ColumnDef colToShowDef = columnDefMap.get(colToShow);
colToShowDef.setHidden(false);
if (dimensionSet.contains(colToShow)) {
colToShowDef.setDimension(true);
}
}
for (MeasureDef measureDef : measureDefs) {
if (measuresToShow.contains(measureDef.getMeasure().getName())) {
measureDef.setHidden(false);
}
}
}
Set renameColumnName(Set columns) {
return columns.stream().map(x -> {
String[] split = x.split("\\.");
if (split.length == 3) {
return split[1] + "." + split[2];
}
return x;
}).collect(Collectors.toSet());
}
private Map authColumns(NDataModel model, boolean isAdmin, Set tables,
Set columns) {
Map modelColsMap = Maps.newHashMap();
for (TableRef tableRef : model.getAllTables()) {
if (!isAdmin && !tables.contains(tableRef.getTableIdentity())) {
continue;
}
for (TblColRef colRef : tableRef.getColumns()) {
if (isAdmin || columns.contains(colRef.getAliasDotName())
|| columns.contains(colRef.getColumnWithTableAndSchema())) {
ColumnDef columnDef = ColumnDef.builder() //
.role("dimension") //
.tableAlias(tableRef.getAlias()) //
.columnName(colRef.getName()) //
.columnType(colRef.getDatatype()) //
.isHidden(true) //
.isComputedColumn(colRef.getColumnDesc().isComputedColumn()) //
.build();
modelColsMap.put(colRef.getIdentity(), columnDef);
}
}
}
// sync col alias
model.getAllNamedColumns().stream() //
.filter(NDataModel.NamedColumn::isExist) //
.forEach(namedColumn -> {
ColumnDef columnDef = modelColsMap.get(namedColumn.getAliasDotColumn());
if (columnDef != null) {
columnDef.setColumnAlias(namedColumn.getName());
}
});
return modelColsMap;
}
private Set addHasPermissionCCColumn(NDataModel modelDesc, Set columns) {
Set allAuthColumns = Sets.newHashSet();
allAuthColumns.addAll(columns);
List computedColumnDescs = modelDesc.getComputedColumnDescs();
Set computedColumnDescSet = computedColumnDescs.stream().filter(computedColumnDesc -> {
Set normalColumns = convertColNames(modelDesc, computedColumnDesc, Sets.newHashSet());
return columns.containsAll(normalColumns);
}).collect(Collectors.toSet());
computedColumnDescSet.forEach(cc -> allAuthColumns.add(cc.getFullName()));
return allAuthColumns;
}
private Set convertColNames(NDataModel model, ComputedColumnDesc computedColumnDesc,
Set normalColumns) {
Set normalCols = convertCCToNormalCols(model, computedColumnDesc, normalColumns);
Set newAuthColumns = Sets.newHashSet();
model.getAllTables().forEach(tableRef -> {
List collect = tableRef.getColumns().stream()
.filter(column -> normalCols.contains(column.getCanonicalName())).collect(Collectors.toList());
collect.forEach(x -> newAuthColumns.add(x.getAliasDotName()));
});
return newAuthColumns;
}
private Set convertCCToNormalCols(NDataModel model, ComputedColumnDesc computedColumnDesc,
Set normalColumns) {
Set ccUsedColsWithModel = ComputedColumnUtil.getCCUsedColsWithModel(model, computedColumnDesc);
Set allCCols = model.getComputedColumnDescs().stream().map(ComputedColumnDesc::getIdentName)
.collect(Collectors.toSet());
ccUsedColsWithModel.forEach(x -> {
if (!allCCols.contains(x)) {
normalColumns.add(x);
}
});
model.getComputedColumnDescs().stream().filter(desc -> ccUsedColsWithModel.contains(desc.getIdentName()))
.forEach(x -> convertCCToNormalCols(model, x, normalColumns));
return normalColumns;
}
private Set getHierarchies(IndexPlan indexPlan) {
Set hierarchies = Sets.newHashSet();
if (indexPlan.getRuleBasedIndex() == null) {
return hierarchies;
}
Set hierarchyNameSet = Sets.newHashSet();
for (NAggregationGroup group : indexPlan.getRuleBasedIndex().getAggregationGroups()) {
SelectRule rule = group.getSelectRule();
if (rule == null) {
continue;
}
for (Integer[] hierarchyIds : rule.hierarchyDims) {
if (ArrayUtils.isNotEmpty(hierarchyIds)) {
String[] hierarchyNames = Arrays.stream(hierarchyIds)
.map(id -> indexPlan.getModel().getColumnNameByColumnId(id)).toArray(String[]::new);
String hierarchyNamesJoined = String.join(",", hierarchyNames);
if (!hierarchyNameSet.contains(hierarchyNamesJoined)) {
hierarchies.add(hierarchyNames);
hierarchyNameSet.add(hierarchyNamesJoined);
}
}
}
}
return hierarchies;
}
private JoinTreeNode generateJoinTree(List joinTables, String factTable) {
Map> joinTreeMap = new HashMap<>();
for (JoinTableDesc joinTable : joinTables) {
String[] fks = joinTable.getJoin().getForeignKey();
String leftTableName = fks[0].substring(0, fks[0].indexOf('.'));
if (joinTreeMap.containsKey(leftTableName)) {
joinTreeMap.get(leftTableName).add(joinTable);
} else {
List rightTables = new LinkedList<>();
rightTables.add(joinTable);
joinTreeMap.put(leftTableName, rightTables);
}
}
return createJoinTree(factTable, joinTreeMap);
}
private JoinTreeNode createJoinTree(String factTableName, Map> joinTreeMap) {
JoinTableDesc factTable = new JoinTableDesc();
int dot = factTableName.indexOf('.');
String alias = factTableName.substring(dot + 1);
factTable.setTable(factTableName);
factTable.setAlias(alias);
factTable.setKind(NDataModel.TableKind.FACT);
return buildChildJoinTree(factTable, joinTreeMap);
}
private JoinTreeNode buildChildJoinTree(JoinTableDesc root, Map> joinTreeMap) {
JoinTreeNode joinTree = new JoinTreeNode();
joinTree.setValue(root);
List childTables = joinTreeMap.get(root.getAlias());
if (childTables != null) {
List childNodes = new LinkedList<>();
for (JoinTableDesc childTable : childTables) {
childNodes.add(buildChildJoinTree(childTable, joinTreeMap));
}
joinTree.setChildNodes(childNodes);
}
return joinTree;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy