com.hazelcast.jet.sql.impl.CalciteSqlOptimizer Maven / Gradle / Ivy
/*
* Copyright 2021 Hazelcast Inc.
*
* Licensed under the Hazelcast Community License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://hazelcast.com/hazelcast-community-license
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hazelcast.jet.sql.impl;
import com.hazelcast.cluster.memberselector.MemberSelectors;
import com.hazelcast.jet.core.DAG;
import com.hazelcast.jet.datamodel.Tuple2;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.AlterJobPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateJobPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateMappingPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateSnapshotPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateTypePlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateViewPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DmlPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DropJobPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DropMappingPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DropSnapshotPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DropTypePlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.DropViewPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.IMapDeletePlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.IMapInsertPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.IMapSelectPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.IMapSinkPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.IMapUpdatePlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.SelectPlan;
import com.hazelcast.jet.sql.impl.SqlPlanImpl.ShowStatementPlan;
import com.hazelcast.jet.sql.impl.connector.SqlConnectorCache;
import com.hazelcast.jet.sql.impl.connector.map.MetadataResolver;
import com.hazelcast.jet.sql.impl.connector.virtual.ViewTable;
import com.hazelcast.jet.sql.impl.opt.Conventions;
import com.hazelcast.jet.sql.impl.opt.OptUtils;
import com.hazelcast.jet.sql.impl.opt.WatermarkKeysAssigner;
import com.hazelcast.jet.sql.impl.opt.logical.FullScanLogicalRel;
import com.hazelcast.jet.sql.impl.opt.logical.LogicalRel;
import com.hazelcast.jet.sql.impl.opt.logical.LogicalRules;
import com.hazelcast.jet.sql.impl.opt.logical.SelectByKeyMapLogicalRule;
import com.hazelcast.jet.sql.impl.opt.physical.AssignDiscriminatorToScansRule;
import com.hazelcast.jet.sql.impl.opt.physical.CreateDagVisitor;
import com.hazelcast.jet.sql.impl.opt.physical.DeleteByKeyMapPhysicalRel;
import com.hazelcast.jet.sql.impl.opt.physical.InsertMapPhysicalRel;
import com.hazelcast.jet.sql.impl.opt.physical.PhysicalRel;
import com.hazelcast.jet.sql.impl.opt.physical.PhysicalRules;
import com.hazelcast.jet.sql.impl.opt.physical.RootRel;
import com.hazelcast.jet.sql.impl.opt.physical.SelectByKeyMapPhysicalRel;
import com.hazelcast.jet.sql.impl.opt.physical.SinkMapPhysicalRel;
import com.hazelcast.jet.sql.impl.opt.physical.UpdateByKeyMapPhysicalRel;
import com.hazelcast.jet.sql.impl.parse.QueryConvertResult;
import com.hazelcast.jet.sql.impl.parse.QueryParseResult;
import com.hazelcast.jet.sql.impl.parse.SqlAlterJob;
import com.hazelcast.jet.sql.impl.parse.SqlCreateIndex;
import com.hazelcast.jet.sql.impl.parse.SqlCreateJob;
import com.hazelcast.jet.sql.impl.parse.SqlCreateMapping;
import com.hazelcast.jet.sql.impl.parse.SqlCreateSnapshot;
import com.hazelcast.jet.sql.impl.parse.SqlCreateType;
import com.hazelcast.jet.sql.impl.parse.SqlCreateView;
import com.hazelcast.jet.sql.impl.parse.SqlDropIndex;
import com.hazelcast.jet.sql.impl.parse.SqlDropJob;
import com.hazelcast.jet.sql.impl.parse.SqlDropMapping;
import com.hazelcast.jet.sql.impl.parse.SqlDropSnapshot;
import com.hazelcast.jet.sql.impl.parse.SqlDropType;
import com.hazelcast.jet.sql.impl.parse.SqlDropView;
import com.hazelcast.jet.sql.impl.parse.SqlExplainStatement;
import com.hazelcast.jet.sql.impl.parse.SqlShowStatement;
import com.hazelcast.jet.sql.impl.schema.HazelcastTable;
import com.hazelcast.jet.sql.impl.schema.TableResolverImpl;
import com.hazelcast.jet.sql.impl.schema.TablesStorage;
import com.hazelcast.jet.sql.impl.schema.TypeDefinitionColumn;
import com.hazelcast.logging.ILogger;
import com.hazelcast.security.permission.ActionConstants;
import com.hazelcast.security.permission.MapPermission;
import com.hazelcast.spi.impl.NodeEngine;
import com.hazelcast.sql.SqlColumnMetadata;
import com.hazelcast.sql.SqlRowMetadata;
import com.hazelcast.sql.impl.QueryException;
import com.hazelcast.sql.impl.QueryParameterMetadata;
import com.hazelcast.sql.impl.QueryUtils;
import com.hazelcast.sql.impl.optimizer.OptimizationTask;
import com.hazelcast.sql.impl.optimizer.PlanKey;
import com.hazelcast.sql.impl.optimizer.PlanObjectKey;
import com.hazelcast.sql.impl.optimizer.SqlOptimizer;
import com.hazelcast.sql.impl.optimizer.SqlPlan;
import com.hazelcast.sql.impl.schema.IMapResolver;
import com.hazelcast.sql.impl.schema.Mapping;
import com.hazelcast.sql.impl.schema.MappingField;
import com.hazelcast.sql.impl.schema.TableResolver;
import com.hazelcast.sql.impl.schema.map.AbstractMapTable;
import com.hazelcast.sql.impl.state.QueryResultRegistry;
import com.hazelcast.sql.impl.type.QueryDataType;
import com.hazelcast.org.apache.calcite.plan.Contexts;
import com.hazelcast.org.apache.calcite.plan.Convention;
import com.hazelcast.org.apache.calcite.plan.RelOptCostImpl;
import com.hazelcast.org.apache.calcite.plan.RelOptTable;
import com.hazelcast.org.apache.calcite.plan.RelOptUtil;
import com.hazelcast.org.apache.calcite.plan.RelTraitSet;
import com.hazelcast.org.apache.calcite.plan.hep.HepPlanner;
import com.hazelcast.org.apache.calcite.plan.hep.HepProgramBuilder;
import com.hazelcast.org.apache.calcite.plan.volcano.VolcanoPlanner;
import com.hazelcast.org.apache.calcite.rel.RelNode;
import com.hazelcast.org.apache.calcite.rel.RelShuttleImpl;
import com.hazelcast.org.apache.calcite.rel.core.TableModify;
import com.hazelcast.org.apache.calcite.rel.core.TableModify.Operation;
import com.hazelcast.org.apache.calcite.rel.core.TableScan;
import com.hazelcast.org.apache.calcite.rel.type.RelDataTypeField;
import com.hazelcast.org.apache.calcite.sql.SqlNode;
import com.hazelcast.org.apache.calcite.sql.dialect.PostgresqlSqlDialect;
import com.hazelcast.org.apache.calcite.sql.util.SqlString;
import com.hazelcast.org.apache.calcite.tools.RuleSets;
import javax.annotation.Nullable;
import java.security.Permission;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import static com.hazelcast.jet.datamodel.Tuple2.tuple2;
import static com.hazelcast.jet.sql.impl.SqlPlanImpl.CreateIndexPlan;
import static com.hazelcast.jet.sql.impl.SqlPlanImpl.DropIndexPlan;
import static com.hazelcast.jet.sql.impl.SqlPlanImpl.ExplainStatementPlan;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
/**
* SQL optimizer based on Apache Calcite.
*
* After parsing and initial sql-to-rel conversion is finished, all relational nodes start with {@link Convention#NONE}
* convention. Such nodes are typically referred as "abstract" in Apache Calcite, because they do not have any physical
* properties.
*
* The optimization process is split into two phases - logical and physical. During logical planning we normalize abstract
* nodes and convert them to nodes with {@link Conventions#LOGICAL} convention. These new nodes are Hazelcast-specific
* and hence may have additional properties. For example, at this stage we do filter pushdowns, introduce constrained scans,
* etc.
*
* During physical planning we look for specific physical implementations of logical nodes. Implementation nodes have
* {@link Conventions#PHYSICAL} convention. The process contains the following fundamental steps:
*
* - Choosing proper access methods for scan (normal scan, index scan, etc)
* - Propagating physical properties from children nodes to their parents
* - Choosing proper implementations of parent operators based on physical properties of children
* (local vs. distributed sorting, blocking vs. streaming aggregation, hash join vs. merge join, etc.)
* - Enforcing exchange operators when data movement is necessary
*
*
* Physical optimization stage uses {@link VolcanoPlanner}. This is a rule-based optimizer. However it doesn't share any
* architectural traits with EXODUS/Volcano/Cascades papers, except for the rule-based nature. In classical Cascades algorithm
* [1], the optimization process is performed in a top-down style. Parent operator may request implementations of children
* operators with specific properties. This is not possible in {@code VolcanoPlanner}. Instead, in this planner the rules are
* fired in effectively uncontrollable fashion, thus making propagation of physical properties difficult. To overcome this
* problem we use several techniques that helps us emulate at least some parts of Cascades-style optimization.
*
* First, {@link Conventions#PHYSICAL} convention overrides {@link Convention#canConvertConvention(Convention)} and
* {@link Convention#useAbstractConvertersForConversion(RelTraitSet, RelTraitSet)} methods. Their implementations ensure that
* whenever a new child node with {@code PHYSICAL} convention is created, the rule of the parent {@code LOGICAL} nodes
* will be re-scheduled. Second, physical rules for {@code LOGICAL} nodes iterate over concrete physical implementations of
* inputs and convert logical nodes to physical nodes with proper traits. Combined, these techniques ensure complete exploration
* of a search space and proper propagation of physical properties from child nodes to parent nodes. The downside is that
* the same rule on the same node could be fired multiple times, thus increase the optimization time.
*
* For example, consider the following logical tree:
*
* LogicalFilter
* LogicalScan
*
* By default Apache Calcite will fire a rule on the logical filter first. But at this point we do not know the physical
* properties of {@code LogicalScan} implementations, since they are not produced yet. As a result, we do not know what
* physical properties should be set to the to-be-created {@code PhysicalFilter}. Then Apache Calcite will optimize
* {@code LogicalScan}, producing physical implementations. However, by default these new physical implementations will not
* re-trigger optimization of {@code LogicalFilter}. The result of the optimization will be:
*
* [LogicalFilter, PhysicalFilter(???)]
* [LogicalScan, PhysicalScan(PARTITIONED), PhysicalIndexScan(PARTITIONED, a ASC)]
*
* Notice how we failed to propagate important physical properties to the {@code PhysicalFilter}.
*
* With the above-described techniques we force Apache Calcite to re-optimize the logical parent after a new physical child
* has been created. This way we are able to pull-up physical properties. The result of the optimization will be:
*
* [LogicalFilter, PhysicalFilter(PARTITIONED), PhysicalFilter(PARTITIONED, a ASC)]
* [LogicalScan, PhysicalScan(PARTITIONED), PhysicalIndexScan(PARTITIONED, a ASC)]
*
*
* [1] Efficiency In The Columbia Database Query Optimizer (1998), chapters 2 and 3
*/
public class CalciteSqlOptimizer implements SqlOptimizer {
private final NodeEngine nodeEngine;
private final IMapResolver iMapResolver;
private final List tableResolvers;
private final PlanExecutor planExecutor;
private final TablesStorage tablesStorage;
private final ILogger logger;
public CalciteSqlOptimizer(NodeEngine nodeEngine, QueryResultRegistry resultRegistry) {
this.nodeEngine = nodeEngine;
this.iMapResolver = new MetadataResolver(nodeEngine);
this.tablesStorage = new TablesStorage(nodeEngine);
TableResolverImpl tableResolverImpl = mappingCatalog(nodeEngine, this.tablesStorage);
this.tableResolvers = singletonList(tableResolverImpl);
this.planExecutor = new PlanExecutor(tableResolverImpl, nodeEngine.getHazelcastInstance(), resultRegistry);
this.logger = nodeEngine.getLogger(getClass());
}
private static TableResolverImpl mappingCatalog(NodeEngine nodeEngine, TablesStorage tablesStorage) {
SqlConnectorCache connectorCache = new SqlConnectorCache(nodeEngine);
return new TableResolverImpl(nodeEngine, tablesStorage, connectorCache);
}
@Nullable
@Override
public String mappingDdl(String name) {
Mapping mapping = iMapResolver.resolve(name);
return mapping != null ? SqlCreateMapping.unparse(mapping) : null;
}
@Override
public List tableResolvers() {
return tableResolvers;
}
public TablesStorage tablesStorage() {
return tablesStorage;
}
@Override
public SqlPlan prepare(OptimizationTask task) {
// 1. Prepare context.
int memberCount = nodeEngine.getClusterService().getSize(MemberSelectors.DATA_MEMBER_SELECTOR);
OptimizerContext context = OptimizerContext.create(
task.getSchema(),
task.getSearchPaths(),
task.getArguments(),
memberCount,
iMapResolver);
try {
OptimizerContext.setThreadContext(context);
// 2. Parse SQL string and validate it.
QueryParseResult parseResult = context.parse(task.getSql());
// 3. Create plan.
return createPlan(task, parseResult, context);
} finally {
OptimizerContext.setThreadContext(null);
}
}
@SuppressWarnings("checkstyle:returncount")
private SqlPlan createPlan(
OptimizationTask task,
QueryParseResult parseResult,
OptimizerContext context
) {
// TODO [sasha] : refactor this.
SqlNode node = parseResult.getNode();
PlanKey planKey = new PlanKey(task.getSearchPaths(), task.getSql());
if (node instanceof SqlCreateMapping) {
return toCreateMappingPlan(planKey, (SqlCreateMapping) node);
} else if (node instanceof SqlDropMapping) {
return toDropMappingPlan(planKey, (SqlDropMapping) node);
} else if (node instanceof SqlCreateIndex) {
return toCreateIndexPlan(planKey, (SqlCreateIndex) node);
} else if (node instanceof SqlDropIndex) {
return toDropIndexPlan(planKey, (SqlDropIndex) node);
} else if (node instanceof SqlCreateJob) {
return toCreateJobPlan(planKey, parseResult, context, task.getSql());
} else if (node instanceof SqlAlterJob) {
return toAlterJobPlan(planKey, (SqlAlterJob) node);
} else if (node instanceof SqlDropJob) {
return toDropJobPlan(planKey, (SqlDropJob) node);
} else if (node instanceof SqlCreateSnapshot) {
return toCreateSnapshotPlan(planKey, (SqlCreateSnapshot) node);
} else if (node instanceof SqlDropSnapshot) {
return toDropSnapshotPlan(planKey, (SqlDropSnapshot) node);
} else if (node instanceof SqlCreateView) {
return toCreateViewPlan(planKey, context, (SqlCreateView) node);
} else if (node instanceof SqlDropView) {
return toDropViewPlan(planKey, (SqlDropView) node);
} else if (node instanceof SqlDropType) {
return toDropTypePlan(planKey, (SqlDropType) node);
} else if (node instanceof SqlShowStatement) {
return toShowStatementPlan(planKey, (SqlShowStatement) node);
} else if (node instanceof SqlExplainStatement) {
return toExplainStatementPlan(planKey, context, parseResult);
} else if (node instanceof SqlCreateType) {
return toCreateTypePlan(planKey, (SqlCreateType) node);
} else {
QueryConvertResult convertResult = context.convert(parseResult.getNode());
return toPlan(
planKey,
parseResult.getParameterMetadata(),
convertResult.getRel(),
convertResult.getFieldNames(),
context,
false,
task.getSql());
}
}
private SqlPlan toCreateMappingPlan(PlanKey planKey, SqlCreateMapping sqlCreateMapping) {
List mappingFields = sqlCreateMapping.columns()
.map(field -> new MappingField(field.name(), field.type(), field.externalName()))
.collect(toList());
Mapping mapping = new Mapping(
sqlCreateMapping.nameWithoutSchema(),
sqlCreateMapping.externalName(),
sqlCreateMapping.type(),
mappingFields,
sqlCreateMapping.options()
);
return new CreateMappingPlan(
planKey,
mapping,
sqlCreateMapping.getReplace(),
sqlCreateMapping.ifNotExists(),
planExecutor
);
}
private SqlPlan toDropMappingPlan(PlanKey planKey, SqlDropMapping sqlDropMapping) {
return new DropMappingPlan(planKey, sqlDropMapping.nameWithoutSchema(), sqlDropMapping.ifExists(), planExecutor);
}
private SqlPlan toCreateIndexPlan(PlanKey planKey, SqlCreateIndex sqlCreateIndex) {
return new CreateIndexPlan(
planKey,
sqlCreateIndex.indexName(),
sqlCreateIndex.mapName(),
sqlCreateIndex.type(),
sqlCreateIndex.columns(),
sqlCreateIndex.options(),
sqlCreateIndex.ifNotExists(),
planExecutor
);
}
private SqlPlan toDropIndexPlan(PlanKey planKey, SqlDropIndex sqlDropIndex) {
return new DropIndexPlan(planKey, sqlDropIndex.indexName(), sqlDropIndex.ifExists(), planExecutor);
}
private SqlPlan toCreateJobPlan(PlanKey planKey, QueryParseResult parseResult, OptimizerContext context, String query) {
SqlCreateJob sqlCreateJob = (SqlCreateJob) parseResult.getNode();
SqlNode source = sqlCreateJob.dmlStatement();
QueryParseResult dmlParseResult = new QueryParseResult(source, parseResult.getParameterMetadata());
QueryConvertResult dmlConvertedResult = context.convert(dmlParseResult.getNode());
boolean infiniteRows = OptUtils.isUnbounded(dmlConvertedResult.getRel());
SqlPlanImpl dmlPlan = toPlan(
null,
parseResult.getParameterMetadata(),
dmlConvertedResult.getRel(),
dmlConvertedResult.getFieldNames(),
context,
true,
query);
assert dmlPlan instanceof DmlPlan && ((DmlPlan) dmlPlan).getOperation() == Operation.INSERT;
return new CreateJobPlan(
planKey,
sqlCreateJob.jobConfig(),
sqlCreateJob.ifNotExists(),
(DmlPlan) dmlPlan,
query,
infiniteRows,
planExecutor
);
}
private SqlPlan toAlterJobPlan(PlanKey planKey, SqlAlterJob sqlAlterJob) {
return new AlterJobPlan(planKey, sqlAlterJob.name(), sqlAlterJob.getOperation(), planExecutor);
}
private SqlPlan toDropJobPlan(PlanKey planKey, SqlDropJob sqlDropJob) {
return new DropJobPlan(
planKey,
sqlDropJob.name(),
sqlDropJob.ifExists(),
sqlDropJob.withSnapshotName(),
planExecutor
);
}
private SqlPlan toCreateSnapshotPlan(PlanKey planKey, SqlCreateSnapshot sqlNode) {
return new CreateSnapshotPlan(planKey, sqlNode.getSnapshotName(), sqlNode.getJobName(), planExecutor);
}
private SqlPlan toDropSnapshotPlan(PlanKey planKey, SqlDropSnapshot sqlNode) {
return new DropSnapshotPlan(planKey, sqlNode.getSnapshotName(), sqlNode.isIfExists(), planExecutor);
}
private SqlPlan toCreateViewPlan(PlanKey planKey, OptimizerContext context, SqlCreateView sqlNode) {
SqlString sqlString = sqlNode.getQuery().toSqlString(PostgresqlSqlDialect.DEFAULT);
String sql = sqlString.getSql();
boolean replace = sqlNode.getReplace();
boolean ifNotExists = sqlNode.ifNotExists;
return new CreateViewPlan(
planKey,
context,
sqlNode.name(),
sql,
replace,
ifNotExists,
planExecutor
);
}
private SqlPlan toDropViewPlan(PlanKey planKey, SqlDropView sqlNode) {
return new DropViewPlan(planKey, sqlNode.viewName(), sqlNode.ifExists(), planExecutor);
}
private SqlPlan toDropTypePlan(PlanKey planKey, SqlDropType sqlNode) {
return new DropTypePlan(planKey, sqlNode.typeName(), sqlNode.ifExists(), planExecutor);
}
private SqlPlan toShowStatementPlan(PlanKey planKey, SqlShowStatement sqlNode) {
return new ShowStatementPlan(planKey, sqlNode.getTarget(), planExecutor);
}
private SqlPlan toExplainStatementPlan(
PlanKey planKey,
OptimizerContext context,
QueryParseResult parseResult
) {
SqlNode node = parseResult.getNode();
assert node instanceof SqlExplainStatement;
QueryConvertResult convertResult = context.convert(((SqlExplainStatement) node).getExplicandum());
PhysicalRel physicalRel = optimize(
parseResult.getParameterMetadata(),
convertResult.getRel(),
context,
false
);
return new ExplainStatementPlan(planKey, physicalRel, planExecutor);
}
private SqlPlan toCreateTypePlan(PlanKey planKey, SqlCreateType sqlNode) {
final List columns = sqlNode.columns()
.map(column -> new TypeDefinitionColumn(column.name(), column.type()))
.collect(toList());
return new CreateTypePlan(
planKey,
sqlNode.getName(),
sqlNode.getReplace(),
sqlNode.ifNotExists(),
columns,
sqlNode.options(),
planExecutor
);
}
private SqlPlanImpl toPlan(
PlanKey planKey,
QueryParameterMetadata parameterMetadata,
RelNode rel,
List fieldNames,
OptimizerContext context,
boolean isCreateJob,
String query
) {
PhysicalRel physicalRel = optimize(parameterMetadata, rel, context, isCreateJob);
List permissions = extractPermissions(physicalRel);
if (physicalRel instanceof SelectByKeyMapPhysicalRel) {
assert !isCreateJob;
SelectByKeyMapPhysicalRel select = (SelectByKeyMapPhysicalRel) physicalRel;
SqlRowMetadata rowMetadata = createRowMetadata(
fieldNames,
physicalRel.schema(parameterMetadata).getTypes(),
rel.getRowType().getFieldList()
);
return new IMapSelectPlan(
planKey,
select.objectKey(),
parameterMetadata, select.mapName(),
select.keyCondition(parameterMetadata),
select.rowProjectorSupplier(parameterMetadata),
rowMetadata,
planExecutor,
permissions
);
} else if (physicalRel instanceof InsertMapPhysicalRel) {
assert !isCreateJob;
InsertMapPhysicalRel insert = (InsertMapPhysicalRel) physicalRel;
return new IMapInsertPlan(
planKey,
insert.objectKey(),
parameterMetadata,
insert.mapName(),
insert.entriesFn(),
planExecutor,
permissions
);
} else if (physicalRel instanceof SinkMapPhysicalRel) {
assert !isCreateJob;
SinkMapPhysicalRel sink = (SinkMapPhysicalRel) physicalRel;
return new IMapSinkPlan(
planKey,
sink.objectKey(),
parameterMetadata,
sink.mapName(),
sink.entriesFn(),
planExecutor,
permissions
);
} else if (physicalRel instanceof UpdateByKeyMapPhysicalRel) {
assert !isCreateJob;
UpdateByKeyMapPhysicalRel update = (UpdateByKeyMapPhysicalRel) physicalRel;
return new IMapUpdatePlan(
planKey,
update.objectKey(),
parameterMetadata,
update.mapName(),
update.keyCondition(parameterMetadata),
update.updaterSupplier(parameterMetadata),
planExecutor,
permissions
);
} else if (physicalRel instanceof DeleteByKeyMapPhysicalRel) {
assert !isCreateJob;
DeleteByKeyMapPhysicalRel delete = (DeleteByKeyMapPhysicalRel) physicalRel;
return new IMapDeletePlan(
planKey,
delete.objectKey(),
parameterMetadata,
delete.mapName(),
delete.keyCondition(parameterMetadata),
planExecutor,
permissions
);
} else if (physicalRel instanceof TableModify) {
checkDmlOperationWithView(physicalRel);
Operation operation = ((TableModify) physicalRel).getOperation();
Tuple2> dagAndKeys = createDag(physicalRel, parameterMetadata, context.getUsedViews());
return new DmlPlan(
operation,
planKey,
parameterMetadata,
dagAndKeys.f1(),
dagAndKeys.f0(),
query,
OptUtils.isUnbounded(physicalRel),
planExecutor,
permissions
);
} else {
Tuple2> dagAndKeys = createDag(new RootRel(physicalRel), parameterMetadata,
context.getUsedViews());
SqlRowMetadata rowMetadata = createRowMetadata(
fieldNames,
physicalRel.schema(parameterMetadata).getTypes(),
rel.getRowType().getFieldList()
);
return new SelectPlan(
planKey,
parameterMetadata,
dagAndKeys.f1(),
dagAndKeys.f0(),
query,
OptUtils.isUnbounded(physicalRel),
rowMetadata,
planExecutor,
permissions
);
}
}
private List extractPermissions(PhysicalRel physicalRel) {
List permissions = new ArrayList<>();
physicalRel.accept(new RelShuttleImpl() {
@Override
public RelNode visit(TableScan scan) {
addPermissionForTable(scan.getTable(), ActionConstants.ACTION_READ);
return super.visit(scan);
}
@Override
public RelNode visit(RelNode other) {
addPermissionForTable(other.getTable(), ActionConstants.ACTION_PUT);
return super.visit(other);
}
private void addPermissionForTable(RelOptTable t, String action) {
if (t == null) {
return;
}
HazelcastTable table = t.unwrap(HazelcastTable.class);
if (table != null && table.getTarget() instanceof AbstractMapTable) {
String mapName = ((AbstractMapTable) table.getTarget()).getMapName();
permissions.add(new MapPermission(mapName, action));
}
}
});
return permissions;
}
private PhysicalRel optimize(
QueryParameterMetadata parameterMetadata,
RelNode rel,
OptimizerContext context,
boolean isCreateJob
) {
context.setParameterMetadata(parameterMetadata);
context.setRequiresJob(isCreateJob);
boolean fineLogOn = logger.isFineEnabled();
if (fineLogOn) {
logger.fine("Before logical opt:\n" + RelOptUtil.toString(rel));
}
LogicalRel logicalRel = optimizeLogical(context, rel);
if (fineLogOn) {
logger.fine("After logical opt:\n" + RelOptUtil.toString(logicalRel));
}
LogicalRel logicalRel2 = optimizeIMapKeyedAccess(context, logicalRel);
if (fineLogOn && logicalRel != logicalRel2) {
logger.fine("After IMap keyed access opt:\n" + RelOptUtil.toString(logicalRel2));
}
PhysicalRel physicalRel = optimizePhysical(context, logicalRel2);
physicalRel = uniquifyScans(physicalRel);
if (fineLogOn) {
logger.fine("After physical opt:\n" + RelOptUtil.toString(physicalRel));
}
return physicalRel;
}
/**
* Perform logical optimization.
*
* @param rel Original logical tree.
* @return Optimized logical tree.
*/
private LogicalRel optimizeLogical(OptimizerContext context, RelNode rel) {
return (LogicalRel) context.optimize(
rel,
LogicalRules.getRuleSet(),
OptUtils.toLogicalConvention(rel.getTraitSet())
);
}
private LogicalRel optimizeIMapKeyedAccess(OptimizerContext context, LogicalRel rel) {
if (!(rel instanceof FullScanLogicalRel)) {
return rel;
}
return (LogicalRel) context.optimize(
rel,
RuleSets.ofList(SelectByKeyMapLogicalRule.INSTANCE),
OptUtils.toLogicalConvention(rel.getTraitSet())
);
}
/**
* Perform physical optimization.
* This is where proper access methods and algorithms for joins and aggregations are chosen.
*
* @param rel Optimized logical tree.
* @return Optimized physical tree.
*/
private PhysicalRel optimizePhysical(OptimizerContext context, RelNode rel) {
return (PhysicalRel) context.optimize(
rel,
PhysicalRules.getRuleSet(),
OptUtils.toPhysicalConvention(rel.getTraitSet())
);
}
/**
* Assign a discriminator to each scan in the plan. This is essentially hack
* to make the scans unique. We need this because in the MEMO structure, the
* plan can contain the same instance of a RelNode multiple times, if it's
* identical. This happens if the query, for example, reads the same table
* twice. The {@link WatermarkKeysAssigner} might need to assign a different
* key to two identical scans, and it can't do it if they are the same
* instance.
*/
public static PhysicalRel uniquifyScans(PhysicalRel rel) {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
// Note that we must create a new instance of the rule for each optimization, because
// the rule has a state that is used during the "optimization".
AssignDiscriminatorToScansRule rule = new AssignDiscriminatorToScansRule();
hepProgramBuilder.addRuleInstance(rule);
HepPlanner planner = new HepPlanner(
hepProgramBuilder.build(),
Contexts.empty(),
true,
null,
RelOptCostImpl.FACTORY
);
planner.setRoot(rel);
return (PhysicalRel) planner.findBestExp();
}
private SqlRowMetadata createRowMetadata(
List columnNames,
List columnTypes,
List fields
) {
assert columnNames.size() == columnTypes.size();
assert columnTypes.size() == fields.size();
List columns = new ArrayList<>(columnNames.size());
for (int i = 0; i < columnNames.size(); i++) {
SqlColumnMetadata column = QueryUtils.getColumnMetadata(
columnNames.get(i),
columnTypes.get(i),
fields.get(i).getType().isNullable()
);
columns.add(column);
}
return new SqlRowMetadata(columns);
}
private Tuple2> createDag(
PhysicalRel physicalRel,
QueryParameterMetadata parameterMetadata,
Set usedViews
) {
WatermarkKeysAssigner wmKeysAssigner = new WatermarkKeysAssigner(physicalRel);
// we should assign watermark keys also for bounded jobs, but due to the
// issue in key assigner we only do it for unbounded
// See https://github.com/hazelcast/hazelcast/issues/21984
if (OptUtils.isUnbounded(physicalRel)) {
wmKeysAssigner.assignWatermarkKeys();
logger.finest("Watermark keys assigned");
}
CreateDagVisitor visitor = new CreateDagVisitor(nodeEngine, parameterMetadata, wmKeysAssigner, usedViews);
physicalRel.accept(visitor);
visitor.optimizeFinishedDag();
return tuple2(visitor.getDag(), visitor.getObjectKeys());
}
private void checkDmlOperationWithView(PhysicalRel rel) {
HazelcastTable table = Objects.requireNonNull(rel.getTable()).unwrap(HazelcastTable.class);
if (table.getTarget() instanceof ViewTable) {
throw QueryException.error("DML operations not supported for views");
}
}
}