com.hazelcast.org.apache.calcite.tools.Programs Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hazelcast.org.apache.calcite.tools;
import com.hazelcast.org.apache.calcite.adapter.enumerable.EnumerableRules;
import com.hazelcast.org.apache.calcite.config.CalciteConnectionConfig;
import com.hazelcast.org.apache.calcite.config.CalciteSystemProperty;
import com.hazelcast.org.apache.calcite.plan.RelOptCostImpl;
import com.hazelcast.org.apache.calcite.plan.RelOptLattice;
import com.hazelcast.org.apache.calcite.plan.RelOptMaterialization;
import com.hazelcast.org.apache.calcite.plan.RelOptPlanner;
import com.hazelcast.org.apache.calcite.plan.RelOptRule;
import com.hazelcast.org.apache.calcite.plan.RelOptRules;
import com.hazelcast.org.apache.calcite.plan.RelOptUtil;
import com.hazelcast.org.apache.calcite.plan.RelTraitSet;
import com.hazelcast.org.apache.calcite.plan.hep.HepMatchOrder;
import com.hazelcast.org.apache.calcite.plan.hep.HepPlanner;
import com.hazelcast.org.apache.calcite.plan.hep.HepProgram;
import com.hazelcast.org.apache.calcite.plan.hep.HepProgramBuilder;
import com.hazelcast.org.apache.calcite.rel.RelNode;
import com.hazelcast.org.apache.calcite.rel.core.Calc;
import com.hazelcast.org.apache.calcite.rel.core.RelFactories;
import com.hazelcast.org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
import com.hazelcast.org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
import com.hazelcast.org.apache.calcite.rel.metadata.RelMetadataProvider;
import com.hazelcast.org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
import com.hazelcast.org.apache.calcite.rel.rules.AggregateReduceFunctionsRule;
import com.hazelcast.org.apache.calcite.rel.rules.AggregateStarTableRule;
import com.hazelcast.org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
import com.hazelcast.org.apache.calcite.rel.rules.FilterJoinRule;
import com.hazelcast.org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import com.hazelcast.org.apache.calcite.rel.rules.FilterTableScanRule;
import com.hazelcast.org.apache.calcite.rel.rules.JoinAssociateRule;
import com.hazelcast.org.apache.calcite.rel.rules.JoinCommuteRule;
import com.hazelcast.org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
import com.hazelcast.org.apache.calcite.rel.rules.JoinToMultiJoinRule;
import com.hazelcast.org.apache.calcite.rel.rules.LoptOptimizeJoinRule;
import com.hazelcast.org.apache.calcite.rel.rules.MatchRule;
import com.hazelcast.org.apache.calcite.rel.rules.MultiJoinOptimizeBushyRule;
import com.hazelcast.org.apache.calcite.rel.rules.ProjectMergeRule;
import com.hazelcast.org.apache.calcite.rel.rules.SemiJoinRule;
import com.hazelcast.org.apache.calcite.rel.rules.SortProjectTransposeRule;
import com.hazelcast.org.apache.calcite.rel.rules.SubQueryRemoveRule;
import com.hazelcast.org.apache.calcite.sql2rel.RelDecorrelator;
import com.hazelcast.org.apache.calcite.sql2rel.RelFieldTrimmer;
import com.hazelcast.org.apache.calcite.sql2rel.SqlToRelConverter;
import com.hazelcast.com.google.common.collect.ImmutableList;
import com.hazelcast.com.google.common.collect.ImmutableSet;
import com.hazelcast.com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* Utilities for creating {@link Program}s.
*/
public class Programs {
@Deprecated // to be removed before 2.0
public static final ImmutableList CALC_RULES = RelOptRules.CALC_RULES;
/** Program that converts filters and projects to {@link Calc}s. */
public static final Program CALC_PROGRAM =
calc(DefaultRelMetadataProvider.INSTANCE);
/** Program that expands sub-queries. */
public static final Program SUB_QUERY_PROGRAM =
subQuery(DefaultRelMetadataProvider.INSTANCE);
public static final ImmutableSet RULE_SET =
ImmutableSet.of(
EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE,
EnumerableRules.ENUMERABLE_JOIN_RULE,
EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE,
EnumerableRules.ENUMERABLE_CORRELATE_RULE,
EnumerableRules.ENUMERABLE_PROJECT_RULE,
EnumerableRules.ENUMERABLE_FILTER_RULE,
EnumerableRules.ENUMERABLE_AGGREGATE_RULE,
EnumerableRules.ENUMERABLE_SORT_RULE,
EnumerableRules.ENUMERABLE_LIMIT_RULE,
EnumerableRules.ENUMERABLE_UNION_RULE,
EnumerableRules.ENUMERABLE_INTERSECT_RULE,
EnumerableRules.ENUMERABLE_MINUS_RULE,
EnumerableRules.ENUMERABLE_TABLE_MODIFICATION_RULE,
EnumerableRules.ENUMERABLE_VALUES_RULE,
EnumerableRules.ENUMERABLE_WINDOW_RULE,
EnumerableRules.ENUMERABLE_MATCH_RULE,
SemiJoinRule.PROJECT,
SemiJoinRule.JOIN,
MatchRule.INSTANCE,
CalciteSystemProperty.COMMUTE.value()
? JoinAssociateRule.INSTANCE
: ProjectMergeRule.INSTANCE,
AggregateStarTableRule.INSTANCE,
AggregateStarTableRule.INSTANCE2,
FilterTableScanRule.INSTANCE,
FilterProjectTransposeRule.INSTANCE,
FilterJoinRule.FILTER_ON_JOIN,
AggregateExpandDistinctAggregatesRule.INSTANCE,
AggregateReduceFunctionsRule.INSTANCE,
FilterAggregateTransposeRule.INSTANCE,
JoinCommuteRule.INSTANCE,
JoinPushThroughJoinRule.RIGHT,
JoinPushThroughJoinRule.LEFT,
SortProjectTransposeRule.INSTANCE);
// private constructor for utility class
private Programs() {}
/** Creates a program that executes a rule set. */
public static Program of(RuleSet ruleSet) {
return new RuleSetProgram(ruleSet);
}
/** Creates a list of programs based on an array of rule sets. */
public static List listOf(RuleSet... ruleSets) {
return Lists.transform(Arrays.asList(ruleSets), Programs::of);
}
/** Creates a list of programs based on a list of rule sets. */
public static List listOf(List ruleSets) {
return Lists.transform(ruleSets, Programs::of);
}
/** Creates a program from a list of rules. */
public static Program ofRules(RelOptRule... rules) {
return of(RuleSets.ofList(rules));
}
/** Creates a program from a list of rules. */
public static Program ofRules(Iterable rules) {
return of(RuleSets.ofList(rules));
}
/** Creates a program that executes a sequence of programs. */
public static Program sequence(Program... programs) {
return new SequenceProgram(ImmutableList.copyOf(programs));
}
/** Creates a program that executes a list of rules in a HEP planner. */
public static Program hep(Iterable rules,
boolean noDag, RelMetadataProvider metadataProvider) {
final HepProgramBuilder builder = HepProgram.builder();
for (RelOptRule rule : rules) {
builder.addRuleInstance(rule);
}
return of(builder.build(), noDag, metadataProvider);
}
/** Creates a program that executes a {@link HepProgram}. */
public static Program of(final HepProgram hepProgram, final boolean noDag,
final RelMetadataProvider metadataProvider) {
return (planner, rel, requiredOutputTraits, materializations, lattices) -> {
final HepPlanner hepPlanner = new HepPlanner(hepProgram,
null, noDag, null, RelOptCostImpl.FACTORY);
List list = new ArrayList<>();
if (metadataProvider != null) {
list.add(metadataProvider);
}
hepPlanner.registerMetadataProviders(list);
for (RelOptMaterialization materialization : materializations) {
hepPlanner.addMaterialization(materialization);
}
for (RelOptLattice lattice : lattices) {
hepPlanner.addLattice(lattice);
}
RelMetadataProvider plannerChain =
ChainedRelMetadataProvider.of(list);
rel.getCluster().setMetadataProvider(plannerChain);
hepPlanner.setRoot(rel);
return hepPlanner.findBestExp();
};
}
/** Creates a program that invokes heuristic join-order optimization
* (via {@link com.hazelcast.org.apache.calcite.rel.rules.JoinToMultiJoinRule},
* {@link com.hazelcast.org.apache.calcite.rel.rules.MultiJoin} and
* {@link com.hazelcast.org.apache.calcite.rel.rules.LoptOptimizeJoinRule})
* if there are 6 or more joins (7 or more relations). */
public static Program heuristicJoinOrder(
final Iterable rules,
final boolean bushy, final int minJoinCount) {
return (planner, rel, requiredOutputTraits, materializations, lattices) -> {
final int joinCount = RelOptUtil.countJoins(rel);
final Program program;
if (joinCount < minJoinCount) {
program = ofRules(rules);
} else {
// Create a program that gathers together joins as a MultiJoin.
final HepProgram hep = new HepProgramBuilder()
.addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
.addMatchOrder(HepMatchOrder.BOTTOM_UP)
.addRuleInstance(JoinToMultiJoinRule.INSTANCE)
.build();
final Program program1 =
of(hep, false, DefaultRelMetadataProvider.INSTANCE);
// Create a program that contains a rule to expand a MultiJoin
// into heuristically ordered joins.
// We use the rule set passed in, but remove JoinCommuteRule and
// JoinPushThroughJoinRule, because they cause exhaustive search.
final List list = Lists.newArrayList(rules);
list.removeAll(
ImmutableList.of(JoinCommuteRule.INSTANCE,
JoinAssociateRule.INSTANCE,
JoinPushThroughJoinRule.LEFT,
JoinPushThroughJoinRule.RIGHT));
list.add(bushy
? MultiJoinOptimizeBushyRule.INSTANCE
: LoptOptimizeJoinRule.INSTANCE);
final Program program2 = ofRules(list);
program = sequence(program1, program2);
}
return program.run(
planner, rel, requiredOutputTraits, materializations, lattices);
};
}
public static Program calc(RelMetadataProvider metadataProvider) {
return hep(RelOptRules.CALC_RULES, true, metadataProvider);
}
@Deprecated // to be removed before 2.0
public static Program subquery(RelMetadataProvider metadataProvider) {
return subQuery(metadataProvider);
}
public static Program subQuery(RelMetadataProvider metadataProvider) {
final HepProgramBuilder builder = HepProgram.builder();
builder.addRuleCollection(
ImmutableList.of(SubQueryRemoveRule.FILTER,
SubQueryRemoveRule.PROJECT,
SubQueryRemoveRule.JOIN));
return of(builder.build(), true, metadataProvider);
}
public static Program getProgram() {
return (planner, rel, requiredOutputTraits, materializations, lattices) ->
null;
}
/** Returns the standard program used by Prepare. */
public static Program standard() {
return standard(DefaultRelMetadataProvider.INSTANCE);
}
/** Returns the standard program with user metadata provider. */
public static Program standard(RelMetadataProvider metadataProvider) {
final Program program1 =
(planner, rel, requiredOutputTraits, materializations, lattices) -> {
planner.setRoot(rel);
for (RelOptMaterialization materialization : materializations) {
planner.addMaterialization(materialization);
}
for (RelOptLattice lattice : lattices) {
planner.addLattice(lattice);
}
final RelNode rootRel2 =
rel.getTraitSet().equals(requiredOutputTraits)
? rel
: planner.changeTraits(rel, requiredOutputTraits);
assert rootRel2 != null;
planner.setRoot(rootRel2);
final RelOptPlanner planner2 = planner.chooseDelegate();
final RelNode rootRel3 = planner2.findBestExp();
assert rootRel3 != null : "could not implement exp";
return rootRel3;
};
return sequence(subQuery(metadataProvider),
new DecorrelateProgram(),
new TrimFieldsProgram(),
program1,
// Second planner pass to do physical "tweaks". This the first time
// that EnumerableCalcRel is introduced.
calc(metadataProvider));
}
/** Program backed by a {@link RuleSet}. */
static class RuleSetProgram implements Program {
final RuleSet ruleSet;
private RuleSetProgram(RuleSet ruleSet) {
this.ruleSet = ruleSet;
}
public RelNode run(RelOptPlanner planner, RelNode rel,
RelTraitSet requiredOutputTraits,
List materializations,
List lattices) {
planner.clear();
for (RelOptRule rule : ruleSet) {
planner.addRule(rule);
}
for (RelOptMaterialization materialization : materializations) {
planner.addMaterialization(materialization);
}
for (RelOptLattice lattice : lattices) {
planner.addLattice(lattice);
}
if (!rel.getTraitSet().equals(requiredOutputTraits)) {
rel = planner.changeTraits(rel, requiredOutputTraits);
}
planner.setRoot(rel);
return planner.findBestExp();
}
}
/** Program that runs sub-programs, sending the output of the previous as
* input to the next. */
private static class SequenceProgram implements Program {
private final ImmutableList programs;
SequenceProgram(ImmutableList programs) {
this.programs = programs;
}
public RelNode run(RelOptPlanner planner, RelNode rel,
RelTraitSet requiredOutputTraits,
List materializations,
List lattices) {
for (Program program : programs) {
rel = program.run(
planner, rel, requiredOutputTraits, materializations, lattices);
}
return rel;
}
}
/** Program that de-correlates a query.
*
* To work around
* [CALCITE-842]
* Decorrelator gets field offsets confused if fields have been trimmed,
* disable field-trimming in {@link SqlToRelConverter}, and run
* {@link TrimFieldsProgram} after this program. */
private static class DecorrelateProgram implements Program {
public RelNode run(RelOptPlanner planner, RelNode rel,
RelTraitSet requiredOutputTraits,
List materializations,
List lattices) {
final CalciteConnectionConfig config =
planner.getContext().unwrap(CalciteConnectionConfig.class);
if (config != null && config.forceDecorrelate()) {
final RelBuilder relBuilder =
RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null);
return RelDecorrelator.decorrelateQuery(rel, relBuilder);
}
return rel;
}
}
/** Program that trims fields. */
private static class TrimFieldsProgram implements Program {
public RelNode run(RelOptPlanner planner, RelNode rel,
RelTraitSet requiredOutputTraits,
List materializations,
List lattices) {
final RelBuilder relBuilder =
RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null);
return new RelFieldTrimmer(null, relBuilder).trim(rel);
}
}
}