All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.hazelcast.org.apache.calcite.plan.SubstitutionVisitor Maven / Gradle / Ivy

There is a newer version: 5.4.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to you under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in com.hazelcast.com.liance with
 * the License.  You may obtain a copy of the License at
 *
 * http://www.apache.com.hazelcast.org.licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hazelcast.org.apache.calcite.plan;

import com.hazelcast.org.apache.calcite.config.CalciteSystemProperty;
import com.hazelcast.org.apache.calcite.rel.RelNode;
import com.hazelcast.org.apache.calcite.rel.core.Aggregate;
import com.hazelcast.org.apache.calcite.rel.core.AggregateCall;
import com.hazelcast.org.apache.calcite.rel.core.JoinRelType;
import com.hazelcast.org.apache.calcite.rel.core.Project;
import com.hazelcast.org.apache.calcite.rel.core.RelFactories;
import com.hazelcast.org.apache.calcite.rel.mutable.Holder;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableAggregate;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableCalc;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableFilter;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableIntersect;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableJoin;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableMinus;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableRel;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableRelVisitor;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableRels;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableScan;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableSetOp;
import com.hazelcast.org.apache.calcite.rel.mutable.MutableUnion;
import com.hazelcast.org.apache.calcite.rel.type.RelDataType;
import com.hazelcast.org.apache.calcite.rel.type.RelDataTypeField;
import com.hazelcast.org.apache.calcite.rex.RexBuilder;
import com.hazelcast.org.apache.calcite.rex.RexCall;
import com.hazelcast.org.apache.calcite.rex.RexExecutor;
import com.hazelcast.org.apache.calcite.rex.RexExecutorImpl;
import com.hazelcast.org.apache.calcite.rex.RexInputRef;
import com.hazelcast.org.apache.calcite.rex.RexLiteral;
import com.hazelcast.org.apache.calcite.rex.RexLocalRef;
import com.hazelcast.org.apache.calcite.rex.RexNode;
import com.hazelcast.org.apache.calcite.rex.RexOver;
import com.hazelcast.org.apache.calcite.rex.RexProgram;
import com.hazelcast.org.apache.calcite.rex.RexProgramBuilder;
import com.hazelcast.org.apache.calcite.rex.RexShuttle;
import com.hazelcast.org.apache.calcite.rex.RexSimplify;
import com.hazelcast.org.apache.calcite.rex.RexUtil;
import com.hazelcast.org.apache.calcite.rex.RexVisitor;
import com.hazelcast.org.apache.calcite.rex.RexVisitorImpl;
import com.hazelcast.org.apache.calcite.sql.SqlAggFunction;
import com.hazelcast.org.apache.calcite.sql.fun.SqlStdOperatorTable;
import com.hazelcast.org.apache.calcite.tools.RelBuilder;
import com.hazelcast.org.apache.calcite.tools.RelBuilderFactory;
import com.hazelcast.org.apache.calcite.util.ControlFlowException;
import com.hazelcast.org.apache.calcite.util.ImmutableBitSet;
import com.hazelcast.org.apache.calcite.util.Litmus;
import com.hazelcast.org.apache.calcite.util.Pair;
import com.hazelcast.org.apache.calcite.util.Util;
import com.hazelcast.org.apache.calcite.util.mapping.Mapping;
import com.hazelcast.org.apache.calcite.util.mapping.Mappings;
import com.hazelcast.org.apache.calcite.util.trace.CalciteTrace;

import com.hazelcast.com.google.com.hazelcast.com.on.annotations.VisibleForTesting;
import com.hazelcast.com.google.com.hazelcast.com.on.collect.ImmutableList;
import com.hazelcast.com.google.com.hazelcast.com.on.collect.LinkedHashMultimap;
import com.hazelcast.com.google.com.hazelcast.com.on.collect.Multimap;
import com.hazelcast.com.google.com.hazelcast.com.on.collect.Sets;

import com.hazelcast.org.slf4j.Logger;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;

import static com.hazelcast.org.apache.calcite.rex.RexUtil.andNot;
import static com.hazelcast.org.apache.calcite.rex.RexUtil.removeAll;

/**
 * Substitutes part of a tree of relational expressions with another tree.
 *
 * 

The call {@code new SubstitutionVisitor(target, query).go(replacement))} * will return {@code query} with every occurrence of {@code target} replaced * by {@code replacement}.

* *

The following example shows how {@code SubstitutionVisitor} can be used * for materialized view recognition.

* *
    *
  • query = SELECT a, c FROM t WHERE x = 5 AND b = 4
  • *
  • target = SELECT a, b, c FROM t WHERE x = 5
  • *
  • replacement = SELECT * FROM mv
  • *
  • result = SELECT a, c FROM mv WHERE b = 4
  • *
* *

Note that {@code result} uses the materialized view table {@code mv} and a * simplified condition {@code b = 4}.

* *

Uses a bottom-up matching algorithm. Nodes do not need to be identical. * At each level, returns the residue.

* *

The inputs must only include the core relational operators: * {@link com.hazelcast.org.apache.calcite.rel.core.TableScan}, * {@link com.hazelcast.org.apache.calcite.rel.core.Filter}, * {@link com.hazelcast.org.apache.calcite.rel.core.Project}, * {@link com.hazelcast.org.apache.calcite.rel.core.Calc}, * {@link com.hazelcast.org.apache.calcite.rel.core.Join}, * {@link com.hazelcast.org.apache.calcite.rel.core.Union}, * {@link com.hazelcast.org.apache.calcite.rel.core.Intersect}, * {@link com.hazelcast.org.apache.calcite.rel.core.Aggregate}.

*/ public class SubstitutionVisitor { private static final boolean DEBUG = CalciteSystemProperty.DEBUG.value(); private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); protected static final ImmutableList DEFAULT_RULES = ImmutableList.of( TrivialRule.INSTANCE, ScanToCalcUnifyRule.INSTANCE, CalcToCalcUnifyRule.INSTANCE, JoinOnLeftCalcToJoinUnifyRule.INSTANCE, JoinOnRightCalcToJoinUnifyRule.INSTANCE, JoinOnCalcsToJoinUnifyRule.INSTANCE, AggregateToAggregateUnifyRule.INSTANCE, AggregateOnCalcToAggregateUnifyRule.INSTANCE, UnionToUnionUnifyRule.INSTANCE, UnionOnCalcsToUnionUnifyRule.INSTANCE, IntersectToIntersectUnifyRule.INSTANCE, IntersectOnCalcsToIntersectUnifyRule.INSTANCE); /** * Factory for a builder for relational expressions. */ protected final RelBuilder relBuilder; private final ImmutableList rules; private final Map, List> ruleMap = new HashMap<>(); private final RelOptCluster cluster; private final RexSimplify simplify; private final Holder query; private final MutableRel target; /** * Nodes in {@link #target} that have no children. */ final List targetLeaves; /** * Nodes in {@link #query} that have no children. */ final List queryLeaves; final Map replacementMap = new HashMap<>(); final Multimap equivalents = LinkedHashMultimap.create(); /** Workspace while rule is being matched. * Careful, re-entrant! * Assumes no rule needs more than 2 slots. */ protected final MutableRel[] slots = new MutableRel[2]; /** Creates a SubstitutionVisitor with the default rule set. */ public SubstitutionVisitor(RelNode target_, RelNode query_) { this(target_, query_, DEFAULT_RULES, RelFactories.LOGICAL_BUILDER); } /** Creates a SubstitutionVisitor with the default logical builder. */ public SubstitutionVisitor(RelNode target_, RelNode query_, ImmutableList rules) { this(target_, query_, rules, RelFactories.LOGICAL_BUILDER); } public SubstitutionVisitor(RelNode target_, RelNode query_, ImmutableList rules, RelBuilderFactory relBuilderFactory) { this.cluster = target_.getCluster(); final RexExecutor executor = Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR); final RelOptPredicateList predicates = RelOptPredicateList.EMPTY; this.simplify = new RexSimplify(cluster.getRexBuilder(), predicates, executor); this.rules = rules; this.query = Holder.of(MutableRels.toMutable(query_)); this.target = MutableRels.toMutable(target_); this.relBuilder = relBuilderFactory.create(cluster, null); final Set parents = Sets.newIdentityHashSet(); final List allNodes = new ArrayList<>(); final MutableRelVisitor visitor = new MutableRelVisitor() { public void visit(MutableRel node) { parents.add(node.getParent()); allNodes.add(node); super.visit(node); } }; visitor.go(target); // Populate the list of leaves in the tree under "target". // Leaves are all nodes that are not parents. // For determinism, it is important that the list is in scan order. allNodes.removeAll(parents); targetLeaves = ImmutableList.copyOf(allNodes); allNodes.clear(); parents.clear(); visitor.go(query); allNodes.removeAll(parents); queryLeaves = ImmutableList.copyOf(allNodes); } void register(MutableRel result, MutableRel query) { } /** * Maps a condition onto a target. * *

If condition is stronger than target, returns the residue. * If it is equal to target, returns the expression that evaluates to * the constant {@code true}. If it is weaker than target, returns * {@code null}.

* *

The terms satisfy the relation

* *
*
{@code condition = target AND residue}
*
* *

and {@code residue} must be as weak as possible.

* *

Example #1: condition stronger than target

*
    *
  • condition: x = 1 AND y = 2
  • *
  • target: x = 1
  • *
  • residue: y = 2
  • *
* *

Note that residue {@code x > 0 AND y = 2} would also satisfy the * relation {@code condition = target AND residue} but is stronger than * necessary, so we prefer {@code y = 2}.

* *

Example #2: target weaker than condition (valid, but not currently * implemented)

*
    *
  • condition: x = 1
  • *
  • target: x = 1 OR z = 3
  • *
  • residue: x = 1
  • *
* *

Example #3: condition and target are equivalent

*
    *
  • condition: x = 1 AND y = 2
  • *
  • target: y = 2 AND x = 1
  • *
  • residue: TRUE
  • *
* *

Example #4: condition weaker than target

*
    *
  • condition: x = 1
  • *
  • target: x = 1 AND y = 2
  • *
  • residue: null (i.e. no match)
  • *
* *

There are many other possible examples. It amounts to solving * whether {@code condition AND NOT target} can ever evaluate to * true, and therefore is a form of the NP-com.hazelcast.com.lete * Satisfiability * problem.

*/ @VisibleForTesting public static RexNode splitFilter(final RexSimplify simplify, RexNode condition, RexNode target) { final RexBuilder rexBuilder = simplify.rexBuilder; RexNode condition2 = canonizeNode(rexBuilder, condition); RexNode target2 = canonizeNode(rexBuilder, target); // First, try splitting into ORs. // Given target c1 OR c2 OR c3 OR c4 // and condition c2 OR c4 // residue is c2 OR c4 // Also deals with case target [x] condition [x] yields residue [true]. RexNode z = splitOr(rexBuilder, condition2, target2); if (z != null) { return z; } if (isEquivalent(rexBuilder, condition2, target2)) { return rexBuilder.makeLiteral(true); } RexNode x = andNot(rexBuilder, target2, condition2); if (mayBeSatisfiable(x)) { RexNode x2 = RexUtil.com.hazelcast.com.oseConjunction(rexBuilder, ImmutableList.of(condition2, target2)); RexNode r = canonizeNode(rexBuilder, simplify.simplifyUnknownAsFalse(x2)); if (!r.isAlwaysFalse() && isEquivalent(rexBuilder, condition2, r)) { List conjs = RelOptUtil.conjunctions(r); for (RexNode e : RelOptUtil.conjunctions(target2)) { removeAll(conjs, e); } return RexUtil.com.hazelcast.com.oseConjunction(rexBuilder, conjs); } } return null; } /** * Reorders some of the operands in this expression so structural com.hazelcast.com.arison, * i.e., based on string representation, can be more precise. */ private static RexNode canonizeNode(RexBuilder rexBuilder, RexNode condition) { switch (condition.getKind()) { case AND: case OR: { RexCall call = (RexCall) condition; SortedMap newOperands = new TreeMap<>(); for (RexNode operand : call.operands) { operand = canonizeNode(rexBuilder, operand); newOperands.put(operand.toString(), operand); } if (newOperands.size() < 2) { return newOperands.values().iterator().next(); } return rexBuilder.makeCall(call.getOperator(), ImmutableList.copyOf(newOperands.values())); } case EQUALS: case NOT_EQUALS: case LESS_THAN: case GREATER_THAN: case LESS_THAN_OR_EQUAL: case GREATER_THAN_OR_EQUAL: { RexCall call = (RexCall) condition; RexNode left = canonizeNode(rexBuilder, call.getOperands().get(0)); RexNode right = canonizeNode(rexBuilder, call.getOperands().get(1)); call = (RexCall) rexBuilder.makeCall(call.getOperator(), left, right); if (left.toString().com.hazelcast.com.areTo(right.toString()) <= 0) { return call; } return RexUtil.invert(rexBuilder, call); } case PLUS: case TIMES: { RexCall call = (RexCall) condition; RexNode left = canonizeNode(rexBuilder, call.getOperands().get(0)); RexNode right = canonizeNode(rexBuilder, call.getOperands().get(1)); if (left.toString().com.hazelcast.com.areTo(right.toString()) <= 0) { return rexBuilder.makeCall(call.getOperator(), left, right); } RexNode newCall = rexBuilder.makeCall(call.getOperator(), right, left); // new call should not be used if its inferred type is not same as old if (!newCall.getType().equals(call.getType())) { return call; } return newCall; } default: return condition; } } private static RexNode splitOr( final RexBuilder rexBuilder, RexNode condition, RexNode target) { List conditions = RelOptUtil.disjunctions(condition); int conditionsLength = conditions.size(); int targetsLength = 0; for (RexNode e : RelOptUtil.disjunctions(target)) { removeAll(conditions, e); targetsLength++; } if (conditions.isEmpty() && conditionsLength == targetsLength) { return rexBuilder.makeLiteral(true); } else if (conditions.isEmpty()) { return condition; } return null; } private static boolean isEquivalent(RexBuilder rexBuilder, RexNode condition, RexNode target) { // Example: // e: x = 1 AND y = 2 AND z = 3 AND NOT (x = 1 AND y = 2) // disjunctions: {x = 1, y = 2, z = 3} // notDisjunctions: {x = 1 AND y = 2} final Set conditionDisjunctions = new HashSet<>( RexUtil.strings(RelOptUtil.conjunctions(condition))); final Set targetDisjunctions = new HashSet<>( RexUtil.strings(RelOptUtil.conjunctions(target))); if (conditionDisjunctions.equals(targetDisjunctions)) { return true; } return false; } /** * Returns whether a boolean expression ever returns true. * *

This method may give false positives. For instance, it will say * that {@code x = 5 AND x > 10} is satisfiable, because at present it * cannot prove that it is not.

*/ public static boolean mayBeSatisfiable(RexNode e) { // Example: // e: x = 1 AND y = 2 AND z = 3 AND NOT (x = 1 AND y = 2) // disjunctions: {x = 1, y = 2, z = 3} // notDisjunctions: {x = 1 AND y = 2} final List disjunctions = new ArrayList<>(); final List notDisjunctions = new ArrayList<>(); RelOptUtil.decomposeConjunction(e, disjunctions, notDisjunctions); // If there is a single FALSE or NOT TRUE, the whole expression is // always false. for (RexNode disjunction : disjunctions) { switch (disjunction.getKind()) { case LITERAL: if (!RexLiteral.booleanValue(disjunction)) { return false; } } } for (RexNode disjunction : notDisjunctions) { switch (disjunction.getKind()) { case LITERAL: if (RexLiteral.booleanValue(disjunction)) { return false; } } } // If one of the not-disjunctions is a disjunction that is wholly // contained in the disjunctions list, the expression is not // satisfiable. // // Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable // Example #2. x AND y AND NOT (x AND y) - not satisfiable // Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable for (RexNode notDisjunction : notDisjunctions) { final List disjunctions2 = RelOptUtil.conjunctions(notDisjunction); if (disjunctions.containsAll(disjunctions2)) { return false; } } return true; } public RelNode go0(RelNode replacement_) { assert false; // not called MutableRel replacement = MutableRels.toMutable(replacement_); assert equalType( "target", target, "replacement", replacement, Litmus.THROW); replacementMap.put(target, replacement); final UnifyResult unifyResult = matchRecurse(target); if (unifyResult == null) { return null; } final MutableRel node0 = unifyResult.result; MutableRel node = node0; // replaceAncestors(node0); if (DEBUG) { System.out.println("Convert: query:\n" + query.deep() + "\nunify.query:\n" + unifyResult.call.query.deep() + "\nunify.result:\n" + unifyResult.result.deep() + "\nunify.target:\n" + unifyResult.call.target.deep() + "\nnode0:\n" + node0.deep() + "\nnode:\n" + node.deep()); } return MutableRels.fromMutable(node, relBuilder); } /** * Returns a list of all possible rels that result from substituting the * matched RelNode with the replacement RelNode within the query. * *

For example, the substitution result of A join B, while A and B * are both a qualified match for replacement R, is R join B, R join R, * A join R. */ public List go(RelNode replacement_) { List> matches = go(MutableRels.toMutable(replacement_)); if (matches.isEmpty()) { return ImmutableList.of(); } List sub = new ArrayList<>(); sub.add(MutableRels.fromMutable(query.getInput(), relBuilder)); reverseSubstitute(relBuilder, query, matches, sub, 0, matches.size()); return sub; } /** * Substitutes the query with replacement whenever possible but meanwhile * keeps track of all the substitutions and their original rel before * replacement, so that in later processing stage, the replacement can be * recovered individually to produce a list of all possible rels with * substitution in different places. */ private List> go(MutableRel replacement) { assert equalType( "target", target, "replacement", replacement, Litmus.THROW); final List queryDescendants = MutableRels.descendants(query); final List targetDescendants = MutableRels.descendants(target); // Populate "equivalents" with (q, t) for each query descendant q and // target descendant t that are equal. final Map map = new HashMap<>(); for (MutableRel queryDescendant : queryDescendants) { map.put(queryDescendant, queryDescendant); } for (MutableRel targetDescendant : targetDescendants) { MutableRel queryDescendant = map.get(targetDescendant); if (queryDescendant != null) { assert rowTypesAreEquivalent( queryDescendant, targetDescendant, Litmus.THROW); equivalents.put(queryDescendant, targetDescendant); } } map.clear(); final List attempted = new ArrayList<>(); List> substitutions = new ArrayList<>(); for (;;) { int count = 0; MutableRel queryDescendant = query; outer: while (queryDescendant != null) { for (Replacement r : attempted) { if (r.stopTrying && queryDescendant == r.after) { // This node has been replaced by previous iterations in the // hope to match its ancestors and stopTrying indicates // there's no need to be matched again. queryDescendant = MutableRels.preOrderTraverseNext(queryDescendant); continue outer; } } final MutableRel next = MutableRels.preOrderTraverseNext(queryDescendant); final MutableRel childOrNext = queryDescendant.getInputs().isEmpty() ? next : queryDescendant.getInputs().get(0); for (MutableRel targetDescendant : targetDescendants) { for (UnifyRule rule : applicableRules(queryDescendant, targetDescendant)) { UnifyRuleCall call = rule.match(this, queryDescendant, targetDescendant); if (call != null) { final UnifyResult result = rule.apply(call); if (result != null) { ++count; attempted.add( new Replacement(result.call.query, result.result, result.stopTrying)); result.call.query.replaceInParent(result.result); // Replace previous equivalents with new equivalents, higher up // the tree. for (int i = 0; i < rule.slotCount; i++) { Collection equi = equivalents.get(slots[i]); if (!equi.isEmpty()) { equivalents.remove(slots[i], equi.iterator().next()); } } assert rowTypesAreEquivalent(result.result, result.call.query, Litmus.THROW); equivalents.put(result.result, result.call.query); if (targetDescendant == target) { // A real substitution happens. We purge the attempted // replacement list and add them into substitution list. // Meanwhile we stop matching the descendants and jump // to the next subtree in pre-order traversal. if (!target.equals(replacement)) { Replacement r = replace( query.getInput(), target, replacement.clone()); assert r != null : rule + "should have returned a result containing the target."; attempted.add(r); } substitutions.add(ImmutableList.copyOf(attempted)); attempted.clear(); queryDescendant = next; continue outer; } // We will try walking the query tree all over again to see // if there can be any substitutions after the replacement // attempt. break outer; } } } } queryDescendant = childOrNext; } // Quit the entire loop if: // 1) we have walked the entire query tree with one or more successful // substitutions, thus count != 0 && attempted.isEmpty(); // 2) we have walked the entire query tree but have made no replacement // attempt, thus count == 0 && attempted.isEmpty(); // 3) we had done some replacement attempt in a previous walk, but in // this one we have not found any potential matches or substitutions, // thus count == 0 && !attempted.isEmpty(). if (count == 0 || attempted.isEmpty()) { break; } } if (!attempted.isEmpty()) { // We had done some replacement attempt in the previous walk, but that // did not lead to any substitutions in this walk, so we need to recover // the replacement. undoReplacement(attempted); } return substitutions; } /** * Equivalence checking for row types, but except for the field names. */ private boolean rowTypesAreEquivalent( MutableRel rel0, MutableRel rel1, Litmus litmus) { if (rel0.rowType.getFieldCount() != rel1.rowType.getFieldCount()) { return litmus.fail("Mismatch for column count: [{}]", Pair.of(rel0, rel1)); } for (Pair pair : Pair.zip(rel0.rowType.getFieldList(), rel1.rowType.getFieldList())) { if (!pair.left.getType().equals(pair.right.getType())) { return litmus.fail("Mismatch for column type: [{}]", Pair.of(rel0, rel1)); } } return litmus.succeed(); } /** * Represents a replacement action: before → after. * {@code stopTrying} indicates whether there's no need * to do matching for the same query node again. */ static class Replacement { final MutableRel before; final MutableRel after; final boolean stopTrying; Replacement(MutableRel before, MutableRel after) { this(before, after, true); } Replacement(MutableRel before, MutableRel after, boolean stopTrying) { this.before = before; this.after = after; this.stopTrying = stopTrying; } } /** Within a relational expression {@code query}, replaces occurrences of * {@code find} with {@code replace}. * *

Assumes relational expressions (and their descendants) are not null. * Does not handle cycles. */ public static Replacement replace(MutableRel query, MutableRel find, MutableRel replace) { if (find.equals(replace)) { // Short-cut com.hazelcast.com.on case. return null; } assert equalType("find", find, "replace", replace, Litmus.THROW); return replaceRecurse(query, find, replace); } /** Helper for {@link #replace}. */ private static Replacement replaceRecurse(MutableRel query, MutableRel find, MutableRel replace) { if (find.equals(query)) { query.replaceInParent(replace); return new Replacement(query, replace); } for (MutableRel input : query.getInputs()) { Replacement r = replaceRecurse(input, find, replace); if (r != null) { return r; } } return null; } private static void undoReplacement(List replacement) { for (int i = replacement.size() - 1; i >= 0; i--) { Replacement r = replacement.get(i); r.after.replaceInParent(r.before); } } private static void redoReplacement(List replacement) { for (Replacement r : replacement) { r.before.replaceInParent(r.after); } } private static void reverseSubstitute(RelBuilder relBuilder, Holder query, List> matches, List sub, int replaceCount, int maxCount) { if (matches.isEmpty()) { return; } final List> rem = matches.subList(1, matches.size()); reverseSubstitute(relBuilder, query, rem, sub, replaceCount, maxCount); undoReplacement(matches.get(0)); if (++replaceCount < maxCount) { sub.add(MutableRels.fromMutable(query.getInput(), relBuilder)); } reverseSubstitute(relBuilder, query, rem, sub, replaceCount, maxCount); redoReplacement(matches.get(0)); } private UnifyResult matchRecurse(MutableRel target) { assert false; // not called final List targetInputs = target.getInputs(); MutableRel queryParent = null; for (MutableRel targetInput : targetInputs) { UnifyResult unifyResult = matchRecurse(targetInput); if (unifyResult == null) { return null; } queryParent = unifyResult.call.query.replaceInParent(unifyResult.result); } if (targetInputs.isEmpty()) { for (MutableRel queryLeaf : queryLeaves) { for (UnifyRule rule : applicableRules(queryLeaf, target)) { final UnifyResult x = apply(rule, queryLeaf, target); if (x != null) { if (DEBUG) { System.out.println("Rule: " + rule + "\nQuery:\n" + queryParent + (x.call.query != queryParent ? "\nQuery (original):\n" + queryParent : "") + "\nTarget:\n" + target.deep() + "\nResult:\n" + x.result.deep() + "\n"); } return x; } } } } else { assert queryParent != null; for (UnifyRule rule : applicableRules(queryParent, target)) { final UnifyResult x = apply(rule, queryParent, target); if (x != null) { if (DEBUG) { System.out.println( "Rule: " + rule + "\nQuery:\n" + queryParent.deep() + (x.call.query != queryParent ? "\nQuery (original):\n" + queryParent.toString() : "") + "\nTarget:\n" + target.deep() + "\nResult:\n" + x.result.deep() + "\n"); } return x; } } } if (DEBUG) { System.out.println( "Unify failed:" + "\nQuery:\n" + queryParent.toString() + "\nTarget:\n" + target.toString() + "\n"); } return null; } private UnifyResult apply(UnifyRule rule, MutableRel query, MutableRel target) { final UnifyRuleCall call = new UnifyRuleCall(rule, query, target, ImmutableList.of()); return rule.apply(call); } private List applicableRules(MutableRel query, MutableRel target) { final Class queryClass = query.getClass(); final Class targetClass = target.getClass(); final Pair key = Pair.of(queryClass, targetClass); List list = ruleMap.get(key); if (list == null) { final ImmutableList.Builder builder = ImmutableList.builder(); for (UnifyRule rule : rules) { //noinspection unchecked if (mightMatch(rule, queryClass, targetClass)) { builder.add(rule); } } list = builder.build(); ruleMap.put(key, list); } return list; } private static boolean mightMatch(UnifyRule rule, Class queryClass, Class targetClass) { return rule.queryOperand.clazz.isAssignableFrom(queryClass) && rule.targetOperand.clazz.isAssignableFrom(targetClass); } /** Exception thrown to exit a matcher. Not really an error. */ protected static class MatchFailed extends ControlFlowException { @SuppressWarnings("ThrowableInstanceNeverThrown") public static final MatchFailed INSTANCE = new MatchFailed(); } /** Rule that attempts to match a query relational expression * against a target relational expression. * *

The rule declares the query and target types; this allows the * engine to fire only a few rules in a given context.

*/ protected abstract static class UnifyRule { protected final int slotCount; protected final Operand queryOperand; protected final Operand targetOperand; protected UnifyRule(int slotCount, Operand queryOperand, Operand targetOperand) { this.slotCount = slotCount; this.queryOperand = queryOperand; this.targetOperand = targetOperand; } /** *

Applies this rule to a particular node in a query. The goal is * to convert {@code query} into {@code target}. Before the rule is * invoked, Calcite has made sure that query's children are equivalent * to target's children. * *

There are 3 possible outcomes:

* *
    * *
  • {@code query} already exactly matches {@code target}; returns * {@code target}
  • * *
  • {@code query} is sufficiently close to a match for * {@code target}; returns {@code target}
  • * *
  • {@code query} cannot be made to match {@code target}; returns * null
  • * *
* *

REVIEW: Is possible that we match query PLUS one or more of its * ancestors?

* * @param call Input parameters */ protected abstract UnifyResult apply(UnifyRuleCall call); protected UnifyRuleCall match(SubstitutionVisitor visitor, MutableRel query, MutableRel target) { if (queryOperand.matches(visitor, query)) { if (targetOperand.matches(visitor, target)) { return visitor.new UnifyRuleCall(this, query, target, copy(visitor.slots, slotCount)); } } return null; } protected ImmutableList copy(E[] slots, int slotCount) { // Optimize if there are 0 or 1 slots. switch (slotCount) { case 0: return ImmutableList.of(); case 1: return ImmutableList.of(slots[0]); default: return ImmutableList.copyOf(slots).subList(0, slotCount); } } } /** * Arguments to an application of a {@link UnifyRule}. */ protected class UnifyRuleCall { protected final UnifyRule rule; public final MutableRel query; public final MutableRel target; protected final ImmutableList slots; public UnifyRuleCall(UnifyRule rule, MutableRel query, MutableRel target, ImmutableList slots) { this.rule = Objects.requireNonNull(rule); this.query = Objects.requireNonNull(query); this.target = Objects.requireNonNull(target); this.slots = Objects.requireNonNull(slots); } public UnifyResult result(MutableRel result) { return result(result, true); } public UnifyResult result(MutableRel result, boolean stopTrying) { assert MutableRels.contains(result, target); assert equalType("result", result, "query", query, Litmus.THROW); MutableRel replace = replacementMap.get(target); if (replace != null) { assert false; // replacementMap is always empty // result = replace(result, target, replace); } register(result, query); return new UnifyResult(this, result, stopTrying); } /** * Creates a {@link UnifyRuleCall} based on the parent of {@code query}. */ public UnifyRuleCall create(MutableRel query) { return new UnifyRuleCall(rule, query, target, slots); } public RelOptCluster getCluster() { return cluster; } public RexSimplify getSimplify() { return simplify; } } /** * Result of an application of a {@link UnifyRule} indicating that the * rule successfully matched {@code query} against {@code target} and * generated a {@code result} that is equivalent to {@code query} and * contains {@code target}. {@code stopTrying} indicates whether there's * no need to do matching for the same query node again. */ protected static class UnifyResult { private final UnifyRuleCall call; private final MutableRel result; private final boolean stopTrying; UnifyResult(UnifyRuleCall call, MutableRel result, boolean stopTrying) { this.call = call; assert equalType("query", call.query, "result", result, Litmus.THROW); this.result = result; this.stopTrying = stopTrying; } } /** Abstract base class for implementing {@link UnifyRule}. */ protected abstract static class AbstractUnifyRule extends UnifyRule { public AbstractUnifyRule(Operand queryOperand, Operand targetOperand, int slotCount) { super(slotCount, queryOperand, targetOperand); //noinspection AssertWithSideEffects assert isValid(); } protected boolean isValid() { final SlotCounter slotCounter = new SlotCounter(); slotCounter.visit(queryOperand); assert slotCounter.queryCount == slotCount; assert slotCounter.targetCount == 0; slotCounter.queryCount = 0; slotCounter.visit(targetOperand); assert slotCounter.queryCount == 0; assert slotCounter.targetCount == slotCount; return true; } /** Creates an operand with given inputs. */ protected static Operand operand(Class clazz, Operand... inputOperands) { return new InternalOperand(clazz, ImmutableList.copyOf(inputOperands)); } /** Creates an operand that doesn't check inputs. */ protected static Operand any(Class clazz) { return new AnyOperand(clazz); } /** Creates an operand that matches a relational expression in the query. */ protected static Operand query(int ordinal) { return new QueryOperand(ordinal); } /** Creates an operand that matches a relational expression in the * target. */ protected static Operand target(int ordinal) { return new TargetOperand(ordinal); } } /** Implementation of {@link UnifyRule} that matches if the query is already * equal to the target. * *

Matches scans to the same table, because these will be * {@link MutableScan}s with the same * {@link com.hazelcast.org.apache.calcite.rel.core.TableScan} instance.

*/ private static class TrivialRule extends AbstractUnifyRule { private static final TrivialRule INSTANCE = new TrivialRule(); private TrivialRule() { super(any(MutableRel.class), any(MutableRel.class), 0); } public UnifyResult apply(UnifyRuleCall call) { if (call.query.equals(call.target)) { return call.result(call.target); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a * {@link MutableScan} to a {@link MutableCalc} * which has {@link MutableScan} as child. */ private static class ScanToCalcUnifyRule extends AbstractUnifyRule { public static final ScanToCalcUnifyRule INSTANCE = new ScanToCalcUnifyRule(); private ScanToCalcUnifyRule() { super(any(MutableScan.class), operand(MutableCalc.class, any(MutableScan.class)), 0); } @Override protected UnifyResult apply(UnifyRuleCall call) { final MutableScan query = (MutableScan) call.query; final MutableCalc target = (MutableCalc) call.target; final MutableScan targetInput = (MutableScan) target.getInput(); final Pair> targetExplained = explainCalc(target); final RexNode targetCond = targetExplained.left; final List targetProjs = targetExplained.right; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); if (!query.equals(targetInput) || !targetCond.isAlwaysTrue()) { return null; } final RexShuttle shuttle = getRexShuttle(targetProjs); final List com.hazelcast.com.enProjs; try { com.hazelcast.com.enProjs = (List) shuttle.apply( rexBuilder.identityProjects(query.rowType)); } catch (MatchFailed e) { return null; } if (RexUtil.isIdentity(com.hazelcast.com.enProjs, target.rowType)) { return call.result(target); } else { RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, null, query.rowType, rexBuilder); MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(target, com.hazelcast.com.enRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a * {@link MutableCalc} to a {@link MutableCalc}. * The matching condition is as below: * 1. All columns of query can be expressed by target; * 2. The filtering condition of query must equals to or be weaker than target. */ private static class CalcToCalcUnifyRule extends AbstractUnifyRule { public static final CalcToCalcUnifyRule INSTANCE = new CalcToCalcUnifyRule(); private CalcToCalcUnifyRule() { super(operand(MutableCalc.class, query(0)), operand(MutableCalc.class, target(0)), 1); } public UnifyResult apply(UnifyRuleCall call) { final MutableCalc query = (MutableCalc) call.query; final Pair> queryExplained = explainCalc(query); final RexNode queryCond = queryExplained.left; final List queryProjs = queryExplained.right; final MutableCalc target = (MutableCalc) call.target; final Pair> targetExplained = explainCalc(target); final RexNode targetCond = targetExplained.left; final List targetProjs = targetExplained.right; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); try { final RexShuttle shuttle = getRexShuttle(targetProjs); final RexNode splitted = splitFilter(call.getSimplify(), queryCond, targetCond); final RexNode com.hazelcast.com.enCond; if (splitted != null) { if (splitted.isAlwaysTrue()) { com.hazelcast.com.enCond = null; } else { // Compensate the residual filtering condition. com.hazelcast.com.enCond = shuttle.apply(splitted); } } else if (implies( call.getCluster(), queryCond, targetCond, query.getInput().rowType)) { // Fail to split filtering condition, but implies that target contains // all lines of query, thus just set com.hazelcast.com.ensating filtering condition // as the filtering condition of query. com.hazelcast.com.enCond = shuttle.apply(queryCond); } else { return null; } final List com.hazelcast.com.enProjs = shuttle.apply(queryProjs); if (com.hazelcast.com.enCond == null && RexUtil.isIdentity(com.hazelcast.com.enProjs, target.rowType)) { return call.result(target); } else { final RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, com.hazelcast.com.enCond, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(target, com.hazelcast.com.enRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } } catch (MatchFailed e) { return null; } } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableJoin} * which has {@link MutableCalc} as left child to a {@link MutableJoin}. * We try to pull up the {@link MutableCalc} to top of {@link MutableJoin}, * then match the {@link MutableJoin} in query to {@link MutableJoin} in target. */ private static class JoinOnLeftCalcToJoinUnifyRule extends AbstractUnifyRule { public static final JoinOnLeftCalcToJoinUnifyRule INSTANCE = new JoinOnLeftCalcToJoinUnifyRule(); private JoinOnLeftCalcToJoinUnifyRule() { super( operand(MutableJoin.class, operand(MutableCalc.class, query(0)), query(1)), operand(MutableJoin.class, target(0), target(1)), 2); } @Override protected UnifyResult apply(UnifyRuleCall call) { final MutableJoin query = (MutableJoin) call.query; final MutableCalc qInput0 = (MutableCalc) query.getLeft(); final MutableRel qInput1 = query.getRight(); final Pair> qInput0Explained = explainCalc(qInput0); final RexNode qInput0Cond = qInput0Explained.left; final List qInput0Projs = qInput0Explained.right; final MutableJoin target = (MutableJoin) call.target; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); // Try pulling up MutableCalc only when: // 1. it's inner join. // 2. it's outer join but no filtering condition from MutableCalc. final JoinRelType joinRelType = sameJoinType(query.joinType, target.joinType); if (joinRelType == null) { return null; } if (joinRelType != JoinRelType.INNER && !(joinRelType.isOuterJoin() && qInput0Cond.isAlwaysTrue())) { return null; } // Try pulling up MutableCalc only when Join condition references mapping. final List identityProjects = (List) rexBuilder.identityProjects(qInput1.rowType); if (!referenceByMapping(query.condition, qInput0Projs, identityProjects)) { return null; } final RexNode newQueryJoinCond = new RexShuttle() { @Override public RexNode visitInputRef(RexInputRef inputRef) { final int idx = inputRef.getIndex(); if (idx < fieldCnt(qInput0)) { final int newIdx = ((RexInputRef) qInput0Projs.get(idx)).getIndex(); return new RexInputRef(newIdx, inputRef.getType()); } else { int newIdx = idx - fieldCnt(qInput0) + fieldCnt(qInput0.getInput()); return new RexInputRef(newIdx, inputRef.getType()); } } }.apply(query.condition); final RexNode splitted = splitFilter(call.getSimplify(), newQueryJoinCond, target.condition); // MutableJoin matches only when the conditions are analyzed to be same. if (splitted != null && splitted.isAlwaysTrue()) { final RexNode com.hazelcast.com.enCond = qInput0Cond; final List com.hazelcast.com.enProjs = new ArrayList<>(); for (int i = 0; i < fieldCnt(query); i++) { if (i < fieldCnt(qInput0)) { com.hazelcast.com.enProjs.add(qInput0Projs.get(i)); } else { final int newIdx = i - fieldCnt(qInput0) + fieldCnt(qInput0.getInput()); com.hazelcast.com.enProjs.add( new RexInputRef(newIdx, query.rowType.getFieldList().get(i).getType())); } } final RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, com.hazelcast.com.enCond, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(target, com.hazelcast.com.enRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableJoin} * which has {@link MutableCalc} as right child to a {@link MutableJoin}. * We try to pull up the {@link MutableCalc} to top of {@link MutableJoin}, * then match the {@link MutableJoin} in query to {@link MutableJoin} in target. */ private static class JoinOnRightCalcToJoinUnifyRule extends AbstractUnifyRule { public static final JoinOnRightCalcToJoinUnifyRule INSTANCE = new JoinOnRightCalcToJoinUnifyRule(); private JoinOnRightCalcToJoinUnifyRule() { super( operand(MutableJoin.class, query(0), operand(MutableCalc.class, query(1))), operand(MutableJoin.class, target(0), target(1)), 2); } @Override protected UnifyResult apply(UnifyRuleCall call) { final MutableJoin query = (MutableJoin) call.query; final MutableRel qInput0 = query.getLeft(); final MutableCalc qInput1 = (MutableCalc) query.getRight(); final Pair> qInput1Explained = explainCalc(qInput1); final RexNode qInput1Cond = qInput1Explained.left; final List qInput1Projs = qInput1Explained.right; final MutableJoin target = (MutableJoin) call.target; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); // Try pulling up MutableCalc only when: // 1. it's inner join. // 2. it's outer join but no filtering condition from MutableCalc. final JoinRelType joinRelType = sameJoinType(query.joinType, target.joinType); if (joinRelType == null) { return null; } if (joinRelType != JoinRelType.INNER && !(joinRelType.isOuterJoin() && qInput1Cond.isAlwaysTrue())) { return null; } // Try pulling up MutableCalc only when Join condition references mapping. final List identityProjects = (List) rexBuilder.identityProjects(qInput0.rowType); if (!referenceByMapping(query.condition, identityProjects, qInput1Projs)) { return null; } final RexNode newQueryJoinCond = new RexShuttle() { @Override public RexNode visitInputRef(RexInputRef inputRef) { final int idx = inputRef.getIndex(); if (idx < fieldCnt(qInput0)) { return inputRef; } else { final int newIdx = ((RexInputRef) qInput1Projs.get(idx - fieldCnt(qInput0))) .getIndex() + fieldCnt(qInput0); return new RexInputRef(newIdx, inputRef.getType()); } } }.apply(query.condition); final RexNode splitted = splitFilter(call.getSimplify(), newQueryJoinCond, target.condition); // MutableJoin matches only when the conditions are analyzed to be same. if (splitted != null && splitted.isAlwaysTrue()) { final RexNode com.hazelcast.com.enCond = RexUtil.shift(qInput1Cond, qInput0.rowType.getFieldCount()); final List com.hazelcast.com.enProjs = new ArrayList<>(); for (int i = 0; i < query.rowType.getFieldCount(); i++) { if (i < fieldCnt(qInput0)) { com.hazelcast.com.enProjs.add( new RexInputRef(i, query.rowType.getFieldList().get(i).getType())); } else { final RexNode shifted = RexUtil.shift(qInput1Projs.get(i - fieldCnt(qInput0)), qInput0.rowType.getFieldCount()); com.hazelcast.com.enProjs.add(shifted); } } final RexProgram com.hazelcast.com.ensatingRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, com.hazelcast.com.enCond, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(target, com.hazelcast.com.ensatingRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableJoin} * which has {@link MutableCalc} as children to a {@link MutableJoin}. * We try to pull up the {@link MutableCalc} to top of {@link MutableJoin}, * then match the {@link MutableJoin} in query to {@link MutableJoin} in target. */ private static class JoinOnCalcsToJoinUnifyRule extends AbstractUnifyRule { public static final JoinOnCalcsToJoinUnifyRule INSTANCE = new JoinOnCalcsToJoinUnifyRule(); private JoinOnCalcsToJoinUnifyRule() { super( operand(MutableJoin.class, operand(MutableCalc.class, query(0)), operand(MutableCalc.class, query(1))), operand(MutableJoin.class, target(0), target(1)), 2); } @Override protected UnifyResult apply(UnifyRuleCall call) { final MutableJoin query = (MutableJoin) call.query; final MutableCalc qInput0 = (MutableCalc) query.getLeft(); final MutableCalc qInput1 = (MutableCalc) query.getRight(); final Pair> qInput0Explained = explainCalc(qInput0); final RexNode qInput0Cond = qInput0Explained.left; final List qInput0Projs = qInput0Explained.right; final Pair> qInput1Explained = explainCalc(qInput1); final RexNode qInput1Cond = qInput1Explained.left; final List qInput1Projs = qInput1Explained.right; final MutableJoin target = (MutableJoin) call.target; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); // Try pulling up MutableCalc only when: // 1. it's inner join. // 2. it's outer join but no filtering condition from MutableCalc. final JoinRelType joinRelType = sameJoinType(query.joinType, target.joinType); if (joinRelType == null) { return null; } if (joinRelType != JoinRelType.INNER && !(joinRelType.isOuterJoin() && qInput0Cond.isAlwaysTrue() && qInput1Cond.isAlwaysTrue())) { return null; } if (!referenceByMapping(query.condition, qInput0Projs, qInput1Projs)) { return null; } RexNode newQueryJoinCond = new RexShuttle() { @Override public RexNode visitInputRef(RexInputRef inputRef) { final int idx = inputRef.getIndex(); if (idx < fieldCnt(qInput0)) { final int newIdx = ((RexInputRef) qInput0Projs.get(idx)).getIndex(); return new RexInputRef(newIdx, inputRef.getType()); } else { final int newIdx = ((RexInputRef) qInput1Projs.get(idx - fieldCnt(qInput0))) .getIndex() + fieldCnt(qInput0.getInput()); return new RexInputRef(newIdx, inputRef.getType()); } } }.apply(query.condition); final RexNode splitted = splitFilter(call.getSimplify(), newQueryJoinCond, target.condition); // MutableJoin matches only when the conditions are analyzed to be same. if (splitted != null && splitted.isAlwaysTrue()) { final RexNode qInput1CondShifted = RexUtil.shift(qInput1Cond, fieldCnt(qInput0.getInput())); final RexNode com.hazelcast.com.enCond = RexUtil.com.hazelcast.com.oseConjunction(rexBuilder, ImmutableList.of(qInput0Cond, qInput1CondShifted)); final List com.hazelcast.com.enProjs = new ArrayList<>(); for (int i = 0; i < query.rowType.getFieldCount(); i++) { if (i < fieldCnt(qInput0)) { com.hazelcast.com.enProjs.add(qInput0Projs.get(i)); } else { RexNode shifted = RexUtil.shift(qInput1Projs.get(i - fieldCnt(qInput0)), fieldCnt(qInput0.getInput())); com.hazelcast.com.enProjs.add(shifted); } } final RexProgram com.hazelcast.com.ensatingRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, com.hazelcast.com.enCond, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.ensatingCalc = MutableCalc.of(target, com.hazelcast.com.ensatingRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.ensatingCalc); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableAggregate} * which has {@link MutableCalc} as child to a {@link MutableAggregate}. * We try to pull up the {@link MutableCalc} to top of {@link MutableAggregate}, * then match the {@link MutableAggregate} in query to {@link MutableAggregate} in target. */ private static class AggregateOnCalcToAggregateUnifyRule extends AbstractUnifyRule { public static final AggregateOnCalcToAggregateUnifyRule INSTANCE = new AggregateOnCalcToAggregateUnifyRule(); private AggregateOnCalcToAggregateUnifyRule() { super(operand(MutableAggregate.class, operand(MutableCalc.class, query(0))), operand(MutableAggregate.class, target(0)), 1); } @Override protected UnifyResult apply(UnifyRuleCall call) { final MutableAggregate query = (MutableAggregate) call.query; final MutableCalc qInput = (MutableCalc) query.getInput(); final Pair> qInputExplained = explainCalc(qInput); final RexNode qInputCond = qInputExplained.left; final List qInputProjs = qInputExplained.right; final MutableAggregate target = (MutableAggregate) call.target; final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); final Mappings.TargetMapping mapping = Project.getMapping(fieldCnt(qInput.getInput()), qInputProjs); if (mapping == null) { return null; } if (!qInputCond.isAlwaysTrue()) { try { // Fail the matching when filtering condition references // non-grouping columns in target. qInputCond.accept(new RexVisitorImpl(true) { @Override public Void visitInputRef(RexInputRef inputRef) { if (!target.groupSets.stream() .allMatch(groupSet -> groupSet.get(inputRef.getIndex()))) { throw Util.FoundOne.NULL; } return super.visitInputRef(inputRef); } }); } catch (Util.FoundOne one) { return null; } } final Mapping inverseMapping = mapping.inverse(); final MutableAggregate aggregate2 = permute(query, qInput.getInput(), inverseMapping); final Mappings.TargetMapping mappingForQueryCond = Mappings.target( target.groupSet::indexOf, target.getInput().rowType.getFieldCount(), target.groupSet.cardinality()); final RexNode targetCond = RexUtil.apply(mappingForQueryCond, qInputCond); final MutableRel unifiedAggregate = unifyAggregates(aggregate2, targetCond, target); if (unifiedAggregate == null) { return null; } // Add Project if the mapping breaks order of fields in GroupSet if (!Mappings.keepsOrdering(mapping)) { final List posList = new ArrayList<>(); final int fieldCount = aggregate2.rowType.getFieldCount(); final List> pairs = new ArrayList<>(); final List groupings = aggregate2.groupSet.toList(); for (int i = 0; i < groupings.size(); i++) { pairs.add(Pair.of(mapping.getTarget(groupings.get(i)), i)); } Collections.sort(pairs); pairs.forEach(pair -> posList.add(pair.right)); for (int i = posList.size(); i < fieldCount; i++) { posList.add(i); } final List com.hazelcast.com.enProjs = MutableRels.createProjectExprs(unifiedAggregate, posList); final RexProgram com.hazelcast.com.ensatingRexProgram = RexProgram.create( unifiedAggregate.rowType, com.hazelcast.com.enProjs, null, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(unifiedAggregate, com.hazelcast.com.ensatingRexProgram); if (unifiedAggregate instanceof MutableCalc) { final MutableCalc newCompenCalc = mergeCalc(rexBuilder, com.hazelcast.com.enCalc, (MutableCalc) unifiedAggregate); return tryMergeParentCalcAndGenResult(call, newCompenCalc); } else { return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } } else { return tryMergeParentCalcAndGenResult(call, unifiedAggregate); } } } /** A {@link SubstitutionVisitor.UnifyRule} that matches a * {@link com.hazelcast.org.apache.calcite.rel.core.Aggregate} to a * {@link com.hazelcast.org.apache.calcite.rel.core.Aggregate}, provided * that they have the same child. */ private static class AggregateToAggregateUnifyRule extends AbstractUnifyRule { public static final AggregateToAggregateUnifyRule INSTANCE = new AggregateToAggregateUnifyRule(); private AggregateToAggregateUnifyRule() { super(operand(MutableAggregate.class, query(0)), operand(MutableAggregate.class, target(0)), 1); } public UnifyResult apply(UnifyRuleCall call) { final MutableAggregate query = (MutableAggregate) call.query; final MutableAggregate target = (MutableAggregate) call.target; assert query != target; // in.query can be rewritten in terms of in.target if its groupSet is // a subset, and its aggCalls are a superset. For example: // query: SELECT x, COUNT(b) FROM t GROUP BY x // target: SELECT x, y, SUM(a) AS s, COUNT(b) AS cb FROM t GROUP BY x, y // transforms to // result: SELECT x, SUM(cb) FROM (target) GROUP BY x if (query.getInput() != target.getInput()) { return null; } if (!target.groupSet.contains(query.groupSet)) { return null; } final MutableRel result = unifyAggregates(query, null, target); if (result == null) { return null; } return tryMergeParentCalcAndGenResult(call, result); } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a * {@link MutableUnion} to a {@link MutableUnion} where the query and target * have the same inputs but might not have the same order. */ private static class UnionToUnionUnifyRule extends AbstractUnifyRule { public static final UnionToUnionUnifyRule INSTANCE = new UnionToUnionUnifyRule(); private UnionToUnionUnifyRule() { super(any(MutableUnion.class), any(MutableUnion.class), 0); } public UnifyResult apply(UnifyRuleCall call) { final MutableUnion query = (MutableUnion) call.query; final MutableUnion target = (MutableUnion) call.target; final List queryInputs = new ArrayList<>(query.getInputs()); final List targetInputs = new ArrayList<>(target.getInputs()); if (query.isAll() == target.isAll() && sameRelCollectionNoOrderConsidered(queryInputs, targetInputs)) { return call.result(target); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableUnion} * which has {@link MutableCalc} as child to a {@link MutableUnion}. * We try to pull up the {@link MutableCalc} to top of {@link MutableUnion}, * then match the {@link MutableUnion} in query to {@link MutableUnion} in target. */ private static class UnionOnCalcsToUnionUnifyRule extends AbstractUnifyRule { public static final UnionOnCalcsToUnionUnifyRule INSTANCE = new UnionOnCalcsToUnionUnifyRule(); private UnionOnCalcsToUnionUnifyRule() { super(any(MutableUnion.class), any(MutableUnion.class), 0); } public UnifyResult apply(UnifyRuleCall call) { return setOpApply(call); } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a * {@link MutableIntersect} to a {@link MutableIntersect} where the query and target * have the same inputs but might not have the same order. */ private static class IntersectToIntersectUnifyRule extends AbstractUnifyRule { public static final IntersectToIntersectUnifyRule INSTANCE = new IntersectToIntersectUnifyRule(); private IntersectToIntersectUnifyRule() { super(any(MutableIntersect.class), any(MutableIntersect.class), 0); } public UnifyResult apply(UnifyRuleCall call) { final MutableIntersect query = (MutableIntersect) call.query; final MutableIntersect target = (MutableIntersect) call.target; final List queryInputs = new ArrayList<>(query.getInputs()); final List targetInputs = new ArrayList<>(target.getInputs()); if (query.isAll() == target.isAll() && sameRelCollectionNoOrderConsidered(queryInputs, targetInputs)) { return call.result(target); } return null; } } /** * A {@link SubstitutionVisitor.UnifyRule} that matches a {@link MutableIntersect} * which has {@link MutableCalc} as child to a {@link MutableIntersect}. * We try to pull up the {@link MutableCalc} to top of {@link MutableIntersect}, * then match the {@link MutableIntersect} in query to {@link MutableIntersect} in target. */ private static class IntersectOnCalcsToIntersectUnifyRule extends AbstractUnifyRule { public static final IntersectOnCalcsToIntersectUnifyRule INSTANCE = new IntersectOnCalcsToIntersectUnifyRule(); private IntersectOnCalcsToIntersectUnifyRule() { super(any(MutableIntersect.class), any(MutableIntersect.class), 0); } public UnifyResult apply(UnifyRuleCall call) { return setOpApply(call); } } /** * Applies a AbstractUnifyRule to a particular node in a query. We try to pull up the * {@link MutableCalc} to top of {@link MutableUnion} or {@link MutableIntersect}, this * method not suit for {@link MutableMinus}. * * @param call Input parameters */ private static UnifyResult setOpApply(UnifyRuleCall call) { if (call.query instanceof MutableMinus && call.target instanceof MutableMinus) { return null; } final MutableSetOp query = (MutableSetOp) call.query; final MutableSetOp target = (MutableSetOp) call.target; final List queryInputs = new ArrayList<>(); final List queryGrandInputs = new ArrayList<>(); final List targetInputs = new ArrayList<>(target.getInputs()); final RexBuilder rexBuilder = call.getCluster().getRexBuilder(); for (MutableRel rel : query.getInputs()) { if (rel instanceof MutableCalc) { queryInputs.add((MutableCalc) rel); queryGrandInputs.add(((MutableCalc) rel).getInput()); } else { return null; } } if (query.isAll() && target.isAll() && sameRelCollectionNoOrderConsidered(queryGrandInputs, targetInputs)) { final Pair> queryInputExplained0 = explainCalc(queryInputs.get(0)); for (int i = 1; i < queryGrandInputs.size(); i++) { final Pair> queryInputExplained = explainCalc(queryInputs.get(i)); // Matching fails when filtering conditions are not equal or projects are not equal. if (!splitFilter(call.getSimplify(), queryInputExplained0.left, queryInputExplained.left).isAlwaysTrue()) { return null; } for (Pair pair : Pair.zip( queryInputExplained0.right, queryInputExplained.right)) { if (!pair.left.equals(pair.right)) { return null; } } } List projectExprs = MutableRels.createProjects(target, queryInputExplained0.right); final RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, projectExprs, queryInputExplained0.left, query.rowType, rexBuilder); final MutableCalc com.hazelcast.com.enCalc = MutableCalc.of(target, com.hazelcast.com.enRexProgram); return tryMergeParentCalcAndGenResult(call, com.hazelcast.com.enCalc); } return null; } /** Check if list0 and list1 contains the same nodes -- order is not considered. */ private static boolean sameRelCollectionNoOrderConsidered( List list0, List list1) { if (list0.size() != list1.size()) { return false; } for (MutableRel rel: list0) { int index = list1.indexOf(rel); if (index == -1) { return false; } else { list1.remove(index); } } return true; } private static int fieldCnt(MutableRel rel) { return rel.rowType.getFieldCount(); } /** Explain filtering condition and projections from MutableCalc. */ private static Pair> explainCalc(MutableCalc calc) { final RexShuttle shuttle = getExpandShuttle(calc.program); final RexNode condition = shuttle.apply(calc.program.getCondition()); final List projects = new ArrayList<>(); for (RexNode rex: shuttle.apply(calc.program.getProjectList())) { projects.add(rex); } if (condition == null) { return Pair.of(calc.cluster.getRexBuilder().makeLiteral(true), projects); } else { return Pair.of(condition, projects); } } /** * Generate result by merging parent and child if they are both MutableCalc. * Otherwise result is the child itself. */ private static UnifyResult tryMergeParentCalcAndGenResult( UnifyRuleCall call, MutableRel child) { final MutableRel parent = call.query.getParent(); if (child instanceof MutableCalc && parent instanceof MutableCalc) { final MutableCalc mergedCalc = mergeCalc(call.getCluster().getRexBuilder(), (MutableCalc) parent, (MutableCalc) child); if (mergedCalc != null) { // Note that property of stopTrying in the result is false // and this query node deserves further matching iterations. return call.create(parent).result(mergedCalc, false); } } return call.result(child); } /** Merge two MutableCalc together. */ private static MutableCalc mergeCalc( RexBuilder rexBuilder, MutableCalc topCalc, MutableCalc bottomCalc) { RexProgram topProgram = topCalc.program; if (RexOver.containsOver(topProgram)) { return null; } RexProgram mergedProgram = RexProgramBuilder.mergePrograms( topCalc.program, bottomCalc.program, rexBuilder); assert mergedProgram.getOutputRowType() == topProgram.getOutputRowType(); return MutableCalc.of(bottomCalc.getInput(), mergedProgram); } private static RexShuttle getExpandShuttle(RexProgram rexProgram) { return new RexShuttle() { @Override public RexNode visitLocalRef(RexLocalRef localRef) { return rexProgram.expandLocalRef(localRef); } }; } /** Check if condition cond0 implies cond1. */ private static boolean implies( RelOptCluster cluster, RexNode cond0, RexNode cond1, RelDataType rowType) { RexExecutorImpl rexImpl = (RexExecutorImpl) (cluster.getPlanner().getExecutor()); RexImplicationChecker rexImplicationChecker = new RexImplicationChecker(cluster.getRexBuilder(), rexImpl, rowType); return rexImplicationChecker.implies(cond0, cond1); } /** Check if join condition only references RexInputRef. */ private static boolean referenceByMapping( RexNode joinCondition, List... projectsOfInputs) { List projects = new ArrayList<>(); for (List projectsOfInput: projectsOfInputs) { projects.addAll(projectsOfInput); } try { RexVisitor rexVisitor = new RexVisitorImpl(true) { @Override public Void visitInputRef(RexInputRef inputRef) { if (!(projects.get(inputRef.getIndex()) instanceof RexInputRef)) { throw Util.FoundOne.NULL; } return super.visitInputRef(inputRef); } }; joinCondition.accept(rexVisitor); } catch (Util.FoundOne e) { return false; } return true; } private static JoinRelType sameJoinType(JoinRelType type0, JoinRelType type1) { if (type0 == type1) { return type0; } else { return null; } } public static MutableAggregate permute(MutableAggregate aggregate, MutableRel input, Mapping mapping) { ImmutableBitSet groupSet = Mappings.apply(mapping, aggregate.groupSet); ImmutableList groupSets = Mappings.apply2(mapping, aggregate.groupSets); List aggregateCalls = Util.transform(aggregate.aggCalls, call -> call.transform(mapping)); return MutableAggregate.of(input, groupSet, groupSets, aggregateCalls); } public static MutableRel unifyAggregates(MutableAggregate query, RexNode targetCond, MutableAggregate target) { MutableRel result; RexBuilder rexBuilder = query.cluster.getRexBuilder(); if (query.groupSets.equals(target.groupSets)) { // Same level of aggregation. Generate a project. final List projects = new ArrayList<>(); final int groupCount = query.groupSet.cardinality(); for (int i = 0; i < groupCount; i++) { projects.add(i); } for (AggregateCall aggregateCall : query.aggCalls) { int i = target.aggCalls.indexOf(aggregateCall); if (i < 0) { return null; } projects.add(groupCount + i); } List com.hazelcast.com.enProjs = MutableRels.createProjectExprs(target, projects); RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, com.hazelcast.com.enProjs, targetCond, query.rowType, rexBuilder); result = MutableCalc.of(target, com.hazelcast.com.enRexProgram); } else if (target.getGroupType() == Aggregate.Group.SIMPLE) { // Query is coarser level of aggregation. Generate an aggregate. final Map map = new HashMap<>(); target.groupSet.forEach(k -> map.put(k, map.size())); for (int c : query.groupSet) { if (!map.containsKey(c)) { return null; } } final ImmutableBitSet groupSet = query.groupSet.permute(map); ImmutableList groupSets = null; if (query.getGroupType() != Aggregate.Group.SIMPLE) { groupSets = ImmutableBitSet.ORDERING.immutableSortedCopy( ImmutableBitSet.permute(query.groupSets, map)); } final List aggregateCalls = new ArrayList<>(); for (AggregateCall aggregateCall : query.aggCalls) { if (aggregateCall.isDistinct()) { return null; } int i = target.aggCalls.indexOf(aggregateCall); if (i < 0) { return null; } aggregateCalls.add( AggregateCall.create(getRollup(aggregateCall.getAggregation()), aggregateCall.isDistinct(), aggregateCall.isApproximate(), aggregateCall.ignoreNulls(), ImmutableList.of(target.groupSet.cardinality() + i), -1, aggregateCall.collation, aggregateCall.type, aggregateCall.name)); } if (targetCond != null && !targetCond.isAlwaysTrue()) { RexProgram com.hazelcast.com.enRexProgram = RexProgram.create( target.rowType, rexBuilder.identityProjects(target.rowType), targetCond, target.rowType, rexBuilder); result = MutableAggregate.of( MutableCalc.of(target, com.hazelcast.com.enRexProgram), groupSet, groupSets, aggregateCalls); } else { result = MutableAggregate.of( target, groupSet, groupSets, aggregateCalls); } } else { return null; } return result; } public static SqlAggFunction getRollup(SqlAggFunction aggregation) { if (aggregation == SqlStdOperatorTable.SUM || aggregation == SqlStdOperatorTable.MIN || aggregation == SqlStdOperatorTable.MAX || aggregation == SqlStdOperatorTable.SUM0 || aggregation == SqlStdOperatorTable.ANY_VALUE) { return aggregation; } else if (aggregation == SqlStdOperatorTable.COUNT) { return SqlStdOperatorTable.SUM0; } else { return null; } } /** Builds a shuttle that stores a list of expressions, and can map incoming * expressions to references to them. */ private static RexShuttle getRexShuttle(List rexNodes) { final Map map = new HashMap<>(); for (RexNode e : rexNodes) { map.put(e, map.size()); } return new RexShuttle() { @Override public RexNode visitInputRef(RexInputRef ref) { final Integer integer = map.get(ref); if (integer != null) { return new RexInputRef(integer, ref.getType()); } throw MatchFailed.INSTANCE; } @Override public RexNode visitCall(RexCall call) { final Integer integer = map.get(call); if (integer != null) { return new RexInputRef(integer, call.getType()); } return super.visitCall(call); } @Override public RexNode visitLiteral(RexLiteral literal) { final Integer integer = map.get(literal); if (integer != null) { return new RexInputRef(integer, literal.getType()); } return super.visitLiteral(literal); } }; } /** Returns if one rel is weaker than another. */ protected boolean isWeaker(MutableRel rel0, MutableRel rel) { if (rel0 == rel || equivalents.get(rel0).contains(rel)) { return false; } if (!(rel0 instanceof MutableFilter) || !(rel instanceof MutableFilter)) { return false; } if (!rel.rowType.equals(rel0.rowType)) { return false; } final MutableRel rel0input = ((MutableFilter) rel0).getInput(); final MutableRel relinput = ((MutableFilter) rel).getInput(); if (rel0input != relinput && !equivalents.get(rel0input).contains(relinput)) { return false; } return implies(rel0.cluster, ((MutableFilter) rel0).condition, ((MutableFilter) rel).condition, rel.rowType); } /** Returns whether two relational expressions have the same row-type. */ public static boolean equalType(String desc0, MutableRel rel0, String desc1, MutableRel rel1, Litmus litmus) { return RelOptUtil.equal(desc0, rel0.rowType, desc1, rel1.rowType, litmus); } /** Operand to a {@link UnifyRule}. */ protected abstract static class Operand { protected final Class clazz; protected Operand(Class clazz) { this.clazz = clazz; } public abstract boolean matches(SubstitutionVisitor visitor, MutableRel rel); public boolean isWeaker(SubstitutionVisitor visitor, MutableRel rel) { return false; } } /** Operand to a {@link UnifyRule} that matches a relational expression of a * given type. It has zero or more child operands. */ private static class InternalOperand extends Operand { private final List inputs; InternalOperand(Class clazz, List inputs) { super(clazz); this.inputs = inputs; } @Override public boolean matches(SubstitutionVisitor visitor, MutableRel rel) { return clazz.isInstance(rel) && allMatch(visitor, inputs, rel.getInputs()); } @Override public boolean isWeaker(SubstitutionVisitor visitor, MutableRel rel) { return clazz.isInstance(rel) && allWeaker(visitor, inputs, rel.getInputs()); } private static boolean allMatch(SubstitutionVisitor visitor, List operands, List rels) { if (operands.size() != rels.size()) { return false; } for (Pair pair : Pair.zip(operands, rels)) { if (!pair.left.matches(visitor, pair.right)) { return false; } } return true; } private static boolean allWeaker( SubstitutionVisitor visitor, List operands, List rels) { if (operands.size() != rels.size()) { return false; } for (Pair pair : Pair.zip(operands, rels)) { if (!pair.left.isWeaker(visitor, pair.right)) { return false; } } return true; } } /** Operand to a {@link UnifyRule} that matches a relational expression of a * given type. */ private static class AnyOperand extends Operand { AnyOperand(Class clazz) { super(clazz); } @Override public boolean matches(SubstitutionVisitor visitor, MutableRel rel) { return clazz.isInstance(rel); } } /** Operand that assigns a particular relational expression to a variable. * *

It is applied to a descendant of the query, writes the operand into the * slots array, and always matches. * There is a corresponding operand of type {@link TargetOperand} that checks * whether its relational expression, a descendant of the target, is * equivalent to this {@code QueryOperand}'s relational expression. */ private static class QueryOperand extends Operand { private final int ordinal; protected QueryOperand(int ordinal) { super(MutableRel.class); this.ordinal = ordinal; } @Override public boolean matches(SubstitutionVisitor visitor, MutableRel rel) { visitor.slots[ordinal] = rel; return true; } } /** Operand that checks that a relational expression matches the corresponding * relational expression that was passed to a {@link QueryOperand}. */ private static class TargetOperand extends Operand { private final int ordinal; protected TargetOperand(int ordinal) { super(MutableRel.class); this.ordinal = ordinal; } @Override public boolean matches(SubstitutionVisitor visitor, MutableRel rel) { final MutableRel rel0 = visitor.slots[ordinal]; assert rel0 != null : "QueryOperand should have been called first"; return rel0 == rel || visitor.equivalents.get(rel0).contains(rel); } @Override public boolean isWeaker(SubstitutionVisitor visitor, MutableRel rel) { final MutableRel rel0 = visitor.slots[ordinal]; assert rel0 != null : "QueryOperand should have been called first"; return visitor.isWeaker(rel0, rel); } } /** Visitor that counts how many {@link QueryOperand} and * {@link TargetOperand} in an operand tree. */ private static class SlotCounter { int queryCount; int targetCount; void visit(Operand operand) { if (operand instanceof QueryOperand) { ++queryCount; } else if (operand instanceof TargetOperand) { ++targetCount; } else if (operand instanceof AnyOperand) { // nothing } else { for (Operand input : ((InternalOperand) operand).inputs) { visit(input); } } } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy