org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.optimizer.calcite;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.plan.RelOptUtil.InputFinder;
import org.apache.calcite.plan.RelOptUtil.InputReferencedVisitor;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexDynamicParam;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexLocalRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexPatternFieldRef;
import org.apache.calcite.rex.RexTableInputRef;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitor;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import com.facebook.presto.hive.$internal.org.slf4j.Logger;
import com.facebook.presto.hive.$internal.org.slf4j.LoggerFactory;
import com.facebook.presto.hive.$internal.com.google.common.base.Function;
import com.facebook.presto.hive.$internal.com.google.common.collect.ImmutableList;
import com.facebook.presto.hive.$internal.com.google.common.collect.ImmutableMap;
import com.facebook.presto.hive.$internal.com.google.common.collect.ImmutableMap.Builder;
import com.facebook.presto.hive.$internal.com.google.common.collect.ImmutableSet;
import com.facebook.presto.hive.$internal.com.google.common.collect.Lists;
import com.facebook.presto.hive.$internal.com.google.common.collect.Maps;
import com.facebook.presto.hive.$internal.com.google.common.collect.Sets;
/**
* Generic utility functions needed for Calcite based Hive CBO.
*/
public class HiveCalciteUtil {
private static final Logger LOG = LoggerFactory.getLogger(HiveCalciteUtil.class);
/**
* Get list of virtual columns from the given list of projections.
*
*
* @param exps
* list of rex nodes representing projections
* @return List of Virtual Columns, will not be null.
*/
public static List getVirtualCols(List exps) {
List vCols = new ArrayList();
for (int i = 0; i < exps.size(); i++) {
if (!(exps.get(i) instanceof RexInputRef)) {
vCols.add(i);
}
}
return vCols;
}
public static boolean validateASTForUnsupportedTokens(ASTNode ast) {
if (ParseUtils.containsTokenOfType(ast, HiveParser.TOK_CHARSETLITERAL, HiveParser.TOK_TABLESPLITSAMPLE)) {
return false;
} else {
return true;
}
}
public static List getProjsFromBelowAsInputRef(final RelNode rel) {
List projectList = Lists.transform(rel.getRowType().getFieldList(),
new Function() {
@Override
public RexNode apply(RelDataTypeField field) {
return rel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex());
}
});
return projectList;
}
public static List translateBitSetToProjIndx(ImmutableBitSet projBitSet) {
List projIndxLst = new ArrayList();
for (int i = 0; i < projBitSet.length(); i++) {
if (projBitSet.get(i)) {
projIndxLst.add(i);
}
}
return projIndxLst;
}
/**
* Push any equi join conditions that are not column references as Projections
* on top of the children.
*
* @param factory
* Project factory to use.
* @param inputRels
* inputs to a join
* @param leftJoinKeys
* expressions for LHS of join key
* @param rightJoinKeys
* expressions for RHS of join key
* @param systemColCount
* number of system columns, usually zero. These columns are
* projected at the leading edge of the output row.
* @param leftKeys
* on return this contains the join key positions from the new
* project rel on the LHS.
* @param rightKeys
* on return this contains the join key positions from the new
* project rel on the RHS.
* @return the join condition after the equi expressions pushed down.
*/
public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, RelNode[] inputRels,
List leftJoinKeys, List rightJoinKeys, int systemColCount,
List leftKeys, List rightKeys) {
RelNode leftRel = inputRels[0];
RelNode rightRel = inputRels[1];
RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
RexNode outJoinCond = null;
int origLeftInputSize = leftRel.getRowType().getFieldCount();
int origRightInputSize = rightRel.getRowType().getFieldCount();
List newLeftFields = new ArrayList();
List newLeftFieldNames = new ArrayList();
List newRightFields = new ArrayList();
List newRightFieldNames = new ArrayList();
int leftKeyCount = leftJoinKeys.size();
int i;
for (i = 0; i < origLeftInputSize; i++) {
final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
newLeftFieldNames.add(field.getName());
}
for (i = 0; i < origRightInputSize; i++) {
final RelDataTypeField field = rightRel.getRowType().getFieldList().get(i);
newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
newRightFieldNames.add(field.getName());
}
ImmutableBitSet.Builder origColEqCondsPosBuilder = ImmutableBitSet.builder();
int newKeyCount = 0;
List> origColEqConds = new ArrayList>();
for (i = 0; i < leftKeyCount; i++) {
RexNode leftKey = leftJoinKeys.get(i);
RexNode rightKey = rightJoinKeys.get(i);
if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
origColEqConds.add(Pair.of(((RexInputRef) leftKey).getIndex(),
((RexInputRef) rightKey).getIndex()));
origColEqCondsPosBuilder.set(i);
} else {
newLeftFields.add(leftKey);
newLeftFieldNames.add(null);
newRightFields.add(rightKey);
newRightFieldNames.add(null);
newKeyCount++;
}
}
ImmutableBitSet origColEqCondsPos = origColEqCondsPosBuilder.build();
for (i = 0; i < origColEqConds.size(); i++) {
Pair p = origColEqConds.get(i);
int condPos = origColEqCondsPos.nth(i);
RexNode leftKey = leftJoinKeys.get(condPos);
RexNode rightKey = rightJoinKeys.get(condPos);
leftKeys.add(p.left);
rightKeys.add(p.right);
RexNode cond = rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
rexBuilder.makeInputRef(rightKey.getType(), systemColCount + origLeftInputSize
+ newKeyCount + p.right));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
if (newKeyCount == 0) {
return outJoinCond;
}
int newLeftOffset = systemColCount + origLeftInputSize;
int newRightOffset = systemColCount + origLeftInputSize + origRightInputSize + newKeyCount;
for (i = 0; i < newKeyCount; i++) {
leftKeys.add(origLeftInputSize + i);
rightKeys.add(origRightInputSize + i);
RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
rexBuilder.makeInputRef(newLeftFields.get(origLeftInputSize + i).getType(), newLeftOffset + i),
rexBuilder.makeInputRef(newRightFields.get(origRightInputSize + i).getType(), newRightOffset + i));
if (outJoinCond == null) {
outJoinCond = cond;
} else {
outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond, cond);
}
}
// added project if need to produce new keys than the original input
// fields
if (newKeyCount > 0) {
leftRel = factory.createProject(leftRel, newLeftFields,
SqlValidatorUtil.uniquify(newLeftFieldNames));
rightRel = factory.createProject(rightRel, newRightFields,
SqlValidatorUtil.uniquify(newRightFieldNames));
}
inputRels[0] = leftRel;
inputRels[1] = rightRel;
return outJoinCond;
}
/**
* JoinPredicateInfo represents Join condition; JoinPredicate Info uses
* JoinLeafPredicateInfo to represent individual conjunctive elements in the
* predicate.
* JoinPredicateInfo = JoinLeafPredicateInfo1 and JoinLeafPredicateInfo2...
*
* JoinPredicateInfo:
* 1. preserves the order of conjuctive elements for
* equi-join(equiJoinPredicateElements)
* 2. Stores set of projection indexes from left and right child which is part
* of equi join keys; the indexes are both in child and Join node schema.
* 3. Keeps a map of projection indexes that are part of join keys to list of
* conjuctive elements(JoinLeafPredicateInfo) that uses them.
*
*/
public static class JoinPredicateInfo {
private final ImmutableList nonEquiJoinPredicateElements;
private final ImmutableList equiJoinPredicateElements;
private final ImmutableList> projsJoinKeysInChildSchema;
private final ImmutableList> projsJoinKeysInJoinSchema;
private final ImmutableMap> mapOfProjIndxInJoinSchemaToLeafPInfo;
public JoinPredicateInfo(List nonEquiJoinPredicateElements,
List equiJoinPredicateElements,
List> projsJoinKeysInChildSchema,
List> projsJoinKeysInJoinSchema,
Map> mapOfProjIndxInJoinSchemaToLeafPInfo) {
this.nonEquiJoinPredicateElements = ImmutableList.copyOf(nonEquiJoinPredicateElements);
this.equiJoinPredicateElements = ImmutableList.copyOf(equiJoinPredicateElements);
this.projsJoinKeysInChildSchema = ImmutableList
.copyOf(projsJoinKeysInChildSchema);
this.projsJoinKeysInJoinSchema = ImmutableList
.copyOf(projsJoinKeysInJoinSchema);
this.mapOfProjIndxInJoinSchemaToLeafPInfo = ImmutableMap
.copyOf(mapOfProjIndxInJoinSchemaToLeafPInfo);
}
public List getNonEquiJoinPredicateElements() {
return this.nonEquiJoinPredicateElements;
}
public List getEquiJoinPredicateElements() {
return this.equiJoinPredicateElements;
}
public Set getProjsFromLeftPartOfJoinKeysInChildSchema() {
assert projsJoinKeysInChildSchema.size() == 2;
return this.projsJoinKeysInChildSchema.get(0);
}
public Set getProjsFromRightPartOfJoinKeysInChildSchema() {
assert projsJoinKeysInChildSchema.size() == 2;
return this.projsJoinKeysInChildSchema.get(1);
}
public Set getProjsJoinKeysInChildSchema(int i) {
return this.projsJoinKeysInChildSchema.get(i);
}
/**
* NOTE: Join Schema = left Schema + (right Schema offset by
* left.fieldcount). Hence its ok to return projections from left in child
* schema.
*/
public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() {
assert projsJoinKeysInJoinSchema.size() == 2;
return this.projsJoinKeysInJoinSchema.get(0);
}
public Set getProjsFromRightPartOfJoinKeysInJoinSchema() {
assert projsJoinKeysInJoinSchema.size() == 2;
return this.projsJoinKeysInJoinSchema.get(1);
}
public Set getProjsJoinKeysInJoinSchema(int i) {
return this.projsJoinKeysInJoinSchema.get(i);
}
public Map> getMapOfProjIndxToLeafPInfo() {
return this.mapOfProjIndxInJoinSchemaToLeafPInfo;
}
public static JoinPredicateInfo constructJoinPredicateInfo(Join j) throws CalciteSemanticException {
return constructJoinPredicateInfo(j, j.getCondition());
}
public static JoinPredicateInfo constructJoinPredicateInfo(HiveMultiJoin mj) throws CalciteSemanticException {
return constructJoinPredicateInfo(mj, mj.getCondition());
}
public static JoinPredicateInfo constructJoinPredicateInfo(Join j, RexNode predicate) throws CalciteSemanticException {
return constructJoinPredicateInfo(j.getInputs(), j.getSystemFieldList(), predicate);
}
public static JoinPredicateInfo constructJoinPredicateInfo(HiveMultiJoin mj, RexNode predicate) throws CalciteSemanticException {
final List systemFieldList = ImmutableList.of();
return constructJoinPredicateInfo(mj.getInputs(), systemFieldList, predicate);
}
public static JoinPredicateInfo constructJoinPredicateInfo(List inputs,
List systemFieldList, RexNode predicate) throws CalciteSemanticException {
JoinPredicateInfo jpi = null;
JoinLeafPredicateInfo jlpi = null;
List equiLPIList = new ArrayList();
List nonEquiLPIList = new ArrayList();
List> projsJoinKeys = new ArrayList>();
for (int i=0; i projsJoinKeysInput = Sets.newHashSet();
projsJoinKeys.add(projsJoinKeysInput);
}
List> projsJoinKeysInJoinSchema = new ArrayList>();
for (int i=0; i projsJoinKeysInJoinSchemaInput = Sets.newHashSet();
projsJoinKeysInJoinSchema.add(projsJoinKeysInJoinSchemaInput);
}
Map> tmpMapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>();
Map> mapOfProjIndxInJoinSchemaToLeafPInfo = new HashMap>();
List tmpJLPILst = null;
List conjuctiveElements;
// 1. Decompose Join condition to a number of leaf predicates
// (conjuctive elements)
conjuctiveElements = RelOptUtil.conjunctions(predicate);
// 2. Walk through leaf predicates building up JoinLeafPredicateInfo
for (RexNode ce : conjuctiveElements) {
// 2.1 Construct JoinLeafPredicateInfo
jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(inputs, systemFieldList, ce);
// 2.2 Classify leaf predicate as Equi vs Non Equi
if (jlpi.comparisonType.equals(SqlKind.EQUALS)) {
equiLPIList.add(jlpi);
// 2.2.1 Maintain join keys (in child & Join Schema)
// 2.2.2 Update Join Key to JoinLeafPredicateInfo map with keys
for (int i=0; i();
}
tmpJLPILst.add(jlpi);
tmpMapOfProjIndxInJoinSchemaToLeafPInfo.put(projIndx, tmpJLPILst);
}
}
} else {
nonEquiLPIList.add(jlpi);
}
}
// 3. Update Update Join Key to List to use
// ImmutableList
for (Entry> e : tmpMapOfProjIndxInJoinSchemaToLeafPInfo
.entrySet()) {
mapOfProjIndxInJoinSchemaToLeafPInfo.put(e.getKey(), ImmutableList.copyOf(e.getValue()));
}
// 4. Construct JoinPredicateInfo
jpi = new JoinPredicateInfo(nonEquiLPIList, equiLPIList, projsJoinKeys,
projsJoinKeysInJoinSchema, mapOfProjIndxInJoinSchemaToLeafPInfo);
return jpi;
}
}
/**
* JoinLeafPredicateInfo represents leaf predicate in Join condition
* (conjuctive lement).
*
* JoinLeafPredicateInfo:
* 1. Stores list of expressions from left and right child which is part of
* equi join keys.
* 2. Stores set of projection indexes from left and right child which is part
* of equi join keys; the indexes are both in child and Join node schema.
*/
public static class JoinLeafPredicateInfo {
private final SqlKind comparisonType;
private final ImmutableList> joinKeyExprs;
private final ImmutableList> projsJoinKeysInChildSchema;
private final ImmutableList> projsJoinKeysInJoinSchema;
public JoinLeafPredicateInfo(
SqlKind comparisonType,
List> joinKeyExprs,
List> projsJoinKeysInChildSchema,
List> projsJoinKeysInJoinSchema) {
this.comparisonType = comparisonType;
ImmutableList.Builder> joinKeyExprsBuilder =
ImmutableList.builder();
for (int i=0; i> projsJoinKeysInChildSchemaBuilder =
ImmutableList.builder();
for (int i=0; i> projsJoinKeysInJoinSchemaBuilder =
ImmutableList.builder();
for (int i=0; i getJoinExprs(int input) {
return this.joinKeyExprs.get(input);
}
public Set getProjsFromLeftPartOfJoinKeysInChildSchema() {
assert projsJoinKeysInChildSchema.size() == 2;
return this.projsJoinKeysInChildSchema.get(0);
}
public Set getProjsFromRightPartOfJoinKeysInChildSchema() {
assert projsJoinKeysInChildSchema.size() == 2;
return this.projsJoinKeysInChildSchema.get(1);
}
public Set getProjsJoinKeysInChildSchema(int input) {
return this.projsJoinKeysInChildSchema.get(input);
}
public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() {
assert projsJoinKeysInJoinSchema.size() == 2;
return this.projsJoinKeysInJoinSchema.get(0);
}
public Set getProjsFromRightPartOfJoinKeysInJoinSchema() {
assert projsJoinKeysInJoinSchema.size() == 2;
return this.projsJoinKeysInJoinSchema.get(1);
}
public Set getProjsJoinKeysInJoinSchema(int input) {
return this.projsJoinKeysInJoinSchema.get(input);
}
// We create the join predicate info object. The object contains the join condition,
// split accordingly. If the join condition is not part of the equi-join predicate,
// the returned object will be typed as SQLKind.OTHER.
private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(List inputs,
List systemFieldList, RexNode pe) throws CalciteSemanticException {
JoinLeafPredicateInfo jlpi = null;
List filterNulls = new ArrayList();
List> joinExprs = new ArrayList>();
for (int i=0; i());
}
// 1. Split leaf join predicate to expressions from left, right
RexNode otherConditions = HiveRelOptUtil.splitHiveJoinCondition(systemFieldList, inputs, pe,
joinExprs, filterNulls, null);
if (otherConditions.isAlwaysTrue()) {
// 2. Collect child projection indexes used
List> projsJoinKeysInChildSchema =
new ArrayList>();
for (int i=0; i projsFromInputJoinKeysInChildSchema = ImmutableSet.builder();
InputReferencedVisitor irvLeft = new InputReferencedVisitor();
irvLeft.apply(joinExprs.get(i));
projsFromInputJoinKeysInChildSchema.addAll(irvLeft.inputPosReferenced);
projsJoinKeysInChildSchema.add(projsFromInputJoinKeysInChildSchema.build());
}
// 3. Translate projection indexes to join schema, by adding offset.
List> projsJoinKeysInJoinSchema =
new ArrayList>();
// The offset of the first input does not need to change.
projsJoinKeysInJoinSchema.add(projsJoinKeysInChildSchema.get(0));
for (int i=1; i projsFromInputJoinKeysInJoinSchema = ImmutableSet.builder();
for (Integer indx : projsJoinKeysInChildSchema.get(i)) {
projsFromInputJoinKeysInJoinSchema.add(indx + offSet);
}
projsJoinKeysInJoinSchema.add(projsFromInputJoinKeysInJoinSchema.build());
}
// 4. Construct JoinLeafPredicateInfo
jlpi = new JoinLeafPredicateInfo(pe.getKind(), joinExprs,
projsJoinKeysInChildSchema, projsJoinKeysInJoinSchema);
} else {
// 2. Construct JoinLeafPredicateInfo
ImmutableBitSet refCols = InputFinder.bits(pe);
int count = 0;
for (int i=0; i>(), new ArrayList>());
}
return jlpi;
}
}
public static boolean pureLimitRelNode(RelNode rel) {
return limitRelNode(rel) && !orderRelNode(rel);
}
public static boolean pureOrderRelNode(RelNode rel) {
return !limitRelNode(rel) && orderRelNode(rel);
}
public static boolean limitRelNode(RelNode rel) {
if ((rel instanceof Sort) && ((Sort) rel).fetch != null) {
return true;
}
return false;
}
public static boolean orderRelNode(RelNode rel) {
if ((rel instanceof Sort) && !((Sort) rel).getCollation().getFieldCollations().isEmpty()) {
return true;
}
return false;
}
/**
* Get top level select starting from root. Assumption here is root can only
* be Sort & Project. Also the top project should be at most 2 levels below
* Sort; i.e Sort(Limit)-Sort(OB)-Select
*
* @param rootRel
* @return
*/
public static Pair getTopLevelSelect(final RelNode rootRel) {
RelNode tmpRel = rootRel;
RelNode parentOforiginalProjRel = rootRel;
HiveProject originalProjRel = null;
while (tmpRel != null) {
if (tmpRel instanceof HiveProject) {
originalProjRel = (HiveProject) tmpRel;
break;
}
parentOforiginalProjRel = tmpRel;
tmpRel = tmpRel.getInput(0);
}
return (new Pair(parentOforiginalProjRel, originalProjRel));
}
public static boolean isComparisonOp(RexCall call) {
return call.getKind().belongsTo(SqlKind.COMPARISON);
}
public static final Function REX_STR_FN = new Function() {
public String apply(RexNode r) {
return r.toString();
}
};
public static ImmutableList getPredsNotPushedAlready(RelNode inp, List predsToPushDown) {
return getPredsNotPushedAlready(Sets.newHashSet(), inp, predsToPushDown);
}
/**
* Given a list of predicates to push down, this methods returns the set of predicates
* that still need to be pushed. Predicates need to be pushed because 1) their String
* representation is not included in input set of predicates to exclude, or 2) they are
* already in the subtree rooted at the input node.
* This method updates the set of predicates to exclude with the String representation
* of the predicates in the output and in the subtree.
*
* @param predicatesToExclude String representation of predicates that should be excluded
* @param inp root of the subtree
* @param predsToPushDown candidate predicates to push down through the subtree
* @return list of predicates to push down
*/
public static ImmutableList getPredsNotPushedAlready(Set predicatesToExclude,
RelNode inp, List predsToPushDown) {
// Bail out if there is nothing to push
if (predsToPushDown.isEmpty()) {
return ImmutableList.of();
}
// Build map to not convert multiple times, further remove already included predicates
Map stringToRexNode = Maps.newLinkedHashMap();
for (RexNode r : predsToPushDown) {
String rexNodeString = r.toString();
if (predicatesToExclude.add(rexNodeString)) {
stringToRexNode.put(rexNodeString, r);
}
}
if (stringToRexNode.isEmpty()) {
return ImmutableList.of();
}
// Finally exclude preds that are already in the subtree as given by the metadata provider
// Note: this is the last step, trying to avoid the expensive call to the metadata provider
// if possible
Set predicatesInSubtree = Sets.newHashSet();
final RelMetadataQuery mq = inp.getCluster().getMetadataQuery();
for (RexNode pred : mq.getPulledUpPredicates(inp).pulledUpPredicates) {
predicatesInSubtree.add(pred.toString());
predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN));
}
final ImmutableList.Builder newConjuncts = ImmutableList.builder();
for (Entry e : stringToRexNode.entrySet()) {
if (predicatesInSubtree.add(e.getKey())) {
newConjuncts.add(e.getValue());
}
}
predicatesToExclude.addAll(predicatesInSubtree);
return newConjuncts.build();
}
public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
RexNode typeSafeRex = rex;
if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
RexBuilder rb = cluster.getRexBuilder();
List fixedPredElems = new ArrayList();
RelDataType commonType = cluster.getTypeFactory().leastRestrictive(
RexUtil.types(((RexCall) rex).getOperands()));
for (RexNode rn : ((RexCall) rex).getOperands()) {
fixedPredElems.add(rb.ensureType(commonType, rn, true));
}
typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
}
return typeSafeRex;
}
public static boolean isDeterministic(RexNode expr) {
boolean deterministic = true;
RexVisitor visitor = new RexVisitorImpl(true) {
@Override
public Void visitCall(org.apache.calcite.rex.RexCall call) {
if (!call.getOperator().isDeterministic()) {
throw new Util.FoundOne(call);
}
return super.visitCall(call);
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
deterministic = false;
}
return deterministic;
}
private static class DeterMinisticFuncVisitorImpl extends RexVisitorImpl {
protected DeterMinisticFuncVisitorImpl() {
super(true);
}
@Override
public Void visitCall(org.apache.calcite.rex.RexCall call) {
if (!call.getOperator().isDeterministic()) {
throw new Util.FoundOne(call);
}
return super.visitCall(call);
}
@Override
public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
throw new Util.FoundOne(correlVariable);
}
@Override
public Void visitLocalRef(RexLocalRef localRef) {
throw new Util.FoundOne(localRef);
}
@Override
public Void visitOver(RexOver over) {
throw new Util.FoundOne(over);
}
@Override
public Void visitDynamicParam(RexDynamicParam dynamicParam) {
throw new Util.FoundOne(dynamicParam);
}
@Override
public Void visitRangeRef(RexRangeRef rangeRef) {
throw new Util.FoundOne(rangeRef);
}
@Override
public Void visitFieldAccess(RexFieldAccess fieldAccess) {
throw new Util.FoundOne(fieldAccess);
}
}
public static boolean isDeterministicFuncOnLiterals(RexNode expr) {
boolean deterministicFuncOnLiterals = true;
RexVisitor visitor = new DeterMinisticFuncVisitorImpl() {
@Override
public Void visitInputRef(RexInputRef inputRef) {
throw new Util.FoundOne(inputRef);
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
deterministicFuncOnLiterals = false;
}
return deterministicFuncOnLiterals;
}
public List getDeterministicFuncWithSingleInputRef(List exprs,
final Set validInputRefs) {
List determExprsWithSingleRef = new ArrayList();
for (RexNode e : exprs) {
if (isDeterministicFuncWithSingleInputRef(e, validInputRefs)) {
determExprsWithSingleRef.add(e);
}
}
return determExprsWithSingleRef;
}
public static boolean isDeterministicFuncWithSingleInputRef(RexNode expr,
final Set validInputRefs) {
boolean deterministicFuncWithSingleInputRef = true;
RexVisitor visitor = new DeterMinisticFuncVisitorImpl() {
Set inputRefs = new HashSet();
@Override
public Void visitInputRef(RexInputRef inputRef) {
if (validInputRefs.contains(inputRef.getIndex())) {
inputRefs.add(inputRef.getIndex());
if (inputRefs.size() > 1) {
throw new Util.FoundOne(inputRef);
}
} else {
throw new Util.FoundOne(inputRef);
}
return null;
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
deterministicFuncWithSingleInputRef = false;
}
return deterministicFuncWithSingleInputRef;
}
public static ImmutableMap getColInfoMap(List hiveCols,
int startIndx) {
Builder bldr = ImmutableMap. builder();
int indx = startIndx;
for (T ci : hiveCols) {
bldr.put(indx, ci);
indx++;
}
return bldr.build();
}
public static ImmutableSet shiftVColsSet(Set hiveVCols, int shift) {
ImmutableSet.Builder bldr = ImmutableSet. builder();
for (Integer pos : hiveVCols) {
bldr.add(shift + pos);
}
return bldr.build();
}
public static ImmutableMap getVColsMap(List hiveVCols,
int startIndx) {
Builder bldr = ImmutableMap. builder();
int indx = startIndx;
for (VirtualColumn vc : hiveVCols) {
bldr.put(indx, vc);
indx++;
}
return bldr.build();
}
public static ImmutableMap getColNameIndxMap(List tableFields) {
Builder bldr = ImmutableMap. builder();
int indx = 0;
for (FieldSchema fs : tableFields) {
bldr.put(fs.getName(), indx);
indx++;
}
return bldr.build();
}
public static ImmutableMap getRowColNameIndxMap(List rowFields) {
Builder bldr = ImmutableMap. builder();
int indx = 0;
for (RelDataTypeField rdt : rowFields) {
bldr.put(rdt.getName(), indx);
indx++;
}
return bldr.build();
}
public static ImmutableList getInputRef(List inputRefs, RelNode inputRel) {
ImmutableList.Builder bldr = ImmutableList. builder();
for (int i : inputRefs) {
bldr.add(new RexInputRef(i, inputRel.getRowType().getFieldList().get(i).getType()));
}
return bldr.build();
}
public static ExprNodeDesc getExprNode(Integer inputRefIndx, RelNode inputRel,
ExprNodeConverter exprConv) {
ExprNodeDesc exprNode = null;
RexNode rexInputRef = new RexInputRef(inputRefIndx, inputRel.getRowType()
.getFieldList().get(inputRefIndx).getType());
exprNode = rexInputRef.accept(exprConv);
return exprNode;
}
public static List getExprNodes(List inputRefs, RelNode inputRel,
String inputTabAlias) {
List exprNodes = new ArrayList();
List rexInputRefs = getInputRef(inputRefs, inputRel);
List exprs = inputRel.getChildExps();
// TODO: Change ExprNodeConverter to be independent of Partition Expr
ExprNodeConverter exprConv = new ExprNodeConverter(inputTabAlias, inputRel.getRowType(),
new HashSet(), inputRel.getCluster().getTypeFactory());
for (int index = 0; index < rexInputRefs.size(); index++) {
// The following check is only a guard against failures.
// TODO: Knowing which expr is constant in GBY's aggregation function
// arguments could be better done using Metadata provider of Calcite.
//check the corresponding expression in exprs to see if it is literal
if (exprs != null && index < exprs.size() && exprs.get(inputRefs.get(index)) instanceof RexLiteral) {
//because rexInputRefs represent ref expr corresponding to value in inputRefs it is used to get
// corresponding index
ExprNodeDesc exprNodeDesc = exprConv.visitLiteral((RexLiteral) exprs.get(inputRefs.get(index)));
exprNodes.add(exprNodeDesc);
} else {
RexNode iRef = rexInputRefs.get(index);
exprNodes.add(iRef.accept(exprConv));
}
}
return exprNodes;
}
public static List getFieldNames(List inputRefs, RelNode inputRel) {
List fieldNames = new ArrayList();
List schemaNames = inputRel.getRowType().getFieldNames();
for (Integer iRef : inputRefs) {
fieldNames.add(schemaNames.get(iRef));
}
return fieldNames;
}
public static AggregateCall createSingleArgAggCall(String funcName, RelOptCluster cluster,
PrimitiveTypeInfo typeInfo, Integer pos, RelDataType aggFnRetType) {
ImmutableList.Builder aggArgRelDTBldr = new ImmutableList.Builder();
aggArgRelDTBldr.add(TypeConverter.convert(typeInfo, cluster.getTypeFactory()));
SqlAggFunction aggFunction = SqlFunctionConverter.getCalciteAggFn(funcName, false,
aggArgRelDTBldr.build(), aggFnRetType);
List argList = new ArrayList();
argList.add(pos);
return AggregateCall.create(aggFunction, false, argList, -1, aggFnRetType, null);
}
/**
* Is the expression usable for query materialization.
*/
public static boolean isMaterializable(RexNode expr) {
return (checkMaterializable(expr) == null);
}
/**
* Check if the expression is usable for query materialization, returning the first failing expression.
*/
public static RexCall checkMaterializable(RexNode expr) {
boolean deterministic = true;
RexCall failingCall = null;
if (expr == null) {
return null;
}
RexVisitor visitor = new RexVisitorImpl(true) {
@Override
public Void visitCall(org.apache.calcite.rex.RexCall call) {
// non-deterministic functions as well as runtime constants are not materializable.
if (!call.getOperator().isDeterministic() || call.getOperator().isDynamicFunction()) {
throw new Util.FoundOne(call);
}
return super.visitCall(call);
}
};
try {
expr.accept(visitor);
} catch (Util.FoundOne e) {
failingCall = (RexCall) e.getNode();
}
return failingCall;
}
public static HiveTableFunctionScan createUDTFForSetOp(RelOptCluster cluster, RelNode input)
throws SemanticException {
RelTraitSet traitSet = TraitsUtil.getDefaultTraitSet(cluster);
List originalInputRefs = Lists.transform(input.getRowType().getFieldList(),
new Function() {
@Override
public RexNode apply(RelDataTypeField input) {
return new RexInputRef(input.getIndex(), input.getType());
}
});
ImmutableList.Builder argTypeBldr = ImmutableList. builder();
for (int i = 0; i < originalInputRefs.size(); i++) {
argTypeBldr.add(originalInputRefs.get(i).getType());
}
RelDataType retType = input.getRowType();
String funcName = "replicate_rows";
FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName);
SqlOperator calciteOp = SqlFunctionConverter.getCalciteOperator(funcName, fi.getGenericUDTF(),
argTypeBldr.build(), retType);
// Hive UDTF only has a single input
List list = new ArrayList<>();
list.add(input);
RexNode rexNode = cluster.getRexBuilder().makeCall(calciteOp, originalInputRefs);
return HiveTableFunctionScan.create(cluster, traitSet, list, rexNode, null, retType, null);
}
// this will create a project which will project out the column in positions
public static HiveProject createProjectWithoutColumn(RelNode input, Set positions)
throws CalciteSemanticException {
List originalInputRefs = Lists.transform(input.getRowType().getFieldList(),
new Function() {
@Override
public RexNode apply(RelDataTypeField input) {
return new RexInputRef(input.getIndex(), input.getType());
}
});
List copyInputRefs = new ArrayList<>();
for (int i = 0; i < originalInputRefs.size(); i++) {
if (!positions.contains(i)) {
copyInputRefs.add(originalInputRefs.get(i));
}
}
return HiveProject.create(input, copyInputRefs, null);
}
/**
* Walks over an expression and determines whether it is constant.
*/
public static class ConstantFinder implements RexVisitor {
@Override
public Boolean visitLiteral(RexLiteral literal) {
return true;
}
@Override
public Boolean visitInputRef(RexInputRef inputRef) {
return false;
}
@Override
public Boolean visitTableInputRef(RexTableInputRef inputRef) {
return false;
}
@Override
public Boolean visitLocalRef(RexLocalRef localRef) {
throw new RuntimeException("Not expected to be called.");
}
@Override
public Boolean visitOver(RexOver over) {
return false;
}
@Override
public Boolean visitCorrelVariable(RexCorrelVariable correlVariable) {
return false;
}
@Override
public Boolean visitDynamicParam(RexDynamicParam dynamicParam) {
return false;
}
@Override
public Boolean visitCall(RexCall call) {
// Constant if operator is deterministic and all operands are
// constant.
return call.getOperator().isDeterministic()
&& RexVisitorImpl.visitArrayAnd(this, call.getOperands());
}
@Override
public Boolean visitRangeRef(RexRangeRef rangeRef) {
return false;
}
@Override
public Boolean visitFieldAccess(RexFieldAccess fieldAccess) {
// ".FIELD" is constant iff "" is constant.
return fieldAccess.getReferenceExpr().accept(this);
}
@Override
public Boolean visitSubQuery(RexSubQuery subQuery) {
// it seems that it is not used by anything.
return false;
}
@Override
public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
return false;
}
}
public static Set getInputRefs(RexNode expr) {
InputRefsCollector irefColl = new InputRefsCollector(true);
expr.accept(irefColl);
return irefColl.getInputRefSet();
}
private static class InputRefsCollector extends RexVisitorImpl {
private final Set inputRefSet = new HashSet();
private InputRefsCollector(boolean deep) {
super(deep);
}
@Override
public Void visitInputRef(RexInputRef inputRef) {
inputRefSet.add(inputRef.getIndex());
return null;
}
public Set getInputRefSet() {
return inputRefSet;
}
}
}