org.apache.druid.sql.calcite.planner.QueryHandler Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.sql.calcite.planner;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import org.apache.calcite.DataContext;
import org.apache.calcite.interpreter.BindableConvention;
import org.apache.calcite.interpreter.BindableRel;
import org.apache.calcite.interpreter.Bindables;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.logical.LogicalSort;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.sql.SqlExplain;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.tools.ValidationException;
import org.apache.calcite.util.Pair;
import org.apache.druid.error.DruidException;
import org.apache.druid.error.InvalidSqlInput;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.guava.BaseSequence;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.query.Query;
import org.apache.druid.server.QueryResponse;
import org.apache.druid.server.security.Action;
import org.apache.druid.server.security.Resource;
import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.sql.calcite.planner.querygen.DruidQueryGenerator;
import org.apache.druid.sql.calcite.rel.DruidConvention;
import org.apache.druid.sql.calcite.rel.DruidQuery;
import org.apache.druid.sql.calcite.rel.DruidRel;
import org.apache.druid.sql.calcite.rel.DruidUnionRel;
import org.apache.druid.sql.calcite.rel.logical.DruidLogicalConvention;
import org.apache.druid.sql.calcite.rel.logical.DruidLogicalNode;
import org.apache.druid.sql.calcite.run.EngineFeature;
import org.apache.druid.sql.calcite.run.QueryMaker;
import org.apache.druid.sql.calcite.table.DruidTable;
import org.apache.druid.utils.Throwables;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Abstract base class for handlers that revolve around queries: SELECT,
* INSERT and REPLACE. This class handles the common SELECT portion of the statement.
*/
public abstract class QueryHandler extends SqlStatementHandler.BaseStatementHandler
{
static final EmittingLogger log = new EmittingLogger(QueryHandler.class);
protected SqlExplain explain;
private boolean isPrepared;
protected RelRoot rootQueryRel;
private PrepareResult prepareResult;
protected RexBuilder rexBuilder;
public QueryHandler(HandlerContext handlerContext, SqlExplain explain)
{
super(handlerContext);
this.explain = explain;
}
protected SqlNode validate(SqlNode root)
{
CalcitePlanner planner = handlerContext.planner();
SqlNode validatedQueryNode;
try {
validatedQueryNode = planner.validate(rewriteParameters(root));
}
catch (ValidationException e) {
throw DruidPlanner.translateException(e);
}
final SqlValidator validator = planner.getValidator();
SqlResourceCollectorShuttle resourceCollectorShuttle = new SqlResourceCollectorShuttle(
validator,
handlerContext.plannerContext()
);
validatedQueryNode.accept(resourceCollectorShuttle);
resourceActions = resourceCollectorShuttle.getResourceActions();
return validatedQueryNode;
}
private SqlNode rewriteParameters(SqlNode original)
{
// Uses {@link SqlParameterizerShuttle} to rewrite {@link SqlNode} to swap out any
// {@link org.apache.calcite.sql.SqlDynamicParam} early for their {@link SqlLiteral}
// replacement.
//
// Parameter replacement is done only if the client provides parameter values.
// If this is a PREPARE-only, then there will be no values even if the statement contains
// parameters. If this is a PLAN, then we'll catch later the case that the statement
// contains parameters, but no values were provided.
PlannerContext plannerContext = handlerContext.plannerContext();
if (plannerContext.getParameters().isEmpty()) {
return original;
} else {
return original.accept(new SqlParameterizerShuttle(plannerContext));
}
}
@Override
public void prepare()
{
if (isPrepared) {
return;
}
isPrepared = true;
SqlNode validatedQueryNode = validatedQueryNode();
rootQueryRel = handlerContext.planner().rel(validatedQueryNode);
handlerContext.hook().captureQueryRel(rootQueryRel);
final RelDataTypeFactory typeFactory = rootQueryRel.rel.getCluster().getTypeFactory();
final SqlValidator validator = handlerContext.planner().getValidator();
final RelDataType parameterTypes = validator.getParameterRowType(validatedQueryNode);
handlerContext.hook().captureParameterTypes(parameterTypes);
final RelDataType returnedRowType;
if (explain != null) {
handlerContext.plannerContext().setExplainAttributes(explainAttributes());
returnedRowType = getExplainStructType(typeFactory);
} else {
returnedRowType = returnedRowType();
}
prepareResult = new PrepareResult(rootQueryRel.validatedRowType, returnedRowType, parameterTypes);
}
@Override
public PrepareResult prepareResult()
{
return prepareResult;
}
protected abstract SqlNode validatedQueryNode();
protected abstract RelDataType returnedRowType();
private static RelDataType getExplainStructType(RelDataTypeFactory typeFactory)
{
return typeFactory.createStructType(
ImmutableList.of(
Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR),
Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR),
Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR)
),
ImmutableList.of("PLAN", "RESOURCES", "ATTRIBUTES")
);
}
@Override
public PlannerResult plan()
{
prepare();
final Set bindableTables = getBindableTables(rootQueryRel.rel);
// the planner's type factory is not available until after parsing
rexBuilder = new RexBuilder(handlerContext.planner().getTypeFactory());
try {
if (!bindableTables.isEmpty()) {
// Consider BINDABLE convention when necessary. Used for metadata tables.
if (!handlerContext.plannerContext().featureAvailable(EngineFeature.ALLOW_BINDABLE_PLAN)) {
throw InvalidSqlInput.exception(
"Cannot query table(s) [%s] with SQL engine [%s]",
bindableTables.stream()
.map(table -> Joiner.on(".").join(table.getQualifiedName()))
.collect(Collectors.joining(", ")),
handlerContext.engine().name()
);
}
return planWithBindableConvention();
} else {
// Druid convention is used whenever there are no tables that require BINDABLE.
return planForDruid();
}
}
catch (RelOptPlanner.CannotPlanException e) {
throw buildSQLPlanningError(e);
}
catch (RuntimeException e) {
if (e instanceof DruidException) {
throw e;
}
// Calcite throws a Runtime exception as the result of an IllegalTargetException
// as the result of invoking a method dynamically, when that method throws an
// exception. Unwrap the exception if this exception is from Calcite.
RelOptPlanner.CannotPlanException cpe = Throwables.getCauseOfType(e, RelOptPlanner.CannotPlanException.class);
if (cpe != null) {
throw buildSQLPlanningError(cpe);
}
DruidException de = Throwables.getCauseOfType(e, DruidException.class);
if (de != null) {
throw de;
}
// Exceptions during rule evaluations could be wrapped inside a RuntimeException by VolcanoRuleCall class.
// This block will extract a user-friendly message from the exception chain.
if (e.getMessage() != null
&& e.getCause() != null
&& e.getCause().getMessage() != null
&& e.getMessage().startsWith("Error while applying rule")) {
throw DruidException.forPersona(DruidException.Persona.ADMIN)
.ofCategory(DruidException.Category.UNCATEGORIZED)
.build(e, "%s", e.getCause().getMessage());
}
throw DruidPlanner.translateException(e);
}
catch (Exception e) {
// Not sure what this is. Should it have been translated sooner?
throw DruidPlanner.translateException(e);
}
}
@Override
public ExplainAttributes explainAttributes()
{
return new ExplainAttributes(
"SELECT",
null,
null,
null,
null
);
}
private static Set getBindableTables(final RelNode relNode)
{
class HasBindableVisitor extends RelVisitor
{
private final Set found = new HashSet<>();
@Override
public void visit(RelNode node, int ordinal, RelNode parent)
{
if (node instanceof TableScan) {
RelOptTable table = node.getTable();
if (table.unwrap(ScannableTable.class) != null && table.unwrap(DruidTable.class) == null) {
found.add(table);
return;
}
}
super.visit(node, ordinal, parent);
}
}
final HasBindableVisitor visitor = new HasBindableVisitor();
visitor.go(relNode);
return visitor.found;
}
/**
* Construct a {@link PlannerResult} for a fall-back 'bindable' rel, for
* things that are not directly translatable to native Druid queries such
* as system tables and just a general purpose (but definitely not optimized)
* fall-back.
*
* See {@link #planWithDruidConvention} which will handle things which are
* directly translatable to native Druid queries.
*
* The bindable path handles parameter substitution of any values not
* bound by the earlier steps.
*/
private PlannerResult planWithBindableConvention()
{
CalcitePlanner planner = handlerContext.planner();
BindableRel bindableRel = (BindableRel) planner.transform(
CalciteRulesManager.BINDABLE_CONVENTION_RULES,
planner.getEmptyTraitSet().replace(BindableConvention.INSTANCE).plus(rootQueryRel.collation),
rootQueryRel.rel
);
if (!rootQueryRel.isRefTrivial()) {
// Add a projection on top to accommodate root.fields.
final List projects = new ArrayList<>();
final RexBuilder rexBuilder = bindableRel.getCluster().getRexBuilder();
for (int field : Pair.left(rootQueryRel.fields)) {
projects.add(rexBuilder.makeInputRef(bindableRel, field));
}
bindableRel = new Bindables.BindableProject(
bindableRel.getCluster(),
bindableRel.getTraitSet(),
bindableRel,
projects,
rootQueryRel.validatedRowType
);
}
handlerContext.hook().captureBindableRel(bindableRel);
PlannerContext plannerContext = handlerContext.plannerContext();
if (explain != null) {
return planExplanation(rootQueryRel, bindableRel, false);
} else {
final BindableRel theRel = bindableRel;
final DataContext dataContext = plannerContext.createDataContext(
planner.getTypeFactory(),
plannerContext.getParameters()
);
final Supplier> resultsSupplier = () -> {
final Enumerable> enumerable = theRel.bind(dataContext);
final Enumerator> enumerator = enumerable.enumerator();
return QueryResponse.withEmptyContext(
Sequences.withBaggage(new BaseSequence<>(
new BaseSequence.IteratorMaker