io.questdb.griffin.model.QueryModel Maven / Gradle / Ivy
Show all versions of core Show documentation
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (c) 2014-2019 Appsicle
* Copyright (c) 2019-2020 QuestDB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/
package io.questdb.griffin.model;
import io.questdb.cairo.sql.Function;
import io.questdb.std.*;
import io.questdb.std.str.CharSink;
import java.util.ArrayDeque;
import static io.questdb.griffin.SqlKeywords.isAndKeyword;
public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sinkable {
public static final QueryModelFactory FACTORY = new QueryModelFactory();
public static final int ORDER_DIRECTION_ASCENDING = 0;
public static final int ORDER_DIRECTION_DESCENDING = 1;
public static final String NO_ROWID_MARKER = "*!*";
public static final int JOIN_INNER = 1;
public static final int JOIN_OUTER = 2;
public static final int JOIN_CROSS = 3;
public static final int JOIN_ASOF = 4;
public static final int JOIN_SPLICE = 5;
public static final int JOIN_LT = 6;
public static final String SUB_QUERY_ALIAS_PREFIX = "_xQdbA";
public static final int SELECT_MODEL_NONE = 0;
public static final int SELECT_MODEL_CHOOSE = 1;
public static final int SELECT_MODEL_VIRTUAL = 2;
public static final int SELECT_MODEL_ANALYTIC = 3;
public static final int SELECT_MODEL_GROUP_BY = 4;
public static final int SELECT_MODEL_DISTINCT = 5;
public static final int UNION_MODEL_ALL = 0;
public static final int UNION_MODEL_DISTINCT = 1;
private static final ObjList modelTypeName = new ObjList<>();
private final ObjList bottomUpColumns = new ObjList<>();
private final CharSequenceHashSet topDownNameSet = new CharSequenceHashSet();
private final ObjList topDownColumns = new ObjList<>();
private final CharSequenceObjHashMap aliasToColumnNameMap = new CharSequenceObjHashMap<>();
private final CharSequenceObjHashMap columnNameToAliasMap = new CharSequenceObjHashMap<>();
private final CharSequenceObjHashMap aliasToColumnMap = new CharSequenceObjHashMap<>();
private final ObjList bottomUpColumnNames = new ObjList<>();
private final ObjList joinModels = new ObjList<>();
private final ObjList orderBy = new ObjList<>();
private final IntList orderByDirection = new IntList();
private final IntHashSet dependencies = new IntHashSet();
private final IntList orderedJoinModels1 = new IntList();
private final IntList orderedJoinModels2 = new IntList();
private final CharSequenceIntHashMap aliasIndexes = new CharSequenceIntHashMap();
private final ObjList expressionModels = new ObjList<>();
// collect frequency of column names from each join model
// and check if any of columns with frequency > 0 are selected
// column name frequency of 1 corresponds to map value 0
// column name frequency of 0 corresponds to map value -1
// list of "and" concatenated expressions
private final ObjList parsedWhere = new ObjList<>();
private final IntHashSet parsedWhereConsts = new IntHashSet();
private final ArrayDeque sqlNodeStack = new ArrayDeque<>();
private final CharSequenceIntHashMap orderHash = new CharSequenceIntHashMap(4, 0.5, -1);
private final ObjList joinColumns = new ObjList<>(4);
private final CharSequenceObjHashMap withClauses = new CharSequenceObjHashMap<>();
private final ObjList sampleByFill = new ObjList<>();
private final ObjList latestBy = new ObjList<>();
private final ObjList orderByAdvice = new ObjList<>();
private final IntList orderByDirectionAdvice = new IntList();
private ExpressionNode whereClause;
private ExpressionNode postJoinWhereClause;
private ExpressionNode constWhereClause;
private QueryModel nestedModel;
private ExpressionNode tableName;
private long tableVersion;
private Function tableNameFunction;
private ExpressionNode alias;
private ExpressionNode timestamp;
private ExpressionNode sampleBy;
private JoinContext context;
private ExpressionNode joinCriteria;
private int joinType;
private int joinKeywordPosition;
private IntList orderedJoinModels = orderedJoinModels2;
private ExpressionNode limitLo;
private ExpressionNode limitHi;
private int selectModelType = SELECT_MODEL_NONE;
private boolean nestedModelIsSubQuery = false;
private boolean distinct = false;
private QueryModel unionModel;
private int unionModelType;
private int modelPosition = 0;
private int orderByAdviceMnemonic;
private QueryModel() {
joinModels.add(this);
}
public boolean addAliasIndex(ExpressionNode node, int index) {
return aliasIndexes.put(node.token, index);
}
public void addBottomUpColumn(QueryColumn column) {
bottomUpColumns.add(column);
addField(column);
}
public void addDependency(int index) {
dependencies.add(index);
}
public void addExpressionModel(ExpressionNode node) {
assert node.queryModel != null;
expressionModels.add(node);
}
public void addField(QueryColumn column) {
final CharSequence alias = column.getAlias();
final ExpressionNode ast = column.getAst();
assert alias != null;
aliasToColumnNameMap.put(alias, ast.token);
columnNameToAliasMap.put(ast.token, alias);
bottomUpColumnNames.add(alias);
aliasToColumnMap.put(alias, column);
}
public void addJoinColumn(ExpressionNode node) {
joinColumns.add(node);
}
public void addJoinModel(QueryModel model) {
joinModels.add(model);
}
public void addLatestBy(ExpressionNode latestBy) {
this.latestBy.add(latestBy);
}
public void addOrderBy(ExpressionNode node, int direction) {
orderBy.add(node);
orderByDirection.add(direction);
}
public void addParsedWhereNode(ExpressionNode node) {
parsedWhere.add(node);
}
public void addSampleByFill(ExpressionNode sampleByFill) {
this.sampleByFill.add(sampleByFill);
}
public void addTopDownColumn(QueryColumn column, CharSequence alias) {
if (topDownNameSet.add(alias)) {
topDownColumns.add(column);
}
}
public void addWithClause(CharSequence name, WithClauseModel model) {
withClauses.put(name, model);
}
public void clear() {
bottomUpColumns.clear();
aliasToColumnNameMap.clear();
joinModels.clear();
joinModels.add(this);
clearSampleBy();
orderBy.clear();
orderByDirection.clear();
dependencies.clear();
parsedWhere.clear();
whereClause = null;
constWhereClause = null;
nestedModel = null;
tableName = null;
alias = null;
latestBy.clear();
joinCriteria = null;
joinType = JOIN_INNER;
joinKeywordPosition = 0;
orderedJoinModels1.clear();
orderedJoinModels2.clear();
parsedWhereConsts.clear();
aliasIndexes.clear();
postJoinWhereClause = null;
context = null;
orderedJoinModels = orderedJoinModels2;
limitHi = null;
limitLo = null;
timestamp = null;
sqlNodeStack.clear();
joinColumns.clear();
withClauses.clear();
selectModelType = SELECT_MODEL_NONE;
columnNameToAliasMap.clear();
tableNameFunction = null;
tableVersion = -1;
bottomUpColumnNames.clear();
expressionModels.clear();
distinct = false;
nestedModelIsSubQuery = false;
unionModel = null;
orderHash.clear();
modelPosition = 0;
topDownColumns.clear();
topDownNameSet.clear();
aliasToColumnMap.clear();
}
public void clearColumnMapStructs() {
this.aliasToColumnNameMap.clear();
this.bottomUpColumnNames.clear();
this.aliasToColumnMap.clear();
}
public void clearOrderBy() {
orderBy.clear();
orderByDirection.clear();
}
public void clearSampleBy() {
sampleBy = null;
sampleByFill.clear();
}
public void copyColumnsFrom(QueryModel other) {
clearColumnMapStructs();
this.aliasToColumnMap.putAll(other.aliasToColumnMap);
ObjList columnNames = other.bottomUpColumnNames;
this.bottomUpColumnNames.addAll(columnNames);
for (int i = 0, n = columnNames.size(); i < n; i++) {
final CharSequence name = columnNames.getQuick(i);
this.aliasToColumnNameMap.put(name, name);
}
}
public void copyOrderByAdvice(ObjList orderByAdvice) {
this.orderByAdvice.clear();
this.orderByAdvice.addAll(orderByAdvice);
}
public void copyOrderByDirectionAdvice(IntList orderByDirection) {
this.orderByDirectionAdvice.clear();
this.orderByDirectionAdvice.addAll(orderByDirection);
}
public ExpressionNode getAlias() {
return alias;
}
public void setAlias(ExpressionNode alias) {
this.alias = alias;
}
public int getAliasIndex(CharSequence column, int start, int end) {
int index = aliasIndexes.keyIndex(column, start, end);
if (index < 0) {
return aliasIndexes.valueAt(index);
}
return -1;
}
public CharSequenceObjHashMap getAliasToColumnMap() {
return aliasToColumnMap;
}
public CharSequenceObjHashMap getAliasToColumnNameMap() {
return aliasToColumnNameMap;
}
public ObjList getBottomUpColumnNames() {
return bottomUpColumnNames;
}
public CharSequenceObjHashMap getColumnNameToAliasMap() {
return columnNameToAliasMap;
}
public ObjList getBottomUpColumns() {
return bottomUpColumns;
}
public ExpressionNode getConstWhereClause() {
return constWhereClause;
}
public void setConstWhereClause(ExpressionNode constWhereClause) {
this.constWhereClause = constWhereClause;
}
public JoinContext getContext() {
return context;
}
public void setContext(JoinContext context) {
this.context = context;
}
public IntHashSet getDependencies() {
return dependencies;
}
public ObjList getExpressionModels() {
return expressionModels;
}
public ObjList getJoinColumns() {
return joinColumns;
}
public ExpressionNode getJoinCriteria() {
return joinCriteria;
}
public void setJoinCriteria(ExpressionNode joinCriteria) {
this.joinCriteria = joinCriteria;
}
public int getJoinKeywordPosition() {
return joinKeywordPosition;
}
public void setJoinKeywordPosition(int position) {
this.joinKeywordPosition = position;
}
public ObjList getJoinModels() {
return joinModels;
}
public int getJoinType() {
return joinType;
}
public void setJoinType(int joinType) {
this.joinType = joinType;
}
public ObjList getLatestBy() {
return latestBy;
}
public ExpressionNode getLimitHi() {
return limitHi;
}
public ExpressionNode getLimitLo() {
return limitLo;
}
public int getModelPosition() {
return modelPosition;
}
public void setModelPosition(int modelPosition) {
this.modelPosition = modelPosition;
}
@Override
public int getModelType() {
return ExecutionModel.QUERY;
}
public CharSequence getName() {
if (alias != null) {
return alias.token;
}
if (tableName != null) {
return tableName.token;
}
return null;
}
public QueryModel getNestedModel() {
return nestedModel;
}
public void setNestedModel(QueryModel nestedModel) {
this.nestedModel = nestedModel;
}
public ObjList getOrderBy() {
return orderBy;
}
public ObjList getOrderByAdvice() {
return orderByAdvice;
}
public int getOrderByAdviceMnemonic() {
return orderByAdviceMnemonic;
}
public void setOrderByAdviceMnemonic(int orderByAdviceMnemonic) {
this.orderByAdviceMnemonic = orderByAdviceMnemonic;
}
public IntList getOrderByDirection() {
return orderByDirection;
}
public IntList getOrderByDirectionAdvice() {
return orderByDirectionAdvice;
}
public CharSequenceIntHashMap getOrderHash() {
return orderHash;
}
public IntList getOrderedJoinModels() {
return orderedJoinModels;
}
public void setOrderedJoinModels(IntList that) {
assert that == orderedJoinModels1 || that == orderedJoinModels2;
this.orderedJoinModels = that;
}
public ObjList getParsedWhere() {
return parsedWhere;
}
public ExpressionNode getPostJoinWhereClause() {
return postJoinWhereClause;
}
public void setPostJoinWhereClause(ExpressionNode postJoinWhereClause) {
this.postJoinWhereClause = postJoinWhereClause;
}
public ExpressionNode getSampleBy() {
return sampleBy;
}
public void setSampleBy(ExpressionNode sampleBy) {
this.sampleBy = sampleBy;
}
public ObjList getSampleByFill() {
return sampleByFill;
}
public int getSelectModelType() {
return selectModelType;
}
public void setSelectModelType(int selectModelType) {
this.selectModelType = selectModelType;
}
public ExpressionNode getTableName() {
return tableName;
}
public void setTableName(ExpressionNode tableName) {
this.tableName = tableName;
}
public Function getTableNameFunction() {
return tableNameFunction;
}
public void setTableNameFunction(Function function) {
this.tableNameFunction = function;
}
public long getTableVersion() {
return tableVersion;
}
public void setTableVersion(long tableVersion) {
this.tableVersion = tableVersion;
}
public ExpressionNode getTimestamp() {
return timestamp;
}
public void setTimestamp(ExpressionNode timestamp) {
this.timestamp = timestamp;
}
public ObjList getTopDownColumns() {
return topDownColumns;
}
public ObjList getColumns() {
return topDownColumns.size() > 0 ? topDownColumns : bottomUpColumns;
}
public QueryModel getUnionModel() {
return unionModel;
}
public void setUnionModel(QueryModel unionModel) {
this.unionModel = unionModel;
}
public int getUnionModelType() {
return unionModelType;
}
public void setUnionModelType(int unionModelType) {
this.unionModelType = unionModelType;
}
public ExpressionNode getWhereClause() {
return whereClause;
}
public void setWhereClause(ExpressionNode whereClause) {
this.whereClause = whereClause;
}
public WithClauseModel getWithClause(CharSequence name) {
return withClauses.get(name);
}
public boolean isDistinct() {
return distinct;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isNestedModelIsSubQuery() {
return nestedModelIsSubQuery;
}
public void setNestedModelIsSubQuery(boolean nestedModelIsSubQuery) {
this.nestedModelIsSubQuery = nestedModelIsSubQuery;
}
public boolean isTopDownNameMissing(CharSequence columnName) {
return topDownNameSet.excludes(columnName);
}
public void moveLimitFrom(QueryModel baseModel) {
this.limitLo = baseModel.getLimitLo();
this.limitHi = baseModel.getLimitHi();
baseModel.setLimit(null, null);
}
public void moveSampleByFrom(QueryModel model) {
this.sampleBy = model.sampleBy;
this.sampleByFill.clear();
this.sampleByFill.addAll(model.sampleByFill);
// clear the source
model.clearSampleBy();
}
/**
* Optimiser may be attempting to order join clauses several times.
* Every time ordering takes place optimiser will keep at most two lists:
* one is last known order the other is new order. If new order cost is better
* optimiser will replace last known order with new one.
*
* To facilitate this behaviour the function will always return non-current list.
*
* @return non current order list.
*/
public IntList nextOrderedJoinModels() {
IntList ordered = orderedJoinModels == orderedJoinModels1 ? orderedJoinModels2 : orderedJoinModels1;
ordered.clear();
return ordered;
}
/*
* Splits "where" clauses into "and" chunks
*/
public ObjList parseWhereClause() {
ExpressionNode n = getWhereClause();
// pre-order traversal
sqlNodeStack.clear();
while (!sqlNodeStack.isEmpty() || n != null) {
if (n != null) {
if (isAndKeyword(n.token)) {
if (n.rhs != null) {
sqlNodeStack.push(n.rhs);
}
n = n.lhs;
} else {
addParsedWhereNode(n);
n = null;
}
} else {
n = sqlNodeStack.poll();
}
}
return getParsedWhere();
}
public void removeDependency(int index) {
dependencies.remove(index);
}
public void replaceJoinModel(int pos, QueryModel model) {
joinModels.setQuick(pos, model);
}
public void setLimit(ExpressionNode lo, ExpressionNode hi) {
this.limitLo = lo;
this.limitHi = hi;
}
@Override
public void toSink(CharSink sink) {
toSink0(sink, false);
}
@Override
public CharSequence translateAlias(CharSequence column) {
return aliasToColumnNameMap.get(column);
}
private static void aliasToSink(CharSequence alias, CharSink sink) {
sink.put(' ');
boolean quote = Chars.indexOf(alias, ' ') != -1;
if (quote) {
sink.put('\'').put(alias).put('\'');
} else {
sink.put(alias);
}
}
private String getSelectModelTypeText() {
return modelTypeName.get(selectModelType);
}
private void sinkColumns(CharSink sink, ObjList columns) {
for (int i = 0, n = columns.size(); i < n; i++) {
if (i > 0) {
sink.put(", ");
}
QueryColumn column = columns.getQuick(i);
CharSequence name = column.getName();
CharSequence alias = column.getAlias();
ExpressionNode ast = column.getAst();
if (column instanceof AnalyticColumn || name == null) {
ast.toSink(sink);
if (alias != null) {
aliasToSink(alias, sink);
}
// this can only be analytic column
if (name != null) {
AnalyticColumn ac = (AnalyticColumn) column;
sink.put(" over (");
final ObjList partitionBy = ac.getPartitionBy();
if (partitionBy.size() > 0) {
sink.put("partition by ");
for (int k = 0, z = partitionBy.size(); k < z; k++) {
if (k > 0) {
sink.put(", ");
}
partitionBy.getQuick(k).toSink(sink);
}
}
final ObjList orderBy = ac.getOrderBy();
if (orderBy.size() > 0) {
if (partitionBy.size() > 0) {
sink.put(' ');
}
sink.put("order by ");
for (int k = 0, z = orderBy.size(); k < z; k++) {
if (k > 0) {
sink.put(", ");
}
orderBy.getQuick(k).toSink(sink);
if (ac.getOrderByDirection().getQuick(k) == 1) {
sink.put(" desc");
}
}
}
sink.put(')');
}
} else {
ast.toSink(sink);
// do not repeat alias when it is the same as AST token, provided AST is a literal
if (alias != null && (ast.type != ExpressionNode.LITERAL || !ast.token.equals(alias))) {
aliasToSink(alias, sink);
}
}
}
}
private void toSink0(CharSink sink, boolean joinSlave) {
final boolean hasColumns = this.topDownColumns.size() > 0 || this.bottomUpColumns.size() > 0;
if (hasColumns) {
sink.put(getSelectModelTypeText());
if (this.topDownColumns.size() > 0) {
sink.put(' ');
sink.put('[');
sinkColumns(sink, this.topDownColumns);
sink.put(']');
}
if (this.bottomUpColumns.size() > 0) {
sink.put(' ');
sinkColumns(sink, this.bottomUpColumns);
}
sink.put(" from ");
}
if (tableName != null) {
tableName.toSink(sink);
} else {
sink.put('(');
nestedModel.toSink(sink);
sink.put(')');
}
if (alias != null) {
aliasToSink(alias.token, sink);
}
if (timestamp != null) {
sink.put(" timestamp (");
timestamp.toSink(sink);
sink.put(')');
}
if (getLatestBy().size() > 0) {
sink.put(" latest by ");
for (int i = 0, n = getLatestBy().size(); i < n; i++) {
getLatestBy().getQuick(i).toSink(sink);
}
}
if (orderedJoinModels.size() > 1) {
for (int i = 0, n = orderedJoinModels.size(); i < n; i++) {
QueryModel model = joinModels.getQuick(orderedJoinModels.getQuick(i));
if (model != this) {
switch (model.getJoinType()) {
case JOIN_OUTER:
sink.put(" outer join ");
break;
case JOIN_ASOF:
sink.put(" asof join ");
break;
case JOIN_SPLICE:
sink.put(" splice join ");
break;
case JOIN_CROSS:
sink.put(" cross join ");
break;
case JOIN_LT:
sink.put(" lt join ");
break;
default:
sink.put(" join ");
break;
}
if (model.getWhereClause() != null) {
sink.put('(');
model.toSink0(sink, true);
sink.put(')');
if (model.getAlias() != null) {
aliasToSink(model.getAlias().token, sink);
} else if (model.getTableName() != null) {
aliasToSink(model.getTableName().token, sink);
}
} else {
model.toSink0(sink, true);
}
JoinContext jc = model.getContext();
if (jc != null && jc.aIndexes.size() > 0) {
// join clause
sink.put(" on ");
for (int k = 0, z = jc.aIndexes.size(); k < z; k++) {
if (k > 0) {
sink.put(" and ");
}
jc.aNodes.getQuick(k).toSink(sink);
sink.put(" = ");
jc.bNodes.getQuick(k).toSink(sink);
}
}
if (model.getPostJoinWhereClause() != null) {
sink.put(" post-join-where ");
model.getPostJoinWhereClause().toSink(sink);
}
}
}
}
if (whereClause != null) {
sink.put(" where ");
whereClause.toSink(sink);
}
if (constWhereClause != null) {
sink.put(" const-where ");
constWhereClause.toSink(sink);
}
if (!joinSlave && postJoinWhereClause != null) {
sink.put(" post-join-where ");
postJoinWhereClause.toSink(sink);
}
if (sampleBy != null) {
sink.put(" sample by ");
sampleBy.toSink(sink);
final int fillCount = sampleByFill.size();
if (fillCount > 0) {
sink.put(" fill(");
sink.put(sampleByFill.getQuick(0));
if (fillCount > 1) {
for (int i = 1; i < fillCount; i++) {
sink.put(',');
sink.put(sampleByFill.getQuick(i));
}
}
sink.put(')');
}
}
if (orderHash.size() > 0 && orderBy.size() > 0) {
sink.put(" order by ");
ObjList columnNames = orderHash.keys();
for (int i = 0, n = columnNames.size(); i < n; i++) {
if (i > 0) {
sink.put(", ");
}
CharSequence key = columnNames.getQuick(i);
sink.put(key);
if (orderHash.get(key) == 1) {
sink.put(" desc");
}
}
}
if (getLimitLo() != null || getLimitHi() != null) {
sink.put(" limit ");
if (getLimitLo() != null) {
getLimitLo().toSink(sink);
}
if (getLimitHi() != null) {
sink.put(',');
getLimitHi().toSink(sink);
}
}
if (unionModel != null) {
sink.put(" union ");
if (unionModelType == QueryModel.UNION_MODEL_ALL) {
sink.put("all ");
}
unionModel.toSink0(sink, false);
}
}
public static final class QueryModelFactory implements ObjectFactory {
@Override
public QueryModel newInstance() {
return new QueryModel();
}
}
static {
modelTypeName.extendAndSet(SELECT_MODEL_NONE, "select");
modelTypeName.extendAndSet(SELECT_MODEL_CHOOSE, "select-choose");
modelTypeName.extendAndSet(SELECT_MODEL_VIRTUAL, "select-virtual");
modelTypeName.extendAndSet(SELECT_MODEL_ANALYTIC, "select-analytic");
modelTypeName.extendAndSet(SELECT_MODEL_GROUP_BY, "select-group-by");
modelTypeName.extendAndSet(SELECT_MODEL_DISTINCT, "select-distinct");
}
}