org.apache.lens.cube.parse.ExpressionResolver Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lens.cube.parse;
import static java.util.stream.Collectors.toSet;
import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
import java.util.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.antlr.runtime.CommonToken;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
/**
* Replaces expression with its AST in all query ASTs
*/
@Slf4j
class ExpressionResolver implements ContextRewriter {
static class ExpressionContext {
@Getter
private final ExprColumn exprCol;
@Getter
private final AbstractBaseTable srcTable;
@Getter
private final String srcAlias;
@Getter
private Set allExprs = new LinkedHashSet();
private Set directlyAvailableIn = new HashSet();
@Getter
private Map> evaluableExpressions = new HashMap<>();
private boolean hasMeasures = false;
boolean hasMeasures() {
return hasMeasures;
}
ExpressionContext(CubeQueryContext cubeql, ExprColumn exprCol, AbstractBaseTable srcTable, String srcAlias)
throws LensException {
this.srcTable = srcTable;
this.exprCol = exprCol;
this.srcAlias = srcAlias;
for (ExprSpec es : exprCol.getExpressionSpecs()) {
allExprs.add(new ExprSpecContext(es, cubeql));
}
resolveColumnsAndAlias(cubeql);
log.debug("All exprs for {} are {}", exprCol.getName(), allExprs);
}
private void resolveColumnsAndAlias(CubeQueryContext cubeql) throws LensException {
for (ExprSpecContext esc : allExprs) {
esc.resolveColumns(cubeql);
esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
cubeql.addOptionalDimTable(table, null, false, null, false,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
}
}
resolveColumnsAndReplaceAlias(cubeql, allExprs);
}
private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set exprs)
throws LensException {
Set nestedExpressions = new LinkedHashSet();
for (ExprSpecContext esc : exprs) {
for (Map.Entry> entry : esc.getTblAliasToColumns().entrySet()) {
if (entry.getKey().equals(CubeQueryContext.DEFAULT_TABLE)) {
continue;
}
AbstractBaseTable baseTable = (AbstractBaseTable) cubeql.getCubeTableForAlias(entry.getKey());
Set exprCols = new HashSet();
for (String col : entry.getValue()) {
// col is an expression
if (baseTable.getExpressionNames().contains(col)) {
exprCols.add(col);
}
}
// get all combinations of expression replaced with inner exprs AST.
addAllNestedExpressions(cubeql, esc, baseTable, nestedExpressions, exprCols);
}
}
for (ExprSpecContext esc : nestedExpressions) {
esc.resolveColumns(cubeql);
esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
cubeql.addOptionalDimTable(table, null, false, null, false,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
}
}
exprs.addAll(nestedExpressions);
}
private void addAllNestedExpressions(CubeQueryContext cubeql, ExprSpecContext baseEsc, AbstractBaseTable baseTable,
Set nestedExpressions, Set exprCols) throws LensException {
for (String col : exprCols) {
Set replacedExpressions = new LinkedHashSet();
for (ExprSpec es : baseTable.getExpressionByName(col).getExpressionSpecs()) {
ASTNode finalAST = MetastoreUtil.copyAST(baseEsc.getFinalAST());
replaceColumnInAST(finalAST, col, es.copyASTNode());
ExprSpecContext replacedESC = new ExprSpecContext(baseEsc, es, finalAST, cubeql);
nestedExpressions.add(replacedESC);
replacedExpressions.add(replacedESC);
}
Set remaining = new LinkedHashSet(exprCols);
remaining.remove(col);
for (ExprSpecContext replacedESC : replacedExpressions) {
addAllNestedExpressions(cubeql, replacedESC, baseTable, nestedExpressions, remaining);
}
}
}
void addDirectlyAvailable(CandidateTable cTable) {
log.debug("Directly available in {}", cTable);
directlyAvailableIn.add(cTable);
}
void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws LensException {
// add optional dimensions involved in expressions
for (String table : esc.getTblAliasToColumns().keySet()) {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
cubeql.addOptionalExprDimTable(table, exprCol.getName(), srcAlias, cTable,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
}
evaluableExpressions.computeIfAbsent(cTable, k -> new LinkedHashSet<>()).add(esc);
}
Set getAllASTNodes() {
Set allAST = new HashSet();
for (ExprSpecContext esc : allExprs) {
allAST.add(esc.finalAST);
}
return allAST;
}
boolean hasAggregates() {
for (ExprSpecContext esc : allExprs) {
if (HQLParser.hasAggregate(esc.finalAST)) {
return true;
}
}
return false;
}
boolean isEvaluable(CandidateTable cTable) {
return directlyAvailableIn.contains(cTable)
|| (evaluableExpressions.get(cTable) != null && !evaluableExpressions.get(cTable).isEmpty());
}
}
static class ExprSpecContext extends TracksQueriedColumns implements TrackDenormContext {
private Set exprSpecs = new LinkedHashSet<>();
@Getter
@Setter
private ASTNode finalAST;
@Getter
private Set exprDims = new HashSet<>();
@Getter
@Setter
private DenormalizationResolver.DenormalizationContext deNormCtx;
ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException {
// replaces table names in expression with aliases in the query
finalAST = replaceAlias(exprSpec.copyASTNode(), cubeql);
exprSpecs.add(exprSpec);
}
ExprSpecContext(ExprSpecContext nested, ExprSpec current, ASTNode node,
CubeQueryContext cubeql) throws LensException {
exprSpecs.addAll(nested.exprSpecs);
exprSpecs.add(current);
finalAST = replaceAlias(node, cubeql);
}
void replaceAliasInAST(CubeQueryContext cubeql)
throws LensException {
AliasReplacer.extractTabAliasForCol(cubeql, this);
finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias());
}
void resolveColumns(CubeQueryContext cubeql) throws LensException {
// finds all columns and table aliases in the expression
ColumnResolver.getColsForTree(cubeql, finalAST, this, false);
}
Date getStartTime() {
Set startTimes = new HashSet<>();
for (ExprSpec es : exprSpecs) {
if (es.getStartTime() != null) {
startTimes.add(es.getStartTime());
}
}
if (!startTimes.isEmpty()) {
return Collections.max(startTimes);
}
return null;
}
Date getEndTime() {
Set endTimes = new HashSet<>();
for (ExprSpec es : exprSpecs) {
if (es.getEndTime() != null) {
endTimes.add(es.getEndTime());
}
}
if (!endTimes.isEmpty()) {
return Collections.min(endTimes);
}
return null;
}
boolean isValidInTimeRange(final TimeRange range) {
return isValidFrom(range.getFromDate()) && isValidTill(range.getToDate());
}
boolean isValidFrom(@NonNull final Date date) {
return (getStartTime() == null) || (date.equals(getStartTime()) || date.after(getStartTime()));
}
boolean isValidTill(@NonNull final Date date) {
return (getEndTime() == null) || (date.equals(getEndTime()) || date.before(getEndTime()));
}
public String toString() {
return HQLParser.getString(finalAST);
}
}
@RequiredArgsConstructor
@ToString
static class PickedExpression {
private final String srcAlias;
private final ExprSpecContext pickedCtx;
private transient ASTNode reWrittenAST = null;
/*
Initialized rewrittenAST as copy of final AST if boolean is passed. Copy would be required if finalAST gets
modified because of denormalization context.
Otherwise, it is final AST reference, without any copy.
*/
void initRewrittenAST(boolean copyFinal) {
if (copyFinal) {
reWrittenAST = MetastoreUtil.copyAST(pickedCtx.getFinalAST());
} else {
reWrittenAST = pickedCtx.getFinalAST();
}
}
ASTNode getRewrittenAST() {
return reWrittenAST;
}
}
static class ExpressionResolverContext {
@Getter
private Map> allExprsQueried = new HashMap<>();
private Map> pickedExpressions = new HashMap<>();
@Getter
private Map>> pickedExpressionsPerCandidate = new HashMap<>();
private Map nonPickedExpressionsForCandidate = new HashMap();
private final CubeQueryContext cubeql;
private boolean replacedHavingExpressions = false;
ExpressionResolverContext(CubeQueryContext cubeql) {
this.cubeql = cubeql;
}
void addExpressionQueried(ExpressionContext expr) {
allExprsQueried.computeIfAbsent(expr.getExprCol().getName().toLowerCase(), k -> new LinkedHashSet<>()).add(expr);
}
boolean isQueriedExpression(String column) {
return allExprsQueried.containsKey(column);
}
boolean hasAggregates() {
for (Set ecSet : allExprsQueried.values()) {
for (ExpressionContext ec : ecSet) {
if (ec.hasAggregates()) {
return true;
}
}
}
return false;
}
ExpressionContext getExpressionContext(String expr, String alias) {
if (allExprsQueried.get(expr) != null) {
for (ExpressionContext ec : allExprsQueried.get(expr)) {
if (ec.getSrcAlias().equals(alias)) {
return ec;
}
}
}
return null;
}
boolean hasMeasures(String expr, CubeInterface cube) {
String alias = cubeql.getAliasForTableName(cube.getName());
ExpressionContext ec = getExpressionContext(expr, alias);
boolean hasMeasures = false;
for (ExprSpecContext esc : ec.allExprs) {
if (esc.getTblAliasToColumns().get(alias) != null) {
for (String cubeCol : esc.getTblAliasToColumns().get(alias)) {
if (cube.getMeasureByName(cubeCol) != null) {
hasMeasures = true;
break;
}
}
}
}
ec.hasMeasures = hasMeasures;
return hasMeasures;
}
//updates all expression specs which are evaluable
void updateEvaluables(String expr, CandidateTable cTable)
throws LensException {
String alias = cubeql.getAliasForTableName(cTable.getBaseTable().getName());
ExpressionContext ec = getExpressionContext(expr, alias);
if (cTable.getColumns().contains(expr)) {
// expression is directly materialized in candidate table
log.debug("{} is directly evaluable in {}", expr, cTable);
ec.addDirectlyAvailable(cTable);
return;
}
for (ExprSpecContext esc : ec.allExprs) {
if (esc.getTblAliasToColumns().get(alias) == null) {
log.debug("{} = {} is evaluable in {}", expr, esc, cTable);
ec.addEvaluable(cubeql, cTable, esc);
} else {
Set columns = esc.getTblAliasToColumns().get(alias);
boolean isEvaluable = true;
for (String col : columns) {
if (!cTable.getColumns().contains(col.toLowerCase())) {
if (!esc.getDeNormCtx().addRefUsage(cubeql, cTable, col, cTable.getBaseTable().getName())) {
// check if it is available as reference, if not expression is not evaluable
log.debug("{} = {} is not evaluable in {}", expr, esc, cTable);
isEvaluable = false;
break;
}
}
}
if (isEvaluable) {
log.debug("{} = {} is evaluable in {}", expr, esc, cTable);
ec.addEvaluable(cubeql, cTable, esc);
}
}
}
}
// checks if expr is evaluable
boolean isEvaluable(String expr, CandidateTable cTable) {
ExpressionContext ec = getExpressionContext(expr, cubeql.getAliasForTableName(cTable.getBaseTable().getName()));
return ec.isEvaluable(cTable);
}
Set rewriteExprCtx(CubeQueryContext cubeql, DimHQLContext sc, Map dimsToQuery)
throws LensException {
Set exprDims = new HashSet();
log.info("Picking expressions for candidate {} ", sc);
if (!allExprsQueried.isEmpty()) {
// pick expressions for fact
if (sc.getStorageCandidate() != null) {
pickExpressionsForTable(sc.getStorageCandidate());
}
// pick expressions for dimensions
if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
for (CandidateDim cdim : dimsToQuery.values()) {
pickExpressionsForTable(cdim);
}
}
log.debug("Picked expressions: {}", pickedExpressions);
for (Set peSet : pickedExpressions.values()) {
for (PickedExpression pe : peSet) {
exprDims.addAll(pe.pickedCtx.exprDims);
pe.initRewrittenAST(pe.pickedCtx.deNormCtx.hasReferences());
exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql,
sc.getStorageCandidate(), dimsToQuery, pe.getRewrittenAST()));
}
}
// Replace picked expressions in all the base trees
replacePickedExpressions(sc);
}
pickedExpressionsPerCandidate.put(sc, Maps.newHashMap(pickedExpressions));
pickedExpressions.clear();
nonPickedExpressionsForCandidate.clear();
return exprDims;
}
private void replacePickedExpressions(DimHQLContext sc)
throws LensException {
QueryAST queryAST = sc.getQueryAst();
replaceAST(cubeql, queryAST.getSelectAST());
if (sc.getStorageCandidate() != null) {
replaceAST(cubeql, sc.getQueryAst().getWhereAST());
} else {
replaceAST(cubeql, queryAST.getWhereAST());
}
replaceAST(cubeql, queryAST.getJoinAST());
replaceAST(cubeql, queryAST.getGroupByAST());
// Resolve having expression for StorageCandidate
replaceAST(cubeql, queryAST.getOrderByAST());
}
public void replaceHavingExpressions() throws LensException {
replaceHavingExpressions(pickedExpressionsPerCandidate);
}
public void replaceHavingExpressions(
Map>> pickedExpressionsPerCandidate) throws LensException {
if (cubeql.getHavingAST() != null && !replacedHavingExpressions) {
HQLParser.bft(cubeql.getHavingAST(), visited -> {
ASTNode node1 = visited.getNode();
int childcount = node1.getChildCount();
for (int i = 0; i < childcount; i++) {
ASTNode current = (ASTNode) node1.getChild(i);
if (current.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
// For example 'select fact.id, dim2.id ...'
// Right child is the column name, left child.ident is table name
ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) current.getChild(1);
String column = colIdent.getText().toLowerCase();
Optional exprOptional = pickedExpressionsPerCandidate.values().stream()
.filter(x -> x.containsKey(column)).map(x -> x.get(column)).flatMap(Collection::stream)
.filter(x -> x.srcAlias.equals(tabident.getText().toLowerCase())).findFirst();
if (exprOptional.isPresent()) {
PickedExpression expr = exprOptional.get();
node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql));
}
}
}
});
replacedHavingExpressions = true;
}
}
private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws LensException {
if (node == null) {
return;
}
// Traverse the tree and resolve expression columns
HQLParser.bft(node, visited -> {
ASTNode node1 = visited.getNode();
int childcount = node1.getChildCount();
for (int i = 0; i < childcount; i++) {
ASTNode current = (ASTNode) node1.getChild(i);
if (current.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
// For example 'select fact.id, dim2.id ...'
// Right child is the column name, left child.ident is table name
ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) current.getChild(1);
String column = colIdent.getText().toLowerCase();
if (pickedExpressions.containsKey(column)) {
assert tabident != null;
PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase());
if (expr != null) {
node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql));
}
} else if (nonPickedExpressionsForCandidate.containsKey(column)) {
node1.setChild(i, nonPickedExpressionsForCandidate.get(column));
}
}
}
});
}
private PickedExpression getPickedExpression(String column, String alias) {
Set peSet = pickedExpressions.get(column);
if (peSet != null && !peSet.isEmpty()) {
for (PickedExpression picked : peSet) {
if (picked.srcAlias.equals(alias)) {
return picked;
}
}
}
return null;
}
private void pickExpressionsForTable(CandidateTable cTable) throws LensException {
for (Map.Entry> ecEntry : allExprsQueried.entrySet()) {
Set ecSet = ecEntry.getValue();
for (ExpressionContext ec : ecSet) {
if (ec.getSrcTable().getName().equals(cTable.getBaseTable().getName())) {
if (!ec.directlyAvailableIn.contains(cTable)) {
log.debug("{} is not directly evaluable in {}", ec, cTable);
if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) {
// pick first evaluable expression
pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new HashSet<>())
.add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next()));
} else {
nonPickedExpressionsForCandidate.put(ecEntry.getKey(), getDefaultExpr(getExprAst(ec)));
}
}
}
}
}
}
private ASTNode getExprAst(ExpressionContext expressionContext) {
Set scSet = CandidateUtil.getStorageCandidates(cubeql.getCandidates());
Set storageTableNames = new HashSet<>();
Set expressionContexts = Sets.newLinkedHashSet();
expressionContexts.add(expressionContext);
for (StorageCandidate sc : scSet) {
storageTableNames.add(sc.getStorageTable());
ExpressionContext ex = sc.getCubeQueryContext().getExprCtx().getExpressionContext(expressionContext.
getExprCol().getName(), expressionContext.getSrcAlias());
if (ex != null) {
expressionContexts.add(ex);
}
}
for (ExpressionContext ec : expressionContexts) {
for (CandidateTable table : ec.evaluableExpressions.keySet()) {
if (storageTableNames.contains(table.getStorageTable())) {
return MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
}
}
}
return null;
}
private ASTNode getDefaultExpr(ASTNode node) {
if (HQLParser.isAggregateAST(node)) {
node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier, "0.0")));
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
getDefaultExpr(child);
}
return node;
}
void pruneExpressions() {
for (Set ecSet : allExprsQueried.values()) {
for (ExpressionContext ec : ecSet) {
Set removedEsc = new HashSet();
for(Iterator iterator = ec.getAllExprs().iterator(); iterator.hasNext();) {
ExprSpecContext esc = iterator.next();
boolean removed = false;
// Go over expression dims and remove expression involving dimensions for which candidate tables are
// not there
for (Dimension exprDim : esc.exprDims) {
if (cubeql.getCandidateDims().get(exprDim) == null || cubeql.getCandidateDims().get(exprDim).isEmpty()) {
log.info("Removing expression {} as {} it does not have any candidate tables", esc, exprDim);
iterator.remove();
removedEsc.add(esc);
removed = true;
break;
}
}
if (removed) {
continue;
}
// Remove expressions for which denormalized columns are no more reachable
esc.getDeNormCtx().pruneReferences(cubeql);
if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
&& esc.getDeNormCtx().getTableToRefCols().keySet().containsAll(
ec.getEvaluableExpressions().keySet().stream().map(Named::getName).collect(toSet()))
&& esc.getDeNormCtx().getTableToRefCols().keySet().stream()
.map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty)) {
log.info("Removing expression {} as all tables have non reachable fields", esc);
iterator.remove();
removedEsc.add(esc);
removed = true;
}
if (removed) {
continue;
}
//remove expressions which are not valid in the timerange queried
// If an expression is defined as
// ex = a + b // from t1 to t2;
// ex = c + d // from t2 to t3
// With range queried, invalid expressions will be removed
// If range is including more than one expression, queries can be unioned as an improvement at later time.
// But for now, they are not eligible expressions
for (TimeRange range : cubeql.getTimeRanges()) {
if (!esc.isValidInTimeRange(range)) {
log.info("Removing expression {} as it is not valid in timerange queried", esc);
iterator.remove();
removedEsc.add(esc);
removed = true;
break;
}
}
if (removed) {
continue;
}
// Go over expressions and remove expression containing unavailable columns in timerange
// In the example above,
// if ex = a +b ; and a is not available in timerange queried, it will be removed.
for (TimeRange range : cubeql.getTimeRanges()) {
boolean toRemove = false;
for (Map.Entry> entry : esc.getTblAliasToColumns().entrySet()) {
if (CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(entry.getKey())) {
continue;
}
AbstractBaseTable baseTable = (AbstractBaseTable) cubeql.getCubeTableForAlias(entry.getKey());
for (String col : entry.getValue()) {
if (!baseTable.getColumnByName(col).isColumnAvailableInTimeRange(range)) {
toRemove = true;
break;
}
}
if (toRemove) {
break;
}
}
if (toRemove) {
log.info("Removing expression {} as its columns are unavailable in timerange queried", esc);
iterator.remove();
removedEsc.add(esc);
removed = true;
break;
}
}
}
for (Set evalSet : ec.evaluableExpressions.values()) {
evalSet.removeAll(removedEsc);
}
}
}
}
}
@Override
public void rewriteContext(CubeQueryContext cubeql) throws LensException {
ExpressionResolverContext exprCtx = cubeql.getExprCtx();
if (exprCtx == null) {
exprCtx = new ExpressionResolverContext(cubeql);
cubeql.setExprCtx(exprCtx);
for (Map.Entry> entry : cubeql.getTblAliasToColumns().entrySet()) {
String alias = entry.getKey();
// skip default alias
if (Objects.equals(alias, CubeQueryContext.DEFAULT_TABLE)) {
continue;
}
AbstractCubeTable tbl = cubeql.getCubeTableForAlias(alias);
Set columns = entry.getValue();
for (String column : columns) {
CubeColumn col;
if (tbl instanceof CubeInterface) {
col = ((CubeInterface) tbl).getColumnByName(column);
} else {
col = ((Dimension) tbl).getColumnByName(column);
}
if (col instanceof ExprColumn) {
exprCtx.addExpressionQueried(new ExpressionContext(cubeql, (ExprColumn)col, (AbstractBaseTable)tbl, alias));
}
}
}
Set exprsWithMeasures = new HashSet();
for (String expr : cubeql.getQueriedExprs()) {
if (cubeql.getExprCtx().hasMeasures(expr, cubeql.getCube())) {
// expression has measures
exprsWithMeasures.add(expr);
}
}
cubeql.addQueriedExprsWithMeasures(exprsWithMeasures);
} else {
// prune invalid expressions
cubeql.getExprCtx().pruneExpressions();
// prune candidate facts without any valid expressions
if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) {
for (Map.Entry> ecEntry : exprCtx.allExprsQueried.entrySet()) {
String expr = ecEntry.getKey();
Set ecSet = ecEntry.getValue();
cubeql.getCandidates().removeIf(x-> {
if (ecSet.stream().noneMatch(x::isExpressionEvaluable)) {
log.info("Not considering Candidate :{} as {} is not evaluable", x, expr);
cubeql.addCandidatePruningMsg(x,
CandidateTablePruneCause.expressionNotEvaluable(expr));
return true;
}
return false;
});
}
}
// prune candidate dims without any valid expressions
if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) {
for (Dimension dim : cubeql.getDimensions()) {
for (Iterator i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) {
CandidateDim cdim = i.next();
for (Map.Entry> ecEntry : exprCtx.allExprsQueried.entrySet()) {
Set ecSet = ecEntry.getValue();
for (ExpressionContext ec : ecSet) {
if (ec.getSrcTable().getName().equals(cdim.getBaseTable().getName())) {
if (!ec.isEvaluable(cdim)) {
log.info("Not considering dim table:{} as {} is not evaluable", cdim, ec.exprCol.getName());
cubeql.addDimPruningMsgs(dim, cdim.dimtable,
CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
i.remove();
}
}
}
}
}
}
}
}
}
private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws LensException {
final ASTNode finalAST = MetastoreUtil.copyAST(expr);
HQLParser.bft(finalAST, visited -> {
ASTNode node = visited.getNode();
ASTNode parent = null;
if (visited.getParent() != null) {
parent = visited.getParent().getNode();
}
if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() == DOT)) {
ASTNode current = (ASTNode) node.getChild(0);
if (current.getToken().getType() == Identifier) {
String tableName = current.getToken().getText().toLowerCase();
String alias = cubeql.getAliasForTableName(tableName);
if (!alias.equalsIgnoreCase(tableName)) {
node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias)));
}
}
}
});
return finalAST;
}
private static void replaceColumnInAST(ASTNode expr, final String toReplace, final ASTNode columnAST)
throws LensException {
if (expr == null) {
return;
}
// Traverse the tree and resolve expression columns
HQLParser.bft(expr, visited -> {
ASTNode node = visited.getNode();
int childcount = node.getChildCount();
for (int i = 0; i < childcount; i++) {
ASTNode current = (ASTNode) node.getChild(i);
if (current.getToken().getType() == TOK_TABLE_OR_COL && node.getToken().getType() != DOT) {
// Take child ident.totext
ASTNode ident = (ASTNode) current.getChild(0);
String column = ident.getText().toLowerCase();
if (toReplace.equals(column)) {
node.setChild(i, MetastoreUtil.copyAST(columnAST));
}
} else if (current.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
// For example 'select fact.id, dim2.id ...'
// Right child is the column name, left child.ident is table name
ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) current.getChild(1);
String column = colIdent.getText().toLowerCase();
if (toReplace.equals(column)) {
node.setChild(i, MetastoreUtil.copyAST(columnAST));
}
}
}
});
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy