Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.parse;
import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.unescapeIdentifier;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FUNCTION;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FUNCTIONDI;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FUNCTIONSTAR;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.AbstractMap.SimpleEntry;
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.lib.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
/**
* Implementation of the parse information related to a query block.
*
**/
public class QBParseInfo {
private boolean isSubQ;
private String alias;
private ASTNode joinExpr;
private ASTNode hints;
private ASTNode colAliases;
private List hintList;
private final Map aliasToSrc;
/**
* insclause-0 -> TOK_TAB ASTNode
*/
private final Map nameToDest;
/**
* For 'insert into FOO(x,y) select ...' this stores the
* insclause-0 -> x,y mapping
*/
private final Map> nameToDestSchema;
private final Map nameToSample;
private final Map exprToColumnAlias;
private final Map destToSelExpr;
private final Map destToWhereExpr;
private final Map destToGroupby;
private final Set destRollups;
private final Set destCubes;
private final Set destGroupingSets;
private final Map destToHaving;
private final Map destToQualify;
private final Map destToOpType;
// insertIntoTables/insertOverwriteTables map a table's fullName to its ast;
private final Map insertIntoTables;
private final Map insertOverwriteTables;
private ASTNode queryFromExpr;
private boolean isAnalyzeCommand; // used for the analyze command (statistics)
private boolean isNoScanAnalyzeCommand; // used for the analyze command (statistics) (noscan)
private final Map tableSpecs; // used for statistics
private AnalyzeRewriteContext analyzeRewrite;
/**
* ClusterBy is a short name for both DistributeBy and SortBy.
*/
private final Map destToClusterby;
/**
* DistributeBy controls the hashcode of the row, which determines which
* reducer the rows will go to.
*/
private final Map destToDistributeby;
/**
* SortBy controls the reduce keys, which affects the order of rows that the
* reducer receives.
*/
private final Map destToSortby;
/**
* Maping from table/subquery aliases to all the associated lateral view nodes.
*/
private final Map> aliasToLateralViews;
private final Map destToLateralView;
/* Order by clause */
private final Map destToOrderby;
// Use SimpleEntry to save the offset and rowcount of limit clause
// KEY of SimpleEntry: offset
// VALUE of SimpleEntry: rowcount
private final Map> destToLimit;
private int outerQueryLimit;
// used by GroupBy
private final Map> destToAggregationExprs;
private final Map> destToDistinctFuncExprs;
// used by Windowing
private final Map> destToWindowingExprs;
// is the query insert overwrite directory
private boolean isInsertOverwriteDir = false;
@SuppressWarnings("unused")
private static final Logger LOG = LoggerFactory.getLogger(QBParseInfo.class.getName());
public QBParseInfo(String alias, boolean isSubQ) {
aliasToSrc = new HashMap();
nameToDest = new HashMap();
nameToDestSchema = new HashMap>();
nameToSample = new HashMap();
exprToColumnAlias = new HashMap();
destToLateralView = new HashMap();
destToSelExpr = new LinkedHashMap();
destToWhereExpr = new HashMap();
destToGroupby = new HashMap();
destToHaving = new HashMap();
destToQualify = new HashMap<>();
destToClusterby = new HashMap();
destToDistributeby = new HashMap();
destToSortby = new HashMap();
destToOrderby = new HashMap();
destToLimit = new HashMap>();
destToOpType = new HashMap<>();
insertIntoTables = new HashMap();
insertOverwriteTables = new HashMap();
destRollups = new HashSet();
destCubes = new HashSet();
destGroupingSets = new HashSet();
destToAggregationExprs = new LinkedHashMap>();
destToWindowingExprs = new LinkedHashMap>();
destToDistinctFuncExprs = new HashMap>();
this.alias = StringInternUtils.internIfNotNull(alias);
this.isSubQ = isSubQ;
outerQueryLimit = -1;
aliasToLateralViews = new HashMap>();
tableSpecs = new HashMap();
}
/*
* If a QB is such that the aggregation expressions need to be handled by
* the Windowing PTF; we invoke this function to clear the AggExprs on the dest.
*/
public void clearAggregationExprsForClause(String clause) {
destToAggregationExprs.get(clause).clear();
}
public void setAggregationExprsForClause(String clause, Map aggregationTrees) {
destToAggregationExprs.put(clause, aggregationTrees);
}
public void addAggregationExprsForClause(String clause, Map aggregationTrees) {
if (destToAggregationExprs.containsKey(clause)) {
destToAggregationExprs.get(clause).putAll(aggregationTrees);
} else {
destToAggregationExprs.put(clause, aggregationTrees);
}
}
public void addInsertIntoTable(String fullName, ASTNode ast) {
insertIntoTables.put(fullName.toLowerCase(), ast);
}
public void setDestToOpType(String clause, boolean value) {
destToOpType.put(clause, value);
}
public boolean isDestToOpTypeInsertOverwrite(String clause) {
if (destToOpType.containsKey(clause)) {
return destToOpType.get(clause);
} else {
return false;
}
}
/**
* See also {@link #getInsertOverwriteTables()}
*/
public boolean isInsertIntoTable(String dbName, String table, String branchName) {
String fullName = dbName + "." + table;
if (branchName != null) {
fullName += "." + branchName;
}
return insertIntoTables.containsKey(fullName.toLowerCase());
}
/**
* Check if a table is in the list to be inserted into
* See also {@link #getInsertOverwriteTables()}
* @param fullTableName table name in dbname.tablename format
* @return
*/
public boolean isInsertIntoTable(String fullTableName) {
return insertIntoTables.containsKey(fullTableName.toLowerCase());
}
public void setInsertOverwriteDirectory(boolean isInsertOverwriteDir) {
this.isInsertOverwriteDir = isInsertOverwriteDir;
}
public Map getAggregationExprsForClause(String clause) {
return destToAggregationExprs.get(clause);
}
public void addWindowingExprToClause(String clause, ASTNode windowingExprNode) {
Map windowingExprs = destToWindowingExprs.get(clause);
if ( windowingExprs == null ) {
windowingExprs = new LinkedHashMap();
destToWindowingExprs.put(clause, windowingExprs);
}
windowingExprs.put(windowingExprNode.toStringTree(), windowingExprNode);
}
public Map getWindowingExprsForClause(String clause) {
return destToWindowingExprs.get(clause);
}
public void clearDistinctFuncExprsForClause(String clause) {
List l = destToDistinctFuncExprs.get(clause);
if ( l != null ) {
l.clear();
}
}
public void setDistinctFuncExprsForClause(String clause, List ast) {
destToDistinctFuncExprs.put(clause, ast);
}
public List getDistinctFuncExprsForClause(String clause) {
return destToDistinctFuncExprs.get(clause);
}
public void setSelExprForClause(String clause, ASTNode ast) {
destToSelExpr.put(clause, ast);
}
public void setQueryFromExpr(ASTNode ast) {
queryFromExpr = ast;
}
public void setWhrExprForClause(String clause, ASTNode ast) {
destToWhereExpr.put(clause, ast);
}
public void setHavingExprForClause(String clause, ASTNode ast) {
destToHaving.put(clause, ast);
}
public void setGroupByExprForClause(String clause, ASTNode ast) {
destToGroupby.put(clause, ast);
}
public void setDestForClause(String clause, ASTNode ast) {
nameToDest.put(clause, ast);
}
List setDestSchemaForClause(String clause, List columnList) {
return nameToDestSchema.put(clause, columnList);
}
List getDestSchemaForClause(String clause) {
return nameToDestSchema.get(clause);
}
/**
* Set the Cluster By AST for the clause.
*
* @param clause
* the name of the clause
* @param ast
* the abstract syntax tree
*/
public void setClusterByExprForClause(String clause, ASTNode ast) {
destToClusterby.put(clause, ast);
}
/**
* Set the Distribute By AST for the clause.
*
* @param clause
* the name of the clause
* @param ast
* the abstract syntax tree
*/
public void setDistributeByExprForClause(String clause, ASTNode ast) {
destToDistributeby.put(clause, ast);
}
/**
* Set the Sort By AST for the clause.
*
* @param clause
* the name of the clause
* @param ast
* the abstract syntax tree
*/
public void setSortByExprForClause(String clause, ASTNode ast) {
destToSortby.put(clause, ast);
}
public void setOrderByExprForClause(String clause, ASTNode ast) {
destToOrderby.put(clause, ast);
}
public void setSrcForAlias(String alias, ASTNode ast) {
aliasToSrc.put(alias.toLowerCase(), ast);
}
public Set getClauseNames() {
return destToSelExpr.keySet();
}
public Set getClauseNamesForDest() {
return nameToDest.keySet();
}
public ASTNode getDestForClause(String clause) {
return nameToDest.get(clause);
}
public ASTNode getWhrForClause(String clause) {
return destToWhereExpr.get(clause);
}
public Map getDestToWhereExpr() {
return destToWhereExpr;
}
public ASTNode getGroupByForClause(String clause) {
return destToGroupby.get(clause);
}
public Set getDestRollups() {
return destRollups;
}
public Set getDestCubes() {
return destCubes;
}
public Set getDestGroupingSets() {
return destGroupingSets;
}
public Map getDestToGroupBy() {
return destToGroupby;
}
public ASTNode getHavingForClause(String clause) {
return destToHaving.get(clause);
}
public Map getDestToHaving() {
return destToHaving;
}
public ASTNode getSelForClause(String clause) {
return destToSelExpr.get(clause);
}
public ASTNode getQueryFrom() {
return queryFromExpr;
}
/**
* Get the Cluster By AST for the clause.
*
* @param clause
* the name of the clause
* @return the abstract syntax tree
*/
public ASTNode getClusterByForClause(String clause) {
return destToClusterby.get(clause);
}
public Map getDestToClusterBy() {
return destToClusterby;
}
/**
* Get the Distribute By AST for the clause.
*
* @param clause
* the name of the clause
* @return the abstract syntax tree
*/
public ASTNode getDistributeByForClause(String clause) {
return destToDistributeby.get(clause);
}
public Map getDestToDistributeBy() {
return destToDistributeby;
}
/**
* Get the Sort By AST for the clause.
*
* @param clause
* the name of the clause
* @return the abstract syntax tree
*/
public ASTNode getSortByForClause(String clause) {
return destToSortby.get(clause);
}
public ASTNode getOrderByForClause(String clause) {
return destToOrderby.get(clause);
}
public Map getDestToSortBy() {
return destToSortby;
}
public Map getDestToOrderBy() {
return destToOrderby;
}
public ASTNode getSrcForAlias(String alias) {
return aliasToSrc.get(alias.toLowerCase());
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
public boolean getIsSubQ() {
return isSubQ;
}
public void setIsSubQ(boolean isSubQ) {
this.isSubQ = isSubQ;
}
public ASTNode getJoinExpr() {
return joinExpr;
}
public void setJoinExpr(ASTNode joinExpr) {
this.joinExpr = joinExpr;
}
public TableSample getTabSample(String alias) {
return nameToSample.get(alias.toLowerCase());
}
public void setTabSample(String alias, TableSample tableSample) {
nameToSample.put(alias.toLowerCase(), tableSample);
}
public String getExprToColumnAlias(ASTNode expr) {
return exprToColumnAlias.get(expr);
}
public Map getAllExprToColumnAlias() {
return exprToColumnAlias;
}
public boolean hasExprToColumnAlias(ASTNode expr) {
return exprToColumnAlias.containsKey(expr);
}
public void setExprToColumnAlias(ASTNode expr, String alias) {
exprToColumnAlias.put(expr, StringInternUtils.internIfNotNull(alias));
}
public void setDestLimit(String dest, Integer offset, Integer limit) {
destToLimit.put(dest, new SimpleEntry<>(offset, limit));
}
public Integer getDestLimit(String dest) {
return destToLimit.get(dest) == null ? null : destToLimit.get(dest).getValue();
}
public Integer getDestLimitOffset(String dest) {
return destToLimit.get(dest) == null ? 0 : destToLimit.get(dest).getKey();
}
/**
* @return the outerQueryLimit
*/
public int getOuterQueryLimit() {
return outerQueryLimit;
}
/**
* @param outerQueryLimit
* the outerQueryLimit to set
*/
public void setOuterQueryLimit(int outerQueryLimit) {
this.outerQueryLimit = outerQueryLimit;
}
public boolean isTopLevelSimpleSelectStarQuery() {
if (alias != null || destToSelExpr.size() != 1 || !isSimpleSelectQuery()) {
return false;
}
for (ASTNode selExprs : destToSelExpr.values()) {
if (selExprs.getChildCount() != 1) {
return false;
}
Tree sel = selExprs.getChild(0).getChild(0);
if (sel == null || sel.getType() != HiveParser.TOK_ALLCOLREF) {
return false;
}
}
return true;
}
// for fast check of possible existence of RS (will be checked again in SimpleFetchOptimizer)
public boolean isSimpleSelectQuery() {
if (joinExpr != null || !destToOrderby.isEmpty() || !destToSortby.isEmpty()
|| !destToGroupby.isEmpty() || !destToClusterby.isEmpty() || !destToDistributeby.isEmpty()
|| !destRollups.isEmpty() || !destCubes.isEmpty() || !destGroupingSets.isEmpty()
|| !destToHaving.isEmpty()) {
return false;
}
for (Map entry : destToAggregationExprs.values()) {
if (entry != null && !entry.isEmpty()) {
return false;
}
}
for (Map entry : destToWindowingExprs.values()) {
if (entry != null && !entry.isEmpty()) {
return false;
}
}
for (List ct : destToDistinctFuncExprs.values()) {
if (!ct.isEmpty()) {
return false;
}
}
// exclude insert queries
for (ASTNode v : nameToDest.values()) {
if (!(v.getChild(0).getType() == HiveParser.TOK_TMP_FILE)) {
return false;
}
}
return true;
}
public void setHints(ASTNode hint) {
hints = hint;
}
public void setHintList(List hintList) {
this.hintList = hintList;
}
public List getHintList() {
return hintList;
}
public ASTNode getHints() {
return hints;
}
public Map> getAliasToLateralViews() {
return aliasToLateralViews;
}
public List getLateralViewsForAlias(String alias) {
return aliasToLateralViews.get(alias.toLowerCase());
}
public void addLateralViewForAlias(String alias, ASTNode lateralView) {
List lateralViews = aliasToLateralViews.get(alias);
if (lateralViews == null) {
lateralViews = new ArrayList();
aliasToLateralViews.put(alias, lateralViews);
}
lateralViews.add(lateralView);
}
public void setIsAnalyzeCommand(boolean isAnalyzeCommand) {
this.isAnalyzeCommand = isAnalyzeCommand;
}
public boolean isAnalyzeCommand() {
return isAnalyzeCommand;
}
public void addTableSpec(String tName, TableSpec tSpec) {
tableSpecs.put(tName, tSpec);
}
public TableSpec getTableSpec(String tName) {
return tableSpecs.get(tName);
}
/**
* This method is used only for the analyze command to get the partition specs
*/
public TableSpec getTableSpec() {
Iterator tName = tableSpecs.keySet().iterator();
return tableSpecs.get(tName.next());
}
public Map> getDestToLimit() {
return destToLimit;
}
public Map> getDestToAggregationExprs() {
return destToAggregationExprs;
}
public Map> getDestToDistinctFuncExprs() {
return destToDistinctFuncExprs;
}
public Map getNameToSample() {
return nameToSample;
}
public Map getDestToLateralView() {
return destToLateralView;
}
public void setQualifyExprForClause(String dest, ASTNode ast) {
destToQualify.put(dest, ast);
}
public ASTNode getQualifyExprForClause(String dest) {
return destToQualify.get(dest);
}
public boolean hasQualifyClause() {
return !destToQualify.isEmpty();
}
protected static enum ClauseType {
CLUSTER_BY_CLAUSE,
DISTRIBUTE_BY_CLAUSE,
ORDER_BY_CLAUSE,
SORT_BY_CLAUSE
}
public AnalyzeRewriteContext getAnalyzeRewrite() {
return analyzeRewrite;
}
public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) {
this.analyzeRewrite = analyzeRewrite;
}
/**
* @return the isNoScanAnalyzeCommand
*/
public boolean isNoScanAnalyzeCommand() {
return isNoScanAnalyzeCommand;
}
/**
* @param isNoScanAnalyzeCommand the isNoScanAnalyzeCommand to set
*/
public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) {
this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand;
}
/**
* See also {@link #isInsertIntoTable(String)}
*/
public Map getInsertOverwriteTables() {
return insertOverwriteTables;
}
public boolean hasInsertTables() {
return this.insertIntoTables.size() > 0 || this.insertOverwriteTables.size() > 0;
}
public boolean isInsertOverwriteDirectory() {
return isInsertOverwriteDir;
}
/**
* Check whether all the expressions in the select clause are aggregate function calls.
* This method starts iterating through the AST nodes representing the expressions in the select clause stored in
* this object. An expression is considered to be an aggregate function call if:
*
*
the AST node type is either TOK_FUNCTION, TOK_FUNCTIONDI or TOK_FUNCTIONSTAR
*
the first child of the node is the name of the function
*
function is registered in Hive
*
the registered function with the specified name is a Generic User Defined Aggregate
*
* If any of the mentioned criteria fails to match to the current expression this function returns false.
* @return true if all the expressions in the select clause are aggregate function calls.
* @throws SemanticException - thrown when {@link FunctionRegistry#getFunctionInfo} fails.
*/
public boolean isFullyAggregate() throws SemanticException {
for (ASTNode selectClause : destToSelExpr.values()) {
for (Node node : selectClause.getChildren()) {
ASTNode selexpr = (ASTNode) node;
Tree expressionTypeToken = selexpr.getChild(0);
int selectExprType = expressionTypeToken.getType();
if (selectExprType != TOK_FUNCTION && selectExprType != TOK_FUNCTIONDI && selectExprType != TOK_FUNCTIONSTAR) {
return false;
}
if (expressionTypeToken.getChild(0).getType() != HiveParser.Identifier) {
return false;
}
String functionName = unescapeIdentifier(expressionTypeToken.getChild(0).getText());
FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo(functionName);
if (functionInfo == null) {
return false;
}
if (functionInfo.getGenericUDAFResolver() == null) {
return false;
}
}
}
return true;
}
public ASTNode getColAliases() {
return colAliases;
}
public void setColAliases(ASTNode colAliases) {
this.colAliases = colAliases;
}
}
