Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.asterix.translator.AqlPlusExpressionToPlanTranslator Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.asterix.translator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.asterix.aql.base.Clause;
import org.apache.asterix.aql.base.Expression;
import org.apache.asterix.aql.base.Expression.Kind;
import org.apache.asterix.aql.expression.CallExpr;
import org.apache.asterix.aql.expression.CompactStatement;
import org.apache.asterix.aql.expression.ConnectFeedStatement;
import org.apache.asterix.aql.expression.CreateDataverseStatement;
import org.apache.asterix.aql.expression.CreateFeedPolicyStatement;
import org.apache.asterix.aql.expression.CreateFeedStatement;
import org.apache.asterix.aql.expression.CreateFunctionStatement;
import org.apache.asterix.aql.expression.CreateIndexStatement;
import org.apache.asterix.aql.expression.CreatePrimaryFeedStatement;
import org.apache.asterix.aql.expression.CreateSecondaryFeedStatement;
import org.apache.asterix.aql.expression.DatasetDecl;
import org.apache.asterix.aql.expression.DataverseDecl;
import org.apache.asterix.aql.expression.DataverseDropStatement;
import org.apache.asterix.aql.expression.DeleteStatement;
import org.apache.asterix.aql.expression.DisconnectFeedStatement;
import org.apache.asterix.aql.expression.DistinctClause;
import org.apache.asterix.aql.expression.DropStatement;
import org.apache.asterix.aql.expression.FLWOGRExpression;
import org.apache.asterix.aql.expression.FeedDropStatement;
import org.apache.asterix.aql.expression.FeedPolicyDropStatement;
import org.apache.asterix.aql.expression.FieldAccessor;
import org.apache.asterix.aql.expression.FieldBinding;
import org.apache.asterix.aql.expression.ForClause;
import org.apache.asterix.aql.expression.FunctionDecl;
import org.apache.asterix.aql.expression.FunctionDropStatement;
import org.apache.asterix.aql.expression.GbyVariableExpressionPair;
import org.apache.asterix.aql.expression.GroupbyClause;
import org.apache.asterix.aql.expression.Identifier;
import org.apache.asterix.aql.expression.IfExpr;
import org.apache.asterix.aql.expression.IndexAccessor;
import org.apache.asterix.aql.expression.IndexDropStatement;
import org.apache.asterix.aql.expression.InsertStatement;
import org.apache.asterix.aql.expression.JoinClause;
import org.apache.asterix.aql.expression.LetClause;
import org.apache.asterix.aql.expression.LimitClause;
import org.apache.asterix.aql.expression.ListConstructor;
import org.apache.asterix.aql.expression.ListConstructor.Type;
import org.apache.asterix.aql.expression.LiteralExpr;
import org.apache.asterix.aql.expression.LoadStatement;
import org.apache.asterix.aql.expression.MetaVariableClause;
import org.apache.asterix.aql.expression.MetaVariableExpr;
import org.apache.asterix.aql.expression.NodeGroupDropStatement;
import org.apache.asterix.aql.expression.NodegroupDecl;
import org.apache.asterix.aql.expression.OperatorExpr;
import org.apache.asterix.aql.expression.OperatorType;
import org.apache.asterix.aql.expression.OrderbyClause;
import org.apache.asterix.aql.expression.OrderbyClause.OrderModifier;
import org.apache.asterix.aql.expression.OrderedListTypeDefinition;
import org.apache.asterix.aql.expression.QuantifiedExpression;
import org.apache.asterix.aql.expression.QuantifiedExpression.Quantifier;
import org.apache.asterix.aql.expression.QuantifiedPair;
import org.apache.asterix.aql.expression.Query;
import org.apache.asterix.aql.expression.RecordConstructor;
import org.apache.asterix.aql.expression.RecordTypeDefinition;
import org.apache.asterix.aql.expression.SetStatement;
import org.apache.asterix.aql.expression.TypeDecl;
import org.apache.asterix.aql.expression.TypeDropStatement;
import org.apache.asterix.aql.expression.TypeReferenceExpression;
import org.apache.asterix.aql.expression.UnaryExpr;
import org.apache.asterix.aql.expression.UnaryExpr.Sign;
import org.apache.asterix.aql.expression.UnionExpr;
import org.apache.asterix.aql.expression.UnorderedListTypeDefinition;
import org.apache.asterix.aql.expression.UpdateClause;
import org.apache.asterix.aql.expression.UpdateStatement;
import org.apache.asterix.aql.expression.VariableExpr;
import org.apache.asterix.aql.expression.WhereClause;
import org.apache.asterix.aql.expression.WriteStatement;
import org.apache.asterix.aql.expression.visitor.IAqlPlusExpressionVisitor;
import org.apache.asterix.aql.util.FunctionUtils;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.FileSplitDataSink;
import org.apache.asterix.metadata.declared.FileSplitSinkId;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.functions.AsterixFunctionInfo;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.Counter;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
/**
* Each visit returns a pair of an operator and a variable. The variable
* corresponds to the new column, if any, added to the tuple flow. E.g., for
* Unnest, the column is the variable bound to the elements in the list, for
* Subplan it is null. The first argument of a visit method is the expression
* which is translated. The second argument of a visit method is the tuple
* source for the current subtree.
*/
public class AqlPlusExpressionToPlanTranslator extends AbstractAqlTranslator implements
IAqlPlusExpressionVisitor, Mutable> {
private static final Logger LOGGER = Logger.getLogger(AqlPlusExpressionToPlanTranslator.class.getName());
private class MetaScopeLogicalVariable {
private HashMap map = new HashMap();
public VariableReferenceExpression getVariableReferenceExpression(Identifier id) throws AsterixException {
LogicalVariable var = map.get(id);
LOGGER.fine("get:" + id + ":" + var);
if (var == null) {
throw new AsterixException("Identifier " + id + " not found in AQL+ meta-scope.");
}
return new VariableReferenceExpression(var);
}
public void put(Identifier id, LogicalVariable var) {
LOGGER.fine("put:" + id + ":" + var);
map.put(id, var);
}
}
private class MetaScopeILogicalOperator {
private HashMap map = new HashMap();
public ILogicalOperator get(Identifier id) throws AsterixException {
ILogicalOperator op = map.get(id);
if (op == null) {
throw new AsterixException("Identifier " + id + " not found in AQL+ meta-scope.");
}
return op;
}
public void put(Identifier id, ILogicalOperator op) {
LOGGER.fine("put:" + id + ":" + op);
map.put(id, op);
}
}
private final JobId jobId;
private TranslationContext context;
private String outputDatasetName;
private ICompiledDmlStatement stmt;
private AqlMetadataProvider metadataProvider;
private MetaScopeLogicalVariable metaScopeExp = new MetaScopeLogicalVariable();
private MetaScopeILogicalOperator metaScopeOp = new MetaScopeILogicalOperator();
private static LogicalVariable METADATA_DUMMY_VAR = new LogicalVariable(-1);
public AqlPlusExpressionToPlanTranslator(JobId jobId, AqlMetadataProvider metadataProvider,
Counter currentVarCounter, String outputDatasetName, ICompiledDmlStatement stmt) {
this.jobId = jobId;
this.metadataProvider = metadataProvider;
this.context = new TranslationContext(currentVarCounter);
this.outputDatasetName = outputDatasetName;
this.stmt = stmt;
this.context.setTopFlwor(false);
}
public int getVarCounter() {
return context.getVarCounter();
}
public ILogicalPlan translate(Query expr) throws AlgebricksException, AsterixException {
return translate(expr, null);
}
public ILogicalPlan translate(Query expr, AqlMetadataProvider metadata) throws AlgebricksException,
AsterixException {
IDataFormat format = metadata.getFormat();
if (format == null) {
throw new AlgebricksException("Data format has not been set.");
}
format.registerRuntimeFunctions();
Pair p = expr.accept(this, new MutableObject(
new EmptyTupleSourceOperator()));
ArrayList> globalPlanRoots = new ArrayList>();
boolean isTransactionalWrite = false;
ILogicalOperator topOp = p.first;
ProjectOperator project = (ProjectOperator) topOp;
LogicalVariable resVar = project.getVariables().get(0);
if (outputDatasetName == null) {
List> writeExprList = new ArrayList>(1);
writeExprList.add(new MutableObject(new VariableReferenceExpression(resVar)));
FileSplitSinkId fssi = new FileSplitSinkId(metadata.getOutputFile());
FileSplitDataSink sink = new FileSplitDataSink(fssi, null);
topOp = new WriteOperator(writeExprList, sink);
topOp.getInputs().add(new MutableObject(project));
} else {
Dataset dataset = metadata.findDataset(stmt.getDataverseName(), outputDatasetName);
if (dataset == null) {
throw new AlgebricksException("Cannot find dataset " + outputDatasetName);
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
throw new AlgebricksException("Cannot write output to an external dataset.");
}
ARecordType itemType = (ARecordType) metadata.findType(dataset.getDataverseName(),
dataset.getItemTypeName());
List> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
ArrayList vars = new ArrayList();
ArrayList> exprs = new ArrayList>();
List> varRefsForLoading = new ArrayList>();
for (List partitioningKey : partitioningKeys) {
Triple partitioner = format
.partitioningEvaluatorFactory(itemType, partitioningKey);
AbstractFunctionCallExpression f = partitioner.second.cloneExpression();
f.substituteVar(METADATA_DUMMY_VAR, resVar);
exprs.add(new MutableObject(f));
LogicalVariable v = context.newVar();
vars.add(v);
varRefsForLoading.add(new MutableObject(new VariableReferenceExpression(v)));
}
AssignOperator assign = new AssignOperator(vars, exprs);
assign.getInputs().add(new MutableObject(project));
}
globalPlanRoots.add(new MutableObject(topOp));
ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
return plan;
}
public ILogicalPlan translate(List clauses) throws AlgebricksException, AsterixException {
if (clauses == null) {
return null;
}
Mutable opRef = new MutableObject(new EmptyTupleSourceOperator());
Pair p = null;
for (Clause c : clauses) {
p = c.accept(this, opRef);
opRef = new MutableObject(p.first);
}
ArrayList> globalPlanRoots = new ArrayList>();
ILogicalOperator topOp = p.first;
globalPlanRoots.add(new MutableObject(topOp));
ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
return plan;
}
@Override
public Pair visitForClause(ForClause fc, Mutable tupSource)
throws AsterixException {
LogicalVariable v = context.newVar(fc.getVarExpr());
Expression inExpr = fc.getInExpr();
Pair> eo = aqlExprToAlgExpression(inExpr, tupSource);
ILogicalOperator returnedOp;
if (fc.getPosVarExpr() == null) {
returnedOp = new UnnestOperator(v, new MutableObject(makeUnnestExpression(eo.first)));
} else {
LogicalVariable pVar = context.newVar(fc.getPosVarExpr());
returnedOp = new UnnestOperator(v, new MutableObject(makeUnnestExpression(eo.first)),
pVar, BuiltinType.AINT32, new AqlPositionWriter());
}
returnedOp.getInputs().add(eo.second);
return new Pair(returnedOp, v);
}
@Override
public Pair visitLetClause(LetClause lc, Mutable tupSource)
throws AsterixException {
LogicalVariable v;
ILogicalOperator returnedOp;
switch (lc.getBindingExpr().getKind()) {
case VARIABLE_EXPRESSION: {
v = context.newVar(lc.getVarExpr());
LogicalVariable prev = context.getVar(((VariableExpr) lc.getBindingExpr()).getVar().getId());
returnedOp = new AssignOperator(v, new MutableObject(
new VariableReferenceExpression(prev)));
returnedOp.getInputs().add(tupSource);
break;
}
default: {
Pair> eo = aqlExprToAlgExpression(lc.getBindingExpr(),
tupSource);
v = context.newVar(lc.getVarExpr());
returnedOp = new AssignOperator(v, new MutableObject(eo.first));
returnedOp.getInputs().add(eo.second);
break;
}
}
return new Pair(returnedOp, v);
}
@Override
public Pair visitFlworExpression(FLWOGRExpression flwor,
Mutable tupSource) throws AsterixException {
Mutable flworPlan = tupSource;
boolean isTop = context.isTopFlwor();
if (isTop) {
context.setTopFlwor(false);
}
for (Clause c : flwor.getClauseList()) {
Pair pC = c.accept(this, flworPlan);
flworPlan = new MutableObject(pC.first);
}
Expression r = flwor.getReturnExpr();
boolean noFlworClause = flwor.noForClause();
if (r.getKind() == Kind.VARIABLE_EXPRESSION) {
VariableExpr v = (VariableExpr) r;
LogicalVariable var = context.getVar(v.getVar().getId());
return produceFlwrResult(noFlworClause, isTop, flworPlan, var);
} else {
Mutable baseOp = new MutableObject(flworPlan.getValue());
Pair rRes = r.accept(this, baseOp);
ILogicalOperator rOp = rRes.first;
ILogicalOperator resOp;
if (expressionNeedsNoNesting(r)) {
baseOp.setValue(flworPlan.getValue());
resOp = rOp;
} else {
SubplanOperator s = new SubplanOperator(rOp);
s.getInputs().add(flworPlan);
resOp = s;
baseOp.setValue(new NestedTupleSourceOperator(new MutableObject(s)));
}
Mutable resOpRef = new MutableObject(resOp);
return produceFlwrResult(noFlworClause, isTop, resOpRef, rRes.second);
}
}
@Override
public Pair visitFieldAccessor(FieldAccessor fa,
Mutable tupSource) throws AsterixException {
Pair> p = aqlExprToAlgExpression(fa.getExpr(), tupSource);
LogicalVariable v = context.newVar();
AbstractFunctionCallExpression fldAccess = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME));
fldAccess.getArguments().add(new MutableObject(p.first));
ILogicalExpression faExpr = new ConstantExpression(new AsterixConstantValue(new AString(fa.getIdent()
.getValue())));
fldAccess.getArguments().add(new MutableObject(faExpr));
AssignOperator a = new AssignOperator(v, new MutableObject(fldAccess));
a.getInputs().add(p.second);
return new Pair(a, v);
}
@Override
public Pair visitIndexAccessor(IndexAccessor ia,
Mutable tupSource) throws AsterixException {
Pair> p = aqlExprToAlgExpression(ia.getExpr(), tupSource);
LogicalVariable v = context.newVar();
AbstractFunctionCallExpression f;
if (ia.isAny()) {
f = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.ANY_COLLECTION_MEMBER));
f.getArguments().add(new MutableObject(p.first));
} else {
Pair> indexPair = aqlExprToAlgExpression(ia.getIndexExpr(),
tupSource);
f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM));
f.getArguments().add(new MutableObject(p.first));
f.getArguments().add(new MutableObject(indexPair.first));
}
AssignOperator a = new AssignOperator(v, new MutableObject(f));
a.getInputs().add(p.second);
return new Pair(a, v);
}
@Override
public Pair visitCallExpr(CallExpr fcall, Mutable tupSource)
throws AsterixException {
LogicalVariable v = context.newVar();
FunctionSignature signature = fcall.getFunctionSignature();
List> args = new ArrayList>();
Mutable topOp = tupSource;
for (Expression expr : fcall.getExprList()) {
switch (expr.getKind()) {
case VARIABLE_EXPRESSION: {
LogicalVariable var = context.getVar(((VariableExpr) expr).getVar().getId());
args.add(new MutableObject(new VariableReferenceExpression(var)));
break;
}
case LITERAL_EXPRESSION: {
LiteralExpr val = (LiteralExpr) expr;
args.add(new MutableObject(new ConstantExpression(new AsterixConstantValue(
ConstantHelper.objectFromLiteral(val.getValue())))));
break;
}
default: {
Pair> eo = aqlExprToAlgExpression(expr, topOp);
AbstractLogicalOperator o1 = (AbstractLogicalOperator) eo.second.getValue();
args.add(new MutableObject(eo.first));
if (o1 != null && !(o1.getOperatorTag() == LogicalOperatorTag.ASSIGN && hasOnlyChild(o1, topOp))) {
topOp = eo.second;
}
break;
}
}
}
FunctionIdentifier fi = new FunctionIdentifier(AlgebricksBuiltinFunctions.ALGEBRICKS_NS, signature.getName());
AsterixFunctionInfo afi = AsterixBuiltinFunctions.lookupFunction(fi);
FunctionIdentifier builtinAquafi = afi == null ? null : afi.getFunctionIdentifier();
if (builtinAquafi != null) {
fi = builtinAquafi;
} else {
fi = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, signature.getName());
FunctionIdentifier builtinAsterixFi = AsterixBuiltinFunctions.getBuiltinFunctionIdentifier(fi);
if (builtinAsterixFi != null) {
fi = builtinAsterixFi;
}
}
AbstractFunctionCallExpression f;
if (AsterixBuiltinFunctions.isBuiltinAggregateFunction(fi)) {
f = AsterixBuiltinFunctions.makeAggregateFunctionExpression(fi, args);
} else if (AsterixBuiltinFunctions.isBuiltinUnnestingFunction(fi)) {
UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(
FunctionUtils.getFunctionInfo(fi), args);
ufce.setReturnsUniqueValues(AsterixBuiltinFunctions.returnsUniqueValues(fi));
f = ufce;
} else {
f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fi), args);
}
AssignOperator op = new AssignOperator(v, new MutableObject(f));
if (topOp != null) {
op.getInputs().add(topOp);
}
return new Pair(op, v);
}
@Override
public Pair visitFunctionDecl(FunctionDecl fd,
Mutable tupSource) {
// TODO Auto-generated method stub
throw new NotImplementedException();
}
@Override
public Pair visitGroupbyClause(GroupbyClause gc,
Mutable tupSource) throws AsterixException {
GroupByOperator gOp = new GroupByOperator();
Mutable topOp = tupSource;
for (GbyVariableExpressionPair ve : gc.getGbyPairList()) {
LogicalVariable v;
VariableExpr vexpr = ve.getVar();
if (vexpr != null) {
v = context.newVar(vexpr);
} else {
v = context.newVar();
}
Pair> eo = aqlExprToAlgExpression(ve.getExpr(), topOp);
gOp.addGbyExpression(v, eo.first);
topOp = eo.second;
}
for (GbyVariableExpressionPair ve : gc.getDecorPairList()) {
LogicalVariable v;
VariableExpr vexpr = ve.getVar();
if (vexpr != null) {
v = context.newVar(vexpr);
} else {
v = context.newVar();
}
Pair> eo = aqlExprToAlgExpression(ve.getExpr(), topOp);
gOp.addDecorExpression(v, eo.first);
topOp = eo.second;
}
gOp.getInputs().add(topOp);
for (VariableExpr var : gc.getWithVarList()) {
LogicalVariable aggVar = context.newVar();
LogicalVariable oldVar = context.getVar(var);
List> flArgs = new ArrayList>(1);
flArgs.add(new MutableObject(new VariableReferenceExpression(oldVar)));
AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions.makeAggregateFunctionExpression(
AsterixBuiltinFunctions.LISTIFY, flArgs);
AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar),
(List) mkSingletonArrayList(new MutableObject(fListify)));
agg.getInputs().add(
new MutableObject(new NestedTupleSourceOperator(
new MutableObject(gOp))));
ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject(agg));
gOp.getNestedPlans().add(plan);
// Hide the variable that was part of the "with", replacing it with
// the one bound by the aggregation op.
context.setVar(var, aggVar);
}
gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint());
return new Pair(gOp, null);
}
@Override
public Pair visitIfExpr(IfExpr ifexpr, Mutable tupSource)
throws AsterixException {
// In the most general case, IfThenElse is translated in the following
// way.
//
// We assign the result of the condition to one variable varCond.
// We create one subplan which contains the plan for the "then" branch,
// on top of which there is a selection whose condition is varCond.
// Similarly, we create one subplan for the "else" branch, in which the
// selection is not(varCond).
// Finally, we concatenate the results. (??)
Pair pCond = ifexpr.getCondExpr().accept(this, tupSource);
ILogicalOperator opCond = pCond.first;
LogicalVariable varCond = pCond.second;
SubplanOperator sp = new SubplanOperator();
Mutable nestedSource = new MutableObject(new NestedTupleSourceOperator(
new MutableObject(sp)));
Pair pThen = ifexpr.getThenExpr().accept(this, nestedSource);
SelectOperator sel1 = new SelectOperator(new MutableObject(new VariableReferenceExpression(
varCond)), false, null);
sel1.getInputs().add(new MutableObject(pThen.first));
Pair pElse = ifexpr.getElseExpr().accept(this, nestedSource);
AbstractFunctionCallExpression notVarCond = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), new MutableObject(
new VariableReferenceExpression(varCond)));
SelectOperator sel2 = new SelectOperator(new MutableObject(notVarCond), false, null);
sel2.getInputs().add(new MutableObject(pElse.first));
ILogicalPlan p1 = new ALogicalPlanImpl(new MutableObject(sel1));
sp.getNestedPlans().add(p1);
ILogicalPlan p2 = new ALogicalPlanImpl(new MutableObject(sel2));
sp.getNestedPlans().add(p2);
Mutable opCondRef = new MutableObject(opCond);
sp.getInputs().add(opCondRef);
LogicalVariable resV = context.newVar();
AbstractFunctionCallExpression concatNonNull = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CONCAT_NON_NULL),
new MutableObject(new VariableReferenceExpression(pThen.second)),
new MutableObject(new VariableReferenceExpression(pElse.second)));
AssignOperator a = new AssignOperator(resV, new MutableObject(concatNonNull));
a.getInputs().add(new MutableObject(sp));
return new Pair(a, resV);
}
@Override
public Pair visitLiteralExpr(LiteralExpr l, Mutable tupSource) {
LogicalVariable var = context.newVar();
AssignOperator a = new AssignOperator(var, new MutableObject(new ConstantExpression(
new AsterixConstantValue(ConstantHelper.objectFromLiteral(l.getValue())))));
if (tupSource != null) {
a.getInputs().add(tupSource);
}
return new Pair(a, var);
}
@Override
public Pair visitOperatorExpr(OperatorExpr op,
Mutable tupSource) throws AsterixException {
ArrayList ops = op.getOpList();
int nOps = ops.size();
if (nOps > 0 && (ops.get(0) == OperatorType.AND || ops.get(0) == OperatorType.OR)) {
return visitAndOrOperator(op, tupSource);
}
ArrayList exprs = op.getExprList();
Mutable topOp = tupSource;
ILogicalExpression currExpr = null;
for (int i = 0; i <= nOps; i++) {
Pair> p = aqlExprToAlgExpression(exprs.get(i), topOp);
topOp = p.second;
ILogicalExpression e = p.first;
// now look at the operator
if (i < nOps) {
if (OperatorExpr.opIsComparison(ops.get(i))) {
AbstractFunctionCallExpression c = createComparisonExpression(ops.get(i));
// chain the operators
if (i == 0) {
c.getArguments().add(new MutableObject(e));
currExpr = c;
if (op.isBroadcastOperand(i)) {
BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
bcast.setObject(BroadcastSide.LEFT);
c.getAnnotations().put(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast);
}
} else {
((AbstractFunctionCallExpression) currExpr).getArguments().add(
new MutableObject(e));
c.getArguments().add(new MutableObject(currExpr));
currExpr = c;
if (i == 1 && op.isBroadcastOperand(i)) {
BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
bcast.setObject(BroadcastSide.RIGHT);
c.getAnnotations().put(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast);
}
}
} else {
AbstractFunctionCallExpression f = createFunctionCallExpressionForBuiltinOperator(ops.get(i));
if (i == 0) {
f.getArguments().add(new MutableObject(e));
currExpr = f;
} else {
((AbstractFunctionCallExpression) currExpr).getArguments().add(
new MutableObject(e));
f.getArguments().add(new MutableObject(currExpr));
currExpr = f;
}
}
} else { // don't forget the last expression...
((AbstractFunctionCallExpression) currExpr).getArguments()
.add(new MutableObject(e));
if (i == 1 && op.isBroadcastOperand(i)) {
BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
bcast.setObject(BroadcastSide.RIGHT);
((AbstractFunctionCallExpression) currExpr).getAnnotations().put(
BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast);
}
}
}
LogicalVariable assignedVar = context.newVar();
AssignOperator a = new AssignOperator(assignedVar, new MutableObject(currExpr));
a.getInputs().add(topOp);
return new Pair(a, assignedVar);
}
@Override
public Pair visitOrderbyClause(OrderbyClause oc,
Mutable tupSource) throws AsterixException {
OrderOperator ord = new OrderOperator();
Iterator modifIter = oc.getModifierList().iterator();
Mutable topOp = tupSource;
for (Expression e : oc.getOrderbyList()) {
Pair> p = aqlExprToAlgExpression(e, topOp);
OrderModifier m = modifIter.next();
OrderOperator.IOrder comp = (m == OrderModifier.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
ord.getOrderExpressions()
.add(new Pair>(comp, new MutableObject(
p.first)));
topOp = p.second;
}
ord.getInputs().add(topOp);
if (oc.getNumTuples() > 0) {
ord.getAnnotations().put(OperatorAnnotations.CARDINALITY, oc.getNumTuples());
}
if (oc.getNumFrames() > 0) {
ord.getAnnotations().put(OperatorAnnotations.MAX_NUMBER_FRAMES, oc.getNumFrames());
}
return new Pair(ord, null);
}
@Override
public Pair visitQuantifiedExpression(QuantifiedExpression qe,
Mutable tupSource) throws AsterixException {
Mutable topOp = tupSource;
ILogicalOperator firstOp = null;
Mutable lastOp = null;
for (QuantifiedPair qt : qe.getQuantifiedList()) {
Pair> eo1 = aqlExprToAlgExpression(qt.getExpr(), topOp);
topOp = eo1.second;
LogicalVariable uVar = context.newVar(qt.getVarExpr());
ILogicalOperator u = new UnnestOperator(uVar, new MutableObject(
makeUnnestExpression(eo1.first)));
if (firstOp == null) {
firstOp = u;
}
if (lastOp != null) {
u.getInputs().add(lastOp);
}
lastOp = new MutableObject(u);
}
// We make all the unnest correspond. to quantif. vars. sit on top
// in the hope of enabling joins & other optimiz.
firstOp.getInputs().add(topOp);
topOp = lastOp;
Pair> eo2 = aqlExprToAlgExpression(qe.getSatisfiesExpr(), topOp);
AggregateFunctionCallExpression fAgg;
SelectOperator s;
if (qe.getQuantifier() == Quantifier.SOME) {
s = new SelectOperator(new MutableObject(eo2.first), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.NON_EMPTY_STREAM,
new ArrayList>());
} else { // EVERY
List> satExprList = new ArrayList>(1);
satExprList.add(new MutableObject(eo2.first));
s = new SelectOperator(new MutableObject(new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)), false, null);
s.getInputs().add(eo2.second);
fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.EMPTY_STREAM,
new ArrayList>());
}
LogicalVariable qeVar = context.newVar();
AggregateOperator a = new AggregateOperator(mkSingletonArrayList(qeVar),
(List) mkSingletonArrayList(new MutableObject(fAgg)));
a.getInputs().add(new MutableObject(s));
return new Pair(a, qeVar);
}
@Override
public Pair visitQuery(Query q, Mutable tupSource)
throws AsterixException {
return q.getBody().accept(this, tupSource);
}
@Override
public Pair visitRecordConstructor(RecordConstructor rc,
Mutable tupSource) throws AsterixException {
AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR));
LogicalVariable v1 = context.newVar();
AssignOperator a = new AssignOperator(v1, new MutableObject(f));
Mutable topOp = tupSource;
for (FieldBinding fb : rc.getFbList()) {
Pair> eo1 = aqlExprToAlgExpression(fb.getLeftExpr(), topOp);
f.getArguments().add(new MutableObject(eo1.first));
topOp = eo1.second;
Pair> eo2 = aqlExprToAlgExpression(fb.getRightExpr(), topOp);
f.getArguments().add(new MutableObject(eo2.first));
topOp = eo2.second;
}
a.getInputs().add(topOp);
return new Pair(a, v1);
}
@Override
public Pair visitListConstructor(ListConstructor lc,
Mutable tupSource) throws AsterixException {
FunctionIdentifier fid = (lc.getType() == Type.ORDERED_LIST_CONSTRUCTOR) ? AsterixBuiltinFunctions.ORDERED_LIST_CONSTRUCTOR
: AsterixBuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR;
AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fid));
LogicalVariable v1 = context.newVar();
AssignOperator a = new AssignOperator(v1, new MutableObject(f));
Mutable topOp = tupSource;
for (Expression expr : lc.getExprList()) {
Pair> eo = aqlExprToAlgExpression(expr, topOp);
f.getArguments().add(new MutableObject(eo.first));
topOp = eo.second;
}
a.getInputs().add(topOp);
return new Pair(a, v1);
}
@Override
public Pair visitUnaryExpr(UnaryExpr u, Mutable tupSource)
throws AsterixException {
Expression expr = u.getExpr();
Pair> eo = aqlExprToAlgExpression(expr, tupSource);
LogicalVariable v1 = context.newVar();
AssignOperator a;
if (u.getSign() == Sign.POSITIVE) {
a = new AssignOperator(v1, new MutableObject(eo.first));
} else {
AbstractFunctionCallExpression m = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NUMERIC_UNARY_MINUS));
m.getArguments().add(new MutableObject(eo.first));
a = new AssignOperator(v1, new MutableObject(m));
}
a.getInputs().add(eo.second);
return new Pair(a, v1);
}
@Override
public Pair visitVariableExpr(VariableExpr v, Mutable tupSource) {
// Should we ever get to this method?
LogicalVariable var = context.newVar();
LogicalVariable oldV = context.getVar(v.getVar().getId());
AssignOperator a = new AssignOperator(var, new MutableObject(
new VariableReferenceExpression(oldV)));
a.getInputs().add(tupSource);
return new Pair(a, var);
}
@Override
public Pair visitWhereClause(WhereClause w, Mutable tupSource)
throws AsterixException {
Pair> p = aqlExprToAlgExpression(w.getWhereExpr(), tupSource);
SelectOperator s = new SelectOperator(new MutableObject(p.first), false, null);
s.getInputs().add(p.second);
return new Pair(s, null);
}
@Override
public Pair visitLimitClause(LimitClause lc, Mutable tupSource)
throws AsterixException {
Pair> p1 = aqlExprToAlgExpression(lc.getLimitExpr(), tupSource);
LimitOperator opLim;
Expression offset = lc.getOffset();
if (offset != null) {
Pair> p2 = aqlExprToAlgExpression(offset, p1.second);
opLim = new LimitOperator(p1.first, p2.first);
opLim.getInputs().add(p2.second);
} else {
opLim = new LimitOperator(p1.first);
opLim.getInputs().add(p1.second);
}
return new Pair(opLim, null);
}
@Override
public Pair visitDistinctClause(DistinctClause dc,
Mutable tupSource) throws AsterixException {
List> exprList = new ArrayList>();
Mutable input = null;
for (Expression expr : dc.getDistinctByExpr()) {
Pair> p = aqlExprToAlgExpression(expr, tupSource);
exprList.add(new MutableObject(p.first));
input = p.second;
}
DistinctOperator opDistinct = new DistinctOperator(exprList);
opDistinct.getInputs().add(input);
return new Pair(opDistinct, null);
}
@Override
public Pair visitUnionExpr(UnionExpr unionExpr,
Mutable tupSource) throws AsterixException {
Mutable ts = tupSource;
ILogicalOperator lastOp = null;
LogicalVariable lastVar = null;
boolean first = true;
for (Expression e : unionExpr.getExprs()) {
if (first) {
first = false;
} else {
ts = new MutableObject(new EmptyTupleSourceOperator());
}
Pair p1 = e.accept(this, ts);
if (lastOp == null) {
lastOp = p1.first;
lastVar = p1.second;
} else {
LogicalVariable unnestVar1 = context.newVar();
UnnestOperator unnest1 = new UnnestOperator(unnestVar1, new MutableObject(
makeUnnestExpression(new VariableReferenceExpression(lastVar))));
unnest1.getInputs().add(new MutableObject(lastOp));
LogicalVariable unnestVar2 = context.newVar();
UnnestOperator unnest2 = new UnnestOperator(unnestVar2, new MutableObject(
makeUnnestExpression(new VariableReferenceExpression(p1.second))));
unnest2.getInputs().add(new MutableObject(p1.first));
List> varMap = new ArrayList>(
1);
LogicalVariable resultVar = context.newVar();
Triple triple = new Triple(
unnestVar1, unnestVar2, resultVar);
varMap.add(triple);
UnionAllOperator unionOp = new UnionAllOperator(varMap);
unionOp.getInputs().add(new MutableObject(unnest1));
unionOp.getInputs().add(new MutableObject(unnest2));
lastVar = resultVar;
lastOp = unionOp;
}
}
LogicalVariable aggVar = context.newVar();
ArrayList aggregVars = new ArrayList(1);
aggregVars.add(aggVar);
List> afcExprs = new ArrayList>(1);
afcExprs.add(new MutableObject(new VariableReferenceExpression(lastVar)));
AggregateFunctionCallExpression afc = AsterixBuiltinFunctions.makeAggregateFunctionExpression(
AsterixBuiltinFunctions.LISTIFY, afcExprs);
ArrayList> aggregExprs = new ArrayList>(1);
aggregExprs.add(new MutableObject(afc));
AggregateOperator agg = new AggregateOperator(aggregVars, aggregExprs);
agg.getInputs().add(new MutableObject(lastOp));
return new Pair(agg, aggVar);
}
private AbstractFunctionCallExpression createComparisonExpression(OperatorType t) {
FunctionIdentifier fi = operatorTypeToFunctionIdentifier(t);
IFunctionInfo finfo = FunctionUtils.getFunctionInfo(fi);
return new ScalarFunctionCallExpression(finfo);
}
private FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) {
switch (t) {
case EQ: {
return AlgebricksBuiltinFunctions.EQ;
}
case NEQ: {
return AlgebricksBuiltinFunctions.NEQ;
}
case GT: {
return AlgebricksBuiltinFunctions.GT;
}
case GE: {
return AlgebricksBuiltinFunctions.GE;
}
case LT: {
return AlgebricksBuiltinFunctions.LT;
}
case LE: {
return AlgebricksBuiltinFunctions.LE;
}
default: {
throw new IllegalStateException();
}
}
}
private AbstractFunctionCallExpression createFunctionCallExpressionForBuiltinOperator(OperatorType t)
throws AsterixException {
FunctionIdentifier fid = null;
switch (t) {
case PLUS: {
fid = AlgebricksBuiltinFunctions.NUMERIC_ADD;
break;
}
case MINUS: {
fid = AsterixBuiltinFunctions.NUMERIC_SUBTRACT;
break;
}
case MUL: {
fid = AsterixBuiltinFunctions.NUMERIC_MULTIPLY;
break;
}
case DIV: {
fid = AsterixBuiltinFunctions.NUMERIC_DIVIDE;
break;
}
case MOD: {
fid = AsterixBuiltinFunctions.NUMERIC_MOD;
break;
}
case IDIV: {
fid = AsterixBuiltinFunctions.NUMERIC_IDIV;
break;
}
case CARET: {
fid = AsterixBuiltinFunctions.CARET;
break;
}
case AND: {
fid = AlgebricksBuiltinFunctions.AND;
break;
}
case OR: {
fid = AlgebricksBuiltinFunctions.OR;
break;
}
case FUZZY_EQ: {
fid = AsterixBuiltinFunctions.FUZZY_EQ;
break;
}
default: {
throw new NotImplementedException("Operator " + t + " is not yet implemented");
}
}
return new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fid));
}
private static boolean hasOnlyChild(ILogicalOperator parent, Mutable childCandidate) {
List> inp = parent.getInputs();
if (inp == null || inp.size() != 1) {
return false;
}
return inp.get(0) == childCandidate;
}
private Pair> aqlExprToAlgExpression(Expression expr,
Mutable topOp) throws AsterixException {
switch (expr.getKind()) {
case VARIABLE_EXPRESSION: {
VariableReferenceExpression ve = new VariableReferenceExpression(context.getVar(((VariableExpr) expr)
.getVar().getId()));
return new Pair>(ve, topOp);
}
case METAVARIABLE_EXPRESSION: {
ILogicalExpression le = metaScopeExp.getVariableReferenceExpression(((VariableExpr) expr).getVar());
return new Pair>(le, topOp);
}
case LITERAL_EXPRESSION: {
LiteralExpr val = (LiteralExpr) expr;
return new Pair>(new ConstantExpression(
new AsterixConstantValue(ConstantHelper.objectFromLiteral(val.getValue()))), topOp);
}
default: {
// Mutable src = new
// Mutable();
// Mutable src = topOp;
if (expressionNeedsNoNesting(expr)) {
Pair 