• Home
• org.apache.pig
• pig
• 0.13.0
• source code
• LogicalPlanGenerator.g

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.apache.pig.parser.LogicalPlanGenerator.g Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * Grammar file for Pig tree parser (for schema alias validation).
 *
 * NOTE: THIS FILE IS BASED ON QueryParser.g, SO IF YOU CHANGE THAT FILE, YOU WILL
 *       PROBABLY NEED TO MAKE CORRESPONDING CHANGES TO THIS FILE AS WELL.
 */

tree grammar LogicalPlanGenerator;

options {
    tokenVocab=QueryParser;
    ASTLabelType=CommonTree;
    output=AST;
    backtrack=true;
}

scope GScope {
    LogicalRelationalOperator currentOp; // Current relational operator that's being built.
}

@header {
package org.apache.pig.parser;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.builtin.GFAny;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.streaming.StreamingCommand;
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec;
import org.apache.pig.impl.util.MultiMap;
import org.apache.pig.impl.util.NumValCarrier;
import org.apache.pig.impl.plan.PlanValidationException;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.expression.AddExpression;
import org.apache.pig.newplan.logical.expression.AndExpression;
import org.apache.pig.newplan.logical.expression.BinCondExpression;
import org.apache.pig.newplan.logical.expression.CastExpression;
import org.apache.pig.newplan.logical.expression.ConstantExpression;
import org.apache.pig.newplan.logical.expression.DereferenceExpression;
import org.apache.pig.newplan.logical.expression.DivideExpression;
import org.apache.pig.newplan.logical.expression.EqualExpression;
import org.apache.pig.newplan.logical.expression.GreaterThanEqualExpression;
import org.apache.pig.newplan.logical.expression.GreaterThanExpression;
import org.apache.pig.newplan.logical.expression.IsNullExpression;
import org.apache.pig.newplan.logical.expression.LessThanEqualExpression;
import org.apache.pig.newplan.logical.expression.LessThanExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.MapLookupExpression;
import org.apache.pig.newplan.logical.expression.ModExpression;
import org.apache.pig.newplan.logical.expression.MultiplyExpression;
import org.apache.pig.newplan.logical.expression.NegativeExpression;
import org.apache.pig.newplan.logical.expression.NotEqualExpression;
import org.apache.pig.newplan.logical.expression.NotExpression;
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.expression.RegexExpression;
import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.expression.SubtractExpression;
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOCube;
import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
import org.apache.pig.newplan.logical.relational.LOLimit;
import org.apache.pig.newplan.logical.relational.LOJoin;
import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LORank;
import org.apache.pig.newplan.logical.relational.LOSplitOutput;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
import org.apache.pig.newplan.logical.relational.LOCogroup.GROUPTYPE;
import org.apache.pig.newplan.logical.relational.LOJoin.JOINTYPE;
import org.apache.pig.FuncSpec;
import org.apache.pig.builtin.PigStreaming;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import java.util.Arrays;
import java.util.Collections;
import java.math.BigInteger;
import java.math.BigDecimal;
}

@members {
private static Log log = LogFactory.getLog( LogicalPlanGenerator.class );

private LogicalPlanBuilder builder = null;

private boolean inForeachPlan = false;

private boolean inNestedCommand = false;

public LogicalPlan getLogicalPlan() {
    return builder.getPlan();
}

public Map getOperators() {
    return builder.getOperators();
}

public String getLastRel() {
    return builder.getLastRel();
}

@Override
protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
throws RecognitionException {
    throw new MismatchedTokenException( ttype, input );
}

@Override
public Object recoverFromMismatchedSet(IntStream input, RecognitionException e, BitSet follow)
throws RecognitionException {
    throw e;
}

public LogicalPlanGenerator(TreeNodeStream input, LogicalPlanBuilder builder) {
    this(input, new RecognizerSharedState());
    this.builder = builder;
}

public LogicalPlanGenerator(TreeNodeStream input, PigContext pigContext, String scope,
    Map fileNameMap) {
    this( input );
    builder = new LogicalPlanBuilder( pigContext, scope, fileNameMap, input );
}

} // End of @members

@rulecatch {
catch(RecognitionException re) {
    throw re;
}
}

query : ^( QUERY statement* )
;

statement
scope {
   // Parsing context
    String alias; // The alias of the current operator, either given or generated by the parser.
    Integer parallel; // Parallelism
    String inputAlias; // The alias of the input operator
    int inputIndex;
}
@init {
    $statement::inputIndex = 0;
}
 : general_statement
 | split_statement
 | realias_statement
 | assert_statement
 | register_statement
;

split_statement : split_clause
;

realias_statement : realias_clause
;

assert_statement : assert_clause
;

register_statement
: ^( REGISTER QUOTEDSTRING (USING IDENTIFIER AS IDENTIFIER)? )
  {
    // registers are handled by QueryParserDriver and are not actually part of the logical plan
    // so we just ignore them here
  }
;

general_statement
: ^( STATEMENT ( alias { $statement::alias = $alias.name; } )? oa = op_clause parallel_clause? )
  {
      Operator op = builder.lookupOperator( $oa.alias );
      builder.setParallel( (LogicalRelationalOperator)op, $statement::parallel );
  }
;

realias_clause
: ^(REALIAS alias IDENTIFIER)
    {
	    Operator op = builder.lookupOperator( $IDENTIFIER.text );
	    if (op==null) {
	        throw new UndefinedAliasException(input,
	            new SourceLocation( (PigParserNode)$IDENTIFIER ), $IDENTIFIER.text);
	    }
	    builder.putOperator( $alias.name, (LogicalRelationalOperator)op );
    }
;

parallel_clause
 : ^( PARALLEL INTEGER )
   {
       $statement::parallel = Integer.parseInt( $INTEGER.text );
   }
;

alias returns[String name]: IDENTIFIER { $name = $IDENTIFIER.text; }
;

op_clause returns[String alias] :
            define_clause
          | load_clause { $alias = $load_clause.alias; }
          | group_clause { $alias = $group_clause.alias; }
          | store_clause { $alias = $store_clause.alias; }
          | filter_clause { $alias = $filter_clause.alias; }
          | distinct_clause { $alias = $distinct_clause.alias; }
          | limit_clause { $alias = $limit_clause.alias; }
          | sample_clause { $alias = $sample_clause.alias; }
          | order_clause { $alias = $order_clause.alias; }
          | rank_clause { $alias = $rank_clause.alias; }
          | cross_clause { $alias = $cross_clause.alias; }
          | join_clause { $alias = $join_clause.alias; }
          | union_clause { $alias = $union_clause.alias; }
          | stream_clause { $alias = $stream_clause.alias; }
          | mr_clause { $alias = $mr_clause.alias; }
          | foreach_clause { $alias = $foreach_clause.alias; }
          | cube_clause { $alias = $cube_clause.alias; }
          | assert_clause { $alias = $assert_clause.alias; }
;

define_clause
 : ^( DEFINE alias cmd[$alias.name] )
   {
       builder.defineCommand( $alias.name, $cmd.command );
   }
 | ^( DEFINE alias func_clause[FunctionType.UNKNOWNFUNC] )
   {
       builder.defineFunction( $alias.name, $func_clause.funcSpec );
   }
;

cmd[String alias] returns[StreamingCommand command]
@init {
    List shipPaths = new ArrayList();
    List cachePaths = new ArrayList();
    SourceLocation loc = new SourceLocation( (PigParserNode)$cmd.start );
}
 : ^( EXECCOMMAND ( ship_clause[shipPaths] | cache_clause[cachePaths] | input_clause | output_clause | error_clause )* )
   {
       $command = builder.buildCommand( loc, builder.unquote( $EXECCOMMAND.text ), shipPaths,
           cachePaths, $input_clause.inputHandleSpecs, $output_clause.outputHandleSpecs,
           $error_clause.dir, $error_clause.limit );
   }
;

ship_clause[List paths]
 : ^( SHIP path_list[$paths]? )
;

path_list[List paths]
 : ( QUOTEDSTRING { $paths.add( builder.unquote( $QUOTEDSTRING.text ) ); } )+
;

cache_clause[List paths]
 : ^( CACHE path_list[$paths] )
;

input_clause returns[List inputHandleSpecs]
@init {
    $inputHandleSpecs = new ArrayList();
}
 : ^( INPUT ( stream_cmd[true] { $inputHandleSpecs.add( $stream_cmd.handleSpec ); } )+ )
;

stream_cmd[boolean in] returns[HandleSpec handleSpec]
@init {
    String handleName = null;
    FuncSpec fs = null;
    String deserializer = PigStreaming.class.getName() + "()";
    byte ft = $in ? FunctionType.PIGTOSTREAMFUNC : FunctionType.STREAMTOPIGFUNC;
}
@after {
    if( fs != null )
        deserializer =  fs.toString();
    $handleSpec = new HandleSpec( handleName, deserializer );
}
 : ^( STDIN { handleName = "stdin"; }
      ( func_clause[ft] { fs = $func_clause.funcSpec; } )? )
 | ^( STDOUT { handleName = "stdout"; }
      ( func_clause[ft] { fs = $func_clause.funcSpec; } )? )
 | ^( QUOTEDSTRING { handleName = builder.unquote( $QUOTEDSTRING.text ); }
      ( func_clause[ft] { fs = $func_clause.funcSpec; } )? )
;

output_clause returns[List outputHandleSpecs]
@init {
    $outputHandleSpecs = new ArrayList();
}
 : ^( OUTPUT ( stream_cmd[false] { $outputHandleSpecs.add( $stream_cmd.handleSpec ); } )+ )
;

error_clause returns[String dir, Integer limit]
@init {
    $limit = StreamingCommand.MAX_TASKS;
}
 : ^( STDERROR
      ( QUOTEDSTRING
        {
            $dir = builder.unquote( $QUOTEDSTRING.text );
        }
        ( INTEGER
          {
              $limit = Integer.parseInt( $INTEGER.text );
          }
        )?
      )?
    )
;

load_clause returns[String alias]
 : ^( LOAD filename func_clause[FunctionType.LOADFUNC]? as_clause? )
  {
      SourceLocation loc = new SourceLocation( (PigParserNode)$load_clause.start );
      $alias = builder.buildLoadOp( loc, $statement::alias,
          $filename.filename, $func_clause.funcSpec, $as_clause.logicalSchema  );
  }
;

filename returns[String filename]
 : QUOTEDSTRING { $filename = builder.unquote( $QUOTEDSTRING.text ); }
;

as_clause returns[LogicalSchema logicalSchema]
 : ^( AS field_def_list )
   {
        LogicalPlanBuilder.setBytearrayForNULLType($field_def_list.schema);
        $logicalSchema = $field_def_list.schema;
   }
;

field_def[NumValCarrier nvc] returns[LogicalFieldSchema fieldSchema]
@init {
    byte datatype = DataType.NULL;
    if ($nvc==null) {
        $nvc=new NumValCarrier();
    }
}
 : ^( FIELD_DEF IDENTIFIER ( type { datatype = $type.datatype;} )? )
   {
           $fieldSchema = new LogicalFieldSchema( $IDENTIFIER.text, $type.logicalSchema, datatype );
   }
 | ^( FIELD_DEF_WITHOUT_IDENTIFIER ( type { datatype = $type.datatype; } ) )
   {
           $fieldSchema = new LogicalFieldSchema ( $nvc.makeNameFromDataType(datatype) , $type.logicalSchema, datatype );
   }
;

field_def_list returns[LogicalSchema schema]
@init {
    $schema = new LogicalSchema();
    NumValCarrier nvc = new NumValCarrier();
}
 : ( field_def[nvc] { $schema.addField( $field_def.fieldSchema ); } )+
;


type returns[Byte datatype, LogicalSchema logicalSchema]
 : simple_type
   {
        $datatype = $simple_type.datatype;
   }
 | tuple_type
   {
       $datatype = DataType.TUPLE;
       $logicalSchema = $tuple_type.logicalSchema;
   }
 | bag_type
   {
       $datatype = DataType.BAG;
       $logicalSchema = $bag_type.logicalSchema;
   }
 | map_type
   {
       $datatype = DataType.MAP;
       $logicalSchema = $map_type.logicalSchema;
   }
;

simple_type returns[byte datatype]
 : BOOLEAN { $datatype = DataType.BOOLEAN; }
 | INT { $datatype = DataType.INTEGER; }
 | LONG { $datatype = DataType.LONG; }
 | FLOAT { $datatype = DataType.FLOAT; }
 | DOUBLE { $datatype = DataType.DOUBLE; }
 | BIGINTEGER { $datatype = DataType.BIGINTEGER; }
 | BIGDECIMAL { $datatype = DataType.BIGDECIMAL; }
 | DATETIME { $datatype = DataType.DATETIME; }
 | CHARARRAY { $datatype = DataType.CHARARRAY; }
 | BYTEARRAY { $datatype = DataType.BYTEARRAY; }
;

tuple_type returns[LogicalSchema logicalSchema]
 : ^( TUPLE_TYPE
      ( field_def_list
        {
            LogicalPlanBuilder.setBytearrayForNULLType($field_def_list.schema);
            $logicalSchema = $field_def_list.schema;
        }
      )?
    )
;

bag_type returns[LogicalSchema logicalSchema]
 : ^( BAG_TYPE IDENTIFIER? tuple_type? )
   {
       if ($tuple_type.logicalSchema!=null && $tuple_type.logicalSchema.size()==1 && $tuple_type.logicalSchema.getField(0).type==DataType.TUPLE) {
           $logicalSchema = $tuple_type.logicalSchema;
       }
       else {
           LogicalSchema s = new LogicalSchema();
           s.addField(new LogicalFieldSchema($IDENTIFIER.text, $tuple_type.logicalSchema, DataType.TUPLE));
           $logicalSchema = s;
       }
   }
;

map_type returns[LogicalSchema logicalSchema]
 : ^( MAP_TYPE type? )
   {
       LogicalSchema s = null;
       if( $type.datatype != null ) {
           s = new LogicalSchema();
           s.addField( new LogicalFieldSchema( null, $type.logicalSchema, $type.datatype ) );
       }
       $logicalSchema = s;
   }
;

func_clause[byte ft] returns[FuncSpec funcSpec]
@init {
    SourceLocation loc = new SourceLocation( (PigParserNode)$func_clause.start );
}
 : ^( FUNC_REF func_name )
   {
       $funcSpec = builder.lookupFunction( $func_name.funcName );
       if( $funcSpec == null )
           $funcSpec = builder.buildFuncSpec( loc, $func_name.funcName, new ArrayList(), $ft );
   }
 | ^( FUNC func_name func_args? )
   {
       $funcSpec = builder.lookupFunction( $func_name.funcName );
       if( $funcSpec == null ) {
           List argList = new ArrayList();
           if( $func_args.args != null )
               argList = $func_args.args;
           $funcSpec = builder.buildFuncSpec( loc, $func_name.funcName, argList, $ft );
       }
   }
;

func_name returns[String funcName]
@init { StringBuilder buf = new StringBuilder(); }
 : p1 = eid { buf.append( $p1.id ); }
      ( ( PERIOD { buf.append( $PERIOD.text ); } | DOLLAR { buf.append( $DOLLAR.text ); } )
      p2 = eid { buf.append( $p2.id ); } )*
   {
       $funcName = buf.toString();
   }
;

func_args returns[List args]
@init { $args = new ArrayList(); }
: ( QUOTEDSTRING { $args.add( builder.unquote( $QUOTEDSTRING.text ) ); }
    | MULTILINE_QUOTEDSTRING { $args.add( builder.unquote( $MULTILINE_QUOTEDSTRING.text ) ); }
  )+
;

// Sets the current operator as CUBE and creates LogicalExpressionPlans based on the user input.
// Ex: x = CUBE inp BY CUBE(a,b), ROLLUP(c,d);
// For the above example this grammar creates LogicalExpressionPlan with ProjectExpression for a,b and c,d dimensions.
// It also outputs the order of operations i.e in this case CUBE operation followed by ROLLUP operation
// These inputs are passed to buildCubeOp methods which then builds the logical plan for CUBE operator.
// If user specifies STAR or RANGE expression for dimensions then it will be expanded inside buildCubeOp.
cube_clause returns[String alias]
scope {
    LOCube cubeOp;
    MultiMap cubePlans;
    List operations;
    int inputIndex;
}
scope GScope;
@init {
    $cube_clause::cubeOp = builder.createCubeOp();
    $GScope::currentOp = $cube_clause::cubeOp;
    $cube_clause::cubePlans = new MultiMap();
    $cube_clause::operations = new ArrayList();
}
 : ^( CUBE cube_item )
   {
       SourceLocation loc = new SourceLocation( (PigParserNode)$cube_clause.start );
       $alias = builder.buildCubeOp( loc, $cube_clause::cubeOp, $statement::alias,
       $statement::inputAlias, $cube_clause::operations, $cube_clause::cubePlans );
   }
;

cube_item
 : rel ( cube_by_clause
   {
       $cube_clause::cubePlans = $cube_by_clause.plans;
       $cube_clause::operations = $cube_by_clause.operations;
   } )
;

cube_by_clause returns[List operations, MultiMap plans]
@init {
    $operations = new ArrayList();
    $plans = new MultiMap();
}
 : ^( BY cube_or_rollup { $operations = $cube_or_rollup.operations; $plans = $cube_or_rollup.plans; })
;

cube_or_rollup returns[List operations, MultiMap plans]
@init {
    $operations = new ArrayList();
    $plans = new MultiMap();
}
 : ( cube_rollup_list
   {
       $operations.add($cube_rollup_list.operation);
       $plans.put( $cube_clause::inputIndex, $cube_rollup_list.plans);
       $cube_clause::inputIndex++;
   } )+
;

cube_rollup_list returns[String operation, List plans]
@init {
    $plans = new ArrayList();
}
 : ^( ( CUBE { $operation = "CUBE"; } | ROLLUP { $operation = "ROLLUP"; } ) cube_by_expr_list { $plans = $cube_by_expr_list.plans; } )
;

cube_by_expr_list returns[List plans]
@init {
    $plans = new ArrayList();
}
 : ( cube_by_expr { $plans.add( $cube_by_expr.plan ); } )+
;

cube_by_expr returns[LogicalExpressionPlan plan]
@init {
    $plan = new LogicalExpressionPlan();
}
 : col_range[$plan]
 | expr[$plan]
 | STAR
   {
       builder.buildProjectExpr( new SourceLocation( (PigParserNode)$STAR ), $plan, $GScope::currentOp, 0, null, -1 );
   }
;

group_clause returns[String alias]
scope {
    MultiMap groupPlans;
    int inputIndex;
    List inputAliases;
    List innerFlags;
}
scope GScope;
@init {
    $GScope::currentOp = builder.createGroupOp();
    $group_clause::groupPlans = new MultiMap();
    $group_clause::inputAliases = new ArrayList();
    $group_clause::innerFlags = new ArrayList();
    GROUPTYPE groupType = GROUPTYPE.REGULAR;
    SourceLocation loc = new SourceLocation( (PigParserNode)$group_clause.start );
    int oldStatementIndex = $statement::inputIndex;
}
@after { $statement::inputIndex = oldStatementIndex; }
 : ^( GROUP group_item+ ( group_type { groupType = $group_type.type; ((LOCogroup)$GScope::currentOp).pinOption(LOCogroup.OPTION_GROUPTYPE); } )? partition_clause? )
   {
       $alias = builder.buildGroupOp( loc, (LOCogroup)$GScope::currentOp, $statement::alias,
           $group_clause::inputAliases, $group_clause::groupPlans, groupType, $group_clause::innerFlags,
           $partition_clause.partitioner );
   }
 | ^( COGROUP group_item+ ( group_type { groupType = $group_type.type;((LOCogroup)$GScope::currentOp).pinOption(LOCogroup.OPTION_GROUPTYPE); } )? partition_clause? )
   {
       $alias = builder.buildGroupOp( loc, (LOCogroup)$GScope::currentOp, $statement::alias,
           $group_clause::inputAliases, $group_clause::groupPlans, groupType, $group_clause::innerFlags,
           $partition_clause.partitioner );
   }
;

group_type returns[GROUPTYPE type]
 : QUOTEDSTRING
   {
       $type =builder.parseGroupType( $QUOTEDSTRING.text, new SourceLocation( (PigParserNode)$QUOTEDSTRING ) );
   }
;

group_item
@init { boolean inner = false; }
 : rel ( join_group_by_clause
         {
             $group_clause::groupPlans.put( $group_clause::inputIndex, $join_group_by_clause.plans );
         }
         | ALL
         {
             LogicalExpressionPlan plan = new LogicalExpressionPlan();
             ConstantExpression ce = new ConstantExpression( plan, "all");
             ce.setLocation( new SourceLocation( (PigParserNode)$ALL ) );
             List plans = new ArrayList( 1 );
             plans.add( plan );
             $group_clause::groupPlans.put( $group_clause::inputIndex, plans );
         }
         | ANY
         {
             LogicalExpressionPlan plan = new LogicalExpressionPlan();
             UserFuncExpression udf = new UserFuncExpression( plan, new FuncSpec( GFAny.class.getName() ) );
             udf.setLocation( new SourceLocation( (PigParserNode)$ANY ) );
             List plans = new ArrayList( 1 );
             plans.add( plan );
             $group_clause::groupPlans.put( $group_clause::inputIndex, plans );
         }
        ) ( INNER { inner =  true; } | OUTER )?
   {
       $group_clause::inputAliases.add( $statement::inputAlias );
       $group_clause::innerFlags.add( inner );
       $group_clause::inputIndex++;
       $statement::inputIndex++;
   }
;

rel
 : alias
   {
       $statement::inputAlias = $alias.name;
   }
 | previous_rel
   {
       $statement::inputAlias = $previous_rel.name;
   }
 | inline_op
;

previous_rel returns[String name] : ARROBA { $name = builder.getLastRel(new SourceLocation((PigParserNode)$ARROBA)); }
;

inline_op
@init {
    String al = $statement::alias;
    $statement::alias = null;
}
@after {
    $statement::alias = al;
}
 : op_clause parallel_clause?
   {
       Operator op = builder.lookupOperator( $op_clause.alias );
       builder.setParallel( (LogicalRelationalOperator)op, $statement::parallel );
       $statement::inputAlias = $op_clause.alias;
   }
;

flatten_generated_item returns[LogicalExpressionPlan plan, boolean flattenFlag, LogicalSchema schema]
@init {
    $plan = new LogicalExpressionPlan();
}
 : ( flatten_clause[$plan] { $flattenFlag = true; }
   | col_range[$plan]
   | expr[$plan]
   | STAR
     {
         builder.buildProjectExpr( new SourceLocation( (PigParserNode)$STAR ), $plan, $GScope::currentOp,
             $statement::inputIndex, null, -1 );
     }
   )
   ( field_def_list { $schema = $field_def_list.schema; } )?
;

flatten_clause[LogicalExpressionPlan plan]
 : ^( FLATTEN expr[$plan] )
;

store_clause returns[String alias]
 : ^( STORE rel filename func_clause[FunctionType.STOREFUNC]? )
   {
       SourceLocation loc = new SourceLocation( (PigParserNode)$STORE );
       $alias= builder.buildStoreOp( loc, $statement::alias,
          $statement::inputAlias, $filename.filename, $func_clause.funcSpec );
   }
;

assert_clause returns[String alias]
scope GScope;
@init {
    $GScope::currentOp = builder.createFilterOp();
    LogicalExpressionPlan exprPlan = new LogicalExpressionPlan();
}
 : ^( ASSERT rel cond[exprPlan] comment? )
   {
       SourceLocation loc = new SourceLocation( (PigParserNode)$ASSERT );
       $alias= builder.buildAssertOp(loc, (LOFilter)$GScope::currentOp, $statement::alias,
          $statement::inputAlias, $cond.expr, $comment.comment, exprPlan);
   }
;

comment returns[String comment]
 : QUOTEDSTRING { $comment = builder.unquote( $QUOTEDSTRING.text ); }
;

filter_clause returns[String alias]
scope GScope;
@init {
    LogicalExpressionPlan exprPlan = new LogicalExpressionPlan();
    $GScope::currentOp = builder.createFilterOp();
}
 : ^( FILTER rel cond[exprPlan] )
   {
       $alias = builder.buildFilterOp( new SourceLocation( (PigParserNode)$FILTER ),
           (LOFilter)$GScope::currentOp, $statement::alias,
           $statement::inputAlias, exprPlan );
   }
;

cond[LogicalExpressionPlan exprPlan] returns[LogicalExpression expr]
 : ^( OR left = cond[exprPlan] right = cond[exprPlan] )
   {
       $expr = new OrExpression( $exprPlan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$OR ) );
   }
 | ^( AND left = cond[exprPlan] right = cond[exprPlan] )
   {
       $expr = new AndExpression( $exprPlan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$AND ) );
   }
 | ^( NOT c = cond[exprPlan] )
   {
       $expr = new NotExpression( $exprPlan, $c.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$NOT ) );
   }
 | ^( NULL expr[$exprPlan] NOT? )
   {
       $expr = new IsNullExpression( $exprPlan, $expr.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$NULL ) );
       if( $NOT != null ) {
           $expr = new NotExpression( $exprPlan, $expr );
           $expr.setLocation( new SourceLocation( (PigParserNode)$NOT ) );
       }
   }
 | ^( rel_op_eq e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new EqualExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_eq.start ) );
   }
 | ^( rel_op_ne e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new NotEqualExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_ne.start ) );
   }
 | ^( rel_op_lt e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new LessThanExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_lt.start ) );
   }
 | ^( rel_op_lte e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new LessThanEqualExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_lte.start ) );
   }
 | ^( rel_op_gt e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new GreaterThanExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_gt.start ) );
   }
 | ^( rel_op_gte e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new GreaterThanEqualExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$rel_op_gte.start ) );
   }
 | ^( STR_OP_MATCHES e1 = expr[$exprPlan] e2 = expr[$exprPlan] )
   {
       $expr = new RegexExpression( $exprPlan, $e1.expr, $e2.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$STR_OP_MATCHES ) );
   }
 | in_eval[$exprPlan]
   {
       $expr = $in_eval.expr;
   }
 | func_eval[$exprPlan]
   {
       $expr = $func_eval.expr;
   }
 | ^( BOOL_COND e1 = expr[$exprPlan] )
   {
       $expr = $e1.expr;
       $expr.setLocation( new SourceLocation( (PigParserNode)$BOOL_COND ) );
   }
;

in_eval[LogicalExpressionPlan plan] returns[LogicalExpression expr]
@init {
    List lhsExprs = new ArrayList();
    List rhsExprs = new ArrayList();
}
 : ^( IN ( ^( IN_LHS lhs = expr[$plan] ) { lhsExprs.add($lhs.expr); }
           ^( IN_RHS rhs = expr[$plan] ) { rhsExprs.add($rhs.expr); } )+ )
    {
        // Convert IN tree to nested or expressions. Please also see
        // QueryParser.g for how IN tree is constructed from IN expression.
        EqualExpression firstBoolExpr = new EqualExpression(plan, lhsExprs.get(0), rhsExprs.get(0));
        if (lhsExprs.size() == 1) {
            $expr = firstBoolExpr;
        } else {
            OrExpression currOrExpr = null;
            OrExpression prevOrExpr = null;
            for (int i = 1; i < lhsExprs.size(); i++) {
                EqualExpression boolExpr = new EqualExpression(plan, lhsExprs.get(i), rhsExprs.get(i));
                currOrExpr = new OrExpression( $plan, prevOrExpr == null ? firstBoolExpr : prevOrExpr, boolExpr );
                prevOrExpr = currOrExpr;
            }
            $expr = currOrExpr;
        }
        $expr.setLocation( new SourceLocation( (PigParserNode)$in_eval.start ) );
    }
;

func_eval[LogicalExpressionPlan plan] returns[LogicalExpression expr]
@init {
    List args = new ArrayList();
}
 : ^( FUNC_EVAL func_name ( real_arg[$plan] { args.add( $real_arg.expr ); } )* )
   {
       SourceLocation loc = new SourceLocation( (PigParserNode)$func_name.start );
       $expr = builder.buildUDF( loc, $plan, $func_name.funcName, args );
   }
 | ^( INVOKER_FUNC_EVAL package_name=IDENTIFIER function_name=IDENTIFIER is_static=IDENTIFIER ( real_arg[$plan] { args.add( $real_arg.expr ); } )* )
   {
       SourceLocation loc = new SourceLocation( (PigParserNode)$function_name );
       $expr = builder.buildInvokerUDF( loc, $plan, $package_name.text, $function_name.text, Boolean.parseBoolean($is_static.text), args );
   }
;

real_arg [LogicalExpressionPlan plan] returns[LogicalExpression expr]
 : e = expr[$plan] { $expr = $e.expr; }
 | STAR
   {
       $expr = builder.buildProjectExpr( new SourceLocation( (PigParserNode)$STAR ), $plan, $GScope::currentOp,
           $statement::inputIndex, null, -1 );
   }
 | cr = col_range[$plan] { $expr = $cr.expr;}
;

expr[LogicalExpressionPlan plan] returns[LogicalExpression expr]
 : ^( PLUS left = expr[$plan] right = expr[$plan] )
   {
       $expr = new AddExpression( $plan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$PLUS ) );
   }
 | ^( MINUS left = expr[$plan] right = expr[$plan] )
   {
       $expr = new SubtractExpression( $plan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$MINUS ) );
   }
 | ^( STAR left = expr[$plan] right = expr[$plan] )
   {
       $expr = new MultiplyExpression( $plan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$STAR ) );
   }
 | ^( DIV left = expr[$plan] right = expr[$plan] )
   {
       $expr = new DivideExpression( $plan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$DIV ) );
   }
 | ^( PERCENT left = expr[$plan] right = expr[$plan] )
   {
       $expr = new ModExpression( $plan, $left.expr, $right.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$PERCENT ) );
   }
 | const_expr[$plan]
   {
       $expr = $const_expr.expr;
   }
 | var_expr[$plan]
   {
       $expr = $var_expr.expr;
   }
 | ^( NEG e = expr[$plan] )
   {
       $expr = new NegativeExpression( $plan, $e.expr );
       $expr.setLocation( new SourceLocation( (PigParserNode)$e.start ) );
   }
 | ^( CAST_EXPR type_cast e = expr[$plan] ) // cast expr
   {
       $expr = new CastExpression( $plan, $e.expr, $type_cast.fieldSchema );
       $expr.setLocation( new SourceLocation( (PigParserNode)$type_cast.start ) );
   }
 | ^( EXPR_IN_PAREN e = expr[$plan] ) // unary expr
   {
       $expr = $e.expr;
   }
;

type_cast returns[LogicalFieldSchema fieldSchema]
 : simple_type
   {
        $fieldSchema = new LogicalFieldSchema( null, null, $simple_type.datatype );
   }
 | map_type
   {
       $fieldSchema = new LogicalFieldSchema( null, $map_type.logicalSchema, DataType.MAP );
   }
 | tuple_type_cast
   {
       $fieldSchema = new LogicalFieldSchema( null, $tuple_type_cast.logicalSchema, DataType.TUPLE );
   }
 | bag_type_cast
   {
       $fieldSchema = new LogicalFieldSchema( null, $bag_type_cast.logicalSchema, DataType.BAG );
   }
;

tuple_type_cast returns[LogicalSchema logicalSchema]
@init {
    $logicalSchema = new LogicalSchema();
}
 : ^( TUPLE_TYPE_CAST ( type_cast { $logicalSchema.addField( $type_cast.fieldSchema ); } )* )
;

bag_type_cast returns[LogicalSchema logicalSchema]
@init {
    $logicalSchema = new LogicalSchema();
}
 : ^( BAG_TYPE_CAST tuple_type_cast? )
   {
       $logicalSchema.addField( new LogicalFieldSchema( null, $tuple_type_cast.logicalSchema, DataType.TUPLE ) );
   }
;

var_expr[LogicalExpressionPlan plan] returns[LogicalExpression expr]
@init {
    List