org.apache.hadoop.hive.serde2.dynamic_type.thrift_grammar.jjt Maven / Gradle / Ivy

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

options {
 MULTI=true;
 STATIC = false;
 NODE_PREFIX = "DynamicSerDe";
}


PARSER_BEGIN(thrift_grammar)

package org.apache.hadoop.hive.serde2.dynamic_type;

import java.util.*;
import java.io.*;
import java.net.*;
import com.facebook.thrift.protocol.*;
import com.facebook.thrift.transport.*;
import org.apache.hadoop.hive.serde2.dynamic_type.*;

public class thrift_grammar {

    private List include_path = null;

    // for computing the autogenerated field ids in thrift
    private  int field_val;

    // store types and tables
    // separately because one cannot use a table (ie service.method) as a Struct like type.
    protected Map types;
    protected Map tables;

    // system include path
    final private static String default_include_path[] = {  "/usr/local/include","/usr/include","/usr/local/include/thrift/if","/usr/local/include/fb303/if" };

    // need three params to differentiate between this and 2 param method auto generated since
    // some calls in the autogenerated code use null param for 2nd param and thus ambiguous.
    protected thrift_grammar(InputStream is, List include_path, boolean junk) {
        this(is,null);
        this.types = new HashMap () ;
        this.tables = new HashMap () ;
        this.include_path = include_path;
        this.field_val = -1;
    }

    // find the file on the include path
    private static File findFile(String fname, List include_path) {
        for(String path: include_path) {
            final String full = path + "/" + fname;
            File f = new File(full);
            if(f.exists()) {
                return f;
            }
        }
        return null;
    }

    public static void main(String args[]) {
        String filename = null;
        List include_path = new ArrayList();

        for(String path: default_include_path)  {
            include_path.add(path);
        }
        for(int i = 0; i < args.length; i++) {
            String arg = args[i];
            if(arg.equals("--include") && i + 1 < args.length) {
                include_path.add(args[++i]);
            }
            if(arg.equals("--file") && i + 1 < args.length) {
                filename = args[++i];
            }
        }

        InputStream is = System.in;
        if(filename != null) {
            try {
                  is = new FileInputStream(findFile(filename, include_path));
            } catch(IOException e) {
            }
        }
        thrift_grammar t = new thrift_grammar(is,include_path,false);

          try {
            t.Start();
        } catch (Exception e) {
            System.out.println("Parse error.");
            System.out.println(e.getMessage());
            e.printStackTrace();
        }
    }
}

PARSER_END(thrift_grammar)



SKIP :
{
  " "
| "\t"
| "\n"
| "\r"
| <"#"(~["\n"])* ("\n"|"\r"|"\r\n")>
| <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")>
| <"/*" (~["*"])* "*" (~["/"] (~["*"])* "*")* "/">
}


/**
 * HELPER DEFINITIONS, COMMENTS, CONSTANTS, AND WHATNOT
 */

TOKEN:
{
|
 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
 |
 |
 |
 |
 |
 |
 |
 |
 |
 |
 |
 |
 | 

}

TOKEN: {


|
)*"."()+(["e","E"](["+","-"])?()+)?>
|
||"_")(||"."|"_")*>
|
<#LETTER: (["a"-"z", "A"-"Z" ]) >
|
<#DIGIT: ["0"-"9"] >
|

|

}


SimpleNode Start() : {}
{
  HeaderList()  ([CommaOrSemicolon()] Definition())+
 {
    return jjtThis;
  }
}

SimpleNode HeaderList() : {}
{
  (Header())*
 {
    return jjtThis;
  }

}

SimpleNode Header() : {}
{
  Include()
    {
    return jjtThis;
    }
| Namespace()
    {
    return jjtThis;
    }
}

SimpleNode Namespace() : {}
{
    
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
|
 
{
  return jjtThis;
}
}


SimpleNode Include() :  {
 String fname;
 boolean found = false;
}
{
   
   fname=.image
{
    // bugbug somewhat fragile below substring expression
    fname = fname.substring(1,fname.length() - 1);

    // try to find the file on the include path
    File f = thrift_grammar.findFile(fname, this.include_path);
    if(f != null) {
        found = true;
        try {
            FileInputStream fis = new FileInputStream(f);
            thrift_grammar t = new thrift_grammar(fis,this.include_path, false);
            t.Start();
            fis.close();
            found = true;
            // add in what we found to our type and table tables.
            this.tables.putAll(t.tables);
            this.types.putAll(t.types);
        } catch (Exception e) {
            System.out.println("File: " + fname + " - Oops.");
            System.out.println(e.getMessage());
            e.printStackTrace();
        }
    }
    if(!found) {
        throw new RuntimeException("include file not found: " + fname);
    }
    return jjtThis;
}
}


SimpleNode Definition() : {}
{
  Const()
    {
    return jjtThis;
    }
| Service()
    {
    return jjtThis;
    }
| TypeDefinition()
    {
    return jjtThis;
    }
}

SimpleNode TypeDefinition() : {}
{
  Typedef()
    {
   return jjtThis;
    }
| Enum()
    {
   return jjtThis;
    }
| Senum()
    {
   return jjtThis;
    }
| Struct()
    {
   return jjtThis;
    }
| Xception()
    {
   return jjtThis;
    }

}

DynamicSerDeTypedef Typedef() : {}
{
  
  DefinitionType()
  jjtThis.name = .image
    {
        // store the type for later retrieval
        this.types.put(jjtThis.name, jjtThis);
        return jjtThis;
    }
}


// returning void because we ignore this production.
void CommaOrSemicolon() : {}
{
  ","
|
  ";"
{
}
}

SimpleNode Enum() : {}
{
    "{" EnumDefList() "}"
    {
    return jjtThis;
    }
}

SimpleNode EnumDefList() : {}
{
  (EnumDef())+
    {
    return jjtThis;
    }
}

SimpleNode EnumDef() : {}
{
   ["=" ] [CommaOrSemicolon()]
    {
    return jjtThis;
    }
}

SimpleNode Senum() : {}
{
    "{" SenumDefList() "}"
    {
    return jjtThis;
    }
}

SimpleNode SenumDefList() : {}
{
  (SenumDef())+
    {
    return jjtThis;
    }
}

SimpleNode SenumDef() : {}
{
   [CommaOrSemicolon()]
    {
    return jjtThis;
    }
}


SimpleNode Const() : {}
{
   FieldType()  "=" ConstValue() [CommaOrSemicolon()]
    {
    return jjtThis;
    }
}

SimpleNode ConstValue() : {}
{
  
    {
    }
| 
    {
    }
| 
    {
    }
| 
    {
    }
| ConstList()
    {
    }
| ConstMap()
    {
    return jjtThis;
    }
}

SimpleNode ConstList() : {}
{
  "[" ConstListContents() "]"
    {
    return jjtThis;
    }
}

SimpleNode ConstListContents() : {}
{
  (ConstValue() [CommaOrSemicolon()])+
    {
    return jjtThis;
    }
}

SimpleNode ConstMap() : {}
{
  "{" ConstMapContents() "}"
    {
    return jjtThis;
    }
}

SimpleNode ConstMapContents() : {}
{
  (ConstValue() ":" ConstValue() [CommaOrSemicolon()])+
    {
    }
|
    {
    return jjtThis;
    }
}

DynamicSerDeStruct Struct() :  {

}
{
  
  jjtThis.name = .image
  "{"
  FieldList()
  "}"
    {
   this.types.put(jjtThis.name,jjtThis);
    return jjtThis;
    }
}


SimpleNode Xception() : {}
{
    "{" FieldList() "}"
    {
    return jjtThis;
    }
}


SimpleNode Service() : {}
{
  
  
  Extends()
  "{"
  FlagArgs()
  (Function())+
  UnflagArgs()
  "}"
    {
        // at some point, these should be inserted as a "db"
        return jjtThis;
    }
}

SimpleNode FlagArgs() : {}
{
    {
    return jjtThis;
    }
}

SimpleNode UnflagArgs() : {}
{
    {
    return jjtThis;
    }
}

SimpleNode Extends() : {}
{
   
    {
    return jjtThis;
    }
|
    {
    return jjtThis;
    }
}


DynamicSerDeFunction Function() : {}
{
  // metastore ignores async and type
  Async()
  FunctionType()

  // the name of the function/table
  jjtThis.name = .image
  "("
  FieldList()
  ")"
  Throws()
  [CommaOrSemicolon()]

    {
        this.tables.put(jjtThis.name, jjtThis);
        return jjtThis;
    }
}

void Async() : {}
{
  
|
{}
}

void Throws() : {}
{
   "(" FieldList() ")"
|
{}
}


// nothing special - just use the DynamicSerDeFieldList's children methods to access the fields
DynamicSerDeFieldList FieldList() : {
   this.field_val = -1;
}
{
  (Field())*    {
    return jjtThis;
  }
}


DynamicSerDeField Field() : {

 String fidnum = "";
 String fid;
}
{

  // parse the field id which is optional
  [fidnum=.image ":"]

  // is this field required or optional? default is optional
  FieldRequiredness()

  // field type - obviously not optional
  FieldType()

   // the name of the field - not optional
  jjtThis.name = .image

  // does it have = some value?
  FieldValue()

  // take it or leave it
  [CommaOrSemicolon()]

    {
    if(fidnum.length() > 0) {
       int fidInt = Integer.valueOf(fidnum);
       jjtThis.fieldid = fidInt;
    } else {
       jjtThis.fieldid = this.field_val--;
    }
    return jjtThis;
    }
}



DynamicSerDeFieldRequiredness FieldRequiredness() : {}
{
  
    {
      jjtThis.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Required;
      return jjtThis;
    }
| 
    {
      jjtThis.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Optional;
        return jjtThis;
    }
| 
    {
      jjtThis.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable;
        return jjtThis;
    }
|
   {
    return jjtThis;
   }
}

SimpleNode FieldValue() : {}
{
  "="
     ConstValue()
    {
        return jjtThis;
    }
|
{
        return jjtThis;
}
}

SimpleNode DefinitionType() : {}
{
//  BaseType() xxx
  TypeString()
    {
    return jjtThis;
    }
| TypeBool()
    {
    return jjtThis;
    }
| Typei16()
    {
    return jjtThis;
    }
| Typei32()
    {
    return jjtThis;
    }
| Typei64()
    {
    return jjtThis;
    }
| TypeDouble()
    {
    return jjtThis;
    }
|  TypeMap()
    {
    return jjtThis;
    }
| TypeSet()
    {
    return jjtThis;
    }
| TypeList()
    {
    return jjtThis;
    }
}

void FunctionType() : {}
{
  FieldType()
| 
{}
}

DynamicSerDeFieldType FieldType() : {
}

{
  TypeString()
    {
     return jjtThis;
    }
| TypeBool()
    {
    return jjtThis;
    }
| Typei16()
    {
    return jjtThis;
    }
| Typei32()
    {
    return jjtThis;
    }
| Typei64()
    {
    return jjtThis;
    }
| TypeDouble()
    {
    return jjtThis;
    }
|
   TypeMap()
    {
    return jjtThis;
    }
|
   TypeSet()
    {
    return jjtThis;
    }
|
   TypeList()
    {
    return jjtThis;
    }
|
  jjtThis.name = .image
    {

    if (this.types.get(jjtThis.name) == null) {
      System.err.println("ERROR: DDL specifying type " + jjtThis.name + " which has not been defined");  
      throw new RuntimeException("specifying type " + jjtThis.name + " which has not been defined");  
    }
     // lookup the specified type and set this nodes type to it. Precludes forward and self references for now.
    jjtThis.jjtAddChild(this.types.get(jjtThis.name),0);
    return jjtThis;
    }
}

DynamicSerDeTypeString TypeString() : {}
{
  
  {
  return jjtThis;
  }
}

DynamicSerDeTypeByte TypeByte() : {
}
{
   
   {
   return jjtThis;
   }
}

DynamicSerDeTypei16 Typei16() : {
}
{
  
  {
   return jjtThis;
  }
}

DynamicSerDeTypei32 Typei32() : {}
{
  
  {
   return jjtThis;
  }
}

DynamicSerDeTypei64 Typei64() : {}
{
  
  {
   return jjtThis;
  }
}

DynamicSerDeTypeDouble TypeDouble() : {}
{
  
  {
   return jjtThis;
  }
}

DynamicSerDeTypeBool TypeBool() : {}
{
  
  {
   return jjtThis;
  }
}

DynamicSerDeTypeMap TypeMap() : {}
{
  
   "<"
  FieldType()
  ","
  FieldType()
   ">"
    {
    return jjtThis;
    }
}

DynamicSerDeTypeSet TypeSet() : {}
{
  
  "<"

  FieldType()

  ">"
    {
    return jjtThis;
    }
}

DynamicSerDeTypeList TypeList() : {}
{
  
  "<"

  FieldType()

  ">"
    {
    return jjtThis;
    }
}