com.thaiopensource.relaxng.parse.compact.CompactSyntax.jj Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of trang Show documentation
Trang, a multi-format schema converter based on RELAX NG.
The newest version!
options {
  STATIC = false;
  UNICODE_INPUT = true;
  JAVA_UNICODE_ESCAPE = true;
}

PARSER_BEGIN(CompactSyntax)

package com.thaiopensource.relaxng.parse.compact;

import com.thaiopensource.relaxng.parse.Annotations;
import com.thaiopensource.relaxng.parse.BuildException;
import com.thaiopensource.relaxng.parse.CommentList;
import com.thaiopensource.relaxng.parse.Context;
import com.thaiopensource.relaxng.parse.DataPatternBuilder;
import com.thaiopensource.relaxng.parse.Div;
import com.thaiopensource.relaxng.parse.ElementAnnotationBuilder;
import com.thaiopensource.relaxng.parse.Grammar;
import com.thaiopensource.relaxng.parse.GrammarSection;
import com.thaiopensource.relaxng.parse.IllegalSchemaException;
import com.thaiopensource.relaxng.parse.Include;
import com.thaiopensource.relaxng.parse.IncludedGrammar;
import com.thaiopensource.relaxng.parse.SchemaBuilder;
import com.thaiopensource.relaxng.parse.Scope;

import com.thaiopensource.util.Ref;
import com.thaiopensource.xml.util.WellKnownNamespaces;

import com.thaiopensource.util.Localizer;

import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.LocatorImpl;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.io.Reader;

public class CompactSyntax,
                           AnnotationsImpl extends Annotations>
    implements Context {
  private static final int IN_ELEMENT = 0;
  private static final int IN_ATTRIBUTE = 1;
  private static final int IN_ANY_NAME = 2;
  private static final int IN_NS_NAME = 4;

  private String defaultNamespace = SchemaBuilder.INHERIT_NS;
  private String compatibilityPrefix = null;
  private SchemaBuilder sb;
  private String sourceUri;
  private ErrorHandler eh;
  private final Map namespaceMap = new HashMap();
  private final Map datatypesMap = new HashMap();
  private boolean hadError = false;
  private static final Localizer localizer = new Localizer(CompactSyntax.class);
  private final Set attributeNames = new HashSet();
  private boolean annotationsIncludeElements = false;

  final class LocatedString {
    private final String str;
    private final Token tok;

    LocatedString(String str, Token tok) {
      this.str = str;
      this.tok = tok;
    }

    String getString() {
      return str;
    }

    Location getLocation() {
      return makeLocation(tok);
    }

    Token getToken() {
      return tok;
    }

  }

  public CompactSyntax(Reader r, String sourceUri, SchemaBuilder sb, ErrorHandler eh) {
    this(r);
    this.sourceUri = sourceUri;
    this.sb = sb;
    this.eh = eh;
    // this causes the root pattern to have non-null annotations
    // which is useful because it gives a context to trang
    this.topLevelComments = sb.makeCommentList();
  }

  Pattern parse(Scope scope) throws IllegalSchemaException {
    try {
      Pattern p = Input(scope);
      if (!hadError)
        return p;
    }
    catch (ParseException e) {
      error("syntax_error", e.currentToken.next);
    }
    catch (EscapeSyntaxException e) {
      reportEscapeSyntaxException(e);
    }
    throw new IllegalSchemaException();
  }

  Pattern parseInclude(IncludedGrammar g) throws IllegalSchemaException {
    try {
      Pattern p = IncludedGrammar(g);
      if (!hadError)
        return p;
    }
    catch (ParseException e) {
      error("syntax_error", e.currentToken.next);
    }
    catch (EscapeSyntaxException e) {
      reportEscapeSyntaxException(e);
    }
    throw new IllegalSchemaException();
  }

  private void checkNsName(int context, LocatedString ns) {
    if ((context & IN_NS_NAME) != 0)
      error("ns_name_except_contains_ns_name", ns.getToken());
  }

  private void checkAnyName(int context, Token t) {
    if ((context & IN_NS_NAME) != 0)
      error("ns_name_except_contains_any_name", t);
    if ((context & IN_ANY_NAME) != 0)
      error("any_name_except_contains_any_name", t);
  }

  private void error(String key, Token tok) {
    doError(localizer.message(key), tok);
  }

  private void error(String key, String arg, Token tok) {
    doError(localizer.message(key, arg), tok);
  }

  private void error(String key, String arg1, String arg2, Token tok) {
    doError(localizer.message(key, arg1, arg2), tok);
  }

  private void doError(String message, Token tok) {
    hadError = true;
    if (eh != null) {
      LocatorImpl loc = new LocatorImpl();
      loc.setLineNumber(tok.beginLine);
      loc.setColumnNumber(tok.beginColumn);
      loc.setSystemId(sourceUri);
      try {
	eh.error(new SAXParseException(message, loc));
      }
      catch (SAXException se) {
	throw new BuildException(se);
      }
    }
  }

  private void reportEscapeSyntaxException(EscapeSyntaxException e) {
    if (eh != null) {
      LocatorImpl loc = new LocatorImpl();
      loc.setLineNumber(e.getLineNumber());
      loc.setColumnNumber(e.getColumnNumber());
      loc.setSystemId(sourceUri);
      try {
	eh.error(new SAXParseException(localizer.message(e.getKey()), loc));
      }
      catch (SAXException se) {
	throw new BuildException(se);
      }
    }
  }

  private static String unquote(String s) {
    if (s.length() >= 6 && s.charAt(0) == s.charAt(1)) {
      s = s.replace('\u0000', '\n');
      return s.substring(3, s.length() - 3);
    }
    else
      return s.substring(1, s.length() - 1);
  }

  Location makeLocation(Token t) {
    return sb.makeLocation(sourceUri, t.beginLine, t.beginColumn);
  }

  String getCompatibilityPrefix() {
    if (compatibilityPrefix == null) {
      compatibilityPrefix = "a";
      while (namespaceMap.get(compatibilityPrefix) != null)
        compatibilityPrefix = compatibilityPrefix + "a";
    }
    return compatibilityPrefix;
  }

  public String resolveNamespacePrefix(String prefix) {
    String result = namespaceMap.get(prefix);
    if (result == null || result.length() == 0)
      return null;
    return result;
  }

  public Set prefixes() {
    return namespaceMap.keySet();
  }

  public String getBaseUri() {
    return sourceUri;
  }

  public boolean isUnparsedEntity(String entityName) {
    return false;
  }

  public boolean isNotation(String notationName) {
    return false;
  }

  public Context copy() {
    return this;
  }

  private Context getContext() {
    return this;
  }

  private CommentListImpl getComments() {
    return getComments(getTopLevelComments());
  }

  private CommentListImpl topLevelComments;

  private CommentListImpl getTopLevelComments() {
    CommentListImpl tem = topLevelComments;
    topLevelComments = null;
    return tem;
  }

  private void noteTopLevelComments() {
    topLevelComments = getComments(topLevelComments);
  }

  private void topLevelComments(GrammarSection section) {
    section.topLevelComment(getComments(null));
  }

  private Token lastCommentSourceToken = null;

  private CommentListImpl getComments(CommentListImpl comments) {
    Token nextToken = getToken(1);
    if (lastCommentSourceToken != nextToken) {
      if (lastCommentSourceToken == null)
        lastCommentSourceToken = token;
      do {
	lastCommentSourceToken = lastCommentSourceToken.next;
	Token t = lastCommentSourceToken.specialToken;
	if (t != null) {
	  while (t.specialToken != null)
	    t = t.specialToken;
	  if (comments == null)
	    comments = sb.makeCommentList();
	  for (; t != null; t = t.next) {
            String s = mungeComment(t.image);
            Location loc = makeLocation(t);
            if (t.next != null
                && t.next.kind == CompactSyntaxConstants.SINGLE_LINE_COMMENT_CONTINUE) {
              StringBuffer buf = new StringBuffer(s);
              do {
                t = t.next;
                buf.append('\n');
                buf.append(mungeComment(t.image));
              } while (t.next != null
                       && t.next.kind == CompactSyntaxConstants.SINGLE_LINE_COMMENT_CONTINUE);
              s = buf.toString();
            }
	    comments.addComment(s, loc);
          }
	}
      } while (lastCommentSourceToken != nextToken);
    }
    return comments;
  }

  private Pattern afterPatternComments(Pattern p) {
    CommentListImpl comments = getComments(null);
    if (comments == null)
      return p;
    return sb.commentAfterPattern(p, comments);
  }

  private NameClass afterNameClassComments(NameClass nc) {
    CommentListImpl comments = getComments(null);
    if (comments == null)
      return nc;
    return sb.commentAfterNameClass(nc, comments);
  }

  private static String mungeComment(String image) {
    int i = image.indexOf('#') + 1;
    while (i < image.length() && image.charAt(i) == '#')
      i++;
    if (i < image.length() && image.charAt(i) == ' ')
      i++;
    return image.substring(i);
  }

  private AnnotationsImpl getCommentsAsAnnotations() {
    CommentListImpl comments = getComments();
    if (comments == null)
      return null;
    return sb.makeAnnotations(comments, getContext());
  }

  private AnnotationsImpl addCommentsToChildAnnotations(AnnotationsImpl a) {
    CommentListImpl comments = getComments();
    if (comments == null)
      return a;
    if (a == null)
      a = sb.makeAnnotations(null, getContext());
    a.addComment(comments);
    return a;
  }

  private AnnotationsImpl addCommentsToLeadingAnnotations(AnnotationsImpl a) {
    CommentListImpl comments = getComments();
    if (comments == null)
      return a;
    if (a == null)
      return sb.makeAnnotations(comments, getContext());
    a.addLeadingComment(comments);
    return a;
  }

  private AnnotationsImpl getTopLevelCommentsAsAnnotations() {
    CommentListImpl comments = getTopLevelComments();
    if (comments == null)
      return null;
    return sb.makeAnnotations(comments, getContext());
  }

  private void clearAttributeList() {
    attributeNames.clear();
  }

  private void addAttribute(Annotations a, String ns, String localName, String prefix, String value, Token tok) {
    String key = ns + "#" + localName;
    if (attributeNames.contains(key))
      error("duplicate_attribute", ns, localName, tok);
    else {
      attributeNames.add(key);
      a.addAttribute(ns, localName, prefix, value, makeLocation(tok));
    }
  }

  private void checkExcept(Token[] except) {
    if (except[0] != null)
      error("except_missing_parentheses", except[0]);
  }

  private String lookupPrefix(String prefix, Token t) {
    String ns = namespaceMap.get(prefix);
    if (ns == null) {
      error("undeclared_prefix", prefix, t);
      return "#error";
    }
    return ns;
  }
  private String lookupDatatype(String prefix, Token t) {
    String ns = datatypesMap.get(prefix);
    if (ns == null) {
      error("undeclared_prefix", prefix, t);
      return ""; // XXX
    }
    return ns;
  }
}

PARSER_END(CompactSyntax)

Pattern Input(Scope scope) :
{
  Pattern p;
}
{
  Preamble()
  (LOOKAHEAD(TopLevelLookahead()) p = TopLevelGrammar(scope)
   | p = Expr(true, scope, null, null) { p = afterPatternComments(p); } )
  { return p; }
}

void TopLevelLookahead() :
{}
{
   "["
  | Identifier() ("[" | "=" | "&=" | "|=")
  | LookaheadGrammarKeyword()
  | LookaheadBody() LookaheadAfterAnnotations()
  | LookaheadDocumentation() (LookaheadBody())? LookaheadAfterAnnotations()
}

void LookaheadAfterAnnotations() :
{}
{
  Identifier() ("=" | "&=" | "|=")
  | LookaheadGrammarKeyword()
}

void LookaheadGrammarKeyword() :
{}
{
  "start" | "div" | "include"
}

void LookaheadDocumentation() :
{}
{
  (( | ) ()*)+
}

void LookaheadBody() :
{}
{
  "["
   ( | UnprefixedName() | "=" |  | "~" | LookaheadBody() )*
  "]"
}

Pattern IncludedGrammar(IncludedGrammar g) :
{
  AnnotationsImpl a;
  Pattern p;
}
{
  Preamble()
  (LOOKAHEAD(TopLevelLookahead()) a = GrammarBody(g, g, getTopLevelCommentsAsAnnotations())
   | a = AnnotationsImpl() "grammar" "{" a = GrammarBody(g, g, a) { topLevelComments(g); } "}")
  { p = afterPatternComments(g.endIncludedGrammar(sb.makeLocation(sourceUri, 1, 1), a)); }
  
  { return p; }
}

Pattern TopLevelGrammar(Scope scope) :
{
  AnnotationsImpl a = getTopLevelCommentsAsAnnotations();
  Grammar g;
  Pattern p;
}
{
  { g = sb.makeGrammar(scope); }
  a = GrammarBody(g, g, a)
  { p = afterPatternComments(g.endGrammar(sb.makeLocation(sourceUri, 1, 1), a)); }
  
  { return p; }
}

void Preamble() :
{}
{
  (NamespaceDecl() | DatatypesDecl())*
  {
    namespaceMap.put("xml", WellKnownNamespaces.XML);
    if (datatypesMap.get("xsd") == null)
      datatypesMap.put("xsd", WellKnownNamespaces.XML_SCHEMA_DATATYPES);
  }
}

void NamespaceDecl() :
{
  LocatedString prefix = null;
  boolean isDefault = false;
  String namespaceName;
}
{
  { noteTopLevelComments(); }
  (("namespace" prefix = UnprefixedName())
    | ("default" { isDefault = true; }
      "namespace" (prefix = UnprefixedName())?))
  "="
  namespaceName = NamespaceName()
  {
    if (isDefault)
      defaultNamespace = namespaceName;
    if (prefix != null) {
      if (prefix.getString().equals("xmlns"))
        error("xmlns_prefix", prefix.getToken());
      else if (prefix.getString().equals("xml")) {
        if (!namespaceName.equals(WellKnownNamespaces.XML))
          error("xml_prefix_bad_uri", prefix.getToken());
      }
      else if (namespaceName.equals(WellKnownNamespaces.XML))
        error("xml_uri_bad_prefix", prefix.getToken());
      else {
        if (namespaceName.equals(WellKnownNamespaces.RELAX_NG_COMPATIBILITY_ANNOTATIONS))
          compatibilityPrefix = prefix.getString();
        namespaceMap.put(prefix.getString(), namespaceName);
      }
    }
  }
}

String NamespaceName() :
{
  String r;
}
{
  (r = Literal() | "inherit" { r = SchemaBuilder.INHERIT_NS; })
  { return r; }
}

void DatatypesDecl() :
{
  LocatedString prefix;
  String uri;
}
{
  { noteTopLevelComments(); }
  "datatypes" prefix = UnprefixedName() "=" uri = Literal()
  {
    datatypesMap.put(prefix.getString(), uri);
  }
}

Pattern AnnotatedPrimaryExpr(boolean topLevel, Scope scope, Token[] except) :
{
  AnnotationsImpl a;
  Pattern p;
  ElementAnnotation e;
  Token t;
}
{
  a = AnnotationsImpl()
  p = PrimaryExpr(topLevel, scope, a, except)
  ( t =  e = AnnotationElement(false) {
       if (topLevel)
         error("top_level_follow_annotation", t);
       else
         p = sb.annotateAfterPattern(p, e);
     })*
  { return p; }
}


Pattern PrimaryExpr(boolean topLevel, Scope scope, AnnotationsImpl a, Token[] except) :
{
  Pattern p;
}
{
  (p = ElementExpr(scope, a)
   | p = AttributeExpr(scope, a)
   | p = GrammarExpr(scope, a)
   | p = ExternalRefExpr(scope, a)
   | p = ListExpr(scope, a)
   | p = MixedExpr(scope, a)
   | p = ParenExpr(topLevel, scope, a)
   | p = IdentifierExpr(scope, a)
   | p = ParentExpr(scope, a)
   | p = DataExpr(topLevel, scope, a, except)
   | p = ValueExpr(topLevel, a)
   | p = TextExpr(a)
   | p = EmptyExpr(a)
   | p = NotAllowedExpr(a))
  { return p; }
}

Pattern EmptyExpr(AnnotationsImpl a) :
{
  Token t;
}
{
  t = "empty"
  { return sb.makeEmpty(makeLocation(t), a); }
}

Pattern TextExpr(AnnotationsImpl a) :
{
  Token t;
}
{
  t = "text"
  { return sb.makeText(makeLocation(t), a); }
}

Pattern NotAllowedExpr(AnnotationsImpl a) :
{
  Token t;
}
{
  t = "notAllowed"
  { return sb.makeNotAllowed(makeLocation(t), a); }
}

Pattern Expr(boolean topLevel, Scope scope, Token t, AnnotationsImpl a) :
{
  Pattern p;
  List patterns = new ArrayList();
  boolean[] hadOccur = new boolean[1];
  Token[] except = new Token[1];
}
{
  p = UnaryExpr(topLevel, scope, hadOccur, except)
  { patterns.add(p); }
  (
   { checkExcept(except); }
   (t = "|" p = UnaryExpr(topLevel, scope, null, except)
     { patterns.add(p); checkExcept(except); } )+
    { p = sb.makeChoice(patterns, makeLocation(t), a); }
    | (t = "&" p = UnaryExpr(topLevel, scope, null, except)
     { patterns.add(p); checkExcept(except); } )+
    { p = sb.makeInterleave(patterns, makeLocation(t), a); }
    | (t = "," p = UnaryExpr(topLevel, scope, null, except)
     { patterns.add(p); checkExcept(except); } )+
    { p = sb.makeGroup(patterns, makeLocation(t), a); }
  )?
  {
    if (patterns.size() == 1 && a != null) {
      if (hadOccur[0])
        p = sb.annotatePattern(p, a);
      else
        p = sb.makeGroup(patterns, makeLocation(t), a);
    }
    return p;
  }
}

Pattern UnaryExpr(boolean topLevel, Scope scope, boolean[] hadOccur, Token[] except) :
{
  Pattern p;
  Token t;
  ElementAnnotation e;
}
{
  p = AnnotatedPrimaryExpr(topLevel, scope, except)
  (
   {
     if (hadOccur != null) hadOccur[0] = true;
     p = afterPatternComments(p);
   }
   (t = "+" { checkExcept(except); p = sb.makeOneOrMore(p, makeLocation(t), null); }
    | t = "?" { checkExcept(except); p = sb.makeOptional(p, makeLocation(t), null); }
    | t = "*" { checkExcept(except); p = sb.makeZeroOrMore(p, makeLocation(t), null); })
   ( t =  e = AnnotationElement(false) {
        if (topLevel)
          error("top_level_follow_annotation", t);
        else
          p = sb.annotateAfterPattern(p, e);
     } )*
  )?
  { return p; }
}

Pattern ElementExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  NameClass nc;
  Pattern p;
}
{
  t = "element"
  nc = NameClass(IN_ELEMENT, null)
  "{"
  p = Expr(false, scope, null, null)
  { p = afterPatternComments(p); }
  "}"
  { return sb.makeElement(nc, p, makeLocation(t), a); }
}

Pattern AttributeExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  NameClass nc;
  Pattern p;
}
{
  t = "attribute"
  nc = NameClass(IN_ATTRIBUTE, null)
  "{"
  p = Expr(false, scope, null, null)
  { p = afterPatternComments(p); }
  "}"
  { return sb.makeAttribute(nc, p, makeLocation(t), a); }
}

NameClass NameClass(int context, Ref ra) :
{
  AnnotationsImpl a;
  NameClass nc;
}
{
  a = AnnotationsImpl()
  (nc = PrimaryNameClass(context, a) nc = AnnotateAfter(nc) nc = NameClassAlternatives(context, nc, ra)
   | nc = AnyNameExceptClass(context, a, ra)
   | nc = NsNameExceptClass(context, a, ra))
  { return nc; }
}

NameClass AnnotateAfter(NameClass nc) :
{
  ElementAnnotation e;
}
{
  (  e = AnnotationElement(false) { nc = sb.annotateAfterNameClass(nc, e); })*
  { return nc; }
}

NameClass NameClassAlternatives(int context, NameClass nc, Ref ra) :
{
  Token t;
  List nameClasses;
}
{
  (
    {
      nameClasses = new ArrayList();
      nameClasses.add(nc);
    }
    (t = "|" nc = BasicNameClass(context) nc = AnnotateAfter(nc)
    {
      nameClasses.add(nc);
    })+
    {
      AnnotationsImpl a;
      if (ra == null)
        a = null;
      else {
	a = ra.get();
        ra.clear();
      }
      nc = sb.makeNameClassChoice(nameClasses, makeLocation(t), a);
    }
  )?
  { return nc; }
}

NameClass BasicNameClass(int context) :
{
  AnnotationsImpl a;
  NameClass nc;
}
{
  a = AnnotationsImpl()
  (nc = PrimaryNameClass(context, a)
   | nc = OpenNameClass(context, a))
  { return nc; }
}

NameClass PrimaryNameClass(int context, AnnotationsImpl a) :
{
  NameClass nc;
}
{
  (nc = UnprefixedNameClass(context, a)
   | nc = PrefixedNameClass(a)
   | nc = ParenNameClass(context, a))
  { return nc; }
}

NameClass OpenNameClass(int context, AnnotationsImpl a) :
{
  Token t;
  LocatedString ns;
}
{
  ns = NsName() { checkNsName(context, ns); return sb.makeNsName(ns.getString(), ns.getLocation(), a); }
  | t = "*" { checkAnyName(context, t); return sb.makeAnyName(makeLocation(t), a); }
}


NameClass UnprefixedNameClass(int context, AnnotationsImpl a) :
{
  LocatedString name;
}
{
  name = UnprefixedName()
  {
    String ns;
    if ((context & (IN_ATTRIBUTE|IN_ELEMENT)) == IN_ATTRIBUTE)
      ns = "";
    else
      ns = defaultNamespace;
    return sb.makeName(ns, name.getString(), null, name.getLocation(), a);
  }
}

NameClass PrefixedNameClass(AnnotationsImpl a) :
{
  Token t;
}
{
  t = 
  {
    String qn = t.image;
    int colon = qn.indexOf(':');
    String prefix = qn.substring(0, colon);
    return sb.makeName(lookupPrefix(prefix, t), qn.substring(colon + 1), prefix, makeLocation(t), a);
  }
}

NameClass NsNameExceptClass(int context, AnnotationsImpl a, Ref ra) :
{
  LocatedString ns;
  NameClass nc;
}
{
  ns = NsName()
  { checkNsName(context, ns); }
  (nc = ExceptNameClass(context | IN_NS_NAME)
   { nc = sb.makeNsName(ns.getString(), nc, ns.getLocation(), a); }
   nc = AnnotateAfter(nc)
  | { nc = sb.makeNsName(ns.getString(), ns.getLocation(), a); }
    nc = AnnotateAfter(nc)
    nc = NameClassAlternatives(context, nc, ra))
  { return nc; }
}

LocatedString NsName() :
{
  Token t;
}
{
  t = 
  {
    String qn = t.image;
    String prefix = qn.substring(0, qn.length() - 2);
    return new LocatedString(lookupPrefix(prefix, t), t);
  }
}

NameClass AnyNameExceptClass(int context, AnnotationsImpl a, Ref ra) :
{
  Token t;
  NameClass nc;
}
{
  t = "*"
  { checkAnyName(context, t); }
  (nc = ExceptNameClass(context | IN_ANY_NAME)
   { nc = sb.makeAnyName(nc, makeLocation(t), a); }
   nc = AnnotateAfter(nc)
  | { nc = sb.makeAnyName(makeLocation(t), a); }
    nc = AnnotateAfter(nc)
    nc = NameClassAlternatives(context, nc, ra))
  { return nc; }
}

NameClass ParenNameClass(int context, AnnotationsImpl a) :
{
  Token t;
  NameClass nc;
  Ref ra = new Ref(a);
}
{
  t = "(" nc = NameClass(context, ra) { nc = afterNameClassComments(nc); } ")"
  {
    if (ra.get() != null)
      nc = sb.makeNameClassChoice(Collections.singletonList(nc), makeLocation(t), ra.get());
    return nc;
  }
}

NameClass ExceptNameClass(int context) :
{
  NameClass nc;
}
{
  "-" nc = BasicNameClass(context)
  { return nc; }
}

Pattern ListExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  Pattern p;
}
{
  t = "list"
  "{"
  p = Expr(false, scope, null, null)
  { p = afterPatternComments(p); }
  "}"
  { return sb.makeList(p, makeLocation(t), a); }
}

Pattern MixedExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  Pattern p;
}
{
  t = "mixed"
  "{"
  p = Expr(false, scope, null, null)
  { p = afterPatternComments(p); }
  "}"
  { return sb.makeMixed(p, makeLocation(t), a); }
}

Pattern GrammarExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  Grammar g;
}
{
  t = "grammar" { g = sb.makeGrammar(scope); }
  "{" a = GrammarBody(g, g, a) { topLevelComments(g); } "}"
  { return g.endGrammar(makeLocation(t), a); }
}

Pattern ParenExpr(boolean topLevel, Scope scope, AnnotationsImpl a) :
{
  Token t;
  Pattern p;
}
{
  t = "(" p = Expr(topLevel, scope, t, a) { p = afterPatternComments(p); } ")"
  { return p; }
}

AnnotationsImpl GrammarBody(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  ElementAnnotation e;
}
{
  (LOOKAHEAD(2) e = AnnotationElementNotKeyword()
   { if (a == null) a = sb.makeAnnotations(null, getContext()); a.addElement(e); })*
  (GrammarComponent(section, scope))*
  { return a; }
}

void GrammarComponent(GrammarSection section, Scope scope) :
{
  ElementAnnotation e;
  AnnotationsImpl a;
}
{
  (a = AnnotationsImpl()
   (Definition(section, scope, a)
    | Include(section, scope, a)
    | Div(section, scope, a)))
  (LOOKAHEAD(2) e = AnnotationElementNotKeyword() { section.topLevelAnnotation(e); })*
}

void Definition(GrammarSection section, Scope scope, AnnotationsImpl a) :
{}
{
  (Define(section, scope, a) | Start(section, scope, a))
}

void Start(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  Token t;
  GrammarSection.Combine combine;
  Pattern p;
}
{
  t = "start" combine = AssignOp() p = Expr(false, scope, null, null)
  { section.define(GrammarSection.START, combine, p, makeLocation(t), a); }
}

void Define(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  LocatedString name;
  GrammarSection.Combine combine;
  Pattern p;
}
{
  name = Identifier() combine = AssignOp() p = Expr(false, scope, null, null)
  { section.define(name.getString(), combine, p, name.getLocation(), a); }
}

GrammarSection.Combine AssignOp() :
{}
{
  "=" { return null; }
  | "|=" { return GrammarSection.COMBINE_CHOICE; }
  | "&=" { return GrammarSection.COMBINE_INTERLEAVE; }
}

void Include(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  Token t;
  String href;
  String ns;
  Include include = section.makeInclude();
}
{
  t = "include" href = Literal()
  ns = Inherit()
  ("{" a = IncludeBody(include, scope, a) { topLevelComments(include); } "}")?
  {
    try {
      include.endInclude(href, sourceUri, ns, makeLocation(t), a);
    }
    catch (IllegalSchemaException e) { }
  }
}

AnnotationsImpl IncludeBody(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  ElementAnnotation e;
}
{
  (LOOKAHEAD(2) e = AnnotationElementNotKeyword()
   { if (a == null) a = sb.makeAnnotations(null, getContext()); a.addElement(e); })*
  (IncludeComponent(section, scope))*
  { return a; }
}


void IncludeComponent(GrammarSection section, Scope scope) :
{
  ElementAnnotation e;
  AnnotationsImpl a;
}
{
  (a = AnnotationsImpl() (Definition(section, scope, a)
                      | IncludeDiv(section, scope, a)))
  (LOOKAHEAD(2) e = AnnotationElementNotKeyword() { section.topLevelAnnotation(e); })*
}

void Div(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  Token t;
  Div div = section.makeDiv();
}
{
  t = "div" "{" a = GrammarBody(div, scope, a) { topLevelComments(div); } "}"
  { div.endDiv(makeLocation(t), a); }
}

void IncludeDiv(GrammarSection section, Scope scope, AnnotationsImpl a) :
{
  Token t;
  Div div = section.makeDiv();
}
{
  t = "div" "{" a = IncludeBody(div, scope, a) { topLevelComments(div); } "}"
  { div.endDiv(makeLocation(t), a); }
}

Pattern ExternalRefExpr(Scope scope, AnnotationsImpl a) :
{
  Token t;
  String href;
  String ns;
}
{
  t = "external" href = Literal()
  ns = Inherit()
  {
    try {
      return sb.makeExternalRef(href, sourceUri, ns, scope, makeLocation(t), a);
    }
    catch (IllegalSchemaException e) {
      return sb.makeErrorPattern();
    }
  }
}

String Inherit() :
{
  String ns = null;
}
{
  ("inherit" "=" ns = Prefix())?
  {
    if (ns == null)
      ns = defaultNamespace;
    return ns;
  }
}

Pattern ParentExpr(Scope scope, AnnotationsImpl a) :
{
  LocatedString name;
}
{
  "parent" { a = addCommentsToChildAnnotations(a); } name = Identifier()
  {
    return scope.makeParentRef(name.getString(), name.getLocation(), a);
  }
}

Pattern IdentifierExpr(Scope scope, AnnotationsImpl a) :
{
  LocatedString name;
}
{
  name = Identifier()
  {
    return scope.makeRef(name.getString(), name.getLocation(), a);
  }
}

Pattern ValueExpr(boolean topLevel, AnnotationsImpl a) :
{
  LocatedString s;
}
{
  s = LocatedLiteral()
  {
    if (topLevel && annotationsIncludeElements) {
      error("top_level_follow_annotation", s.getToken());
      a = null;
    }
    return sb.makeValue("", "token", s.getString(), getContext(), defaultNamespace, s.getLocation(), a);
  }
}

Pattern DataExpr(boolean topLevel, Scope scope, AnnotationsImpl a, Token[] except) :
{
  Token datatypeToken;
  Location loc;
  String datatype;
  String datatypeUri = null;
  String s = null;
  Pattern e = null;
  DataPatternBuilder dpb;
}
{
  datatypeToken = DatatypeName()
  {
    datatype = datatypeToken.image;
    loc = makeLocation(datatypeToken);
    int colon = datatype.indexOf(':');
    if (colon < 0)
      datatypeUri = "";
    else {
      String prefix = datatype.substring(0, colon);
      datatypeUri = lookupDatatype(prefix, datatypeToken);
      datatype = datatype.substring(colon + 1);
    }
  }
  ((s = Literal()
    {
      if (topLevel && annotationsIncludeElements) {
        error("top_level_follow_annotation", datatypeToken);
        a = null;
      }
      return sb.makeValue(datatypeUri, datatype, s, getContext(), defaultNamespace, loc, a);
    }
  )
  | ( { dpb = sb.makeDataPatternBuilder(datatypeUri, datatype, loc); }
      ( (Params(dpb) (e = Except(scope, except))?)
        | (e = Except(scope, except))?)
      { return e == null ? dpb.makePattern(loc, a) : dpb.makePattern(e, loc, a); }))
}

Token DatatypeName() :
{
  Token t;
}
{
  (t = "string" | t = "token" | t = )
  { return t; }
}

LocatedString Identifier() :
{
  LocatedString s;
  Token t;
}
{
  (t =  { s = new LocatedString(t.image, t); }
  | t =  { s = new LocatedString(t.image.substring(1), t); })
  { return s; }
}

String Prefix() :
{
  Token t;
  String prefix;
}
{
  (t =  { prefix = t.image; }
  | t =  { prefix = t.image.substring(1); }
  | t = Keyword() { prefix = t.image; })
  { return lookupPrefix(prefix, t); }
}

LocatedString UnprefixedName() :
{
  LocatedString s;
  Token t;
}
{
  (s = Identifier()
   | t = Keyword() { s = new LocatedString(t.image, t); })
  { return s; }
}

void Params(DataPatternBuilder dpb) :
{}
{
  "{" (Param(dpb))* "}"
}

void Param(DataPatternBuilder dpb) :
{
  LocatedString name;
  AnnotationsImpl a;
  String value;
}
{
  a = AnnotationsImpl() name = UnprefixedName() "=" { a = addCommentsToLeadingAnnotations(a); } value = Literal()
  { dpb.addParam(name.getString(), value, getContext(), defaultNamespace, name.getLocation(), a); }
}

Pattern Except(Scope scope, Token[] except) :
{
  AnnotationsImpl a;
  Pattern p;
  Token t;
  Token[] innerExcept = new Token[1];
}
{
  t = "-" a = AnnotationsImpl() p = PrimaryExpr(false, scope, a, innerExcept)
  {
    checkExcept(innerExcept);
    except[0] = t;
    return p;
  }
}

ElementAnnotation Documentation() :
{
  CommentListImpl comments = getComments();
  ElementAnnotationBuilder eab;
  Token t;
}
{
  (t =  | t = )
  {
    eab = sb.makeElementAnnotationBuilder(WellKnownNamespaces.RELAX_NG_COMPATIBILITY_ANNOTATIONS,
                                          "documentation",
                                          getCompatibilityPrefix(),
                                          makeLocation(t),
                                          comments,
                                          getContext());
    eab.addText(mungeComment(t.image), makeLocation(t), null);
  }
  (t =  { eab.addText("\n" + mungeComment(t.image), makeLocation(t), null); })*
  { return eab.makeElementAnnotation(); }
}

AnnotationsImpl AnnotationsImpl() :
{
  CommentListImpl comments = getComments();
  AnnotationsImpl a = null;
  ElementAnnotation e;
}
{
  ( { a = sb.makeAnnotations(comments, getContext()); }
    (e = Documentation() { a.addElement(e); })+
    {
      comments = getComments();
      if (comments != null)
        a.addLeadingComment(comments);
    }
  )?
  ("[" { if (a == null) a = sb.makeAnnotations(comments, getContext()); clearAttributeList(); annotationsIncludeElements = false; }
       (LOOKAHEAD(2) PrefixedAnnotationAttribute(a, false) )*
       ( e = AnnotationElement(false) { a.addElement(e); annotationsIncludeElements = true; } )*
       { a.addComment(getComments()); }
  "]")?
  {
    if (a == null && comments != null)
      a = sb.makeAnnotations(comments, getContext());
    return a;
  }
}

void AnnotationAttribute(Annotations a) :
{}
{
  PrefixedAnnotationAttribute(a, true) | UnprefixedAnnotationAttribute(a)
}

void PrefixedAnnotationAttribute(Annotations a, boolean nested) :
{
  Token t;
  String value;
}
{
  t =  "=" value = Literal()
  {
    String qn = t.image;
    int colon = qn.indexOf(':');
    String prefix = qn.substring(0, colon);
    String ns = lookupPrefix(prefix, t);
    if (ns == SchemaBuilder.INHERIT_NS)
      error("inherited_annotation_namespace", t);
    else if (ns.length() == 0 && !nested)
      error("unqualified_annotation_attribute", t);
    else if (ns.equals(WellKnownNamespaces.RELAX_NG) && !nested)
      error("relax_ng_namespace", t);
    /*else if (ns.length() == 0
             && qn.length() - colon - 1 == 5
             && qn.regionMatches(colon + 1, "xmlns", 0, 5))
      error("xmlns_annotation_attribute", t);*/
    else if (ns.equals(WellKnownNamespaces.XMLNS))
      error("xmlns_annotation_attribute_uri", t);
    else {
      if (ns.length() == 0)
	prefix = null;
      addAttribute(a, ns, qn.substring(colon + 1), prefix, value, t);
    }
  }
}

void UnprefixedAnnotationAttribute(Annotations a) :
{
  LocatedString name;
  String value;
}
{
  name = UnprefixedName() "=" value = Literal()
  {
    if (name.getString().equals("xmlns"))
      error("xmlns_annotation_attribute", name.getToken());
    else
      addAttribute(a, "", name.getString(), null, value, name.getToken());
  }
}

ElementAnnotation AnnotationElement(boolean nested) :
{
  ElementAnnotation a;
}
{
  (a = PrefixedAnnotationElement(nested)
   | a = UnprefixedAnnotationElement())
  { return a; }
}

ElementAnnotation AnnotationElementNotKeyword() :
{
  ElementAnnotation a;
}
{
  (a = PrefixedAnnotationElement(false)
   | a = IdentifierAnnotationElement())
  { return a; }
}

ElementAnnotation PrefixedAnnotationElement(boolean nested) :
{
  CommentListImpl comments = getComments();
  Token t;
  ElementAnnotationBuilder eab;
}
{
  t = 
  {
    String qn = t.image;
    int colon = qn.indexOf(':');
    String prefix = qn.substring(0, colon);
    String ns = lookupPrefix(prefix, t);
    if (ns == SchemaBuilder.INHERIT_NS) {
      error("inherited_annotation_namespace", t);
      ns = "";
    }
    else if (!nested && ns.equals(WellKnownNamespaces.RELAX_NG)) {
      error("relax_ng_namespace", t);
      ns = "";
    }
    else {
      if (ns.length() == 0)
        prefix = null;
    }
    eab = sb.makeElementAnnotationBuilder(ns, qn.substring(colon + 1), prefix,
					  makeLocation(t), comments, getContext());
  }
  AnnotationElementContent(eab)
  { return eab.makeElementAnnotation(); }
}

ElementAnnotation UnprefixedAnnotationElement() :
{
  CommentListImpl comments = getComments();
  LocatedString name;
  ElementAnnotationBuilder eab;
}
{
  name = UnprefixedName()
  {
    eab = sb.makeElementAnnotationBuilder("", name.getString(), null,
                                          name.getLocation(), comments, getContext());
  }
  AnnotationElementContent(eab)
  { return eab.makeElementAnnotation(); }
}

ElementAnnotation IdentifierAnnotationElement() :
{
  CommentListImpl comments = getComments();
  LocatedString name;
  ElementAnnotationBuilder eab;
}
{
  name = Identifier()
  {
    eab = sb.makeElementAnnotationBuilder("", name.getString(), null,
                                          name.getLocation(), comments, getContext());
  }
  AnnotationElementContent(eab)
  { return eab.makeElementAnnotation(); }
}

void AnnotationElementContent(ElementAnnotationBuilder eab) :
{
  ElementAnnotation e;
}
{
  "[" { clearAttributeList(); }
  (LOOKAHEAD(2) AnnotationAttribute(eab))*
  ((AnnotationElementLiteral(eab)
    ("~" AnnotationElementLiteral(eab))*)
   | e = AnnotationElement(true) { eab.addElement(e); })*
  { eab.addComment(getComments()); }
  "]"
}

void AnnotationElementLiteral(ElementAnnotationBuilder eab) :
{
  Token t;
  CommentListImpl comments = getComments();
}
{
  t =  { eab.addText(unquote(t.image), makeLocation(t), comments); }
}

String Literal() :
{
  Token t;
  String s;
  StringBuffer buf;
}
{
  t = 
  {
    s = unquote(t.image);
  }
  (
    { buf = new StringBuffer(s); }
    ("~" t =  { buf.append(unquote(t.image)); })+
    { s = buf.toString(); }
  )?
  { return s; }
}

LocatedString LocatedLiteral() :
{
  Token t;
  Token t2;
  String s;
  StringBuffer buf;
}
{
  t = 
  {
    s = unquote(t.image);
  }
  (
    { buf = new StringBuffer(s); }
    ("~" t2 =  { buf.append(unquote(t2.image)); })+
    { s = buf.toString(); }
  )?
  { return new LocatedString(s, t); }
}

Token Keyword() :
{
  Token t;
}
{
  (t = "element"
  | t = "attribute"
  | t = "namespace"
  | t = "list"
  | t = "mixed"
  | t = "grammar"
  | t = "empty"
  | t = "text"
  | t = "parent"
  | t = "external"
  | t = "notAllowed"
  | t = "start"
  | t = "include"
  | t = "default"
  | t = "inherit"
  | t = "string"
  | t = "token"
  | t = "datatypes"
  | t = "div")
  { return t; }
}

<*>
SKIP: {
  < #NEWLINE : [ "\u0000", "\n" ] >
  | < #NOT_NEWLINE : ~[ "\u0000", "\n" ] >
  | < WS: ([ "\u0000", " ", "\n", "\t" ])+ > : DEFAULT
}

TOKEN :
{
  < DOCUMENTATION: "##" ()* > : AFTER_DOCUMENTATION
}


TOKEN :
{
  < DOCUMENTATION_CONTINUE:  ([" ", "\t"])*  >
}

SPECIAL_TOKEN:
{
  < SINGLE_LINE_COMMENT: "#" ()* > : AFTER_SINGLE_LINE_COMMENT
}


TOKEN :
{
  < DOCUMENTATION_AFTER_SINGLE_LINE_COMMENT:  ([" ", "\t"])*  > : AFTER_DOCUMENTATION
}


SPECIAL_TOKEN :
{
  < SINGLE_LINE_COMMENT_CONTINUE:  ([" ", "\t"])*  >
}

TOKEN :
{
  < #BASE_CHAR : [
    "\u0041" - "\u005a",
    "\u0061" - "\u007a",
    "\u00c0" - "\u00d6",
    "\u00d8" - "\u00f6",
    "\u00f8" - "\u00ff",
    "\u0100" - "\u0131",
    "\u0134" - "\u013e",
    "\u0141" - "\u0148",
    "\u014a" - "\u017e",
    "\u0180" - "\u01c3",
    "\u01cd" - "\u01f0",
    "\u01f4" - "\u01f5",
    "\u01fa" - "\u0217",
    "\u0250" - "\u02a8",
    "\u02bb" - "\u02c1",
    "\u0386",
    "\u0388" - "\u038a",
    "\u038c",
    "\u038e" - "\u03a1",
    "\u03a3" - "\u03ce",
    "\u03d0" - "\u03d6",
    "\u03da",
    "\u03dc",
    "\u03de",
    "\u03e0",
    "\u03e2" - "\u03f3",
    "\u0401" - "\u040c",
    "\u040e" - "\u044f",
    "\u0451" - "\u045c",
    "\u045e" - "\u0481",
    "\u0490" - "\u04c4",
    "\u04c7" - "\u04c8",
    "\u04cb" - "\u04cc",
    "\u04d0" - "\u04eb",
    "\u04ee" - "\u04f5",
    "\u04f8" - "\u04f9",
    "\u0531" - "\u0556",
    "\u0559",
    "\u0561" - "\u0586",
    "\u05d0" - "\u05ea",
    "\u05f0" - "\u05f2",
    "\u0621" - "\u063a",
    "\u0641" - "\u064a",
    "\u0671" - "\u06b7",
    "\u06ba" - "\u06be",
    "\u06c0" - "\u06ce",
    "\u06d0" - "\u06d3",
    "\u06d5",
    "\u06e5" - "\u06e6",
    "\u0905" - "\u0939",
    "\u093d",
    "\u0958" - "\u0961",
    "\u0985" - "\u098c",
    "\u098f" - "\u0990",
    "\u0993" - "\u09a8",
    "\u09aa" - "\u09b0",
    "\u09b2",
    "\u09b6" - "\u09b9",
    "\u09dc" - "\u09dd",
    "\u09df" - "\u09e1",
    "\u09f0" - "\u09f1",
    "\u0a05" - "\u0a0a",
    "\u0a0f" - "\u0a10",
    "\u0a13" - "\u0a28",
    "\u0a2a" - "\u0a30",
    "\u0a32" - "\u0a33",
    "\u0a35" - "\u0a36",
    "\u0a38" - "\u0a39",
    "\u0a59" - "\u0a5c",
    "\u0a5e",
    "\u0a72" - "\u0a74",
    "\u0a85" - "\u0a8b",
    "\u0a8d",
    "\u0a8f" - "\u0a91",
    "\u0a93" - "\u0aa8",
    "\u0aaa" - "\u0ab0",
    "\u0ab2" - "\u0ab3",
    "\u0ab5" - "\u0ab9",
    "\u0abd",
    "\u0ae0",
    "\u0b05" - "\u0b0c",
    "\u0b0f" - "\u0b10",
    "\u0b13" - "\u0b28",
    "\u0b2a" - "\u0b30",
    "\u0b32" - "\u0b33",
    "\u0b36" - "\u0b39",
    "\u0b3d",
    "\u0b5c" - "\u0b5d",
    "\u0b5f" - "\u0b61",
    "\u0b85" - "\u0b8a",
    "\u0b8e" - "\u0b90",
    "\u0b92" - "\u0b95",
    "\u0b99" - "\u0b9a",
    "\u0b9c",
    "\u0b9e" - "\u0b9f",
    "\u0ba3" - "\u0ba4",
    "\u0ba8" - "\u0baa",
    "\u0bae" - "\u0bb5",
    "\u0bb7" - "\u0bb9",
    "\u0c05" - "\u0c0c",
    "\u0c0e" - "\u0c10",
    "\u0c12" - "\u0c28",
    "\u0c2a" - "\u0c33",
    "\u0c35" - "\u0c39",
    "\u0c60" - "\u0c61",
    "\u0c85" - "\u0c8c",
    "\u0c8e" - "\u0c90",
    "\u0c92" - "\u0ca8",
    "\u0caa" - "\u0cb3",
    "\u0cb5" - "\u0cb9",
    "\u0cde",
    "\u0ce0" - "\u0ce1",
    "\u0d05" - "\u0d0c",
    "\u0d0e" - "\u0d10",
    "\u0d12" - "\u0d28",
    "\u0d2a" - "\u0d39",
    "\u0d60" - "\u0d61",
    "\u0e01" - "\u0e2e",
    "\u0e30",
    "\u0e32" - "\u0e33",
    "\u0e40" - "\u0e45",
    "\u0e81" - "\u0e82",
    "\u0e84",
    "\u0e87" - "\u0e88",
    "\u0e8a",
    "\u0e8d",
    "\u0e94" - "\u0e97",
    "\u0e99" - "\u0e9f",
    "\u0ea1" - "\u0ea3",
    "\u0ea5",
    "\u0ea7",
    "\u0eaa" - "\u0eab",
    "\u0ead" - "\u0eae",
    "\u0eb0",
    "\u0eb2" - "\u0eb3",
    "\u0ebd",
    "\u0ec0" - "\u0ec4",
    "\u0f40" - "\u0f47",
    "\u0f49" - "\u0f69",
    "\u10a0" - "\u10c5",
    "\u10d0" - "\u10f6",
    "\u1100",
    "\u1102" - "\u1103",
    "\u1105" - "\u1107",
    "\u1109",
    "\u110b" - "\u110c",
    "\u110e" - "\u1112",
    "\u113c",
    "\u113e",
    "\u1140",
    "\u114c",
    "\u114e",
    "\u1150",
    "\u1154" - "\u1155",
    "\u1159",
    "\u115f" - "\u1161",
    "\u1163",
    "\u1165",
    "\u1167",
    "\u1169",
    "\u116d" - "\u116e",
    "\u1172" - "\u1173",
    "\u1175",
    "\u119e",
    "\u11a8",
    "\u11ab",
    "\u11ae" - "\u11af",
    "\u11b7" - "\u11b8",
    "\u11ba",
    "\u11bc" - "\u11c2",
    "\u11eb",
    "\u11f0",
    "\u11f9",
    "\u1e00" - "\u1e9b",
    "\u1ea0" - "\u1ef9",
    "\u1f00" - "\u1f15",
    "\u1f18" - "\u1f1d",
    "\u1f20" - "\u1f45",
    "\u1f48" - "\u1f4d",
    "\u1f50" - "\u1f57",
    "\u1f59",
    "\u1f5b",
    "\u1f5d",
    "\u1f5f" - "\u1f7d",
    "\u1f80" - "\u1fb4",
    "\u1fb6" - "\u1fbc",
    "\u1fbe",
    "\u1fc2" - "\u1fc4",
    "\u1fc6" - "\u1fcc",
    "\u1fd0" - "\u1fd3",
    "\u1fd6" - "\u1fdb",
    "\u1fe0" - "\u1fec",
    "\u1ff2" - "\u1ff4",
    "\u1ff6" - "\u1ffc",
    "\u2126",
    "\u212a" - "\u212b",
    "\u212e",
    "\u2180" - "\u2182",
    "\u3041" - "\u3094",
    "\u30a1" - "\u30fa",
    "\u3105" - "\u312c",
    "\uac00" - "\ud7a3"
    ] >
  | < #IDEOGRAPHIC : [
    "\u4e00" - "\u9fa5",
    "\u3007",
    "\u3021" - "\u3029"
    ] >
  | < #LETTER : ( | ) >
  | < #COMBINING_CHAR : [
    "\u0300" - "\u0345",
    "\u0360" - "\u0361",
    "\u0483" - "\u0486",
    "\u0591" - "\u05a1",
    "\u05a3" - "\u05b9",
    "\u05bb" - "\u05bd",
    "\u05bf",
    "\u05c1" - "\u05c2",
    "\u05c4",
    "\u064b" - "\u0652",
    "\u0670",
    "\u06d6" - "\u06dc",
    "\u06dd" - "\u06df",
    "\u06e0" - "\u06e4",
    "\u06e7" - "\u06e8",
    "\u06ea" - "\u06ed",
    "\u0901" - "\u0903",
    "\u093c",
    "\u093e" - "\u094c",
    "\u094d",
    "\u0951" - "\u0954",
    "\u0962" - "\u0963",
    "\u0981" - "\u0983",
    "\u09bc",
    "\u09be",
    "\u09bf",
    "\u09c0" - "\u09c4",
    "\u09c7" - "\u09c8",
    "\u09cb" - "\u09cd",
    "\u09d7",
    "\u09e2" - "\u09e3",
    "\u0a02",
    "\u0a3c",
    "\u0a3e",
    "\u0a3f",
    "\u0a40" - "\u0a42",
    "\u0a47" - "\u0a48",
    "\u0a4b" - "\u0a4d",
    "\u0a70" - "\u0a71",
    "\u0a81" - "\u0a83",
    "\u0abc",
    "\u0abe" - "\u0ac5",
    "\u0ac7" - "\u0ac9",
    "\u0acb" - "\u0acd",
    "\u0b01" - "\u0b03",
    "\u0b3c",
    "\u0b3e" - "\u0b43",
    "\u0b47" - "\u0b48",
    "\u0b4b" - "\u0b4d",
    "\u0b56" - "\u0b57",
    "\u0b82" - "\u0b83",
    "\u0bbe" - "\u0bc2",
    "\u0bc6" - "\u0bc8",
    "\u0bca" - "\u0bcd",
    "\u0bd7",
    "\u0c01" - "\u0c03",
    "\u0c3e" - "\u0c44",
    "\u0c46" - "\u0c48",
    "\u0c4a" - "\u0c4d",
    "\u0c55" - "\u0c56",
    "\u0c82" - "\u0c83",
    "\u0cbe" - "\u0cc4",
    "\u0cc6" - "\u0cc8",
    "\u0cca" - "\u0ccd",
    "\u0cd5" - "\u0cd6",
    "\u0d02" - "\u0d03",
    "\u0d3e" - "\u0d43",
    "\u0d46" - "\u0d48",
    "\u0d4a" - "\u0d4d",
    "\u0d57",
    "\u0e31",
    "\u0e34" - "\u0e3a",
    "\u0e47" - "\u0e4e",
    "\u0eb1",
    "\u0eb4" - "\u0eb9",
    "\u0ebb" - "\u0ebc",
    "\u0ec8" - "\u0ecd",
    "\u0f18" - "\u0f19",
    "\u0f35",
    "\u0f37",
    "\u0f39",
    "\u0f3e",
    "\u0f3f",
    "\u0f71" - "\u0f84",
    "\u0f86" - "\u0f8b",
    "\u0f90" - "\u0f95",
    "\u0f97",
    "\u0f99" - "\u0fad",
    "\u0fb1" - "\u0fb7",
    "\u0fb9",
    "\u20d0" - "\u20dc",
    "\u20e1",
    "\u302a" - "\u302f",
    "\u3099",
    "\u309a"
    ] >
  | < #DIGIT : [
    "\u0030" - "\u0039",
    "\u0660" - "\u0669",
    "\u06f0" - "\u06f9",
    "\u0966" - "\u096f",
    "\u09e6" - "\u09ef",
    "\u0a66" - "\u0a6f",
    "\u0ae6" - "\u0aef",
    "\u0b66" - "\u0b6f",
    "\u0be7" - "\u0bef",
    "\u0c66" - "\u0c6f",
    "\u0ce6" - "\u0cef",
    "\u0d66" - "\u0d6f",
    "\u0e50" - "\u0e59",
    "\u0ed0" - "\u0ed9",
    "\u0f20" - "\u0f29"
    ] >
  | < #EXTENDER : [
    "\u00b7",
    "\u02d0",
    "\u02d1",
    "\u0387",
    "\u0640",
    "\u0e46",
    "\u0ec6",
    "\u3005",
    "\u3031" - "\u3035",
    "\u309d" - "\u309e",
    "\u30fc" - "\u30fe"
    ] >
  | < #NMSTART : ( | "_") >
  | < #NMCHAR : ( |  |  |  | "." | "-" | "_") >
  | < #NCNAME:  ()* >
}

TOKEN :
{
  < IDENTIFIER:  >
  | < ESCAPED_IDENTIFIER: "\\"  >
  | < PREFIX_STAR:  ":*" >
  | < PREFIXED_NAME:  ":"  >
  | < LITERAL : ("\"" (~["\u0000", "\""])* "\"")
                | ("'" (~["\u0000", "'"])* "'")
                | ("\"\"\"" (~["\""]
                             | ("\"" ~["\""])
                             | ("\"\"" ~["\""]))* "\"\"\"")
                | ("'''" (~["'"]
                             | ("'" ~["'"])
                             | ("''" ~["'"]))* "'''") >
  | < FANNOTATE : ">>" >
}

/* This avoids lexical errors from JavaCC. */
<*>
TOKEN :
{
  < ILLEGAL_CHAR : [ "\u0000" - "\u0008", "\u000b" - "\uffff" ] >
}