All Downloads are FREE. Search and download functionalities are using the official Maven repository.

javacc-7.0.4.classes.templates.TableDrivenTokenManager.template Maven / Gradle / Ivy

public class ${parserName}TokenManager ${superClass} implements ${parserName}Constants {

protected ${STATIC?static :}boolean moveToNextChar() {
  try {
    curChar = input_stream.readChar();
#if IGNORE_CASE
    curChar = (int)Character.toLowerCase((char)curChar);
#fi
  } catch(java.io.IOException e) {
    return false;
  }
  return true;
}

${decls}

#if TOKEN_MANAGER_USES_PARSER && !STATIC

/* Declarations from the TOKEN_MGR_DECLS section */

    /** Constructor with parser. */
    public ${parserName}TokenManager (${parserName} parserArg, ${charStreamName} stream){
    parser = parserArg;
#else
    /** Constructor. */
    public ${parserName}TokenManager(${charStreamName} stream){
#fi

#if STATIC && !USER_CHAR_STREAM
      if (input_stream != null)
        throw new ${LEGACY_EXCEPTION_HANDLING?TokenMgrError:TokenMgrException}("ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.", ${LEGACY_EXCEPTION_HANDLING?TokenMgrError:TokenMgrException}.STATIC_LEXER_ERROR);
#elif !USER_CHAR_STREAM
    #if JAVA_UNICODE_ESCAPE
      if (JavaCharStream.staticFlag)
    #else
      if (${charStreamName}.staticFlag)
    #fi
            throw new ${LEGACY_EXCEPTION_HANDLING?Error:RuntimeException}("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
#fi

    input_stream = stream;
  }

#if TOKEN_MANAGER_USES_PARSER && !STATIC
  /** Constructor with parser. */
  public ${parserName}TokenManager (${parserName} parserArg, ${charStreamName} stream, int lexState){
    ReInit(parserArg, stream);
  }
#else
  /** Constructor. */
  public ${parserName}TokenManager (${charStreamName} stream, int lexState){
    ReInit(stream);
    SwitchTo(lexState);
  }
#fi

  /** Reinitialise parser. */
  
#if TOKEN_MANAGER_USES_PARSER && !STATIC
  public void ReInit(${parserName} parserArg, ${charStreamName} stream)
#else
  ${STATIC?static :}public void ReInit(${charStreamName} stream)
#fi
  {
#if TOKEN_MANAGER_USES_PARSER && !STATIC
	this.parser = parserArg;
#else

#fi

    jjmatchedPos = 0;
    curLexState = defaultLexState;
    input_stream = stream;
  }

  /** Reinitialise parser. */
#if TOKEN_MANAGER_USES_PARSER && !STATIC
  public void ReInit( ${parserName} parserArg, ${charStreamName} stream, int lexState)
#else
  ${STATIC?static :}public void ReInit(${charStreamName} stream, int lexState)
#fi
  
  {
#if TOKEN_MANAGER_USES_PARSER && !STATIC
    ReInit(parserArg, stream);
#else
    ReInit(stream);
#fi
    SwitchTo(lexState);
  }

  /** Switch to specified lex state. */
  public ${STATIC?static :}void SwitchTo(int lexState) {
    curLexState = lexState;
  }

#if !NO_DFA
private ${STATIC?static :}final int jjRunStringLiteralMatch() {
  int curPos = 0;
  final int key = (int)curLexState << 16 | curChar;
  final int[] arr = startAndSize.get(key);
  int startState = jjInitStates[curLexState];
  if (arr != null) {
    int index = arr[0];
    for (int i = 0; i < arr[1]; i++) {
      final int len = stringLiterals[index++];
#if DEBUG_TOKEN_MANAGER
      System.err.println(
          "Looking for string literal match of kind: " +
          stringLiterals[index + len] +
          "; token image: " + tokenImage[stringLiterals[index + len]]);
#fi
      do {
#if DEBUG_TOKEN_MANAGER
      System.err.println("Cur char: '" + (char)curChar + "'");
#fi
        if (curChar != stringLiterals[index + curPos]) break;
        if (++curPos == len) break;
        if (!moveToNextChar()) {
          --curPos;
          break;
        }
      } while(curPos < len);
      if (curPos == len) {
        jjmatchedKind = stringLiterals[index + len];
        jjmatchedPos = curPos;
        startState = stringLiterals[index + len + 1];
#if DEBUG_TOKEN_MANAGER
        System.err.println(
            "Currently matched the first: " + jjmatchedPos +
            " chars as kind: " + stringLiterals[index + len] +
            "; with image: " + tokenImage[jjmatchedKind]);
#fi
        if (!moveToNextChar()) {
          return curPos;
        }
        curPos++;
        break;
      } else {
        index += len + 2;
        input_stream.backup(curPos + 1);
        curPos = 0;
        if (!moveToNextChar()) {
          assert(false);
        }
      }
    }
  } else {
#if DEBUG_TOKEN_MANAGER
      System.err.println(
          "No string literal start with char: '" + (char)curChar + "'");
#fi
  }
  return jjMoveNfa(startState, curPos);
}
#fi

private ${STATIC?static :}  int[] stateSet = new int[${generatedStates}];
private ${STATIC?static :}  int[] newStateSet = new int[${generatedStates}];
private ${STATIC?static :}  final long[] moved = new long[${generatedStates}];
private ${STATIC?static :} long moveIndex = 1L;

private ${STATIC?static :}final int jjMoveNfa(int startState, int curPos) {

  if (startState < 0) {
#if DEBUG_TOKEN_MANAGER
  System.err.println("No NFA state at pos: " + curPos);
#fi
    return curPos;
  }

  // We have a long array indexed by the NFA state number to roughly indicate
  // the input position so when the input reaches part Long.MAX_VALUE (which
  // should be extremely rare), we need to reset them all back to zero.
  if (++moveIndex == Long.MAX_VALUE) {
    for (int i = 0; i < ${generatedStates}; i++) moved[i] = 0L;
    moveIndex = 1L;
  }

  // We initialize the kind to MAX value so that when a match is found, we can
  // simply check if it's less than the current match and store it in that
  // case. This helps implement the 'first occurring' rule properly.
  int kind = Integer.MAX_VALUE;
  int cnt = 0;
  stateSet[cnt++] = startState;
  moved[startState] = moveIndex;
 
  // Some NFA states have epsilon transitions (move on empty string). So we
  // just start with all of them. Note that the nextStates array already adds
  // the epsilon closure. Only the initial state needs to do this explicitly.
  for (int s : jjcompositeState[startState]) { 
    if (moved[s] != moveIndex) {
      stateSet[cnt++] = s;
      moved[s] = moveIndex;
    }
  }

#if DEBUG_TOKEN_MANAGER
  System.err.println("Starting NFA with start state: " + startState);
#fi
  do {
#if DEBUG_TOKEN_MANAGER
    System.err.println("Cur char: '" + (char)curChar + "'");
#fi
    int newCnt = 0;
    if (++moveIndex == Long.MAX_VALUE) {
      for (int i = 0; i < ${generatedStates}; i++) moved[i] = 0L;
      moveIndex = 1L;
    }

    final int vectorIndex = curChar >> 6;
    final long bitpattern = (1L << (curChar & 077));
    do {
      final int state = stateSet[--cnt];
#if DEBUG_TOKEN_MANAGER
      // TODO(sreeni) : Fix it to give better error message instead of UNKNOWN.
      System.err.println(
         "Looking to move from state: " + state + "; for: " +
         (jjmatchKinds[state] != Integer.MAX_VALUE
              ? tokenImage[jjmatchKinds[state]] : " "));
#fi
      if ((jjChars[state][vectorIndex] & bitpattern) != 0L) {
        // Current input character can move this NFA state. So add all the
        // next states of the current states for use with the next input char.
        for (int newState : jjnextStateSet[state]) {
          if (moved[newState] != moveIndex) {
            // We add each state only once.
            newStateSet[newCnt++] = newState;
            moved[newState] = moveIndex;
          }
        }
        final int newKind = jjmatchKinds[state];
        if (kind > newKind) {
          // It's a final state so store the matched kind if it's smaller than
          // what's already matched.
          kind = newKind;
#if DEBUG_TOKEN_MANAGER
          System.err.println(
              "Found a match of kind: " + kind + "; kind: " +
              tokenImage[kind] + " using the first: " + curPos +
              " characters.");
#fi
        }
      }
    } while (cnt > 0);

    if (kind != Integer.MAX_VALUE) {
      // We found a match. So remember the kind and position of the match.
      jjmatchedKind = kind;
      jjmatchedPos = curPos;
      // Reset the kind to max value so we can contine looking for a longer
      // match.
      kind = Integer.MAX_VALUE;
    }

    // Swap the current and next state sets.
    int[] tmp = stateSet;
    stateSet = newStateSet;
    newStateSet = tmp;
    // Reset the number of states.
    cnt = newCnt;
    if (newCnt == 0) {
      // There were no transitions from any of the current states on the
      // current input. So we are done.
#if DEBUG_TOKEN_MANAGER
      System.err.println("Done with NFA at pos: " + curPos);
#fi
      return curPos;
    }
    // Read the next character and try to continue running the NFA.
    if (!moveToNextChar()) {
      // EOF reached!
#if DEBUG_TOKEN_MANAGER
      System.err.println("Reached EOF here at pos: " + curPos);
#fi
      return curPos;
    }
    ++curPos;
  } while (cnt > 0);

  assert(false) :
      "Interal error. Please submit a bug at: http://javacc.java.net.";
  return curPos;
}

private ${STATIC?static :} int defaultLexState = ${defaultLexState};
private ${STATIC?static :} int curLexState = ${defaultLexState};
private ${STATIC?static :} int jjmatchedPos;
private ${STATIC?static :} int jjmatchedKind;
private ${STATIC?static :}final StringBuilder jjimage = new StringBuilder();
private ${STATIC?static :}StringBuilder image = jjimage;
private ${STATIC?static :}int jjimageLen;
private ${STATIC?static :}int lengthOfMatch;
protected ${STATIC?static :}int curChar;
protected ${STATIC?static :} ${charStreamName} input_stream;

public static final boolean isToken(int kind) {
  return (jjtoToken[kind >> 6] & (1L << (kind & 077))) != 0L;
}

public static final boolean isSkip(int kind) {
  return (jjtoSkip[kind >> 6] & (1L << (kind & 077))) != 0L;
}

public static final boolean isSpecial(int kind) {
  return (jjtoSpecial[kind >> 6] & (1L << (kind & 077))) != 0L;
}

public static final boolean isMore(int kind) {
  return (jjtoMore[kind >> 6] & (1L << (kind & 077))) != 0L;
}

protected ${STATIC?static :} Token jjFillToken() {
  final Token t;
  final String curTokenImage;
#if KEEP_LINE_COLUMN
  final int beginLine;
  final int endLine;
  final int beginColumn;
  final int endColumn;
#fi
  if (jjmatchedPos < 0) {
    if (image == null) {
      curTokenImage = "";
    } else {
      curTokenImage = image.toString();
    }
#if KEEP_LINE_COLUMN
    beginLine = endLine = input_stream.getEndLine();
    beginColumn = endColumn = input_stream.getEndColumn();
#fi
  } else {
    String im = jjstrLiteralImages[jjmatchedKind];
    curTokenImage = (im == null) ? input_stream.GetImage() : im;
#if KEEP_LINE_COLUMN
    beginLine = input_stream.getBeginLine();
    beginColumn = input_stream.getBeginColumn();
    endLine = input_stream.getEndLine();
    endColumn = input_stream.getEndColumn();
#fi
  }

#if TOKEN_FACTORY
    t = ${TOKEN_FACTORY}.newToken(jjmatchedKind, curTokenImage);
#elif BINARY_NEW_TOKEN
    t = Token.newToken(jjmatchedKind, curTokenImage);
#else
   t = Token.newToken(jjmatchedKind);
   t.kind = jjmatchedKind;
   t.image = curTokenImage;
#fi

#if KEEP_LINE_COLUMN
   t.beginLine = beginLine;
   t.endLine = endLine;
   t.beginColumn = beginColumn;
   t.endColumn = endColumn;
#fi

   return t;
}

/** Get the next Token. */
public ${STATIC?static :} Token getNextToken() {
  Token specialToken = null;
  Token matchedToken;
  int lastReadPosition = 0;

  EOFLoop:
  for (;;) {
    // First see if we have any input at all.
    try {
      curChar = input_stream.BeginToken();
#if IGNORE_CASE
      curChar = Character.toLowerCase(curChar);
#fi
    } catch(Exception e) {
#if DEBUG_TOKEN_MANAGER
    if (lexStateNames.length > 1) {
      System.err.print("<" + lexStateNames[curLexState] + "> ");
    }
    System.err.println("Reached EOF at " +
                       input_stream.getBeginLine() + ":" +
                       input_stream.getBeginColumn());
#fi
      // No input. So return EOF token.
      jjmatchedKind = EOF;
      jjmatchedPos = -1;
      matchedToken = jjFillToken();
      matchedToken.specialToken = specialToken;
      return matchedToken;
    }
#if DEBUG_TOKEN_MANAGER
    if (lexStateNames.length > 1) {
      System.err.print("<" + lexStateNames[curLexState] + "> ");
    }
    System.err.println("Current input char: '" + (char)curChar + "' at " +
                       input_stream.getBeginLine() + ":" +
                       input_stream.getBeginColumn());
#fi

    // Set matched kind to a MAX VALUE to implement largest, first occuring rule
    // i.e., smallest kind value matched should be used.
    image = jjimage;
    image.setLength(0);
    jjimageLen = 0;

    MoreLoop: for (;;) {
      jjmatchedKind = Integer.MAX_VALUE;
      jjmatchedPos = 0;
#if DEBUG_TOKEN_MANAGER
    if (lexStateNames.length > 1) {
      System.err.print("<" + lexStateNames[curLexState] + "> ");
    }
    System.err.println("Current input char: '" + (char)curChar + "' at " +
                       input_stream.getBeginLine() + ":" +
                       input_stream.getBeginColumn());
#fi
#if !NO_DFA
      lastReadPosition = jjRunStringLiteralMatch();
#else
      lastReadPosition = jjMoveNfa(0, 0);
#fi
      if (jjmatchedPos == 0 && jjmatchedKind > canMatchAnyChar[curLexState]) {
        jjmatchedKind = canMatchAnyChar[curLexState];
#if DEBUG_TOKEN_MANAGER
        System.err.println(
            "Matched current char: '" + (char)curChar +
            "' as a wildcard kind: " + jjmatchedKind);
#fi
      }

      if (jjmatchedKind != Integer.MAX_VALUE) {
        // We have a match!
  
        // Put back any characters looked ahead.
        input_stream.backup(lastReadPosition - jjmatchedPos);
        if (isToken(jjmatchedKind)) {
#if DEBUG_TOKEN_MANAGER
          System.err.println("Returning token.");
#fi
          // Matched kind is a real TOKEN.
          matchedToken = jjFillToken();
          matchedToken.specialToken = specialToken;
          TokenLexicalActions(matchedToken);
          if (jjnewLexState[jjmatchedKind] != -1) {
            curLexState = jjnewLexState[jjmatchedKind];
          }
#if COMMON_TOKEN_ACTION
          CommonTokenAction(matchedToken);
#fi
          return matchedToken;
        } else if (isSkip(jjmatchedKind)) {
#if DEBUG_TOKEN_MANAGER
          System.err.println("Found a SKIP match.");
#fi
          // Matched kind is a SKIP or SPECIAL_TOKEN.
          if (isSpecial(jjmatchedKind)) {
            matchedToken = jjFillToken();
            if (specialToken == null) {
              specialToken = matchedToken;
            } else {
              matchedToken.specialToken = specialToken;
              specialToken = (specialToken.next = matchedToken);
            }
            SkipLexicalActions(matchedToken);
          } else {
            SkipLexicalActions(null);
          }
          if (jjnewLexState[jjmatchedKind] != -1) {
            curLexState = jjnewLexState[jjmatchedKind];
          }
          continue EOFLoop;
        }
#if DEBUG_TOKEN_MANAGER
        System.err.println("Found a MORE match.");
#fi
        // Here it's a MORE.
        MoreLexicalActions();
        if (jjnewLexState[jjmatchedKind] != -1) {
          curLexState = jjnewLexState[jjmatchedKind];
        }
        lastReadPosition = 0;
        jjmatchedKind = 0x7fffffff;
        try {
          curChar = input_stream.readChar();
          continue;
        }
        catch (java.io.IOException e1) { }
      }
      reportError(lastReadPosition);
    }
  }
}

protected ${STATIC?static :} void reportError(int lastReadPosition) {
  int error_line = input_stream.getEndLine();
  int error_column = input_stream.getEndColumn();
  String error_after = null;
  boolean EOFSeen = false;
  try {
    input_stream.readChar();
    input_stream.backup(1);
  } catch (java.io.IOException e1) {
    EOFSeen = true;
    error_after = lastReadPosition <= 1 ? "" : input_stream.GetImage();
    if (curChar == '\n' || curChar == '\r') {
       error_line++;
       error_column = 0;
    }
    else
       error_column++;
  }
  if (!EOFSeen) {
    input_stream.backup(1);
    error_after = lastReadPosition <= 1 ? "" : input_stream.GetImage();
  }
  throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column,
                          error_after, curChar, TokenMgrError.LEXICAL_ERROR);
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy