javacc-7.0.4.src.main.resources.templates.TableDrivenTokenManager.template Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of javacc Show documentation
Show all versions of javacc Show documentation
JavaCC modified by Tencent blueking.
The newest version!
public class ${parserName}TokenManager ${superClass} implements ${parserName}Constants {
protected ${STATIC?static :}boolean moveToNextChar() {
try {
curChar = input_stream.readChar();
#if IGNORE_CASE
curChar = (int)Character.toLowerCase((char)curChar);
#fi
} catch(java.io.IOException e) {
return false;
}
return true;
}
${decls}
#if TOKEN_MANAGER_USES_PARSER && !STATIC
/* Declarations from the TOKEN_MGR_DECLS section */
/** Constructor with parser. */
public ${parserName}TokenManager (${parserName} parserArg, ${charStreamName} stream){
parser = parserArg;
#else
/** Constructor. */
public ${parserName}TokenManager(${charStreamName} stream){
#fi
#if STATIC && !USER_CHAR_STREAM
if (input_stream != null)
throw new ${LEGACY_EXCEPTION_HANDLING?TokenMgrError:TokenMgrException}("ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.", ${LEGACY_EXCEPTION_HANDLING?TokenMgrError:TokenMgrException}.STATIC_LEXER_ERROR);
#elif !USER_CHAR_STREAM
#if JAVA_UNICODE_ESCAPE
if (JavaCharStream.staticFlag)
#else
if (${charStreamName}.staticFlag)
#fi
throw new ${LEGACY_EXCEPTION_HANDLING?Error:RuntimeException}("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
#fi
input_stream = stream;
}
#if TOKEN_MANAGER_USES_PARSER && !STATIC
/** Constructor with parser. */
public ${parserName}TokenManager (${parserName} parserArg, ${charStreamName} stream, int lexState){
ReInit(parserArg, stream);
}
#else
/** Constructor. */
public ${parserName}TokenManager (${charStreamName} stream, int lexState){
ReInit(stream);
SwitchTo(lexState);
}
#fi
/** Reinitialise parser. */
#if TOKEN_MANAGER_USES_PARSER && !STATIC
public void ReInit(${parserName} parserArg, ${charStreamName} stream)
#else
${STATIC?static :}public void ReInit(${charStreamName} stream)
#fi
{
#if TOKEN_MANAGER_USES_PARSER && !STATIC
this.parser = parserArg;
#else
#fi
jjmatchedPos = 0;
curLexState = defaultLexState;
input_stream = stream;
}
/** Reinitialise parser. */
#if TOKEN_MANAGER_USES_PARSER && !STATIC
public void ReInit( ${parserName} parserArg, ${charStreamName} stream, int lexState)
#else
${STATIC?static :}public void ReInit(${charStreamName} stream, int lexState)
#fi
{
#if TOKEN_MANAGER_USES_PARSER && !STATIC
ReInit(parserArg, stream);
#else
ReInit(stream);
#fi
SwitchTo(lexState);
}
/** Switch to specified lex state. */
public ${STATIC?static :}void SwitchTo(int lexState) {
curLexState = lexState;
}
#if !NO_DFA
private ${STATIC?static :}final int jjRunStringLiteralMatch() {
int curPos = 0;
final int key = (int)curLexState << 16 | curChar;
final int[] arr = startAndSize.get(key);
int startState = jjInitStates[curLexState];
if (arr != null) {
int index = arr[0];
for (int i = 0; i < arr[1]; i++) {
final int len = stringLiterals[index++];
#if DEBUG_TOKEN_MANAGER
System.err.println(
"Looking for string literal match of kind: " +
stringLiterals[index + len] +
"; token image: " + tokenImage[stringLiterals[index + len]]);
#fi
do {
#if DEBUG_TOKEN_MANAGER
System.err.println("Cur char: '" + (char)curChar + "'");
#fi
if (curChar != stringLiterals[index + curPos]) break;
if (++curPos == len) break;
if (!moveToNextChar()) {
--curPos;
break;
}
} while(curPos < len);
if (curPos == len) {
jjmatchedKind = stringLiterals[index + len];
jjmatchedPos = curPos;
startState = stringLiterals[index + len + 1];
#if DEBUG_TOKEN_MANAGER
System.err.println(
"Currently matched the first: " + jjmatchedPos +
" chars as kind: " + stringLiterals[index + len] +
"; with image: " + tokenImage[jjmatchedKind]);
#fi
if (!moveToNextChar()) {
return curPos;
}
curPos++;
break;
} else {
index += len + 2;
input_stream.backup(curPos + 1);
curPos = 0;
if (!moveToNextChar()) {
assert(false);
}
}
}
} else {
#if DEBUG_TOKEN_MANAGER
System.err.println(
"No string literal start with char: '" + (char)curChar + "'");
#fi
}
return jjMoveNfa(startState, curPos);
}
#fi
private ${STATIC?static :} int[] stateSet = new int[${generatedStates}];
private ${STATIC?static :} int[] newStateSet = new int[${generatedStates}];
private ${STATIC?static :} final long[] moved = new long[${generatedStates}];
private ${STATIC?static :} long moveIndex = 1L;
private ${STATIC?static :}final int jjMoveNfa(int startState, int curPos) {
if (startState < 0) {
#if DEBUG_TOKEN_MANAGER
System.err.println("No NFA state at pos: " + curPos);
#fi
return curPos;
}
// We have a long array indexed by the NFA state number to roughly indicate
// the input position so when the input reaches part Long.MAX_VALUE (which
// should be extremely rare), we need to reset them all back to zero.
if (++moveIndex == Long.MAX_VALUE) {
for (int i = 0; i < ${generatedStates}; i++) moved[i] = 0L;
moveIndex = 1L;
}
// We initialize the kind to MAX value so that when a match is found, we can
// simply check if it's less than the current match and store it in that
// case. This helps implement the 'first occurring' rule properly.
int kind = Integer.MAX_VALUE;
int cnt = 0;
stateSet[cnt++] = startState;
moved[startState] = moveIndex;
// Some NFA states have epsilon transitions (move on empty string). So we
// just start with all of them. Note that the nextStates array already adds
// the epsilon closure. Only the initial state needs to do this explicitly.
for (int s : jjcompositeState[startState]) {
if (moved[s] != moveIndex) {
stateSet[cnt++] = s;
moved[s] = moveIndex;
}
}
#if DEBUG_TOKEN_MANAGER
System.err.println("Starting NFA with start state: " + startState);
#fi
do {
#if DEBUG_TOKEN_MANAGER
System.err.println("Cur char: '" + (char)curChar + "'");
#fi
int newCnt = 0;
if (++moveIndex == Long.MAX_VALUE) {
for (int i = 0; i < ${generatedStates}; i++) moved[i] = 0L;
moveIndex = 1L;
}
final int vectorIndex = curChar >> 6;
final long bitpattern = (1L << (curChar & 077));
do {
final int state = stateSet[--cnt];
#if DEBUG_TOKEN_MANAGER
// TODO(sreeni) : Fix it to give better error message instead of UNKNOWN.
System.err.println(
"Looking to move from state: " + state + "; for: " +
(jjmatchKinds[state] != Integer.MAX_VALUE
? tokenImage[jjmatchKinds[state]] : " "));
#fi
if ((jjChars[state][vectorIndex] & bitpattern) != 0L) {
// Current input character can move this NFA state. So add all the
// next states of the current states for use with the next input char.
for (int newState : jjnextStateSet[state]) {
if (moved[newState] != moveIndex) {
// We add each state only once.
newStateSet[newCnt++] = newState;
moved[newState] = moveIndex;
}
}
final int newKind = jjmatchKinds[state];
if (kind > newKind) {
// It's a final state so store the matched kind if it's smaller than
// what's already matched.
kind = newKind;
#if DEBUG_TOKEN_MANAGER
System.err.println(
"Found a match of kind: " + kind + "; kind: " +
tokenImage[kind] + " using the first: " + curPos +
" characters.");
#fi
}
}
} while (cnt > 0);
if (kind != Integer.MAX_VALUE) {
// We found a match. So remember the kind and position of the match.
jjmatchedKind = kind;
jjmatchedPos = curPos;
// Reset the kind to max value so we can contine looking for a longer
// match.
kind = Integer.MAX_VALUE;
}
// Swap the current and next state sets.
int[] tmp = stateSet;
stateSet = newStateSet;
newStateSet = tmp;
// Reset the number of states.
cnt = newCnt;
if (newCnt == 0) {
// There were no transitions from any of the current states on the
// current input. So we are done.
#if DEBUG_TOKEN_MANAGER
System.err.println("Done with NFA at pos: " + curPos);
#fi
return curPos;
}
// Read the next character and try to continue running the NFA.
if (!moveToNextChar()) {
// EOF reached!
#if DEBUG_TOKEN_MANAGER
System.err.println("Reached EOF here at pos: " + curPos);
#fi
return curPos;
}
++curPos;
} while (cnt > 0);
assert(false) :
"Interal error. Please submit a bug at: http://javacc.java.net.";
return curPos;
}
private ${STATIC?static :} int defaultLexState = ${defaultLexState};
private ${STATIC?static :} int curLexState = ${defaultLexState};
private ${STATIC?static :} int jjmatchedPos;
private ${STATIC?static :} int jjmatchedKind;
private ${STATIC?static :}final StringBuilder jjimage = new StringBuilder();
private ${STATIC?static :}StringBuilder image = jjimage;
private ${STATIC?static :}int jjimageLen;
private ${STATIC?static :}int lengthOfMatch;
protected ${STATIC?static :}int curChar;
protected ${STATIC?static :} ${charStreamName} input_stream;
public static final boolean isToken(int kind) {
return (jjtoToken[kind >> 6] & (1L << (kind & 077))) != 0L;
}
public static final boolean isSkip(int kind) {
return (jjtoSkip[kind >> 6] & (1L << (kind & 077))) != 0L;
}
public static final boolean isSpecial(int kind) {
return (jjtoSpecial[kind >> 6] & (1L << (kind & 077))) != 0L;
}
public static final boolean isMore(int kind) {
return (jjtoMore[kind >> 6] & (1L << (kind & 077))) != 0L;
}
protected ${STATIC?static :} Token jjFillToken() {
final Token t;
final String curTokenImage;
#if KEEP_LINE_COLUMN
final int beginLine;
final int endLine;
final int beginColumn;
final int endColumn;
#fi
if (jjmatchedPos < 0) {
if (image == null) {
curTokenImage = "";
} else {
curTokenImage = image.toString();
}
#if KEEP_LINE_COLUMN
beginLine = endLine = input_stream.getEndLine();
beginColumn = endColumn = input_stream.getEndColumn();
#fi
} else {
String im = jjstrLiteralImages[jjmatchedKind];
curTokenImage = (im == null) ? input_stream.GetImage() : im;
#if KEEP_LINE_COLUMN
beginLine = input_stream.getBeginLine();
beginColumn = input_stream.getBeginColumn();
endLine = input_stream.getEndLine();
endColumn = input_stream.getEndColumn();
#fi
}
#if TOKEN_FACTORY
t = ${TOKEN_FACTORY}.newToken(jjmatchedKind, curTokenImage);
#elif BINARY_NEW_TOKEN
t = Token.newToken(jjmatchedKind, curTokenImage);
#else
t = Token.newToken(jjmatchedKind);
t.kind = jjmatchedKind;
t.image = curTokenImage;
#fi
#if KEEP_LINE_COLUMN
t.beginLine = beginLine;
t.endLine = endLine;
t.beginColumn = beginColumn;
t.endColumn = endColumn;
#fi
return t;
}
/** Get the next Token. */
public ${STATIC?static :} Token getNextToken() {
Token specialToken = null;
Token matchedToken;
int lastReadPosition = 0;
EOFLoop:
for (;;) {
// First see if we have any input at all.
try {
curChar = input_stream.BeginToken();
#if IGNORE_CASE
curChar = Character.toLowerCase(curChar);
#fi
} catch(Exception e) {
#if DEBUG_TOKEN_MANAGER
if (lexStateNames.length > 1) {
System.err.print("<" + lexStateNames[curLexState] + "> ");
}
System.err.println("Reached EOF at " +
input_stream.getBeginLine() + ":" +
input_stream.getBeginColumn());
#fi
// No input. So return EOF token.
jjmatchedKind = EOF;
jjmatchedPos = -1;
matchedToken = jjFillToken();
matchedToken.specialToken = specialToken;
return matchedToken;
}
#if DEBUG_TOKEN_MANAGER
if (lexStateNames.length > 1) {
System.err.print("<" + lexStateNames[curLexState] + "> ");
}
System.err.println("Current input char: '" + (char)curChar + "' at " +
input_stream.getBeginLine() + ":" +
input_stream.getBeginColumn());
#fi
// Set matched kind to a MAX VALUE to implement largest, first occuring rule
// i.e., smallest kind value matched should be used.
image = jjimage;
image.setLength(0);
jjimageLen = 0;
MoreLoop: for (;;) {
jjmatchedKind = Integer.MAX_VALUE;
jjmatchedPos = 0;
#if DEBUG_TOKEN_MANAGER
if (lexStateNames.length > 1) {
System.err.print("<" + lexStateNames[curLexState] + "> ");
}
System.err.println("Current input char: '" + (char)curChar + "' at " +
input_stream.getBeginLine() + ":" +
input_stream.getBeginColumn());
#fi
#if !NO_DFA
lastReadPosition = jjRunStringLiteralMatch();
#else
lastReadPosition = jjMoveNfa(0, 0);
#fi
if (jjmatchedPos == 0 && jjmatchedKind > canMatchAnyChar[curLexState]) {
jjmatchedKind = canMatchAnyChar[curLexState];
#if DEBUG_TOKEN_MANAGER
System.err.println(
"Matched current char: '" + (char)curChar +
"' as a wildcard kind: " + jjmatchedKind);
#fi
}
if (jjmatchedKind != Integer.MAX_VALUE) {
// We have a match!
// Put back any characters looked ahead.
input_stream.backup(lastReadPosition - jjmatchedPos);
if (isToken(jjmatchedKind)) {
#if DEBUG_TOKEN_MANAGER
System.err.println("Returning token.");
#fi
// Matched kind is a real TOKEN.
matchedToken = jjFillToken();
matchedToken.specialToken = specialToken;
TokenLexicalActions(matchedToken);
if (jjnewLexState[jjmatchedKind] != -1) {
curLexState = jjnewLexState[jjmatchedKind];
}
#if COMMON_TOKEN_ACTION
CommonTokenAction(matchedToken);
#fi
return matchedToken;
} else if (isSkip(jjmatchedKind)) {
#if DEBUG_TOKEN_MANAGER
System.err.println("Found a SKIP match.");
#fi
// Matched kind is a SKIP or SPECIAL_TOKEN.
if (isSpecial(jjmatchedKind)) {
matchedToken = jjFillToken();
if (specialToken == null) {
specialToken = matchedToken;
} else {
matchedToken.specialToken = specialToken;
specialToken = (specialToken.next = matchedToken);
}
SkipLexicalActions(matchedToken);
} else {
SkipLexicalActions(null);
}
if (jjnewLexState[jjmatchedKind] != -1) {
curLexState = jjnewLexState[jjmatchedKind];
}
continue EOFLoop;
}
#if DEBUG_TOKEN_MANAGER
System.err.println("Found a MORE match.");
#fi
// Here it's a MORE.
MoreLexicalActions();
if (jjnewLexState[jjmatchedKind] != -1) {
curLexState = jjnewLexState[jjmatchedKind];
}
lastReadPosition = 0;
jjmatchedKind = 0x7fffffff;
try {
curChar = input_stream.readChar();
continue;
}
catch (java.io.IOException e1) { }
}
reportError(lastReadPosition);
}
}
}
protected ${STATIC?static :} void reportError(int lastReadPosition) {
int error_line = input_stream.getEndLine();
int error_column = input_stream.getEndColumn();
String error_after = null;
boolean EOFSeen = false;
try {
input_stream.readChar();
input_stream.backup(1);
} catch (java.io.IOException e1) {
EOFSeen = true;
error_after = lastReadPosition <= 1 ? "" : input_stream.GetImage();
if (curChar == '\n' || curChar == '\r') {
error_line++;
error_column = 0;
}
else
error_column++;
}
if (!EOFSeen) {
input_stream.backup(1);
error_after = lastReadPosition <= 1 ? "" : input_stream.GetImage();
}
throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column,
error_after, curChar, TokenMgrError.LEXICAL_ERROR);
}