javacc-7.0.4.src.main.resources.templates.TokenizerAlgorithm.template Maven / Gradle / Ivy
// Bit vector to track possible kinds. maxLongs = MAX(kind) / 64 + 1;
private ${STATIC?static :}final long[] jjcurActive = new long[${maxLongs}];
private ${STATIC?static :}final long[] jjnewActive = new long[${maxLongs}];
private ${STATIC?static :}final int jjRunDfa() {
int curPos = 0;
for (int i = 0; i < ${maxLongs}; i++) jjcurActive[i] = 0xffffffffffffffffL;
do {
// Algorithm: Keep trying to match a DFA as long as there is one
// possible and in the end, start the NFA if the string literal token
// match can also be a prefix of non-string literal token.
if (finalStates[curPos].containsKey((char)curChar)) {
// The current input character at the current position marks the end
// of a literal token match e.g., pos == 2 and curChar == 'r' can
// potentially mark the token "for" in Java grammar.
final int[] kindAndNext= finalStates[curPos].get((char)curChar);
for (int i = 0; i < kindAndNext.length; i += 2) {
int kind = kindAndNext[i];
// See if this kind is feasible based on the prefix. So for examaple,
// in Java grammar, pos == 3 and curChar == 'e' can indicate the match
// of one the 4 tokens - "byte", "case", "else", "true".
// The jjcurActive bit mask tracks the feasible ones as we go through
// input. So if the kind is one of the currently active kinds to
// be matched, we match it.
if ((jjcurActive[kind >> 6] & (1L << (kind % 64))) != 0L) {
// Store the kind matched so far.
jjmatchedPos = curPos;
jjmatchedKind = kind;
int nfaState = kindAndNext[i + 1];
if (nfaState != -1) {
return jjStartNfaWithStates_0(curPos, kind, nfaState);
}
}
}
}
// curChar at curPos is a prefix of another string literal token.
// So for example in Java grammar, curChar == 't' and and
// curPos == '2', then this could end up begin the "interface" token.
boolean continueDfa = false;
if (validKinds[curPos].containsKey((char)curChar)) {
int[] validKind = validKinds[curPos].get((char)curChar);
for (final int kind : validKind) {
if ((jjcurActive[kind >> 6] & (1L << (kind % 64))) != 0L) {
continueDfa = true;
break;
}
}
if (continueDfa) {
for (int i = 0; i < ${maxLongs}; i++) jjnewActive[i] = 0L;
for (final int kind : validKind) {
jjnewActive[kind >> 6] |= 1L << (kind % 64);
}
for (int i = 0; i < ${maxLongs}; i++) jjcurActive[i] &= jjnewActive[i];
}
}
if (!continueDfa) {
if (curPos == 0) return jjMoveNfa(0, 0);
return jjStartNfa_0(curPos - 1, jjcurActive);
}
int i = 0;
for (; i < ${maxLongs}; i++) {
if (jjcurActive[i] != 0L) break;
}
if (i == ${maxLongs}) {
// No more NFA or DFS possible so just return. For example, in Java
// grammar, when curPos == 1 and curChar == "|" it will return a
// "||" token as it is not a prefix of any other token.
return curPos + 1;
}
// Move to the next character.
try {
// We cannot update curChar unless we can actually advance the stream.
curChar = input_stream.readChar();
} catch(java.io.IOException e) {
// No more input so curPos and curChar are used to start the NFA.
return jjStopStringLiteralDfa_0(curPos, jjcurActive);
}
curPos++;
} while(true);
}
private ${STATIC?static :}final int jjMoveNfa(int startState, int curPos) {
if (startState<0) {
return curPos + 1;
}
java.util.Stack stateSet = new java.util.Stack();
java.util.Stack newStateSet = new java.util.Stack();
final java.util.BitSet added = new java.util.BitSet(67);
final java.util.BitSet done = new java.util.BitSet(67);
int kind = 0x7fffffff;
stateSet.push(startState);
done.set(startState);
for (int s : jjcompositeState[state]) {
if (!done.get(s)) stateSet.push(s);
done.set(s);
}
do {
added.clear();
done.clear();
do {
final int state = stateSet.pop();
done.set(state);
if (jjChars[state].get(curChar)) {
for (int newState : jjnextStateSet[state]) {
if (added.get(newState)) continue;
added.set(newState);
newStateSet.push(newState);
}
final int newKind = jjmatchKinds[state];
if (kind > newKind) kind = newKind;
}
} while (!stateSet.empty());
if (kind != 0x7fffffff) {
if (jjmatchedPos != curPos || jjmatchedKind > kind) {
jjmatchedKind = kind;
}
jjmatchedPos = curPos;
kind = 0x7fffffff;
}
++curPos;
if (newStateSet.empty()) {
return curPos;
}
java.util.Stack tmp = stateSet;
stateSet = newStateSet;
newStateSet = tmp;
try {
curChar = input_stream.readChar();
} catch(java.io.IOException e) {
return curPos;
}
} while (!stateSet.empty());
assert(false) :
"Interal error. Please submit a bug at: http://javacc.java.net.";
return curPos;
}
#fi
© 2015 - 2024 Weber Informatics LLC | Privacy Policy