javacc-7.0.4.src.main.java.org.javacc.parser.LexGen Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of javacc Show documentation
Show all versions of javacc Show documentation
JavaCC modified by Tencent blueking.
The newest version!
// Copyright 2011 Google Inc. All Rights Reserved.
// Author: [email protected] (Sreeni Viswanadha)
/* Copyright (c) 2006, Sun Microsystems, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the Sun Microsystems, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.javacc.parser;
import java.io.*;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.javacc.utils.OutputFileGenerator;
import static org.javacc.parser.JavaCCGlobals.*;
/**
* Generate lexer.
*/
public class LexGen extends CodeGenerator implements JavaCCParserConstants
{
private static final String DUMP_STATIC_VAR_DECLARATIONS_TEMPLATE_RESOURCE_URL = "/templates/DumpStaticVarDeclarations.template";
private static final String DUMP_DEBUG_METHODS_TEMPLATE_RESOURCE_URL = "/templates/DumpDebugMethods.template";
private static final String BOILERPLATER_METHOD_RESOURCE_URL = "/templates/TokenManagerBoilerPlateMethods.template";
public static String staticString;
public static String tokMgrClassName;
// Hashtable of vectors
static Hashtable allTpsForState = new Hashtable();
public static int lexStateIndex = 0;
static int[] kinds;
public static int maxOrdinal = 1;
public static String lexStateSuffix;
static String[] newLexState;
public static int[] lexStates;
public static boolean[] ignoreCase;
public static Action[] actions;
public static Hashtable initStates = new Hashtable();
public static int stateSetSize;
public static int totalNumStates;
public static int maxLexStates;
public static String[] lexStateName;
static NfaState[] singlesToSkip;
public static long[] toSkip;
public static long[] toSpecial;
public static long[] toMore;
public static long[] toToken;
public static int defaultLexState;
public static RegularExpression[] rexprs;
public static int[] maxLongsReqd;
public static int[] initMatch;
public static int[] canMatchAnyChar;
public static boolean hasEmptyMatch;
public static boolean[] canLoop;
public static boolean[] stateHasActions;
public static boolean hasLoop = false;
public static boolean[] canReachOnMore;
public static boolean[] hasNfa;
public static boolean[] mixed;
public static NfaState initialState;
public static int curKind;
static boolean hasSkipActions = false;
static boolean hasMoreActions = false;
static boolean hasTokenActions = false;
static boolean hasSpecial = false;
static boolean hasSkip = false;
static boolean hasMore = false;
public static RegularExpression curRE;
public static boolean keepLineCol;
public static String errorHandlingClass;
public static TokenizerData tokenizerData;
public static boolean generateDataOnly;
void PrintClassHead()
{
int i, j;
List tn = new ArrayList(toolNames);
tn.add(toolName);
// TODO :: CBA -- Require Unification of output language specific processing into a single Enum class
genCodeLine("/* " + getIdString(tn, tokMgrClassName + getFileExtension(Options.getOutputLanguage())) + " */");
int l = 0, kind;
i = 1;
for (;;)
{
if (cu_to_insertion_point_1.size() <= l)
break;
kind = ((Token)cu_to_insertion_point_1.get(l)).kind;
if(kind == PACKAGE || kind == IMPORT) {
for (; i < cu_to_insertion_point_1.size(); i++) {
kind = ((Token)cu_to_insertion_point_1.get(i)).kind;
if (kind == SEMICOLON ||
kind == ABSTRACT ||
kind == FINAL ||
kind == PUBLIC ||
kind == CLASS ||
kind == INTERFACE)
{
cline = ((Token)(cu_to_insertion_point_1.get(l))).beginLine;
ccol = ((Token)(cu_to_insertion_point_1.get(l))).beginColumn;
for (j = l; j < i; j++) {
printToken((Token)(cu_to_insertion_point_1.get(j)));
}
if (kind == SEMICOLON)
printToken((Token)(cu_to_insertion_point_1.get(j)));
genCodeLine("");
break;
}
}
l = ++i;
}
else
break;
}
genCodeLine("");
genCodeLine("/** Token Manager. */");
if(Options.getSupportClassVisibilityPublic()) {
//genModifier("public ");
genModifier("public ");
}
//genCodeLine("class " + tokMgrClassName + " implements " +
//cu_name + "Constants");
//String superClass = Options.stringValue(Options.USEROPTION__TOKEN_MANAGER_SUPER_CLASS);
genClassStart(null, tokMgrClassName, new String[]{}, new String[]{cu_name + "Constants"});
//genCodeLine("{"); // }
if (token_mgr_decls != null && token_mgr_decls.size() > 0)
{
Token t = (Token)token_mgr_decls.get(0);
boolean commonTokenActionSeen = false;
boolean commonTokenActionNeeded = Options.getCommonTokenAction();
printTokenSetup((Token)token_mgr_decls.get(0));
ccol = 1;
for (j = 0; j < token_mgr_decls.size(); j++)
{
t = (Token)token_mgr_decls.get(j);
if (t.kind == IDENTIFIER &&
commonTokenActionNeeded &&
!commonTokenActionSeen)
commonTokenActionSeen = t.image.equals("CommonTokenAction");
printToken(t);
}
genCodeLine("");
if (commonTokenActionNeeded && !commonTokenActionSeen)
JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +
"But it appears you have not defined the method :\n"+
" " + staticString + "void CommonTokenAction(Token t)\n" +
"in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
}
else if (Options.getCommonTokenAction())
{
JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +
"But you have not defined the method :\n"+
" " + staticString + "void CommonTokenAction(Token t)\n" +
"in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
}
genCodeLine("");
genCodeLine(" /** Debug output. */");
genCodeLine(" public " + staticString + " java.io.PrintStream debugStream = System.out;");
genCodeLine(" /** Set debug output. */");
genCodeLine(" public " + staticString + " void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }");
if(Options.getTokenManagerUsesParser()){
genCodeLine("");
genCodeLine(" public " + cu_name + " parser = null;");
}
}
@SuppressWarnings("unchecked")
protected void writeTemplate(String name, Object... additionalOptions) throws IOException
{
Map options = new HashMap(Options.getOptions());
options.put("maxOrdinal", Integer.valueOf(maxOrdinal));
options.put("maxLexStates", Integer.valueOf(maxLexStates));
options.put("hasEmptyMatch", Boolean.valueOf(hasEmptyMatch));
options.put("hasSkip", Boolean.valueOf(hasSkip));
options.put("hasMore", Boolean.valueOf(hasMore));
options.put("hasSpecial", Boolean.valueOf(hasSpecial));
options.put("hasMoreActions", Boolean.valueOf(hasMoreActions));
options.put("hasSkipActions", Boolean.valueOf(hasSkipActions));
options.put("hasTokenActions", Boolean.valueOf(hasTokenActions));
options.put("stateSetSize", stateSetSize);
options.put("hasActions", hasMoreActions || hasSkipActions || hasTokenActions);
options.put("tokMgrClassName", tokMgrClassName);
int x = 0;
for (int l : maxLongsReqd) x = Math.max(x, l);
options.put("maxLongs", x);
options.put("cu_name", cu_name);
// options.put("", .valueOf(maxOrdinal));
for (int i = 0; i < additionalOptions.length; i++)
{
Object o = additionalOptions[i];
if (o instanceof Map)
{
options.putAll((Map) o);
}
else
{
if (i == additionalOptions.length - 1)
throw new IllegalArgumentException("Must supply pairs of [name value] args");
options.put((String) o, additionalOptions[i+1]);
i++;
}
}
OutputFileGenerator gen = new OutputFileGenerator(name, options);
StringWriter sw = new StringWriter();
gen.generate(new PrintWriter(sw));
sw.close();
genCode(sw.toString());
}
void DumpDebugMethods() throws IOException
{
writeTemplate(DUMP_DEBUG_METHODS_TEMPLATE_RESOURCE_URL);
}
static void BuildLexStatesTable()
{
Iterator it = rexprlist.iterator();
TokenProduction tp;
int i;
String[] tmpLexStateName = new String[lexstate_I2S.size()];
while (it.hasNext())
{
tp = it.next();
List respecs = tp.respecs;
List tps;
for (i = 0; i < tp.lexStates.length; i++)
{
if ((tps = (List)allTpsForState.get(tp.lexStates[i])) == null)
{
tmpLexStateName[maxLexStates++] = tp.lexStates[i];
allTpsForState.put(tp.lexStates[i], tps = new ArrayList());
}
tps.add(tp);
}
if (respecs == null || respecs.size() == 0)
continue;
RegularExpression re;
for (i = 0; i < respecs.size(); i++)
if (maxOrdinal <= (re = ((RegExprSpec)respecs.get(i)).rexp).ordinal)
maxOrdinal = re.ordinal + 1;
}
kinds = new int[maxOrdinal];
toSkip = new long[maxOrdinal / 64 + 1];
toSpecial = new long[maxOrdinal / 64 + 1];
toMore = new long[maxOrdinal / 64 + 1];
toToken = new long[maxOrdinal / 64 + 1];
toToken[0] = 1L;
actions = new Action[maxOrdinal];
actions[0] = actForEof;
hasTokenActions = actForEof != null;
initStates = new Hashtable();
canMatchAnyChar = new int[maxLexStates];
canLoop = new boolean[maxLexStates];
stateHasActions = new boolean[maxLexStates];
lexStateName = new String[maxLexStates];
singlesToSkip = new NfaState[maxLexStates];
System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);
for (i = 0; i < maxLexStates; i++)
canMatchAnyChar[i] = -1;
hasNfa = new boolean[maxLexStates];
mixed = new boolean[maxLexStates];
maxLongsReqd = new int[maxLexStates];
initMatch = new int[maxLexStates];
newLexState = new String[maxOrdinal];
newLexState[0] = nextStateForEof;
hasEmptyMatch = false;
lexStates = new int[maxOrdinal];
ignoreCase = new boolean[maxOrdinal];
rexprs = new RegularExpression[maxOrdinal];
RStringLiteral.allImages = new String[maxOrdinal];
canReachOnMore = new boolean[maxLexStates];
}
static int GetIndex(String name)
{
for (int i = 0; i < lexStateName.length; i++)
if (lexStateName[i] != null && lexStateName[i].equals(name))
return i;
throw new Error(); // Should never come here
}
public static void AddCharToSkip(char c, int kind)
{
singlesToSkip[lexStateIndex].AddChar(c);
singlesToSkip[lexStateIndex].kind = kind;
}
public void start() throws IOException
{
if (!Options.getBuildTokenManager() ||
Options.getUserTokenManager() ||
JavaCCErrors.get_error_count() > 0)
return;
final String codeGeneratorClass = Options.getTokenManagerCodeGenerator();
keepLineCol = Options.getKeepLineColumn();
errorHandlingClass = Options.getTokenMgrErrorClass();
List choices = new ArrayList();
Enumeration e;
TokenProduction tp;
int i, j;
staticString = (Options.getStatic() ? "static " : "");
tokMgrClassName = cu_name + "TokenManager";
if (!generateDataOnly && codeGeneratorClass == null) PrintClassHead();
BuildLexStatesTable();
e = allTpsForState.keys();
boolean ignoring = false;
while (e.hasMoreElements())
{
int startState = -1;
NfaState.ReInit();
RStringLiteral.ReInit();
String key = (String)e.nextElement();
lexStateIndex = GetIndex(key);
lexStateSuffix = "_" + lexStateIndex;
List allTps = (List)allTpsForState.get(key);
initStates.put(key, initialState = new NfaState());
ignoring = false;
singlesToSkip[lexStateIndex] = new NfaState();
singlesToSkip[lexStateIndex].dummy = true;
if (key.equals("DEFAULT"))
defaultLexState = lexStateIndex;
for (i = 0; i < allTps.size(); i++)
{
tp = (TokenProduction)allTps.get(i);
int kind = tp.kind;
boolean ignore = tp.ignoreCase;
List rexps = tp.respecs;
if (i == 0)
ignoring = ignore;
for (j = 0; j < rexps.size(); j++)
{
RegExprSpec respec = (RegExprSpec)rexps.get(j);
curRE = respec.rexp;
rexprs[curKind = curRE.ordinal] = curRE;
lexStates[curRE.ordinal] = lexStateIndex;
ignoreCase[curRE.ordinal] = ignore;
if (curRE.private_rexp)
{
kinds[curRE.ordinal] = -1;
continue;
}
if (!Options.getNoDfa() && curRE instanceof RStringLiteral &&
!((RStringLiteral)curRE).image.equals(""))
{
((RStringLiteral)curRE).GenerateDfa(this, curRE.ordinal);
if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore) {
mixed[lexStateIndex] = true;
}
}
else if (curRE.CanMatchAnyChar())
{
if (canMatchAnyChar[lexStateIndex] == -1 ||
canMatchAnyChar[lexStateIndex] > curRE.ordinal)
canMatchAnyChar[lexStateIndex] = curRE.ordinal;
}
else
{
Nfa temp;
if (curRE instanceof RChoice)
choices.add(curRE);
temp = curRE.GenerateNfa(ignore);
temp.end.isFinal = true;
temp.end.kind = curRE.ordinal;
initialState.AddMove(temp.start);
}
if (kinds.length < curRE.ordinal)
{
int[] tmp = new int[curRE.ordinal + 1];
System.arraycopy(kinds, 0, tmp, 0, kinds.length);
kinds = tmp;
}
//System.out.println(" ordina : " + curRE.ordinal);
kinds[curRE.ordinal] = kind;
if (respec.nextState != null &&
!respec.nextState.equals(lexStateName[lexStateIndex]))
newLexState[curRE.ordinal] = respec.nextState;
if (respec.act != null && respec.act.getActionTokens() != null &&
respec.act.getActionTokens().size() > 0)
actions[curRE.ordinal] = respec.act;
switch(kind)
{
case TokenProduction.SPECIAL :
hasSkipActions |= (actions[curRE.ordinal] != null) ||
(newLexState[curRE.ordinal] != null);
hasSpecial = true;
toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
break;
case TokenProduction.SKIP :
hasSkipActions |= (actions[curRE.ordinal] != null);
hasSkip = true;
toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
break;
case TokenProduction.MORE :
hasMoreActions |= (actions[curRE.ordinal] != null);
hasMore = true;
toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
if (newLexState[curRE.ordinal] != null)
canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;
else
canReachOnMore[lexStateIndex] = true;
break;
case TokenProduction.TOKEN :
hasTokenActions |= (actions[curRE.ordinal] != null);
toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
break;
}
}
}
// Generate a static block for initializing the nfa transitions
NfaState.ComputeClosures();
for (i = 0; i < initialState.epsilonMoves.size(); i++)
((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();
if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))
{
initialState.GenerateCode();
startState = initialState.GenerateInitMoves(this);
}
if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)
{
if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||
(toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
hasSkipActions = true;
else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
hasMoreActions = true;
else
hasTokenActions = true;
if (initMatch[lexStateIndex] == 0 ||
initMatch[lexStateIndex] > initialState.kind)
{
initMatch[lexStateIndex] = initialState.kind;
hasEmptyMatch = true;
}
}
else if (initMatch[lexStateIndex] == 0)
initMatch[lexStateIndex] = Integer.MAX_VALUE;
RStringLiteral.FillSubString();
if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
RStringLiteral.GenerateNfaStartStates(this, initialState);
if (generateDataOnly || codeGeneratorClass != null) {
RStringLiteral.UpdateStringLiteralData(totalNumStates, lexStateIndex);
NfaState.UpdateNfaData(totalNumStates, startState, lexStateIndex,
canMatchAnyChar[lexStateIndex]);
} else {
RStringLiteral.DumpDfaCode(this);
if (hasNfa[lexStateIndex]) {
NfaState.DumpMoveNfa(this);
}
}
totalNumStates += NfaState.generatedStates;
if (stateSetSize < NfaState.generatedStates)
stateSetSize = NfaState.generatedStates;
}
for (i = 0; i < choices.size(); i++)
((RChoice)choices.get(i)).CheckUnmatchability();
CheckEmptyStringMatch();
if (generateDataOnly || codeGeneratorClass != null) {
tokenizerData.setParserName(cu_name);
NfaState.BuildTokenizerData(tokenizerData);
RStringLiteral.BuildTokenizerData(tokenizerData);
int[] newLexStateIndices = new int[maxOrdinal];
StringBuilder tokenMgrDecls = new StringBuilder();
if (token_mgr_decls != null && token_mgr_decls.size() > 0) {
Token t = (Token)token_mgr_decls.get(0);
for (j = 0; j < token_mgr_decls.size(); j++) {
tokenMgrDecls.append(((Token)token_mgr_decls.get(j)).image + " ");
}
}
tokenizerData.setDecls(tokenMgrDecls.toString());
Map actionStrings = new HashMap();
for (i = 0; i < maxOrdinal; i++) {
if (newLexState[i] == null) {
newLexStateIndices[i] = -1;
} else {
newLexStateIndices[i] = GetIndex(newLexState[i]);
}
// For java, we have this but for other languages, eventually we will
// simply have a string.
Action act = actions[i];
if (act == null) continue;
StringBuilder sb = new StringBuilder();
for (int k = 0; k < act.getActionTokens().size(); k++) {
sb.append(((Token)act.getActionTokens().get(k)).image);
sb.append(" ");
}
actionStrings.put(i, sb.toString());
}
tokenizerData.setDefaultLexState(defaultLexState);
tokenizerData.setLexStateNames(lexStateName);
tokenizerData.updateMatchInfo(
actionStrings, newLexStateIndices,
toSkip, toSpecial, toMore, toToken);
if (generateDataOnly) return;
Class codeGenClazz;
TokenManagerCodeGenerator gen;
try {
codeGenClazz = (Class)Class.forName(codeGeneratorClass);
gen = codeGenClazz.newInstance();
} catch(Exception ee) {
JavaCCErrors.semantic_error(
"Cound not load the token manager code generator class: " +
codeGeneratorClass + "\nError: " + ee.getMessage());
return;
}
gen.generateCode(tokenizerData);
gen.finish(tokenizerData);
return;
}
RStringLiteral.DumpStrLiteralImages(this);
DumpFillToken();
NfaState.DumpStateSets(this);
NfaState.DumpNonAsciiMoveMethods(this);
DumpGetNextToken();
if (Options.getDebugTokenManager())
{
NfaState.DumpStatesForKind(this);
DumpDebugMethods();
}
if (hasLoop)
{
genCodeLine(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");
genCodeLine(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");
genCodeLine(staticString + "" + Options.getBooleanType() + "[] jjbeenHere = new " + Options.getBooleanType() + "[" + maxLexStates + "];");
}
DumpSkipActions();
DumpMoreActions();
DumpTokenActions();
NfaState.PrintBoilerPlate(this);
String charStreamName;
if (Options.getUserCharStream())
charStreamName = "CharStream";
else
{
if (Options.getJavaUnicodeEscape())
charStreamName = "JavaCharStream";
else
charStreamName = "SimpleCharStream";
}
writeTemplate(BOILERPLATER_METHOD_RESOURCE_URL,
"charStreamName", charStreamName,
"lexStateNameLength", lexStateName.length,
"defaultLexState", defaultLexState,
"noDfa", Options.getNoDfa(),
"generatedStates", totalNumStates);
DumpStaticVarDeclarations(charStreamName);
genCodeLine(/*{*/ "}");
// TODO :: CBA -- Require Unification of output language specific processing into a single Enum class
String fileName = Options.getOutputDirectory() + File.separator +
tokMgrClassName +
getFileExtension(Options.getOutputLanguage());
if (Options.getBuildParser()) {
saveOutput(fileName);
}
}
static void CheckEmptyStringMatch()
{
int i, j, k, len;
boolean[] seen = new boolean[maxLexStates];
boolean[] done = new boolean[maxLexStates];
String cycle;
String reList;
Outer:
for (i = 0; i < maxLexStates; i++)
{
if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||
canMatchAnyChar[i] != -1)
continue;
done[i] = true;
len = 0;
cycle = "";
reList = "";
for (k = 0; k < maxLexStates; k++)
seen[k] = false;
j = i;
seen[i] = true;
cycle += lexStateName[j] + "-->";
while (newLexState[initMatch[j]] != null)
{
cycle += newLexState[initMatch[j]];
if (seen[j = GetIndex(newLexState[initMatch[j]])])
break;
cycle += "-->";
done[j] = true;
seen[j] = true;
if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||
canMatchAnyChar[j] != -1)
continue Outer;
if (len != 0)
reList += "; ";
reList += "line " + rexprs[initMatch[j]].getLine() + ", column " +
rexprs[initMatch[j]].getColumn();
len++;
}
if (newLexState[initMatch[j]] == null)
cycle += lexStateName[lexStates[initMatch[j]]];
for (k = 0; k < maxLexStates; k++)
canLoop[k] |= seen[k];
hasLoop = true;
if (len == 0)
JavaCCErrors.warning(rexprs[initMatch[i]],
"Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
? "" : (" for " + rexprs[initMatch[i]].label)) +
" can be matched by the empty string (\"\") in lexical state " +
lexStateName[i] + ". This can result in an endless loop of " +
"empty string matches.");
else
{
JavaCCErrors.warning(rexprs[initMatch[i]],
"Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
? "" : (" for " + rexprs[initMatch[i]].label)) +
" can be matched by the empty string (\"\") in lexical state " +
lexStateName[i] + ". This regular expression along with the " +
"regular expressions at " + reList + " forms the cycle \n " +
cycle + "\ncontaining regular expressions with empty matches." +
" This can result in an endless loop of empty string matches.");
}
}
}
private void DumpStaticVarDeclarations(String charStreamName) throws IOException
{
int i;
genCodeLine("");
genCodeLine("/** Lexer state names. */");
genCodeLine("public static final String[] lexStateNames = {");
for (i = 0; i < maxLexStates; i++)
genCodeLine(" \"" + lexStateName[i] + "\",");
genCodeLine("};");
{
genCodeLine("");
genCodeLine("/** Lex State array. */");
genCode("public static final int[] jjnewLexState = {");
for (i = 0; i < maxOrdinal; i++)
{
if (i % 25 == 0)
genCode("\n ");
if (newLexState[i] == null)
genCode("-1, ");
else
genCode(GetIndex(newLexState[i]) + ", ");
}
genCodeLine("\n};");
}
{
// Bit vector for TOKEN
genCode("static final long[] jjtoToken = {");
for (i = 0; i < maxOrdinal / 64 + 1; i++)
{
if (i % 4 == 0)
genCode("\n ");
genCode("0x" + Long.toHexString(toToken[i]) + "L, ");
}
genCodeLine("\n};");
}
{
// Bit vector for SKIP
genCode("static final long[] jjtoSkip = {");
for (i = 0; i < maxOrdinal / 64 + 1; i++)
{
if (i % 4 == 0)
genCode("\n ");
genCode("0x" + Long.toHexString(toSkip[i]) + "L, ");
}
genCodeLine("\n};");
}
{
// Bit vector for SPECIAL
genCode("static final long[] jjtoSpecial = {");
for (i = 0; i < maxOrdinal / 64 + 1; i++)
{
if (i % 4 == 0)
genCode("\n ");
genCode("0x" + Long.toHexString(toSpecial[i]) + "L, ");
}
genCodeLine("\n};");
}
{
// Bit vector for MORE
genCode("static final long[] jjtoMore = {");
for (i = 0; i < maxOrdinal / 64 + 1; i++)
{
if (i % 4 == 0)
genCode("\n ");
genCode("0x" + Long.toHexString(toMore[i]) + "L, ");
}
genCodeLine("\n};");
}
writeTemplate(DUMP_STATIC_VAR_DECLARATIONS_TEMPLATE_RESOURCE_URL,
"charStreamName", charStreamName,
"protected", isJavaLanguage() ? "protected" : "",
"private", isJavaLanguage() ? "private" : "",
"final", isJavaLanguage() ? "final" : "",
"lexStateNameLength", lexStateName.length);
}
// Assumes l != 0L
static char MaxChar(long l)
{
for (int i = 64; i-- > 0; )
if ((l & (1L << i)) != 0L)
return (char)i;
return 0xffff;
}
void DumpFillToken()
{
final double tokenVersion = JavaFiles.getVersion("Token.java");
final boolean hasBinaryNewToken = tokenVersion > 4.09;
genCodeLine(staticString + "protected Token jjFillToken()");
genCodeLine("{");
genCodeLine(" final Token t;");
genCodeLine(" final String curTokenImage;");
if (keepLineCol)
{
genCodeLine(" final int beginLine;");
genCodeLine(" final int endLine;");
genCodeLine(" final int beginColumn;");
genCodeLine(" final int endColumn;");
}
if (hasEmptyMatch)
{
genCodeLine(" if (jjmatchedPos < 0)");
genCodeLine(" {");
genCodeLine(" if (image == null)");
genCodeLine(" curTokenImage = \"\";");
genCodeLine(" else");
genCodeLine(" curTokenImage = image.toString();");
if (keepLineCol)
{
genCodeLine(" beginLine = endLine = input_stream.getEndLine();");
genCodeLine(" beginColumn = endColumn = input_stream.getEndColumn();");
}
genCodeLine(" }");
genCodeLine(" else");
genCodeLine(" {");
genCodeLine(" String im = jjstrLiteralImages[jjmatchedKind];");
genCodeLine(" curTokenImage = (im == null) ? input_stream.GetImage() : im;");
if (keepLineCol)
{
genCodeLine(" beginLine = input_stream.getBeginLine();");
genCodeLine(" beginColumn = input_stream.getBeginColumn();");
genCodeLine(" endLine = input_stream.getEndLine();");
genCodeLine(" endColumn = input_stream.getEndColumn();");
}
genCodeLine(" }");
}
else
{
genCodeLine(" String im = jjstrLiteralImages[jjmatchedKind];");
genCodeLine(" curTokenImage = (im == null) ? input_stream.GetImage() : im;");
if (keepLineCol)
{
genCodeLine(" beginLine = input_stream.getBeginLine();");
genCodeLine(" beginColumn = input_stream.getBeginColumn();");
genCodeLine(" endLine = input_stream.getEndLine();");
genCodeLine(" endColumn = input_stream.getEndColumn();");
}
}
if (Options.getTokenFactory().length() > 0) {
genCodeLine(" t = " + Options.getTokenFactory() + ".newToken(jjmatchedKind, curTokenImage);");
} else if (hasBinaryNewToken)
{
genCodeLine(" t = Token.newToken(jjmatchedKind, curTokenImage);");
}
else
{
genCodeLine(" t = Token.newToken(jjmatchedKind);");
genCodeLine(" t.kind = jjmatchedKind;");
genCodeLine(" t.image = curTokenImage;");
}
if (keepLineCol) {
genCodeLine("");
genCodeLine(" t.beginLine = beginLine;");
genCodeLine(" t.endLine = endLine;");
genCodeLine(" t.beginColumn = beginColumn;");
genCodeLine(" t.endColumn = endColumn;");
}
genCodeLine("");
genCodeLine(" return t;");
genCodeLine("}");
}
void DumpGetNextToken()
{
int i;
genCodeLine("");
genCodeLine(staticString + "int curLexState = " + defaultLexState + ";");
genCodeLine(staticString + "int defaultLexState = " + defaultLexState + ";");
genCodeLine(staticString + "int jjnewStateCnt;");
genCodeLine(staticString + "int jjround;");
genCodeLine(staticString + "int jjmatchedPos;");
genCodeLine(staticString + "int jjmatchedKind;");
genCodeLine("");
genCodeLine("/** Get the next Token. */");
genCodeLine("public " + staticString + "Token getNextToken()" +
" ");
genCodeLine("{");
if (hasSpecial) {
genCodeLine(" Token specialToken = null;");
}
genCodeLine(" Token matchedToken;");
genCodeLine(" int curPos = 0;");
genCodeLine("");
genCodeLine(" EOFLoop :\n for (;;)");
genCodeLine(" {");
genCodeLine(" try");
genCodeLine(" {");
genCodeLine(" curChar = input_stream.BeginToken();");
genCodeLine(" }");
genCodeLine(" catch(Exception e)");
genCodeLine(" {");
if (Options.getDebugTokenManager())
genCodeLine(" debugStream.println(\"Returning the token.\\n\");");
genCodeLine(" jjmatchedKind = 0;");
genCodeLine(" jjmatchedPos = -1;");
genCodeLine(" matchedToken = jjFillToken();");
if (hasSpecial)
genCodeLine(" matchedToken.specialToken = specialToken;");
if (nextStateForEof != null || actForEof != null)
genCodeLine(" TokenLexicalActions(matchedToken);");
if (Options.getCommonTokenAction())
genCodeLine(" CommonTokenAction(matchedToken);");
genCodeLine(" return matchedToken;");
genCodeLine(" }");
if (hasMoreActions || hasSkipActions || hasTokenActions)
{
genCodeLine(" image = jjimage;");
genCodeLine(" image.setLength(0);");
genCodeLine(" jjimageLen = 0;");
}
genCodeLine("");
String prefix = "";
if (hasMore)
{
genCodeLine(" for (;;)");
genCodeLine(" {");
prefix = " ";
}
String endSwitch = "";
String caseStr = "";
// this also sets up the start state of the nfa
if (maxLexStates > 1)
{
genCodeLine(prefix + " switch(curLexState)");
genCodeLine(prefix + " {");
endSwitch = prefix + " }";
caseStr = prefix + " case ";
prefix += " ";
}
prefix += " ";
for(i = 0; i < maxLexStates; i++)
{
if (maxLexStates > 1)
genCodeLine(caseStr + i + ":");
if (singlesToSkip[i].HasTransitions())
{
// added the backup(0) to make JIT happy
genCodeLine(prefix + "try { input_stream.backup(0);");
if (singlesToSkip[i].asciiMoves[0] != 0L &&
singlesToSkip[i].asciiMoves[1] != 0L)
{
genCodeLine(prefix + " while ((curChar < 64" + " && (0x" +
Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
"L & (1L << curChar)) != 0L) || \n" +
prefix + " (curChar >> 6) == 1" +
" && (0x" +
Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
"L & (1L << (curChar & 077))) != 0L)");
}
else if (singlesToSkip[i].asciiMoves[1] == 0L)
{
genCodeLine(prefix + " while (curChar <= " +
(int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +
Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
"L & (1L << curChar)) != 0L)");
}
else if (singlesToSkip[i].asciiMoves[0] == 0L)
{
genCodeLine(prefix + " while (curChar > 63 && curChar <= " +
((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +
" && (0x" +
Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
"L & (1L << (curChar & 077))) != 0L)");
}
if (Options.getDebugTokenManager())
{
genCodeLine(prefix + "{");
genCodeLine(" debugStream.println(" +
(maxLexStates > 1 ?
"\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
"\"Skipping character : \" + " +
errorHandlingClass+".addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \")\");");
}
genCodeLine(prefix + " curChar = input_stream.BeginToken();");
if (Options.getDebugTokenManager())
genCodeLine(prefix + "}");
genCodeLine(prefix + "}");
genCodeLine(prefix + "catch (java.io.IOException e1) { continue EOFLoop; }");
}
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
{
if (Options.getDebugTokenManager())
genCodeLine(" debugStream.println(\" Matched the empty string as \" + tokenImage[" +
initMatch[i] + "] + \" token.\");");
genCodeLine(prefix + "jjmatchedKind = " + initMatch[i] + ";");
genCodeLine(prefix + "jjmatchedPos = -1;");
genCodeLine(prefix + "curPos = 0;");
}
else
{
genCodeLine(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
genCodeLine(prefix + "jjmatchedPos = 0;");
}
if (Options.getDebugTokenManager())
genCodeLine(" debugStream.println(" +
(maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
"\"Current character : \" + " +
errorHandlingClass+".addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") " +
"at line \" + input_stream.getEndLine() + \" column \" + input_stream.getEndColumn());");
genCodeLine(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");
if (canMatchAnyChar[i] != -1)
{
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
genCodeLine(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +
canMatchAnyChar[i] + "))");
else
genCodeLine(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +
canMatchAnyChar[i] + ")");
genCodeLine(prefix + "{");
if (Options.getDebugTokenManager())
genCodeLine(" debugStream.println(\" Current character matched as a \" + tokenImage[" +
canMatchAnyChar[i] + "] + \" token.\");");
genCodeLine(prefix + " jjmatchedKind = " + canMatchAnyChar[i] + ";");
if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
genCodeLine(prefix + " jjmatchedPos = 0;");
genCodeLine(prefix + "}");
}
if (maxLexStates > 1)
genCodeLine(prefix + "break;");
}
if (maxLexStates > 1)
genCodeLine(endSwitch);
else if (maxLexStates == 0)
genCodeLine(" jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
if (maxLexStates > 1)
prefix = " ";
else
prefix = "";
if (maxLexStates > 0)
{
genCodeLine(prefix + " if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
genCodeLine(prefix + " {");
genCodeLine(prefix + " if (jjmatchedPos + 1 < curPos)");
if (Options.getDebugTokenManager())
{
genCodeLine(prefix + " {");
genCodeLine(prefix + " debugStream.println(" +
"\" Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");
}
genCodeLine(prefix + " input_stream.backup(curPos - jjmatchedPos - 1);");
if (Options.getDebugTokenManager())
genCodeLine(prefix + " }");
if (Options.getDebugTokenManager())
{
if (Options.getJavaUnicodeEscape() ||
Options.getUserCharStream())
genCodeLine(" debugStream.println(" +
"\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +
"(\" + "+errorHandlingClass+".addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +
"\") ******\\n\");");
else
genCodeLine(" debugStream.println(" +
"\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +
"(\" + "+errorHandlingClass+".addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +
"\") ******\\n\");");
}
if (hasSkip || hasMore || hasSpecial)
{
genCodeLine(prefix + " if ((jjtoToken[jjmatchedKind >> 6] & " +
"(1L << (jjmatchedKind & 077))) != 0L)");
genCodeLine(prefix + " {");
}
genCodeLine(prefix + " matchedToken = jjFillToken();");
if (hasSpecial)
genCodeLine(prefix + " matchedToken.specialToken = specialToken;");
if (hasTokenActions)
genCodeLine(prefix + " TokenLexicalActions(matchedToken);");
if (maxLexStates > 1)
{
genCodeLine(" if (jjnewLexState[jjmatchedKind] != -1)");
genCodeLine(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
}
if (Options.getCommonTokenAction())
genCodeLine(prefix + " CommonTokenAction(matchedToken);");
genCodeLine(prefix + " return matchedToken;");
if (hasSkip || hasMore || hasSpecial)
{
genCodeLine(prefix + " }");
if (hasSkip || hasSpecial)
{
if (hasMore)
{
genCodeLine(prefix + " else if ((jjtoSkip[jjmatchedKind >> 6] & " +
"(1L << (jjmatchedKind & 077))) != 0L)");
}
else
genCodeLine(prefix + " else");
genCodeLine(prefix + " {");
if (hasSpecial)
{
genCodeLine(prefix + " if ((jjtoSpecial[jjmatchedKind >> 6] & " +
"(1L << (jjmatchedKind & 077))) != 0L)");
genCodeLine(prefix + " {");
genCodeLine(prefix + " matchedToken = jjFillToken();");
genCodeLine(prefix + " if (specialToken == null)");
genCodeLine(prefix + " specialToken = matchedToken;");
genCodeLine(prefix + " else");
genCodeLine(prefix + " {");
genCodeLine(prefix + " matchedToken.specialToken = specialToken;");
genCodeLine(prefix + " specialToken = (specialToken.next = matchedToken);");
genCodeLine(prefix + " }");
if (hasSkipActions)
genCodeLine(prefix + " SkipLexicalActions(matchedToken);");
genCodeLine(prefix + " }");
if (hasSkipActions)
{
genCodeLine(prefix + " else");
genCodeLine(prefix + " SkipLexicalActions(null);");
}
}
else if (hasSkipActions)
genCodeLine(prefix + " SkipLexicalActions(null);");
if (maxLexStates > 1)
{
genCodeLine(" if (jjnewLexState[jjmatchedKind] != -1)");
genCodeLine(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
}
genCodeLine(prefix + " continue EOFLoop;");
genCodeLine(prefix + " }");
}
if (hasMore)
{
if (hasMoreActions)
genCodeLine(prefix + " MoreLexicalActions();");
else if (hasSkipActions || hasTokenActions)
genCodeLine(prefix + " jjimageLen += jjmatchedPos + 1;");
if (maxLexStates > 1)
{
genCodeLine(" if (jjnewLexState[jjmatchedKind] != -1)");
genCodeLine(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
}
genCodeLine(prefix + " curPos = 0;");
genCodeLine(prefix + " jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
genCodeLine(prefix + " try {");
genCodeLine(prefix + " curChar = input_stream.readChar();");
if (Options.getDebugTokenManager())
genCodeLine(" debugStream.println(" +
(maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
"\"Current character : \" + " +
""+errorHandlingClass+".addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") " +
"at line \" + input_stream.getEndLine() + \" column \" + input_stream.getEndColumn());");
genCodeLine(prefix + " continue;");
genCodeLine(prefix + " }");
genCodeLine(prefix + " catch (java.io.IOException e1) { }");
}
}
genCodeLine(prefix + " }");
genCodeLine(prefix + " int error_line = input_stream.getEndLine();");
genCodeLine(prefix + " int error_column = input_stream.getEndColumn();");
genCodeLine(prefix + " String error_after = null;");
genCodeLine(prefix + " " + Options.getBooleanType() + " EOFSeen = false;");
genCodeLine(prefix + " try { input_stream.readChar(); input_stream.backup(1); }");
genCodeLine(prefix + " catch (java.io.IOException e1) {");
genCodeLine(prefix + " EOFSeen = true;");
genCodeLine(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
genCodeLine(prefix + " if (curChar == '\\n' || curChar == '\\r') {");
genCodeLine(prefix + " error_line++;");
genCodeLine(prefix + " error_column = 0;");
genCodeLine(prefix + " }");
genCodeLine(prefix + " else");
genCodeLine(prefix + " error_column++;");
genCodeLine(prefix + " }");
genCodeLine(prefix + " if (!EOFSeen) {");
genCodeLine(prefix + " input_stream.backup(1);");
genCodeLine(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
genCodeLine(prefix + " }");
genCodeLine(prefix + " throw new "+errorHandlingClass+"(" +
"EOFSeen, curLexState, error_line, error_column, error_after, curChar, "+errorHandlingClass+".LEXICAL_ERROR);");
}
if (hasMore)
genCodeLine(prefix + " }");
genCodeLine(" }");
genCodeLine("}");
genCodeLine("");
}
public void DumpSkipActions()
{
Action act;
genCodeLine(staticString + "void SkipLexicalActions(Token matchedToken)");
genCodeLine("{");
genCodeLine(" switch(jjmatchedKind)");
genCodeLine(" {");
Outer:
for (int i = 0; i < maxOrdinal; i++)
{
if ((toSkip[i / 64] & (1L << (i % 64))) == 0L)
continue;
for (;;)
{
if (((act = (Action)actions[i]) == null ||
act.getActionTokens() == null ||
act.getActionTokens().size() == 0) && !canLoop[lexStates[i]])
continue Outer;
genCodeLine(" case " + i + " :");
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
{
genCodeLine(" if (jjmatchedPos == -1)");
genCodeLine(" {");
genCodeLine(" if (jjbeenHere[" + lexStates[i] + "] &&");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() &&");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
genCodeLine(" throw new "+errorHandlingClass+"(" +
"(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
"at line \" + input_stream.getBeginLine() + \", " +
"column \" + input_stream.getBeginColumn() + \".\"), "+errorHandlingClass+".LOOP_DETECTED);");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
genCodeLine(" jjbeenHere[" + lexStates[i] + "] = true;");
genCodeLine(" }");
}
if ((act = (Action)actions[i]) == null ||
act.getActionTokens().size() == 0)
break;
genCode( " image.append");
if (RStringLiteral.allImages[i] != null) {
genCodeLine("(jjstrLiteralImages[" + i + "]);");
genCodeLine(" lengthOfMatch = jjstrLiteralImages[" + i + "].length();");
} else {
genCodeLine("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
}
printTokenSetup((Token)act.getActionTokens().get(0));
ccol = 1;
for (int j = 0; j < act.getActionTokens().size(); j++)
printToken((Token)act.getActionTokens().get(j));
genCodeLine("");
break;
}
genCodeLine(" break;");
}
genCodeLine(" default :");
genCodeLine(" break;");
genCodeLine(" }");
genCodeLine("}");
}
public void DumpMoreActions()
{
Action act;
genCodeLine(staticString + "void MoreLexicalActions()");
genCodeLine("{");
genCodeLine(" jjimageLen += (lengthOfMatch = jjmatchedPos + 1);");
genCodeLine(" switch(jjmatchedKind)");
genCodeLine(" {");
Outer:
for (int i = 0; i < maxOrdinal; i++)
{
if ((toMore[i / 64] & (1L << (i % 64))) == 0L)
continue;
for (;;)
{
if (((act = (Action)actions[i]) == null ||
act.getActionTokens() == null ||
act.getActionTokens().size() == 0) && !canLoop[lexStates[i]])
continue Outer;
genCodeLine(" case " + i + " :");
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
{
genCodeLine(" if (jjmatchedPos == -1)");
genCodeLine(" {");
genCodeLine(" if (jjbeenHere[" + lexStates[i] + "] &&");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() &&");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
genCodeLine(" throw new "+errorHandlingClass+"(" +
"(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
"at line \" + input_stream.getBeginLine() + \", " +
"column \" + input_stream.getBeginColumn() + \".\"), "+errorHandlingClass+".LOOP_DETECTED);");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
genCodeLine(" jjbeenHere[" + lexStates[i] + "] = true;");
genCodeLine(" }");
}
if ((act = (Action)actions[i]) == null ||
act.getActionTokens().size() == 0)
{
break;
}
genCode( " image.append");
if (RStringLiteral.allImages[i] != null)
genCodeLine("(jjstrLiteralImages[" + i + "]);");
else
genCodeLine("(input_stream.GetSuffix(jjimageLen));");
genCodeLine(" jjimageLen = 0;");
printTokenSetup((Token)act.getActionTokens().get(0));
ccol = 1;
for (int j = 0; j < act.getActionTokens().size(); j++)
printToken((Token)act.getActionTokens().get(j));
genCodeLine("");
break;
}
genCodeLine(" break;");
}
genCodeLine(" default :");
genCodeLine(" break;");
genCodeLine(" }");
genCodeLine("}");
}
public void DumpTokenActions()
{
Action act;
int i;
genCodeLine(staticString + "void TokenLexicalActions(Token matchedToken)");
genCodeLine("{");
genCodeLine(" switch(jjmatchedKind)");
genCodeLine(" {");
Outer:
for (i = 0; i < maxOrdinal; i++)
{
if ((toToken[i / 64] & (1L << (i % 64))) == 0L)
continue;
for (;;)
{
if (((act = (Action)actions[i]) == null ||
act.getActionTokens() == null ||
act.getActionTokens().size() == 0) && !canLoop[lexStates[i]])
continue Outer;
genCodeLine(" case " + i + " :");
if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
{
genCodeLine(" if (jjmatchedPos == -1)");
genCodeLine(" {");
genCodeLine(" if (jjbeenHere[" + lexStates[i] + "] &&");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() &&");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
genCodeLine(" throw new "+errorHandlingClass+"(" +
"(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
"at line \" + input_stream.getBeginLine() + \", " +
"column \" + input_stream.getBeginColumn() + \".\"), "+errorHandlingClass+".LOOP_DETECTED);");
genCodeLine(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
genCodeLine(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
genCodeLine(" jjbeenHere[" + lexStates[i] + "] = true;");
genCodeLine(" }");
}
if ((act = (Action)actions[i]) == null ||
act.getActionTokens().size() == 0)
break;
if (i == 0)
{
genCodeLine(" image.setLength(0);"); // For EOF no image is there
}
else
{
genCode( " image.append");
if (RStringLiteral.allImages[i] != null) {
genCodeLine("(jjstrLiteralImages[" + i + "]);");
genCodeLine(" lengthOfMatch = jjstrLiteralImages[" + i + "].length();");
} else {
genCodeLine("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
}
}
printTokenSetup((Token)act.getActionTokens().get(0));
ccol = 1;
for (int j = 0; j < act.getActionTokens().size(); j++)
printToken((Token)act.getActionTokens().get(j));
genCodeLine("");
break;
}
genCodeLine(" break;");
}
genCodeLine(" default :");
genCodeLine(" break;");
genCodeLine(" }");
genCodeLine("}");
}
public static void reInit()
{
actions = null;
allTpsForState = new Hashtable();
canLoop = null;
canMatchAnyChar = null;
canReachOnMore = null;
curKind = 0;
curRE = null;
defaultLexState = 0;
errorHandlingClass = null;
hasEmptyMatch = false;
hasLoop = false;
hasMore = false;
hasMoreActions = false;
hasNfa = null;
hasSkip = false;
hasSkipActions = false;
hasSpecial = false;
hasTokenActions = false;
ignoreCase = null;
initMatch = null;
initStates = new Hashtable();
initialState = null;
keepLineCol = false;
kinds = null;
lexStateIndex = 0;
lexStateName = null;
lexStateSuffix = null;
lexStates = null;
maxLexStates = 0;
maxLongsReqd = null;
maxOrdinal = 1;
mixed = null;
newLexState = null;
rexprs = null;
singlesToSkip = null;
stateHasActions = null;
stateSetSize = 0;
staticString = null;
toMore = null;
toSkip = null;
toSpecial = null;
toToken = null;
tokMgrClassName = null;
tokenizerData = new TokenizerData();
generateDataOnly = false;
}
}