org.stringtemplate.v4.compiler.STLexer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of virtdata-lib-realer Show documentation
Show all versions of virtdata-lib-realer Show documentation
With inspiration from other libraries
/*
* [The "BSD license"]
* Copyright (c) 2011 Terence Parr
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.stringtemplate.v4.compiler;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.MismatchedTokenException;
import org.antlr.runtime.NoViableAltException;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenSource;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.misc.ErrorManager;
import org.stringtemplate.v4.misc.Misc;
import java.util.ArrayList;
import java.util.List;
/**
* This class represents the tokenizer for templates. It operates in two modes:
* inside and outside of expressions. It implements the {@link TokenSource}
* interface so it can be used with ANTLR parsers. Outside of expressions, we
* can return these token types: {@link #TEXT}, {@link #INDENT}, {@link #LDELIM}
* (start of expression), {@link #RCURLY} (end of subtemplate), and
* {@link #NEWLINE}. Inside of an expression, this lexer returns all of the
* tokens needed by {@link STParser}. From the parser's point of view, it can
* treat a template as a simple stream of elements.
*
* This class defines the token types and communicates these values to
* {@code STParser.g} via {@code STLexer.tokens} file (which must remain
* consistent).
*/
public class STLexer implements TokenSource {
public static final char EOF = (char)-1; // EOF char
public static final int EOF_TYPE = CharStream.EOF; // EOF token type
/** We build {@code STToken} tokens instead of relying on {@link CommonToken}
* so we can override {@link #toString()}. It just converts token types to
* token names like 23 to {@code "LDELIM"}.
*/
public static class STToken extends CommonToken {
public STToken(CharStream input, int type, int start, int stop) {
super(input, type, DEFAULT_CHANNEL, start, stop);
}
public STToken(int type, String text) { super(type, text); }
@Override
public String toString() {
String channelStr = "";
if ( channel>0 ) {
channelStr=",channel="+channel;
}
String txt = getText();
if ( txt!=null ) txt = Misc.replaceEscapes(txt);
else txt = "";
String tokenName;
if ( type==EOF_TYPE ) tokenName = "EOF";
else tokenName = STParser.tokenNames[type];
return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+ tokenName +">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
}
}
public static final Token SKIP = new STToken(-1, "");
// must follow STLexer.tokens file that STParser.g loads
public static final int RBRACK=17;
public static final int LBRACK=16;
public static final int ELSE=5;
public static final int ELLIPSIS=11;
public static final int LCURLY=20;
public static final int BANG=10;
public static final int EQUALS=12;
public static final int TEXT=22;
public static final int ID=25;
public static final int SEMI=9;
public static final int LPAREN=14;
public static final int IF=4;
public static final int ELSEIF=6;
public static final int COLON=13;
public static final int RPAREN=15;
public static final int COMMA=18;
public static final int RCURLY=21;
public static final int ENDIF=7;
public static final int RDELIM=24;
public static final int SUPER=8;
public static final int DOT=19;
public static final int LDELIM=23;
public static final int STRING=26;
public static final int PIPE=28;
public static final int OR=29;
public static final int AND=30;
public static final int INDENT=31;
public static final int NEWLINE=32;
public static final int AT=33;
public static final int REGION_END=34;
public static final int TRUE=35;
public static final int FALSE=36;
public static final int COMMENT=37;
public static final int SLASH=38;
/** The char which delimits the start of an expression. */
char delimiterStartChar = '<';
/** The char which delimits the end of an expression. */
char delimiterStopChar = '>';
/**
* This keeps track of the current mode of the lexer. Are we inside or
* outside an ST expression?
*/
boolean scanningInsideExpr = false;
/** To be able to properly track the inside/outside mode, we need to
* track how deeply nested we are in some templates. Otherwise, we
* know whether a '}' and the outermost subtemplate to send this
* back to outside mode.
*/
public int subtemplateDepth = 0; // start out *not* in a {...} subtemplate
ErrorManager errMgr;
/** template embedded in a group file? this is the template */
Token templateToken;
CharStream input;
/** current character */
char c;
/** When we started token, track initial coordinates so we can properly
* build token objects.
*/
int startCharIndex;
int startLine;
int startCharPositionInLine;
/** Our lexer routines might have to emit more than a single token. We
* buffer everything through this list.
*/
List tokens = new ArrayList();
public STLexer(CharStream input) { this(STGroup.DEFAULT_ERR_MGR, input, null, '<', '>'); }
public STLexer(ErrorManager errMgr, CharStream input, Token templateToken) {
this(errMgr, input, templateToken, '<', '>');
}
public STLexer(ErrorManager errMgr,
CharStream input,
Token templateToken,
char delimiterStartChar,
char delimiterStopChar)
{
this.errMgr = errMgr;
this.input = input;
c = (char)input.LA(1); // prime lookahead
this.templateToken = templateToken;
this.delimiterStartChar = delimiterStartChar;
this.delimiterStopChar = delimiterStopChar;
}
@Override
public Token nextToken() {
Token t;
if ( tokens.size()>0 ) { t = tokens.remove(0); }
else t = _nextToken();
// System.out.println(t);
return t;
}
/** Consume if {@code x} is next character on the input stream.
*/
public void match(char x) {
if ( c != x ) {
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "expecting '"+x+"', found '"+str(c)+"'", templateToken, e);
}
consume();
}
protected void consume() {
input.consume();
c = (char)input.LA(1);
}
public void emit(Token token) { tokens.add(token); }
public Token _nextToken() {
//System.out.println("nextToken: c="+(char)c+"@"+input.index());
while ( true ) { // lets us avoid recursion when skipping stuff
startCharIndex = input.index();
startLine = input.getLine();
startCharPositionInLine = input.getCharPositionInLine();
if ( c==EOF ) return newToken(EOF_TYPE);
Token t;
if ( scanningInsideExpr ) t = inside();
else t = outside();
if ( t!=SKIP ) return t;
}
}
protected Token outside() {
if ( input.getCharPositionInLine()==0 && (c==' '||c=='\t') ) {
while ( c==' ' || c=='\t' ) consume(); // scarf indent
if ( c!=EOF ) return newToken(INDENT);
return newToken(TEXT);
}
if ( c==delimiterStartChar ) {
consume();
if ( c=='!' ) return COMMENT();
if ( c=='\\' ) return ESCAPE(); // <\\> <\uFFFF> <\n> etc...
scanningInsideExpr = true;
return newToken(LDELIM);
}
if ( c=='\r' ) { consume(); consume(); return newToken(NEWLINE); } // \r\n -> \n
if ( c=='\n') { consume(); return newToken(NEWLINE); }
if ( c=='}' && subtemplateDepth>0 ) {
scanningInsideExpr = true;
subtemplateDepth--;
consume();
return newTokenFromPreviousChar(RCURLY);
}
return mTEXT();
}
protected Token inside() {
while ( true ) {
switch ( c ) {
case ' ': case '\t': case '\n': case '\r':
consume();
return SKIP;
case '.' :
consume();
if ( input.LA(1)=='.' && input.LA(2)=='.' ) {
consume();
match('.');
return newToken(ELLIPSIS);
}
return newToken(DOT);
case ',' : consume(); return newToken(COMMA);
case ':' : consume(); return newToken(COLON);
case ';' : consume(); return newToken(SEMI);
case '(' : consume(); return newToken(LPAREN);
case ')' : consume(); return newToken(RPAREN);
case '[' : consume(); return newToken(LBRACK);
case ']' : consume(); return newToken(RBRACK);
case '=' : consume(); return newToken(EQUALS);
case '!' : consume(); return newToken(BANG);
case '/' : consume(); return newToken(SLASH);
case '@' :
consume();
if ( c=='e' && input.LA(2)=='n' && input.LA(3)=='d' ) {
consume(); consume(); consume();
return newToken(REGION_END);
}
return newToken(AT);
case '"' : return mSTRING();
case '&' : consume(); match('&'); return newToken(AND); // &&
case '|' : consume(); match('|'); return newToken(OR); // ||
case '{' : return subTemplate();
default:
if ( c==delimiterStopChar ) {
consume();
scanningInsideExpr =false;
return newToken(RDELIM);
}
if ( isIDStartLetter(c) ) {
Token id = mID();
String name = id.getText();
if ( name.equals("if") ) return newToken(IF);
else if ( name.equals("endif") ) return newToken(ENDIF);
else if ( name.equals("else") ) return newToken(ELSE);
else if ( name.equals("elseif") ) return newToken(ELSEIF);
else if ( name.equals("super") ) return newToken(SUPER);
else if ( name.equals("true") ) return newToken(TRUE);
else if ( name.equals("false") ) return newToken(FALSE);
return id;
}
RecognitionException re =
new NoViableAltException("",0,0,input);
re.line = startLine;
re.charPositionInLine = startCharPositionInLine;
errMgr.lexerError(input.getSourceName(), "invalid character '"+str(c)+"'", templateToken, re);
if (c==EOF) {
return newToken(EOF_TYPE);
}
consume();
}
}
}
Token subTemplate() {
// look for "{ args ID (',' ID)* '|' ..."
subtemplateDepth++;
int m = input.mark();
int curlyStartChar = startCharIndex;
int curlyLine = startLine;
int curlyPos = startCharPositionInLine;
List argTokens = new ArrayList();
consume();
Token curly = newTokenFromPreviousChar(LCURLY);
WS();
argTokens.add( mID() );
WS();
while ( c==',' ) {
consume();
argTokens.add( newTokenFromPreviousChar(COMMA) );
WS();
argTokens.add( mID() );
WS();
}
WS();
if ( c=='|' ) {
consume();
argTokens.add( newTokenFromPreviousChar(PIPE) );
if ( isWS(c) ) consume(); // ignore a single whitespace after |
//System.out.println("matched args: "+argTokens);
for (Token t : argTokens) emit(t);
input.release(m);
scanningInsideExpr = false;
startCharIndex = curlyStartChar; // reset state
startLine = curlyLine;
startCharPositionInLine = curlyPos;
return curly;
}
input.rewind(m);
startCharIndex = curlyStartChar; // reset state
startLine = curlyLine;
startCharPositionInLine = curlyPos;
consume();
scanningInsideExpr = false;
return curly;
}
Token ESCAPE() {
startCharIndex = input.index();
startCharPositionInLine = input.getCharPositionInLine();
consume(); // kill \\
if ( c=='u') return UNICODE();
String text;
switch ( c ) {
case '\\' : LINEBREAK(); return SKIP;
case 'n' : text = "\n"; break;
case 't' : text = "\t"; break;
case ' ' : text = " "; break;
default :
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "invalid escaped char: '"+str(c)+"'", templateToken, e);
consume();
match(delimiterStopChar);
return SKIP;
}
consume();
Token t = newToken(TEXT, text, input.getCharPositionInLine()-2);
match(delimiterStopChar);
return t;
}
Token UNICODE() {
consume();
char[] chars = new char[4];
if ( !isUnicodeLetter(c) ) {
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e);
}
chars[0] = c;
consume();
if ( !isUnicodeLetter(c) ) {
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e);
}
chars[1] = c;
consume();
if ( !isUnicodeLetter(c) ) {
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e);
}
chars[2] = c;
consume();
if ( !isUnicodeLetter(c) ) {
NoViableAltException e = new NoViableAltException("",0,0,input);
errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e);
}
chars[3] = c;
// ESCAPE kills >
char uc = (char)Integer.parseInt(new String(chars), 16);
Token t = newToken(TEXT, String.valueOf(uc), input.getCharPositionInLine()-6);
consume();
match(delimiterStopChar);
return t;
}
Token mTEXT() {
boolean modifiedText = false;
StringBuilder buf = new StringBuilder();
while ( c != EOF && c != delimiterStartChar ) {
if ( c=='\r' || c=='\n') break;
if ( c=='}' && subtemplateDepth>0 ) break;
if ( c=='\\' ) {
if ( input.LA(2)=='\\' ) { // convert \\ to \
consume(); consume(); buf.append('\\');
modifiedText = true;
continue;
}
if ( input.LA(2)==delimiterStartChar ||
input.LA(2)=='}' )
{
modifiedText = true;
consume(); // toss out \ char
buf.append(c); consume();
}
else {
buf.append(c);
consume();
}
continue;
}
buf.append(c);
consume();
}
if ( modifiedText ) return newToken(TEXT, buf.toString());
else return newToken(TEXT);
}
/**
* ID : ('a'..'z'|'A'..'Z'|'_'|'/')
* ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'/')*
* ;
*
*/
Token mID() {
// called from subTemplate; so keep resetting position during speculation
startCharIndex = input.index();
startLine = input.getLine();
startCharPositionInLine = input.getCharPositionInLine();
consume();
while ( isIDLetter(c) ) {
consume();
}
return newToken(ID);
}
/**
* STRING : '"'
* ( '\\' '"'
* | '\\' ~'"'
* | ~('\\'|'"')
* )*
* '"'
* ;
*
*/
Token mSTRING() {
//{setText(getText().substring(1, getText().length()-1));}
boolean sawEscape = false;
StringBuilder buf = new StringBuilder();
buf.append(c); consume();
while ( c != '"' ) {
if ( c=='\\' ) {
sawEscape = true;
consume();
switch ( c ) {
case 'n' : buf.append('\n'); break;
case 'r' : buf.append('\r'); break;
case 't' : buf.append('\t'); break;
default : buf.append(c); break;
}
consume();
continue;
}
buf.append(c);
consume();
if ( c==EOF ) {
RecognitionException re =
new MismatchedTokenException((int)'"', input);
re.line = input.getLine();
re.charPositionInLine = input.getCharPositionInLine();
errMgr.lexerError(input.getSourceName(), "EOF in string", templateToken, re);
break;
}
}
buf.append(c);
consume();
if ( sawEscape ) return newToken(STRING, buf.toString());
else return newToken(STRING);
}
void WS() {
while ( c==' ' || c=='\t' || c=='\n' || c=='\r' ) consume();
}
Token COMMENT() {
match('!');
while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
if (c==EOF) {
RecognitionException re =
new MismatchedTokenException((int)'!', input);
re.line = input.getLine();
re.charPositionInLine = input.getCharPositionInLine();
errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at " +
startLine+":"+startCharPositionInLine+": '!"+
delimiterStopChar+"' missing", templateToken, re);
break;
}
consume();
}
consume(); consume(); // grab !>
return newToken(COMMENT);
}
void LINEBREAK() {
match('\\'); // only kill 2nd \ as ESCAPE() kills first one
match(delimiterStopChar);
while ( c==' ' || c=='\t' ) consume(); // scarf WS after <\\>
if ( c==EOF ) {
RecognitionException re = new RecognitionException(input);
re.line = input.getLine();
re.charPositionInLine = input.getCharPositionInLine();
errMgr.lexerError(input.getSourceName(), "Missing newline after newline escape <\\\\>",
templateToken, re);
return;
}
if ( c=='\r' ) consume();
match('\n');
while ( c==' ' || c=='\t' ) consume(); // scarf any indent
}
public static boolean isIDStartLetter(char c) { return isIDLetter(c); }
public static boolean isIDLetter(char c) { return c>='a'&&c<='z' || c>='A'&&c<='Z' || c>='0'&&c<='9' || c=='-' || c=='_'; }
public static boolean isWS(char c) { return c==' ' || c=='\t' || c=='\n' || c=='\r'; }
public static boolean isUnicodeLetter(char c) { return c>='a'&&c<='f' || c>='A'&&c<='F' || c>='0'&&c<='9'; }
public Token newToken(int ttype) {
STToken t = new STToken(input, ttype, startCharIndex, input.index()-1);
t.setLine(startLine);
t.setCharPositionInLine(startCharPositionInLine);
return t;
}
public Token newTokenFromPreviousChar(int ttype) {
STToken t = new STToken(input, ttype, input.index()-1, input.index()-1);
t.setLine(input.getLine());
t.setCharPositionInLine(input.getCharPositionInLine()-1);
return t;
}
public Token newToken(int ttype, String text, int pos) {
STToken t = new STToken(ttype, text);
t.setStartIndex(startCharIndex);
t.setStopIndex(input.index()-1);
t.setLine(input.getLine());
t.setCharPositionInLine(pos);
return t;
}
public Token newToken(int ttype, String text) {
STToken t = new STToken(ttype, text);
t.setStartIndex(startCharIndex);
t.setStopIndex(input.index()-1);
t.setLine(startLine);
t.setCharPositionInLine(startCharPositionInLine);
return t;
}
// public String getErrorHeader() {
// return startLine+":"+startCharPositionInLine;
// }
//
@Override
public String getSourceName() {
return "no idea";
}
public static String str(int c) {
if ( c==EOF ) return "";
return String.valueOf((char)c);
}
}