antlr.tree_format.TreeFormatLexer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of Treepat Show documentation
Show all versions of Treepat Show documentation
Treepat is a language to recognise patterns in trees in a similar way as regular expressions recognize patterns in strings. Treepat includes analogous operators to regex union, concatenation, and closure, which are extended to the notion of trees.
// Generated from antlr/tree_format/TreeFormat.g4 by ANTLR 4.1
package antlr.tree_format;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.*;
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public class TreeFormatLexer extends Lexer {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public static final int
COLON=1, STRING=2, MARKLESS_STRING=3, NEWLINE=4, WS=5;
public static String[] modeNames = {
"DEFAULT_MODE"
};
public static final String[] tokenNames = {
"",
"':'", "STRING", "MARKLESS_STRING", "NEWLINE", "WS"
};
public static final String[] ruleNames = {
"COLON", "STRING", "MARKLESS_STRING", "NEWLINE", "WS", "SPACES"
};
// A queue where extra tokens are pushed on (see the NEWLINE lexer rule).
private java.util.LinkedList tokens = new java.util.LinkedList<>();
// The stack that keeps track of the indentation level.
private java.util.Stack indents = new java.util.Stack<>();
// The amount of opened braces, brackets and parenthesis.
private int opened = 0;
// The most recently produced token.
private Token lastToken = null;
@Override
public void emit(Token t) {
super.setToken(t);
tokens.offer(t);
}
@Override
public Token nextToken() {
// Check if the end-of-file is ahead and there are still some DEDENTS expected.
if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
// Remove any trailing EOF tokens from our buffer.
for (int i = tokens.size() - 1; i >= 0; i--) {
if (tokens.get(i).getType() == EOF) {
tokens.remove(i);
}
}
// First emit an extra line break that serves as the end of the statement.
this.emit(commonToken(TreeFormatParser.NEWLINE, "\n"));
// Now emit as much DEDENT tokens as needed.
while (!indents.isEmpty()) {
this.emit(createDedent());
indents.pop();
}
// Put the EOF back on the token stream.
this.emit(commonToken(TreeFormatParser.EOF, ""));
}
Token next = super.nextToken();
if (next.getChannel() == Token.DEFAULT_CHANNEL) {
// Keep track of the last token on the default channel.
this.lastToken = next;
}
return tokens.isEmpty() ? next : tokens.poll();
}
private Token createDedent() {
CommonToken dedent = commonToken(TreeFormatParser.DEDENT, "");
dedent.setLine(this.lastToken.getLine());
return dedent;
}
private CommonToken commonToken(int type, String text) {
int stop = this.getCharIndex() - 1;
int start = text.isEmpty() ? stop : stop - text.length() + 1;
return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop);
}
// Calculates the indentation of the provided spaces, taking the
// following rules into account:
//
// "Tabs are replaced (from left to right) by one to eight spaces
// such that the total number of characters up to and including
// the replacement is a multiple of eight [...]"
//
// -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
static int getIndentationCount(String spaces) {
int count = 0;
for (char ch : spaces.toCharArray()) {
switch (ch) {
case '\t':
count += 8 - (count % 8);
break;
default:
// A normal space char.
count++;
}
}
return count;
}
boolean atStartOfInput() {
return super.getCharPositionInLine() == 0 && super.getLine() == 1;
}
public TreeFormatLexer(CharStream input) {
super(input);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
@Override
public String getGrammarFileName() { return "TreeFormat.g4"; }
@Override
public String[] getTokenNames() { return tokenNames; }
@Override
public String[] getRuleNames() { return ruleNames; }
@Override
public String[] getModeNames() { return modeNames; }
@Override
public ATN getATN() { return _ATN; }
@Override
public void action(RuleContext _localctx, int ruleIndex, int actionIndex) {
switch (ruleIndex) {
case 3: NEWLINE_action((RuleContext)_localctx, actionIndex); break;
case 4: WS_action((RuleContext)_localctx, actionIndex); break;
}
}
private void NEWLINE_action(RuleContext _localctx, int actionIndex) {
switch (actionIndex) {
case 0:
String newLine = getText().replaceAll("[^\r\n]+", "");
String spaces = getText().replaceAll("[\r\n]+", "");
int next = _input.LA(1);
if (opened > 0 || next == '\r' || next == '\n' || next == '#') {
// If we're inside a list or on a blank line, ignore all indents,
// dedents and line breaks.
skip();
}
else {
emit(commonToken(NEWLINE, newLine));
int indent = getIndentationCount(spaces);
int previous = indents.isEmpty() ? 0 : indents.peek();
if (indent == previous) {
// skip indents of the same size as the present indent-size
skip();
}
else if (indent > previous) {
indents.push(indent);
emit(commonToken(TreeFormatParser.INDENT, spaces));
}
else {
// Possibly emit more than 1 DEDENT token.
while(!indents.isEmpty() && indents.peek() > indent) {
this.emit(createDedent());
indents.pop();
}
}
}
break;
}
}
private void WS_action(RuleContext _localctx, int actionIndex) {
switch (actionIndex) {
case 1: skip(); break;
}
}
@Override
public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
switch (ruleIndex) {
case 3: return NEWLINE_sempred((RuleContext)_localctx, predIndex);
}
return true;
}
private boolean NEWLINE_sempred(RuleContext _localctx, int predIndex) {
switch (predIndex) {
case 0: return atStartOfInput();
}
return true;
}
public static final String _serializedATN =
"\3\uacf5\uee8c\u4f5d\u8b0d\u4a45\u78bd\u1b2f\u3378\2\7;\b\1\4\2\t\2\4"+
"\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\3\2\3\2\3\3\3\3\7\3\24\n\3\f\3"+
"\16\3\27\13\3\3\3\3\3\3\4\6\4\34\n\4\r\4\16\4\35\3\5\3\5\3\5\5\5#\n\5"+
"\3\5\3\5\5\5\'\n\5\3\5\5\5*\n\5\5\5,\n\5\3\5\3\5\3\6\6\6\61\n\6\r\6\16"+
"\6\62\3\6\3\6\3\7\6\78\n\7\r\7\16\79\2\b\3\3\1\5\4\1\7\5\1\t\6\2\13\7"+
"\3\r\2\1\3\2\5\3\2$$\6\2\62;C\\aac|\4\2\13\13\"\"A\2\3\3\2\2\2\2\5\3\2"+
"\2\2\2\7\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\3\17\3\2\2\2\5\21\3\2\2\2\7"+
"\33\3\2\2\2\t+\3\2\2\2\13\60\3\2\2\2\r\67\3\2\2\2\17\20\7<\2\2\20\4\3"+
"\2\2\2\21\25\7$\2\2\22\24\n\2\2\2\23\22\3\2\2\2\24\27\3\2\2\2\25\23\3"+
"\2\2\2\25\26\3\2\2\2\26\30\3\2\2\2\27\25\3\2\2\2\30\31\7$\2\2\31\6\3\2"+
"\2\2\32\34\t\3\2\2\33\32\3\2\2\2\34\35\3\2\2\2\35\33\3\2\2\2\35\36\3\2"+
"\2\2\36\b\3\2\2\2\37 \6\5\2\2 ,\5\r\7\2!#\7\17\2\2\"!\3\2\2\2\"#\3\2\2"+
"\2#$\3\2\2\2$\'\7\f\2\2%\'\4\16\17\2&\"\3\2\2\2&%\3\2\2\2\')\3\2\2\2("+
"*\5\r\7\2)(\3\2\2\2)*\3\2\2\2*,\3\2\2\2+\37\3\2\2\2+&\3\2\2\2,-\3\2\2"+
"\2-.\b\5\2\2.\n\3\2\2\2/\61\5\r\7\2\60/\3\2\2\2\61\62\3\2\2\2\62\60\3"+
"\2\2\2\62\63\3\2\2\2\63\64\3\2\2\2\64\65\b\6\3\2\65\f\3\2\2\2\668\t\4"+
"\2\2\67\66\3\2\2\289\3\2\2\29\67\3\2\2\29:\3\2\2\2:\16\3\2\2\2\13\2\25"+
"\35\"&)+\629";
public static final ATN _ATN =
ATNSimulator.deserialize(_serializedATN.toCharArray());
static {
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) {
_decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i);
}
}
}