ai.vespa.schemals.parser.indexinglanguage.Token Maven / Gradle / Ivy
/*
* Generated by: CongoCC Parser Generator. Token.java
*/
package ai.vespa.schemals.parser.indexinglanguage;
import ai.vespa.schemals.parser.indexinglanguage.ast.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
public class Token implements CharSequence, Node.TerminalNode {
public enum TokenType implements Node.NodeType {
EOF, _TOKEN_1, _TOKEN_2, _TOKEN_3, _TOKEN_4, NL, ADD, SUB, MUL, DIV, MOD,
EQ, NE, LT, LE, GT, GE, CHOICE, PIPE, LCURLY, RCURLY, LPAREN, RPAREN, DOT,
COMMA, COLON, SCOLON, ATTRIBUTE, BASE64_DECODE, BASE64_ENCODE, BINARIZE, BUSY_WAIT,
CASE, CASE_DEFAULT, CLEAR_STATE, CREATE_IF_NON_EXISTENT, ECHO, ELSE, EMBED,
EXACT, FLATTEN, FOR_EACH, GET_FIELD, GET_VAR, GUARD, HASH, HEX_DECODE, HEX_ENCODE,
HOST_NAME, IF, INDEX, INPUT, JOIN, LOWER_CASE, MAX_LENGTH, MAX_OCCURRENCES,
MAX_TOKEN_LENGTH, NGRAM, NORMALIZE, NOW, OPTIMIZE_PREDICATE, PACK_BITS, PASSTHROUGH,
RANDOM, REMOVE_IF_ZERO, SELECT_INPUT, SET_LANGUAGE, SET_VAR, SLEEP, SPLIT,
STEM, SUBSTRING, SUMMARY, SWITCH, THIS, TOKENIZE, TO_ARRAY, TO_BYTE, TO_DOUBLE,
TO_FLOAT, TO_INT, TO_LONG, TO_POS, TO_EPOCH_SECOND, TO_STRING, TO_WSET, TO_BOOL,
TRIM, ZCURVE, TRUE, FALSE, UNDERSCORE, COMMENT, INTEGER, LONG, DOUBLE, FLOAT,
STRING, IDENTIFIER, DUMMY, INVALID;
public boolean isUndefined() {
return this == DUMMY;
}
public boolean isInvalid() {
return this == INVALID;
}
public boolean isEOF() {
return this == EOF;
}
}
private IndexingParserLexer tokenSource;
private TokenType type = TokenType.DUMMY;
private int beginOffset;
private int endOffset;
private boolean unparsed;
private Node parent;
/**
* It would be extremely rare that an application
* programmer would use this method. It needs to
* be public because it is part of the ai.vespa.schemals.parser.indexinglanguage.Node interface.
*/
public void setBeginOffset(int beginOffset) {
this.beginOffset = beginOffset;
}
/**
* It would be extremely rare that an application
* programmer would use this method. It needs to
* be public because it is part of the ai.vespa.schemals.parser.indexinglanguage.Node interface.
*/
public void setEndOffset(int endOffset) {
this.endOffset = endOffset;
}
/**
* @return the IndexingParserLexer object that handles
* location info for the tokens.
*/
public IndexingParserLexer getTokenSource() {
return this.tokenSource;
}
/**
* It should be exceedingly rare that an application
* programmer needs to use this method.
*/
public void setTokenSource(TokenSource tokenSource) {
this.tokenSource = (IndexingParserLexer) tokenSource;
}
public boolean isInvalid() {
return getType().isInvalid();
}
/**
* Return the TokenType of this Token object
*/
@Override
public TokenType getType() {
return type;
}
protected void setType(TokenType type) {
this.type = type;
}
/**
* @return whether this Token represent actual input or was it inserted somehow?
*/
public boolean isVirtual() {
return virtual || type == TokenType.EOF;
}
/**
* @return Did we skip this token in parsing?
*/
public boolean isSkipped() {
return skipped;
}
private boolean virtual;
private boolean skipped;
private boolean dirty;
void setVirtual(boolean virtual) {
this.virtual = virtual;
if (virtual) dirty = true;
}
void setSkipped(boolean skipped) {
this.skipped = skipped;
if (skipped) dirty = true;
}
public boolean isDirty() {
return dirty;
}
public void setDirty(boolean dirty) {
this.dirty = dirty;
}
public int getBeginOffset() {
return beginOffset;
}
public int getEndOffset() {
return endOffset;
}
/**
* @return the string image of the token.
*/
@Override
/**
* @return the next _cached_ regular (i.e. parsed) token
* or null
*/
public final Token getNext() {
return getNextParsedToken();
}
/**
* @return the previous regular (i.e. parsed) token
* or null
*/
public final Token getPrevious() {
Token result = previousCachedToken();
while (result != null && result.isUnparsed()) {
result = result.previousCachedToken();
}
return result;
}
/**
* @return the next regular (i.e. parsed) token
*/
private Token getNextParsedToken() {
Token result = nextCachedToken();
while (result != null && result.isUnparsed()) {
result = result.nextCachedToken();
}
return result;
}
/**
* @return the next token of any sort (parsed or unparsed or invalid)
*/
public Token nextCachedToken() {
if (getType() == TokenType.EOF) return null;
IndexingParserLexer tokenSource = getTokenSource();
return tokenSource != null ? (Token) tokenSource.nextCachedToken(getEndOffset()) : null;
}
public Token previousCachedToken() {
if (getTokenSource() == null) return null;
return (Token) getTokenSource().previousCachedToken(getBeginOffset());
}
Token getPreviousToken() {
return previousCachedToken();
}
public Token replaceType(TokenType type) {
Token result = newToken(type, getTokenSource(), getBeginOffset(), getEndOffset());
getTokenSource().cacheToken(result);
return result;
}
public String getSource() {
if (type == TokenType.EOF) return "";
IndexingParserLexer ts = getTokenSource();
return ts == null ? null : ts.getText(getBeginOffset(), getEndOffset());
}
protected Token() {
}
public Token(TokenType type, IndexingParserLexer tokenSource, int beginOffset, int endOffset) {
this.type = type;
this.tokenSource = tokenSource;
this.beginOffset = beginOffset;
this.endOffset = endOffset;
}
public boolean isUnparsed() {
return unparsed;
}
public void setUnparsed(boolean unparsed) {
this.unparsed = unparsed;
}
/**
* @return An iterator of the tokens preceding this one.
*/
public Iterator precedingTokens() {
return new Iterator() {
Token currentPoint = Token.this;
public boolean hasNext() {
return currentPoint.previousCachedToken() != null;
}
public Token next() {
Token previous = currentPoint.previousCachedToken();
if (previous == null) throw new java.util.NoSuchElementException("No previous token!");
return currentPoint = previous;
}
};
}
/**
* @return a list of the unparsed tokens preceding this one in the order they appear in the input
*/
public List precedingUnparsedTokens() {
List result = new ArrayList<>();
Token t = this.previousCachedToken();
while (t != null && t.isUnparsed()) {
result.add(t);
t = t.previousCachedToken();
}
Collections.reverse(result);
return result;
}
/**
* @return An iterator of the (cached) tokens that follow this one.
*/
public Iterator followingTokens() {
return new java.util.Iterator() {
Token currentPoint = Token.this;
public boolean hasNext() {
return currentPoint.nextCachedToken() != null;
}
public Token next() {
Token next = currentPoint.nextCachedToken();
if (next == null) throw new java.util.NoSuchElementException("No next token!");
return currentPoint = next;
}
};
}
public void copyLocationInfo(Token from) {
setTokenSource(from.getTokenSource());
setBeginOffset(from.getBeginOffset());
setEndOffset(from.getEndOffset());
}
public void copyLocationInfo(Token start, Token end) {
setTokenSource(start.getTokenSource());
if (tokenSource == null) setTokenSource(end.getTokenSource());
setBeginOffset(start.getBeginOffset());
setEndOffset(end.getEndOffset());
}
public static Token newToken(TokenType type, IndexingParserLexer tokenSource, int beginOffset, int endOffset) {
switch(type) {
case TO_EPOCH_SECOND :
return new TO_EPOCH_SECOND(TokenType.TO_EPOCH_SECOND, tokenSource, beginOffset, endOffset);
case ADD :
return new ADD(TokenType.ADD, tokenSource, beginOffset, endOffset);
case SLEEP :
return new SLEEP(TokenType.SLEEP, tokenSource, beginOffset, endOffset);
case UNDERSCORE :
return new UNDERSCORE(TokenType.UNDERSCORE, tokenSource, beginOffset, endOffset);
case INPUT :
return new INPUT(TokenType.INPUT, tokenSource, beginOffset, endOffset);
case CLEAR_STATE :
return new CLEAR_STATE(TokenType.CLEAR_STATE, tokenSource, beginOffset, endOffset);
case ECHO :
return new ECHO(TokenType.ECHO, tokenSource, beginOffset, endOffset);
case TRIM :
return new TRIM(TokenType.TRIM, tokenSource, beginOffset, endOffset);
case SET_LANGUAGE :
return new SET_LANGUAGE(TokenType.SET_LANGUAGE, tokenSource, beginOffset, endOffset);
case SUBSTRING :
return new SUBSTRING(TokenType.SUBSTRING, tokenSource, beginOffset, endOffset);
case OPTIMIZE_PREDICATE :
return new OPTIMIZE_PREDICATE(TokenType.OPTIMIZE_PREDICATE, tokenSource, beginOffset, endOffset);
case LT :
return new LT(TokenType.LT, tokenSource, beginOffset, endOffset);
case JOIN :
return new JOIN(TokenType.JOIN, tokenSource, beginOffset, endOffset);
case LOWER_CASE :
return new LOWER_CASE(TokenType.LOWER_CASE, tokenSource, beginOffset, endOffset);
case PIPE :
return new PIPE(TokenType.PIPE, tokenSource, beginOffset, endOffset);
case CREATE_IF_NON_EXISTENT :
return new CREATE_IF_NON_EXISTENT(TokenType.CREATE_IF_NON_EXISTENT, tokenSource, beginOffset, endOffset);
case PASSTHROUGH :
return new PASSTHROUGH(TokenType.PASSTHROUGH, tokenSource, beginOffset, endOffset);
case SUMMARY :
return new SUMMARY(TokenType.SUMMARY, tokenSource, beginOffset, endOffset);
case GUARD :
return new GUARD(TokenType.GUARD, tokenSource, beginOffset, endOffset);
case ELSE :
return new ELSE(TokenType.ELSE, tokenSource, beginOffset, endOffset);
case TO_BOOL :
return new TO_BOOL(TokenType.TO_BOOL, tokenSource, beginOffset, endOffset);
case GET_VAR :
return new GET_VAR(TokenType.GET_VAR, tokenSource, beginOffset, endOffset);
case INDEX :
return new INDEX(TokenType.INDEX, tokenSource, beginOffset, endOffset);
case SELECT_INPUT :
return new SELECT_INPUT(TokenType.SELECT_INPUT, tokenSource, beginOffset, endOffset);
case TO_STRING :
return new TO_STRING(TokenType.TO_STRING, tokenSource, beginOffset, endOffset);
case IF :
return new IF(TokenType.IF, tokenSource, beginOffset, endOffset);
case BASE64_ENCODE :
return new BASE64_ENCODE(TokenType.BASE64_ENCODE, tokenSource, beginOffset, endOffset);
case INTEGER :
return new INTEGER(TokenType.INTEGER, tokenSource, beginOffset, endOffset);
case SUB :
return new SUB(TokenType.SUB, tokenSource, beginOffset, endOffset);
case ATTRIBUTE :
return new ATTRIBUTE(TokenType.ATTRIBUTE, tokenSource, beginOffset, endOffset);
case MUL :
return new MUL(TokenType.MUL, tokenSource, beginOffset, endOffset);
case SET_VAR :
return new SET_VAR(TokenType.SET_VAR, tokenSource, beginOffset, endOffset);
case LPAREN :
return new LPAREN(TokenType.LPAREN, tokenSource, beginOffset, endOffset);
case DOT :
return new DOT(TokenType.DOT, tokenSource, beginOffset, endOffset);
case TRUE :
return new TRUE(TokenType.TRUE, tokenSource, beginOffset, endOffset);
case EXACT :
return new EXACT(TokenType.EXACT, tokenSource, beginOffset, endOffset);
case RPAREN :
return new RPAREN(TokenType.RPAREN, tokenSource, beginOffset, endOffset);
case BUSY_WAIT :
return new BUSY_WAIT(TokenType.BUSY_WAIT, tokenSource, beginOffset, endOffset);
case EQ :
return new EQ(TokenType.EQ, tokenSource, beginOffset, endOffset);
case TO_POS :
return new TO_POS(TokenType.TO_POS, tokenSource, beginOffset, endOffset);
case MAX_LENGTH :
return new MAX_LENGTH(TokenType.MAX_LENGTH, tokenSource, beginOffset, endOffset);
case CASE :
return new CASE(TokenType.CASE, tokenSource, beginOffset, endOffset);
case HASH :
return new HASH(TokenType.HASH, tokenSource, beginOffset, endOffset);
case FOR_EACH :
return new FOR_EACH(TokenType.FOR_EACH, tokenSource, beginOffset, endOffset);
case TO_ARRAY :
return new TO_ARRAY(TokenType.TO_ARRAY, tokenSource, beginOffset, endOffset);
case NE :
return new NE(TokenType.NE, tokenSource, beginOffset, endOffset);
case NOW :
return new NOW(TokenType.NOW, tokenSource, beginOffset, endOffset);
case BASE64_DECODE :
return new BASE64_DECODE(TokenType.BASE64_DECODE, tokenSource, beginOffset, endOffset);
case THIS :
return new THIS(TokenType.THIS, tokenSource, beginOffset, endOffset);
case SWITCH :
return new SWITCH(TokenType.SWITCH, tokenSource, beginOffset, endOffset);
case SCOLON :
return new SCOLON(TokenType.SCOLON, tokenSource, beginOffset, endOffset);
case FLATTEN :
return new FLATTEN(TokenType.FLATTEN, tokenSource, beginOffset, endOffset);
case NL :
return new NL(TokenType.NL, tokenSource, beginOffset, endOffset);
case HEX_ENCODE :
return new HEX_ENCODE(TokenType.HEX_ENCODE, tokenSource, beginOffset, endOffset);
case GET_FIELD :
return new GET_FIELD(TokenType.GET_FIELD, tokenSource, beginOffset, endOffset);
case FLOAT :
return new FLOAT(TokenType.FLOAT, tokenSource, beginOffset, endOffset);
case REMOVE_IF_ZERO :
return new REMOVE_IF_ZERO(TokenType.REMOVE_IF_ZERO, tokenSource, beginOffset, endOffset);
case EMBED :
return new EMBED(TokenType.EMBED, tokenSource, beginOffset, endOffset);
case NGRAM :
return new NGRAM(TokenType.NGRAM, tokenSource, beginOffset, endOffset);
case PACK_BITS :
return new PACK_BITS(TokenType.PACK_BITS, tokenSource, beginOffset, endOffset);
case SPLIT :
return new SPLIT(TokenType.SPLIT, tokenSource, beginOffset, endOffset);
case COMMENT :
return new COMMENT(TokenType.COMMENT, tokenSource, beginOffset, endOffset);
case CHOICE :
return new CHOICE(TokenType.CHOICE, tokenSource, beginOffset, endOffset);
case CASE_DEFAULT :
return new CASE_DEFAULT(TokenType.CASE_DEFAULT, tokenSource, beginOffset, endOffset);
case BINARIZE :
return new BINARIZE(TokenType.BINARIZE, tokenSource, beginOffset, endOffset);
case HEX_DECODE :
return new HEX_DECODE(TokenType.HEX_DECODE, tokenSource, beginOffset, endOffset);
case HOST_NAME :
return new HOST_NAME(TokenType.HOST_NAME, tokenSource, beginOffset, endOffset);
case MAX_TOKEN_LENGTH :
return new MAX_TOKEN_LENGTH(TokenType.MAX_TOKEN_LENGTH, tokenSource, beginOffset, endOffset);
case TO_INT :
return new TO_INT(TokenType.TO_INT, tokenSource, beginOffset, endOffset);
case ZCURVE :
return new ZCURVE(TokenType.ZCURVE, tokenSource, beginOffset, endOffset);
case DOUBLE :
return new DOUBLE(TokenType.DOUBLE, tokenSource, beginOffset, endOffset);
case IDENTIFIER :
return new IDENTIFIER(TokenType.IDENTIFIER, tokenSource, beginOffset, endOffset);
case GE :
return new GE(TokenType.GE, tokenSource, beginOffset, endOffset);
case RCURLY :
return new RCURLY(TokenType.RCURLY, tokenSource, beginOffset, endOffset);
case NORMALIZE :
return new NORMALIZE(TokenType.NORMALIZE, tokenSource, beginOffset, endOffset);
case LONG :
return new LONG(TokenType.LONG, tokenSource, beginOffset, endOffset);
case COMMA :
return new COMMA(TokenType.COMMA, tokenSource, beginOffset, endOffset);
case TOKENIZE :
return new TOKENIZE(TokenType.TOKENIZE, tokenSource, beginOffset, endOffset);
case MOD :
return new MOD(TokenType.MOD, tokenSource, beginOffset, endOffset);
case RANDOM :
return new RANDOM(TokenType.RANDOM, tokenSource, beginOffset, endOffset);
case LCURLY :
return new LCURLY(TokenType.LCURLY, tokenSource, beginOffset, endOffset);
case COLON :
return new COLON(TokenType.COLON, tokenSource, beginOffset, endOffset);
case GT :
return new GT(TokenType.GT, tokenSource, beginOffset, endOffset);
case STEM :
return new STEM(TokenType.STEM, tokenSource, beginOffset, endOffset);
case DIV :
return new DIV(TokenType.DIV, tokenSource, beginOffset, endOffset);
case TO_FLOAT :
return new TO_FLOAT(TokenType.TO_FLOAT, tokenSource, beginOffset, endOffset);
case TO_WSET :
return new TO_WSET(TokenType.TO_WSET, tokenSource, beginOffset, endOffset);
case TO_BYTE :
return new TO_BYTE(TokenType.TO_BYTE, tokenSource, beginOffset, endOffset);
case TO_LONG :
return new TO_LONG(TokenType.TO_LONG, tokenSource, beginOffset, endOffset);
case LE :
return new LE(TokenType.LE, tokenSource, beginOffset, endOffset);
case STRING :
return new STRING(TokenType.STRING, tokenSource, beginOffset, endOffset);
case FALSE :
return new FALSE(TokenType.FALSE, tokenSource, beginOffset, endOffset);
case MAX_OCCURRENCES :
return new MAX_OCCURRENCES(TokenType.MAX_OCCURRENCES, tokenSource, beginOffset, endOffset);
case TO_DOUBLE :
return new TO_DOUBLE(TokenType.TO_DOUBLE, tokenSource, beginOffset, endOffset);
case INVALID :
return new InvalidToken(tokenSource, beginOffset, endOffset);
default :
return new Token(type, tokenSource, beginOffset, endOffset);
}
}
public String getLocation() {
return getInputSource() + ":" + getBeginLine() + ":" + getBeginColumn();
}
public Node getParent() {
return parent;
}
public void setParent(Node parent) {
this.parent = parent;
}
public boolean isEmpty() {
return length() == 0;
}
public int length() {
return endOffset - beginOffset;
}
public CharSequence subSequence(int start, int end) {
return getTokenSource().subSequence(beginOffset + start, beginOffset + end);
}
public char charAt(int offset) {
return getTokenSource().charAt(beginOffset + offset);
}
/**
* @deprecated Use toString() instead
*/
@Deprecated
public String getImage() {
return getSource();
}
@Override
public String toString() {
return getSource();
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy