org.antlr.v4.runtime.CommonToken Maven / Gradle / Ivy
/*
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;
import java.io.Serializable;
public class CommonToken implements WritableToken, Serializable {
/**
* An empty {@link Pair} which is used as the default value of
* {@link #source} for tokens that do not have a source.
*/
protected static final Pair EMPTY_SOURCE =
new Pair(null, null);
/**
* This is the backing field for {@link #getType} and {@link #setType}.
*/
protected int type;
/**
* This is the backing field for {@link #getLine} and {@link #setLine}.
*/
protected int line;
/**
* This is the backing field for {@link #getCharPositionInLine} and
* {@link #setCharPositionInLine}.
*/
protected int charPositionInLine = -1; // set to invalid position
/**
* This is the backing field for {@link #getChannel} and
* {@link #setChannel}.
*/
protected int channel=DEFAULT_CHANNEL;
/**
* This is the backing field for {@link #getTokenSource} and
* {@link #getInputStream}.
*
*
* These properties share a field to reduce the memory footprint of
* {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
* the same source and input stream share a reference to the same
* {@link Pair} containing these values.
*/
protected Pair source;
/**
* This is the backing field for {@link #getText} when the token text is
* explicitly set in the constructor or via {@link #setText}.
*
* @see #getText()
*/
protected String text;
/**
* This is the backing field for {@link #getTokenIndex} and
* {@link #setTokenIndex}.
*/
protected int index = -1;
/**
* This is the backing field for {@link #getStartIndex} and
* {@link #setStartIndex}.
*/
protected int start;
/**
* This is the backing field for {@link #getStopIndex} and
* {@link #setStopIndex}.
*/
protected int stop;
/**
* Constructs a new {@link CommonToken} with the specified token type.
*
* @param type The token type.
*/
public CommonToken(int type) {
this.type = type;
this.source = EMPTY_SOURCE;
}
public CommonToken(Pair source, int type, int channel, int start, int stop) {
this.source = source;
this.type = type;
this.channel = channel;
this.start = start;
this.stop = stop;
if (source.a != null) {
this.line = source.a.getLine();
this.charPositionInLine = source.a.getCharPositionInLine();
}
}
/**
* Constructs a new {@link CommonToken} with the specified token type and
* text.
*
* @param type The token type.
* @param text The text of the token.
*/
public CommonToken(int type, String text) {
this.type = type;
this.channel = DEFAULT_CHANNEL;
this.text = text;
this.source = EMPTY_SOURCE;
}
/**
* Constructs a new {@link CommonToken} as a copy of another {@link Token}.
*
*
* If {@code oldToken} is also a {@link CommonToken} instance, the newly
* constructed token will share a reference to the {@link #text} field and
* the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
* be assigned the result of calling {@link #getText}, and {@link #source}
* will be constructed from the result of {@link Token#getTokenSource} and
* {@link Token#getInputStream}.
*
* @param oldToken The token to copy.
*/
public CommonToken(Token oldToken) {
type = oldToken.getType();
line = oldToken.getLine();
index = oldToken.getTokenIndex();
charPositionInLine = oldToken.getCharPositionInLine();
channel = oldToken.getChannel();
start = oldToken.getStartIndex();
stop = oldToken.getStopIndex();
if (oldToken instanceof CommonToken) {
text = ((CommonToken)oldToken).text;
source = ((CommonToken)oldToken).source;
}
else {
text = oldToken.getText();
source = new Pair(oldToken.getTokenSource(), oldToken.getInputStream());
}
}
@Override
public int getType() {
return type;
}
@Override
public void setLine(int line) {
this.line = line;
}
@Override
public String getText() {
if ( text!=null ) {
return text;
}
CharStream input = getInputStream();
if ( input==null ) return null;
int n = input.size();
if ( start";
}
}
/**
* Explicitly set the text for this token. If {code text} is not
* {@code null}, then {@link #getText} will return this value rather than
* extracting the text from the input.
*
* @param text The explicit text of the token, or {@code null} if the text
* should be obtained from the input along with the start and stop indexes
* of the token.
*/
@Override
public void setText(String text) {
this.text = text;
}
@Override
public int getLine() {
return line;
}
@Override
public int getCharPositionInLine() {
return charPositionInLine;
}
@Override
public void setCharPositionInLine(int charPositionInLine) {
this.charPositionInLine = charPositionInLine;
}
@Override
public int getChannel() {
return channel;
}
@Override
public void setChannel(int channel) {
this.channel = channel;
}
@Override
public void setType(int type) {
this.type = type;
}
@Override
public int getStartIndex() {
return start;
}
public void setStartIndex(int start) {
this.start = start;
}
@Override
public int getStopIndex() {
return stop;
}
public void setStopIndex(int stop) {
this.stop = stop;
}
@Override
public int getTokenIndex() {
return index;
}
@Override
public void setTokenIndex(int index) {
this.index = index;
}
@Override
public TokenSource getTokenSource() {
return source.a;
}
@Override
public CharStream getInputStream() {
return source.b;
}
@Override
public String toString() {
return toString(null);
}
public String toString(Recognizer r) {
String channelStr = "";
if ( channel>0 ) {
channelStr=",channel="+channel;
}
String txt = getText();
if ( txt!=null ) {
txt = txt.replace("\n","\\n");
txt = txt.replace("\r","\\r");
txt = txt.replace("\t","\\t");
}
else {
txt = "";
}
String typeString = String.valueOf(type);
if ( r!=null ) {
typeString = r.getVocabulary().getDisplayName(type);
}
return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+typeString+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
}
}