All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.antlr.v4.runtime.CommonToken Maven / Gradle / Ivy

/*
 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
 * Use of this file is governed by the BSD 3-clause license that
 * can be found in the LICENSE.txt file in the project root.
 */
package org.antlr.v4.runtime;

import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;

import java.io.Serializable;

public class CommonToken implements WritableToken, Serializable {
	/**
	 * An empty {@link Pair} which is used as the default value of
	 * {@link #source} for tokens that do not have a source.
	 */
	protected static final Pair EMPTY_SOURCE =
		new Pair(null, null);

	/**
	 * This is the backing field for {@link #getType} and {@link #setType}.
	 */
	protected int type;

	/**
	 * This is the backing field for {@link #getLine} and {@link #setLine}.
	 */
	protected int line;

	/**
	 * This is the backing field for {@link #getCharPositionInLine} and
	 * {@link #setCharPositionInLine}.
	 */
	protected int charPositionInLine = -1; // set to invalid position

	/**
	 * This is the backing field for {@link #getChannel} and
	 * {@link #setChannel}.
	 */
	protected int channel=DEFAULT_CHANNEL;

	/**
	 * This is the backing field for {@link #getTokenSource} and
	 * {@link #getInputStream}.
	 *
	 * 

* These properties share a field to reduce the memory footprint of * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from * the same source and input stream share a reference to the same * {@link Pair} containing these values.

*/ protected Pair source; /** * This is the backing field for {@link #getText} when the token text is * explicitly set in the constructor or via {@link #setText}. * * @see #getText() */ protected String text; /** * This is the backing field for {@link #getTokenIndex} and * {@link #setTokenIndex}. */ protected int index = -1; /** * This is the backing field for {@link #getStartIndex} and * {@link #setStartIndex}. */ protected int start; /** * This is the backing field for {@link #getStopIndex} and * {@link #setStopIndex}. */ protected int stop; /** * Constructs a new {@link CommonToken} with the specified token type. * * @param type The token type. */ public CommonToken(int type) { this.type = type; this.source = EMPTY_SOURCE; } public CommonToken(Pair source, int type, int channel, int start, int stop) { this.source = source; this.type = type; this.channel = channel; this.start = start; this.stop = stop; if (source.a != null) { this.line = source.a.getLine(); this.charPositionInLine = source.a.getCharPositionInLine(); } } /** * Constructs a new {@link CommonToken} with the specified token type and * text. * * @param type The token type. * @param text The text of the token. */ public CommonToken(int type, String text) { this.type = type; this.channel = DEFAULT_CHANNEL; this.text = text; this.source = EMPTY_SOURCE; } /** * Constructs a new {@link CommonToken} as a copy of another {@link Token}. * *

* If {@code oldToken} is also a {@link CommonToken} instance, the newly * constructed token will share a reference to the {@link #text} field and * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will * be assigned the result of calling {@link #getText}, and {@link #source} * will be constructed from the result of {@link Token#getTokenSource} and * {@link Token#getInputStream}.

* * @param oldToken The token to copy. */ public CommonToken(Token oldToken) { type = oldToken.getType(); line = oldToken.getLine(); index = oldToken.getTokenIndex(); charPositionInLine = oldToken.getCharPositionInLine(); channel = oldToken.getChannel(); start = oldToken.getStartIndex(); stop = oldToken.getStopIndex(); if (oldToken instanceof CommonToken) { text = ((CommonToken)oldToken).text; source = ((CommonToken)oldToken).source; } else { text = oldToken.getText(); source = new Pair(oldToken.getTokenSource(), oldToken.getInputStream()); } } @Override public int getType() { return type; } @Override public void setLine(int line) { this.line = line; } @Override public String getText() { if ( text!=null ) { return text; } CharStream input = getInputStream(); if ( input==null ) return null; int n = input.size(); if ( start"; } } /** * Explicitly set the text for this token. If {code text} is not * {@code null}, then {@link #getText} will return this value rather than * extracting the text from the input. * * @param text The explicit text of the token, or {@code null} if the text * should be obtained from the input along with the start and stop indexes * of the token. */ @Override public void setText(String text) { this.text = text; } @Override public int getLine() { return line; } @Override public int getCharPositionInLine() { return charPositionInLine; } @Override public void setCharPositionInLine(int charPositionInLine) { this.charPositionInLine = charPositionInLine; } @Override public int getChannel() { return channel; } @Override public void setChannel(int channel) { this.channel = channel; } @Override public void setType(int type) { this.type = type; } @Override public int getStartIndex() { return start; } public void setStartIndex(int start) { this.start = start; } @Override public int getStopIndex() { return stop; } public void setStopIndex(int stop) { this.stop = stop; } @Override public int getTokenIndex() { return index; } @Override public void setTokenIndex(int index) { this.index = index; } @Override public TokenSource getTokenSource() { return source.a; } @Override public CharStream getInputStream() { return source.b; } @Override public String toString() { return toString(null); } public String toString(Recognizer r) { String channelStr = ""; if ( channel>0 ) { channelStr=",channel="+channel; } String txt = getText(); if ( txt!=null ) { txt = txt.replace("\n","\\n"); txt = txt.replace("\r","\\r"); txt = txt.replace("\t","\\t"); } else { txt = ""; } String typeString = String.valueOf(type); if ( r!=null ) { typeString = r.getVocabulary().getDisplayName(type); } return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+typeString+">"+channelStr+","+line+":"+getCharPositionInLine()+"]"; } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy