org.antlr.v4.runtime.CommonTokenStream Maven / Gradle / Ivy
/*
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
package org.antlr.v4.runtime;
/**
* This class extends {@link BufferedTokenStream} with functionality to filter
* token streams to tokens on a particular channel (tokens where
* {@link Token#getChannel} returns a particular value).
*
*
* This token stream provides access to all tokens by index or when calling
* methods like {@link #getText}. The channel filtering is only used for code
* accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
* {@link #LB}.
*
*
* By default, tokens are placed on the default channel
* ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
* {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
* call {@link Lexer#setChannel}.
*
*
*
* Note: lexer rules which use the {@code ->skip} lexer command or call
* {@link Lexer#skip} do not produce tokens at all, so input text matched by
* such a rule will not be available as part of the token stream, regardless of
* channel.
we
*/
public class CommonTokenStream extends BufferedTokenStream {
/**
* Specifies the channel to use for filtering tokens.
*
*
* The default value is {@link Token#DEFAULT_CHANNEL}, which matches the
* default channel assigned to tokens created by the lexer.
*/
protected int channel = Token.DEFAULT_CHANNEL;
/**
* Constructs a new {@link CommonTokenStream} using the specified token
* source and the default token channel ({@link Token#DEFAULT_CHANNEL}).
*
* @param tokenSource The token source.
*/
public CommonTokenStream(TokenSource tokenSource) {
super(tokenSource);
}
/**
* Constructs a new {@link CommonTokenStream} using the specified token
* source and filtering tokens to the specified channel. Only tokens whose
* {@link Token#getChannel} matches {@code channel} or have the
* {@link Token#getType} equal to {@link Token#EOF} will be returned by the
* token stream lookahead methods.
*
* @param tokenSource The token source.
* @param channel The channel to use for filtering tokens.
*/
public CommonTokenStream(TokenSource tokenSource, int channel) {
this(tokenSource);
this.channel = channel;
}
@Override
protected int adjustSeekIndex(int i) {
return nextTokenOnChannel(i, channel);
}
@Override
protected Token LB(int k) {
if ( k==0 || (p-k)<0 ) return null;
int i = p;
int n = 1;
// find k good tokens looking backwards
while ( n<=k && i>0 ) {
// skip off-channel tokens
i = previousTokenOnChannel(i - 1, channel);
n++;
}
if ( i<0 ) return null;
return tokens.get(i);
}
@Override
public Token LT(int k) {
//System.out.println("enter LT("+k+")");
lazyInit();
if ( k == 0 ) return null;
if ( k < 0 ) return LB(-k);
int i = p;
int n = 1; // we know tokens[p] is a good one
// find k good tokens
while ( nrange ) range = i;
return tokens.get(i);
}
/** Count EOF just once. */
public int getNumberOfOnChannelTokens() {
int n = 0;
fill();
for (int i = 0; i < tokens.size(); i++) {
Token t = tokens.get(i);
if ( t.getChannel()==channel ) n++;
if ( t.getType()==Token.EOF ) break;
}
return n;
}
}