All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.antlr.v4.runtime.CommonTokenStream Maven / Gradle / Ivy

Go to download

Statistical sampling library for use in virtdata libraries, based on apache commons math 4

There is a newer version: 5.17.0
Show newest version
/*
 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
 * Use of this file is governed by the BSD 3-clause license that
 * can be found in the LICENSE.txt file in the project root.
 */

package org.antlr.v4.runtime;

/**
 * This class extends {@link BufferedTokenStream} with functionality to filter
 * token streams to tokens on a particular channel (tokens where
 * {@link Token#getChannel} returns a particular value).
 *
 * 

* This token stream provides access to all tokens by index or when calling * methods like {@link #getText}. The channel filtering is only used for code * accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and * {@link #LB}.

* *

* By default, tokens are placed on the default channel * ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the * {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to * call {@link Lexer#setChannel}. *

* *

* Note: lexer rules which use the {@code ->skip} lexer command or call * {@link Lexer#skip} do not produce tokens at all, so input text matched by * such a rule will not be available as part of the token stream, regardless of * channel.

we */ public class CommonTokenStream extends BufferedTokenStream { /** * Specifies the channel to use for filtering tokens. * *

* The default value is {@link Token#DEFAULT_CHANNEL}, which matches the * default channel assigned to tokens created by the lexer.

*/ protected int channel = Token.DEFAULT_CHANNEL; /** * Constructs a new {@link CommonTokenStream} using the specified token * source and the default token channel ({@link Token#DEFAULT_CHANNEL}). * * @param tokenSource The token source. */ public CommonTokenStream(TokenSource tokenSource) { super(tokenSource); } /** * Constructs a new {@link CommonTokenStream} using the specified token * source and filtering tokens to the specified channel. Only tokens whose * {@link Token#getChannel} matches {@code channel} or have the * {@link Token#getType} equal to {@link Token#EOF} will be returned by the * token stream lookahead methods. * * @param tokenSource The token source. * @param channel The channel to use for filtering tokens. */ public CommonTokenStream(TokenSource tokenSource, int channel) { this(tokenSource); this.channel = channel; } @Override protected int adjustSeekIndex(int i) { return nextTokenOnChannel(i, channel); } @Override protected Token LB(int k) { if ( k==0 || (p-k)<0 ) return null; int i = p; int n = 1; // find k good tokens looking backwards while ( n<=k && i>0 ) { // skip off-channel tokens i = previousTokenOnChannel(i - 1, channel); n++; } if ( i<0 ) return null; return tokens.get(i); } @Override public Token LT(int k) { //System.out.println("enter LT("+k+")"); lazyInit(); if ( k == 0 ) return null; if ( k < 0 ) return LB(-k); int i = p; int n = 1; // we know tokens[p] is a good one // find k good tokens while ( nrange ) range = i; return tokens.get(i); } /** Count EOF just once. */ public int getNumberOfOnChannelTokens() { int n = 0; fill(); for (int i = 0; i < tokens.size(); i++) { Token t = tokens.get(i); if ( t.getChannel()==channel ) n++; if ( t.getType()==Token.EOF ) break; } return n; } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy