All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.antlr.v4.runtime.BufferedTokenStream Maven / Gradle / Ivy

Go to download

Statistical sampling library for use in virtdata libraries, based on apache commons math 4

There is a newer version: 5.17.0
Show newest version
/*
 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
 * Use of this file is governed by the BSD 3-clause license that
 * can be found in the LICENSE.txt file in the project root.
 */

package org.antlr.v4.runtime;

import org.antlr.v4.runtime.misc.Interval;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * This implementation of {@link TokenStream} loads tokens from a
 * {@link TokenSource} on-demand, and places the tokens in a buffer to provide
 * access to any previous token by index.
 *
 * 

* This token stream ignores the value of {@link Token#getChannel}. If your * parser requires the token stream filter tokens to only those on a particular * channel, such as {@link Token#DEFAULT_CHANNEL} or * {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a * {@link CommonTokenStream}.

*/ public class BufferedTokenStream implements TokenStream { /** * The {@link TokenSource} from which tokens for this stream are fetched. */ protected TokenSource tokenSource; /** * A collection of all tokens fetched from the token source. The list is * considered a complete view of the input once {@link #fetchedEOF} is set * to {@code true}. */ protected List tokens = new ArrayList(100); /** * The index into {@link #tokens} of the current token (next token to * {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be * {@link #LT LT(1)}. * *

This field is set to -1 when the stream is first constructed or when * {@link #setTokenSource} is called, indicating that the first token has * not yet been fetched from the token source. For additional information, * see the documentation of {@link IntStream} for a description of * Initializing Methods.

*/ protected int p = -1; /** * Indicates whether the {@link Token#EOF} token has been fetched from * {@link #tokenSource} and added to {@link #tokens}. This field improves * performance for the following cases: * *
    *
  • {@link #consume}: The lookahead check in {@link #consume} to prevent * consuming the EOF symbol is optimized by checking the values of * {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.
  • *
  • {@link #fetch}: The check to prevent adding multiple EOF symbols into * {@link #tokens} is trivial with this field.
  • *
      */ protected boolean fetchedEOF; public BufferedTokenStream(TokenSource tokenSource) { if (tokenSource == null) { throw new NullPointerException("tokenSource cannot be null"); } this.tokenSource = tokenSource; } @Override public TokenSource getTokenSource() { return tokenSource; } @Override public int index() { return p; } @Override public int mark() { return 0; } @Override public void release(int marker) { // no resources to release } /** * This method resets the token stream back to the first token in the * buffer. It is equivalent to calling {@link #seek}{@code (0)}. * * @see #setTokenSource(TokenSource) * @deprecated Use {@code seek(0)} instead. */ @Deprecated public void reset() { seek(0); } @Override public void seek(int index) { lazyInit(); p = adjustSeekIndex(index); } @Override public int size() { return tokens.size(); } @Override public void consume() { boolean skipEofCheck; if (p >= 0) { if (fetchedEOF) { // the last token in tokens is EOF. skip check if p indexes any // fetched token except the last. skipEofCheck = p < tokens.size() - 1; } else { // no EOF token in tokens. skip check if p indexes a fetched token. skipEofCheck = p < tokens.size(); } } else { // not yet initialized skipEofCheck = false; } if (!skipEofCheck && LA(1) == EOF) { throw new IllegalStateException("cannot consume EOF"); } if (sync(p + 1)) { p = adjustSeekIndex(p + 1); } } /** Make sure index {@code i} in tokens has a token. * * @return {@code true} if a token is located at index {@code i}, otherwise * {@code false}. * @see #get(int i) */ protected boolean sync(int i) { assert i >= 0; int n = i - tokens.size() + 1; // how many more elements we need? //System.out.println("sync("+i+") needs "+n); if ( n > 0 ) { int fetched = fetch(n); return fetched >= n; } return true; } /** Add {@code n} elements to buffer. * * @return The actual number of elements added to the buffer. */ protected int fetch(int n) { if (fetchedEOF) { return 0; } for (int i = 0; i < n; i++) { Token t = tokenSource.nextToken(); if ( t instanceof WritableToken ) { ((WritableToken)t).setTokenIndex(tokens.size()); } tokens.add(t); if ( t.getType()==Token.EOF ) { fetchedEOF = true; return i + 1; } } return n; } @Override public Token get(int i) { if ( i < 0 || i >= tokens.size() ) { throw new IndexOutOfBoundsException("token index "+i+" out of range 0.."+(tokens.size()-1)); } return tokens.get(i); } /** Get all tokens from start..stop inclusively */ public List get(int start, int stop) { if ( start<0 || stop<0 ) return null; lazyInit(); List subset = new ArrayList(); if ( stop>=tokens.size() ) stop = tokens.size()-1; for (int i = start; i <= stop; i++) { Token t = tokens.get(i); if ( t.getType()==Token.EOF ) break; subset.add(t); } return subset; } @Override public int LA(int i) { return LT(i).getType(); } protected Token LB(int k) { if ( (p-k)<0 ) return null; return tokens.get(p-k); } @Override public Token LT(int k) { lazyInit(); if ( k==0 ) return null; if ( k < 0 ) return LB(-k); int i = p + k - 1; sync(i); if ( i >= tokens.size() ) { // return EOF token // EOF must be last token return tokens.get(tokens.size()-1); } // if ( i>range ) range = i; return tokens.get(i); } /** * Allowed derived classes to modify the behavior of operations which change * the current stream position by adjusting the target token index of a seek * operation. The default implementation simply returns {@code i}. If an * exception is thrown in this method, the current stream index should not be * changed. * *

      For example, {@link CommonTokenStream} overrides this method to ensure that * the seek target is always an on-channel token.

      * * @param i The target token index. * @return The adjusted target token index. */ protected int adjustSeekIndex(int i) { return i; } protected final void lazyInit() { if (p == -1) { setup(); } } protected void setup() { sync(0); p = adjustSeekIndex(0); } /** Reset this token stream by setting its token source. */ public void setTokenSource(TokenSource tokenSource) { this.tokenSource = tokenSource; tokens.clear(); p = -1; fetchedEOF = false; } public List getTokens() { return tokens; } public List getTokens(int start, int stop) { return getTokens(start, stop, null); } /** Given a start and stop index, return a List of all tokens in * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. */ public List getTokens(int start, int stop, Set types) { lazyInit(); if ( start<0 || stop>=tokens.size() || stop<0 || start>=tokens.size() ) { throw new IndexOutOfBoundsException("start "+start+" or stop "+stop+ " not in 0.."+(tokens.size()-1)); } if ( start>stop ) return null; // list = tokens[start:stop]:{T t, t.getType() in types} List filteredTokens = new ArrayList(); for (int i=start; i<=stop; i++) { Token t = tokens.get(i); if ( types==null || types.contains(t.getType()) ) { filteredTokens.add(t); } } if ( filteredTokens.isEmpty() ) { filteredTokens = null; } return filteredTokens; } public List getTokens(int start, int stop, int ttype) { HashSet s = new HashSet(ttype); s.add(ttype); return getTokens(start,stop, s); } /** * Given a starting index, return the index of the next token on channel. * Return {@code i} if {@code tokens[i]} is on channel. Return the index of * the EOF token if there are no tokens on channel between {@code i} and * EOF. */ protected int nextTokenOnChannel(int i, int channel) { sync(i); if (i >= size()) { return size() - 1; } Token token = tokens.get(i); while ( token.getChannel()!=channel ) { if ( token.getType()==Token.EOF ) { return i; } i++; sync(i); token = tokens.get(i); } return i; } /** * Given a starting index, return the index of the previous token on * channel. Return {@code i} if {@code tokens[i]} is on channel. Return -1 * if there are no tokens on channel between {@code i} and 0. * *

      * If {@code i} specifies an index at or after the EOF token, the EOF token * index is returned. This is due to the fact that the EOF token is treated * as though it were on every channel.

      */ protected int previousTokenOnChannel(int i, int channel) { sync(i); if (i >= size()) { // the EOF token is on every channel return size() - 1; } while (i >= 0) { Token token = tokens.get(i); if (token.getType() == Token.EOF || token.getChannel() == channel) { return i; } i--; } return i; } /** Collect all tokens on specified channel to the right of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or * EOF. If channel is -1, find any non default channel token. */ public List getHiddenTokensToRight(int tokenIndex, int channel) { lazyInit(); if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); } int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL); int to; int from = tokenIndex+1; // if none onchannel to right, nextOnChannel=-1 so set to = last token if ( nextOnChannel == -1 ) to = size()-1; else to = nextOnChannel; return filterForChannel(from, to, channel); } /** Collect all hidden tokens (any off-default channel) to the right of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL * or EOF. */ public List getHiddenTokensToRight(int tokenIndex) { return getHiddenTokensToRight(tokenIndex, -1); } /** Collect all tokens on specified channel to the left of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. * If channel is -1, find any non default channel token. */ public List getHiddenTokensToLeft(int tokenIndex, int channel) { lazyInit(); if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); } if (tokenIndex == 0) { // obviously no tokens can appear before the first token return null; } int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL); if ( prevOnChannel == tokenIndex - 1 ) return null; // if none onchannel to left, prevOnChannel=-1 then from=0 int from = prevOnChannel+1; int to = tokenIndex-1; return filterForChannel(from, to, channel); } /** Collect all hidden tokens (any off-default channel) to the left of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. */ public List getHiddenTokensToLeft(int tokenIndex) { return getHiddenTokensToLeft(tokenIndex, -1); } protected List filterForChannel(int from, int to, int channel) { List hidden = new ArrayList(); for (int i=from; i<=to; i++) { Token t = tokens.get(i); if ( channel==-1 ) { if ( t.getChannel()!= Lexer.DEFAULT_TOKEN_CHANNEL ) hidden.add(t); } else { if ( t.getChannel()==channel ) hidden.add(t); } } if ( hidden.size()==0 ) return null; return hidden; } @Override public String getSourceName() { return tokenSource.getSourceName(); } /** Get the text of all tokens in this buffer. */ @Override public String getText() { return getText(Interval.of(0,size()-1)); } @Override public String getText(Interval interval) { int start = interval.a; int stop = interval.b; if ( start<0 || stop<0 ) return ""; fill(); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { Token t = tokens.get(i); if ( t.getType()==Token.EOF ) break; buf.append(t.getText()); } return buf.toString(); } @Override public String getText(RuleContext ctx) { return getText(ctx.getSourceInterval()); } @Override public String getText(Token start, Token stop) { if ( start!=null && stop!=null ) { return getText(Interval.of(start.getTokenIndex(), stop.getTokenIndex())); } return ""; } /** Get all tokens from lexer until EOF */ public void fill() { lazyInit(); final int blockSize = 1000; while (true) { int fetched = fetch(blockSize); if (fetched < blockSize) { return; } } } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy