com.aliasi.test.unit.tokenizer.ConstantTokenizer Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of aliasi-lingpipe Show documentation

This is the original Lingpipe: http://alias-i.com/lingpipe/web/download.html There were not made any changes to the source code.

There is a newer version: 4.1.2-JL1.0

Show newest version

package com.aliasi.test.unit.tokenizer;

import com.aliasi.tokenizer.Tokenizer;

public class ConstantTokenizer extends Tokenizer {
    private final String[] mTokens;
    private final String[] mWhitespaces;
    private int mNextToken = 0;
    public ConstantTokenizer(String[] tokens,
                             String[] whitespaces) {
        if (whitespaces.length != tokens.length + 1)
            throw new IllegalArgumentException("lengths bad");
        mTokens = tokens;
        mWhitespaces = whitespaces;
    }
    @Override
    public String nextToken() {
        if (mNextToken >= mTokens.length) {
            return null;
        }
        return mTokens[mNextToken++];
    }
    @Override
    public String nextWhitespace() {
        return mWhitespaces[mNextToken];
    }
}