org.deeplearning4j.word2vec.tokenizer.DefaultTokenizer Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of deeplearning4j-scaleout-akka-word2vec Show documentation

The newest version!

package org.deeplearning4j.word2vec.tokenizer;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

public class DefaultTokenizer implements Tokenizer {

	public DefaultTokenizer(String tokens) {
		tokenizer = new StringTokenizer(tokens);
	}
	
	private StringTokenizer tokenizer;
	
	
	@Override
	public boolean hasMoreTokens() {
		return tokenizer.hasMoreTokens();
	}

	@Override
	public int countTokens() {
		return tokenizer.countTokens();
	}

	@Override
	public String nextToken() {
		return tokenizer.nextToken();
	}

	@Override
	public List getTokens() {
		List tokens = new ArrayList();
		while(hasMoreTokens()) {
			tokens.add(nextToken());
		}
		return tokens;
	}
	
	

	
}