ai.idylnlp.dl4j.IdylNLPTokenizerFactory Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of idylnlp-dl4j Show documentation
Idyl NLP for DeepLearning4j
The newest version!
/*******************************************************************************
 * Copyright 2018 Mountain Fog, Inc.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License.  You may obtain a copy
 * of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 ******************************************************************************/
package ai.idylnlp.dl4j;

import java.io.InputStream;

import org.apache.commons.lang3.NotImplementedException;
import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess;
import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;

/**
 * Implements {@link TokenizerFactory} to make Idyl NLP's tokenizers
 * available to DeepLearning4j's NLP capabilities.
 * 
 * @author Mountain Fog, Inc.
 *
 */
public class IdylNLPTokenizerFactory implements TokenizerFactory {

	private ai.idylnlp.model.nlp.Tokenizer tokenizer;
	private TokenPreProcess preProcessor;
	
	/**
	 * Creates a new tokenizer factory.
	 * @param tokenizer An Idyl NLP {@link ai.idylnlp.model.nlp.Tokenizer}.
	 */
	public IdylNLPTokenizerFactory(ai.idylnlp.model.nlp.Tokenizer tokenizer) {
		this.tokenizer = tokenizer;
	}
	
	/**
	 * Creates a new tokenizer factory.
	 * @param tokenizer An Idyl NLP {@link ai.idylnlp.model.nlp.Tokenizer}.
	 */
	public IdylNLPTokenizerFactory(ai.idylnlp.model.nlp.Tokenizer tokenizer, TokenPreProcess preProcessor) {
		this.tokenizer = tokenizer;
		this.preProcessor = preProcessor;
	}
	
	@Override
	public Tokenizer create(String toTokenize) {
		return new IdylNLPTokenizer(tokenizer, preProcessor, toTokenize);
	}

	@Override
	public Tokenizer create(InputStream toTokenize) {
		throw new NotImplementedException("Not yet implemented.");
	}

	@Override
	public void setTokenPreProcessor(TokenPreProcess preProcessor) {
		this.preProcessor = preProcessor;
	}

	@Override
	public TokenPreProcess getTokenPreProcessor() {
		return preProcessor;
	}

}