org.jpmml.sparkml.feature.StopWordsRemoverConverter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pmml-sparkml Show documentation
JPMML Apache Spark ML to PMML converter
The newest version!
/*
 * Copyright (c) 2017 Villu Ruusmann
 *
 * This file is part of JPMML-SparkML
 *
 * JPMML-SparkML is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * JPMML-SparkML is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with JPMML-SparkML.  If not, see .
 */
package org.jpmml.sparkml.feature;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.spark.ml.feature.StopWordsRemover;
import org.jpmml.converter.Feature;
import org.jpmml.sparkml.DocumentFeature;
import org.jpmml.sparkml.MultiFeatureConverter;
import org.jpmml.sparkml.SparkMLEncoder;
import org.jpmml.sparkml.TermUtil;

public class StopWordsRemoverConverter extends MultiFeatureConverter {

	public StopWordsRemoverConverter(StopWordsRemover transformer){
		super(transformer);
	}

	@Override
	public List encodeFeatures(SparkMLEncoder encoder){
		StopWordsRemover transformer = getTransformer();

		boolean caseSensitive = transformer.getCaseSensitive();
		String[] stopWords = transformer.getStopWords();

		InOutMode inputMode = getInputMode();

		List result = new ArrayList<>();

		String[] inputCols = inputMode.getInputCols(transformer);
		for(String inputCol : inputCols){
			DocumentFeature documentFeature = (DocumentFeature)encoder.getOnlyFeature(inputCol);

			Pattern pattern = Pattern.compile(documentFeature.getWordSeparatorRE());

			DocumentFeature.StopWordSet stopWordSet = new DocumentFeature.StopWordSet(caseSensitive);

			for(String stopWord : stopWords){
				String[] stopTokens = pattern.split(stopWord);

				// Skip multi-token stopwords. See https://issues.apache.org/jira/browse/SPARK-18374
				if(stopTokens.length > 1){
					continue;
				} // End if

				if(TermUtil.hasPunctuation(stopWord)){
					throw new IllegalArgumentException("Punctuated stop words (" + stopWord + ") are not supported");
				}

				stopWordSet.add(stopWord);
			}

			documentFeature.addStopWordSet(stopWordSet);

			result.add(documentFeature);
		}

		return result;
	}
}