All Downloads are FREE. Search and download functionalities are using the official Maven repository.

chalk.tools.sentdetect.SentenceDetectorME Maven / Gradle / Ivy

There is a newer version: 1.3.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package chalk.tools.sentdetect;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import nak.core.AbstractModel;
import nak.data.EventStream;
import nak.core.LinearModel;
import nak.core.TrainUtil;
import nak.core.ClassifierUtil;
import chalk.tools.dictionary.Dictionary;
import chalk.tools.sentdetect.lang.Factory;
import chalk.tools.util.ObjectStream;
import chalk.tools.util.Span;
import chalk.tools.util.StringUtil;
import chalk.tools.util.TrainingParameters;
import chalk.tools.util.model.ModelUtil;


/**
 * A sentence detector for splitting up raw text into sentences.
 * 

* A maximum entropy model is used to evaluate the characters ".", "!", and "?" in a * string to determine if they signify the end of a sentence. */ public class SentenceDetectorME implements SentenceDetector { /** * Constant indicates a sentence split. */ public static final String SPLIT ="s"; /** * Constant indicates no sentence split. */ public static final String NO_SPLIT ="n"; // Note: That should be inlined when doing a re-factoring! private static final Double ONE = new Double(1); /** * The maximum entropy model to use to evaluate contexts. */ private LinearModel model; /** * The feature context generator. */ private final SDContextGenerator cgen; /** * The {@link EndOfSentenceScanner} to use when scanning for end of sentence offsets. */ private final EndOfSentenceScanner scanner; /** * The list of probabilities associated with each decision. */ private List sentProbs = new ArrayList(); protected boolean useTokenEnd; /** * Initializes the current instance. * * @param model the {@link SentenceModel} */ public SentenceDetectorME(SentenceModel model) { SentenceDetectorFactory sdFactory = model.getFactory(); this.model = model.getLinearModel(); cgen = sdFactory.getSDContextGenerator(); scanner = sdFactory.getEndOfSentenceScanner(); useTokenEnd = sdFactory.isUseTokenEnd(); } /** * @deprecated Use a {@link SentenceDetectorFactory} to extend * SentenceDetector functionality. */ public SentenceDetectorME(SentenceModel model, Factory factory) { this.model = model.getLinearModel(); // if the model has custom EOS characters set, use this to get the context // generator and the EOS scanner; otherwise use language-specific defaults char[] customEOSCharacters = model.getEosCharacters(); if (customEOSCharacters == null) { cgen = factory.createSentenceContextGenerator(model.getLanguage(), getAbbreviations(model.getAbbreviations())); scanner = factory.createEndOfSentenceScanner(model.getLanguage()); } else { cgen = factory.createSentenceContextGenerator( getAbbreviations(model.getAbbreviations()), customEOSCharacters); scanner = factory.createEndOfSentenceScanner(customEOSCharacters); } useTokenEnd = model.useTokenEnd(); } private static Set getAbbreviations(Dictionary abbreviations) { if(abbreviations == null) { return Collections.emptySet(); } return abbreviations.asStringSet(); } /** * Detect sentences in a String. * * @param s The string to be processed. * * @return A string array containing individual sentences as elements. */ public String[] sentDetect(String s) { Span[] spans = sentPosDetect(s); String sentences[]; if (spans.length != 0) { sentences = new String[spans.length]; for (int si = 0; si < spans.length; si++) { sentences[si] = spans[si].getCoveredText(s).toString(); } } else { sentences = new String[] {}; } return sentences; } private int getFirstWS(String s, int pos) { while (pos < s.length() && !StringUtil.isWhitespace(s.charAt(pos))) pos++; return pos; } private int getFirstNonWS(String s, int pos) { while (pos < s.length() && StringUtil.isWhitespace(s.charAt(pos))) pos++; return pos; } /** * Detect the position of the first words of sentences in a String. * * @param s The string to be processed. * @return A integer array containing the positions of the end index of * every sentence * */ public Span[] sentPosDetect(String s) { sentProbs.clear(); StringBuffer sb = new StringBuffer(s); List enders = scanner.getPositions(s); List positions = new ArrayList(enders.size()); for (int i = 0, end = enders.size(), index = 0; i < end; i++) { Integer candidate = enders.get(i); int cint = candidate; // skip over the leading parts of non-token final delimiters int fws = getFirstWS(s,cint + 1); if (i + 1 < end && enders.get(i + 1) < fws) { continue; } double[] probs = model.eval(cgen.getContext(sb, cint)); String bestOutcome = ClassifierUtil.getBestOutcome(model, probs); if (bestOutcome.equals(SPLIT) && isAcceptableBreak(s, index, cint)) { if (index != cint) { if (useTokenEnd) { positions.add(getFirstNonWS(s, getFirstWS(s,cint + 1))); } else { positions.add(getFirstNonWS(s,cint)); } sentProbs.add(probs[model.getIndex(bestOutcome)]); } index = cint + 1; } } int[] starts = new int[positions.size()]; for (int i = 0; i < starts.length; i++) { starts[i] = positions.get(i); } // string does not contain sentence end positions if (starts.length == 0) { // remove leading and trailing whitespace int start = 0; int end = s.length(); while (start < s.length() && StringUtil.isWhitespace(s.charAt(start))) start++; while (end > 0 && StringUtil.isWhitespace(s.charAt(end - 1))) end--; if ((end - start) > 0) { sentProbs.add(1d); return new Span[] {new Span(start, end)}; } else return new Span[0]; } // Now convert the sent indexes to spans boolean leftover = starts[starts.length - 1] != s.length(); Span[] spans = new Span[leftover? starts.length + 1 : starts.length]; for (int si=0;si 0 && StringUtil.isWhitespace(s.charAt(end-1))) { end--; } spans[si]=new Span(start,end); } if (leftover) { spans[spans.length-1] = new Span(starts[starts.length-1],s.length()); sentProbs.add(ONE); } return spans; } /** * Returns the probabilities associated with the most recent * calls to sentDetect(). * * @return probability for each sentence returned for the most recent * call to sentDetect. If not applicable an empty array is * returned. */ public double[] getSentenceProbabilities() { double[] sentProbArray = new double[sentProbs.size()]; for (int i = 0; i < sentProbArray.length; i++) { sentProbArray[i] = sentProbs.get(i); } return sentProbArray; } /** * Allows subclasses to check an overzealous (read: poorly * trained) model from flagging obvious non-breaks as breaks based * on some boolean determination of a break's acceptability. * *

The implementation here always returns true, which means * that the LinearModel's outcome is taken as is.

* * @param s the string in which the break occurred. * @param fromIndex the start of the segment currently being evaluated * @param candidateIndex the index of the candidate sentence ending * @return true if the break is acceptable */ protected boolean isAcceptableBreak(String s, int fromIndex, int candidateIndex) { return true; } /** * @deprecated Use * {@link #train(String, ObjectStream, SentenceDetectorFactory, TrainingParameters)} * and pass in af {@link SentenceDetectorFactory}. */ public static SentenceModel train(String languageCode, ObjectStream samples, boolean useTokenEnd, Dictionary abbreviations, TrainingParameters mlParams) throws IOException { SentenceDetectorFactory sdFactory = new SentenceDetectorFactory( languageCode, useTokenEnd, abbreviations, null); return train(languageCode, samples, sdFactory, mlParams); } public static SentenceModel train(String languageCode, ObjectStream samples, SentenceDetectorFactory sdFactory, TrainingParameters mlParams) throws IOException { Map manifestInfoEntries = new HashMap(); // TODO: Fix the EventStream to throw exceptions when training goes wrong EventStream eventStream = new SDEventStream(samples, sdFactory.getSDContextGenerator(), sdFactory.getEndOfSentenceScanner()); LinearModel sentModel = TrainUtil.train(eventStream, mlParams.getSettings(), manifestInfoEntries); return new SentenceModel(languageCode, sentModel, manifestInfoEntries, sdFactory); } /** * @deprecated Use * {@link #train(String, ObjectStream, SentenceDetectorFactory, TrainingParameters)} * and pass in af {@link SentenceDetectorFactory}. */ @Deprecated public static SentenceModel train(String languageCode, ObjectStream samples, boolean useTokenEnd, Dictionary abbreviations, int cutoff, int iterations) throws IOException { return train(languageCode, samples, useTokenEnd, abbreviations, ModelUtil.createTrainingParameters(iterations, cutoff)); } /** * @deprecated Use * {@link #train(String, ObjectStream, SentenceDetectorFactory, TrainingParameters)} * and pass in af {@link SentenceDetectorFactory}. */ public static SentenceModel train(String languageCode, ObjectStream samples, boolean useTokenEnd, Dictionary abbreviations) throws IOException { return train(languageCode, samples, useTokenEnd, abbreviations,5,100); } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy