org.apache.lucene.search.spell.DirectSpellChecker Maven / Gradle / Ivy
Show all versions of lucene-suggest Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.search.spell;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Locale;
import java.util.PriorityQueue;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.FuzzyTermsEnum;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
/**
* Simple automaton-based spellchecker.
*
* Candidates are presented directly from the term dictionary, based on
* Levenshtein distance. This is an alternative to {@link SpellChecker}
* if you are using an edit-distance-like metric such as Levenshtein
* or {@link JaroWinklerDistance}.
*
* A practical benefit of this spellchecker is that it requires no additional
* datastructures (neither in RAM nor on disk) to do its work.
*
* @see LevenshteinAutomata
* @see FuzzyTermsEnum
*
* @lucene.experimental
*/
public class DirectSpellChecker {
/** The default StringDistance, Damerau-Levenshtein distance implemented internally
* via {@link LevenshteinAutomata}.
*
* Note: this is the fastest distance metric, because Damerau-Levenshtein is used
* to draw candidates from the term dictionary: this just re-uses the scoring.
*/
public static final StringDistance INTERNAL_LEVENSHTEIN = new LuceneLevenshteinDistance();
/** maximum edit distance for candidate terms */
private int maxEdits = LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE;
/** minimum prefix for candidate terms */
private int minPrefix = 1;
/** maximum number of top-N inspections per suggestion */
private int maxInspections = 5;
/** minimum accuracy for a term to match */
private float accuracy = SpellChecker.DEFAULT_ACCURACY;
/** value in [0..1] (or absolute number >= 1) representing the minimum
* number of documents (of the total) where a term should appear. */
private float thresholdFrequency = 0f;
/** minimum length of a query word to return suggestions */
private int minQueryLength = 4;
/** maximum length of a query word to return suggestions */
private int maxQueryLength = Integer.MAX_VALUE;
/** value in [0..1] (or absolute number >= 1) representing the maximum
* number of documents (of the total) a query term can appear in to
* be corrected. */
private float maxQueryFrequency = 0.01f;
/** true if the spellchecker should lowercase terms */
private boolean lowerCaseTerms = true;
/** the comparator to use */
private Comparator comparator = SuggestWordQueue.DEFAULT_COMPARATOR;
/** the string distance to use */
private StringDistance distance = INTERNAL_LEVENSHTEIN;
/** Creates a DirectSpellChecker with default configuration values */
public DirectSpellChecker() {}
/** Get the maximum number of Levenshtein edit-distances to draw
* candidate terms from. */
public int getMaxEdits() {
return maxEdits;
}
/** Sets the maximum number of Levenshtein edit-distances to draw
* candidate terms from. This value can be 1 or 2. The default is 2.
*
* Note: a large number of spelling errors occur with an edit distance
* of 1, by setting this value to 1 you can increase both performance
* and precision at the cost of recall.
*/
public void setMaxEdits(int maxEdits) {
if (maxEdits < 1 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE)
throw new UnsupportedOperationException("Invalid maxEdits");
this.maxEdits = maxEdits;
}
/**
* Get the minimal number of characters that must match exactly
*/
public int getMinPrefix() {
return minPrefix;
}
/**
* Sets the minimal number of initial characters (default: 1)
* that must match exactly.
*
* This can improve both performance and accuracy of results,
* as misspellings are commonly not the first character.
*/
public void setMinPrefix(int minPrefix) {
this.minPrefix = minPrefix;
}
/**
* Get the maximum number of top-N inspections per suggestion
*/
public int getMaxInspections() {
return maxInspections;
}
/**
* Set the maximum number of top-N inspections (default: 5) per suggestion.
*
* Increasing this number can improve the accuracy of results, at the cost
* of performance.
*/
public void setMaxInspections(int maxInspections) {
this.maxInspections = maxInspections;
}
/**
* Get the minimal accuracy from the StringDistance for a match
*/
public float getAccuracy() {
return accuracy;
}
/**
* Set the minimal accuracy required (default: 0.5f) from a StringDistance
* for a suggestion match.
*/
public void setAccuracy(float accuracy) {
this.accuracy = accuracy;
}
/**
* Get the minimal threshold of documents a term must appear for a match
*/
public float getThresholdFrequency() {
return thresholdFrequency;
}
/**
* Set the minimal threshold of documents a term must appear for a match.
*
* This can improve quality by only suggesting high-frequency terms. Note that
* very high values might decrease performance slightly, by forcing the spellchecker
* to draw more candidates from the term dictionary, but a practical value such
* as 1 can be very useful towards improving quality.
*
* This can be specified as a relative percentage of documents such as 0.5f,
* or it can be specified as an absolute whole document frequency, such as 4f.
* Absolute document frequencies may not be fractional.
*/
public void setThresholdFrequency(float thresholdFrequency) {
if (thresholdFrequency >= 1f && thresholdFrequency != (int) thresholdFrequency)
throw new IllegalArgumentException("Fractional absolute document frequencies are not allowed");
this.thresholdFrequency = thresholdFrequency;
}
/** Get the minimum length of a query term needed to return suggestions */
public int getMinQueryLength() {
return minQueryLength;
}
/**
* Set the minimum length of a query term (default: 4) needed to return suggestions.
*
* Very short query terms will often cause only bad suggestions with any distance
* metric.
*/
public void setMinQueryLength(int minQueryLength) {
if (minQueryLength > this.maxQueryLength)
throw new IllegalArgumentException("minQueryLength must not be greater than maxQueryLength");
this.minQueryLength = minQueryLength;
}
/** Get the maximum length of a query term to return suggestions */
public int getMaxQueryLength() {
return maxQueryLength;
}
/**
* Set the maximum length of a query term to return suggestions.
*
* Long queries can be expensive to process and/or trigger exceptions.
*/
public void setMaxQueryLength(int maxQueryLength) {
if (maxQueryLength < this.minQueryLength)
throw new IllegalArgumentException("maxQueryLength must not be smaller than minQueryLength");
this.maxQueryLength = maxQueryLength;
}
/**
* Get the maximum threshold of documents a query term can appear in order
* to provide suggestions.
*/
public float getMaxQueryFrequency() {
return maxQueryFrequency;
}
/**
* Set the maximum threshold (default: 0.01f) of documents a query term can
* appear in order to provide suggestions.
*
* Very high-frequency terms are typically spelled correctly. Additionally,
* this can increase performance as it will do no work for the common case
* of correctly-spelled input terms.
*
* This can be specified as a relative percentage of documents such as 0.5f,
* or it can be specified as an absolute whole document frequency, such as 4f.
* Absolute document frequencies may not be fractional.
*/
public void setMaxQueryFrequency(float maxQueryFrequency) {
if (maxQueryFrequency >= 1f && maxQueryFrequency != (int) maxQueryFrequency)
throw new IllegalArgumentException("Fractional absolute document frequencies are not allowed");
this.maxQueryFrequency = maxQueryFrequency;
}
/** true if the spellchecker should lowercase terms */
public boolean getLowerCaseTerms() {
return lowerCaseTerms;
}
/**
* True if the spellchecker should lowercase terms (default: true)
*
* This is a convenience method, if your index field has more complicated
* analysis (such as StandardTokenizer removing punctuation), it's probably
* better to turn this off, and instead run your query terms through your
* Analyzer first.
*
* If this option is not on, case differences count as an edit!
*/
public void setLowerCaseTerms(boolean lowerCaseTerms) {
this.lowerCaseTerms = lowerCaseTerms;
}
/**
* Get the current comparator in use.
*/
public Comparator getComparator() {
return comparator;
}
/**
* Set the comparator for sorting suggestions.
* The default is {@link SuggestWordQueue#DEFAULT_COMPARATOR}
*/
public void setComparator(Comparator comparator) {
this.comparator = comparator;
}
/**
* Get the string distance metric in use.
*/
public StringDistance getDistance() {
return distance;
}
/**
* Set the string distance metric.
* The default is {@link #INTERNAL_LEVENSHTEIN}
*
* Note: because this spellchecker draws its candidates from the term
* dictionary using Damerau-Levenshtein, it works best with an edit-distance-like
* string metric. If you use a different metric than the default,
* you might want to consider increasing {@link #setMaxInspections(int)}
* to draw more candidates for your metric to rank.
*/
public void setDistance(StringDistance distance) {
this.distance = distance;
}
/**
* Calls {@link #suggestSimilar(Term, int, IndexReader, SuggestMode)
* suggestSimilar(term, numSug, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX)}
*/
public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir)
throws IOException {
return suggestSimilar(term, numSug, ir, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
}
/**
* Calls {@link #suggestSimilar(Term, int, IndexReader, SuggestMode, float)
* suggestSimilar(term, numSug, ir, suggestMode, this.accuracy)}
*
*/
public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir,
SuggestMode suggestMode) throws IOException {
return suggestSimilar(term, numSug, ir, suggestMode, this.accuracy);
}
/**
* Suggest similar words.
*
*
Unlike {@link SpellChecker}, the similarity used to fetch the most
* relevant terms is an edit distance, therefore typically a low value
* for numSug will work very well.
*
* @param term Term you want to spell check on
* @param numSug the maximum number of suggested words
* @param ir IndexReader to find terms from
* @param suggestMode specifies when to return suggested words
* @param accuracy return only suggested words that match with this similarity
* @return sorted list of the suggested words according to the comparator
* @throws IOException If there is a low-level I/O error.
*/
public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir,
SuggestMode suggestMode, float accuracy) throws IOException {
final CharsRefBuilder spare = new CharsRefBuilder();
String text = term.text();
int textLength = text.codePointCount(0, text.length());
if (textLength < minQueryLength || textLength > maxQueryLength)
return new SuggestWord[0];
if (lowerCaseTerms) {
term = new Term(term.field(), text.toLowerCase(Locale.ROOT));
}
int docfreq = ir.docFreq(term);
if (suggestMode==SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && docfreq > 0) {
return new SuggestWord[0];
}
int maxDoc = ir.maxDoc();
if (maxQueryFrequency >= 1f && docfreq > maxQueryFrequency) {
return new SuggestWord[0];
} else if (docfreq > (int) Math.ceil(maxQueryFrequency * (float)maxDoc)) {
return new SuggestWord[0];
}
if (suggestMode!=SuggestMode.SUGGEST_MORE_POPULAR) docfreq = 0;
if (thresholdFrequency >= 1f) {
docfreq = Math.max(docfreq, (int) thresholdFrequency);
} else if (thresholdFrequency > 0f) {
docfreq = Math.max(docfreq, (int)(thresholdFrequency * (float)maxDoc)-1);
}
Collection terms = null;
int inspections = numSug * maxInspections;
// try ed=1 first, in case we get lucky
terms = suggestSimilar(term, inspections, ir, docfreq, 1, accuracy, spare);
if (maxEdits > 1 && terms.size() < inspections) {
HashSet moreTerms = new HashSet<>();
moreTerms.addAll(terms);
moreTerms.addAll(suggestSimilar(term, inspections, ir, docfreq, maxEdits, accuracy, spare));
terms = moreTerms;
}
// create the suggestword response, sort it, and trim it to size.
SuggestWord suggestions[] = new SuggestWord[terms.size()];
int index = suggestions.length - 1;
for (ScoreTerm s : terms) {
SuggestWord suggestion = new SuggestWord();
if (s.termAsString == null) {
spare.copyUTF8Bytes(s.term);
s.termAsString = spare.toString();
}
suggestion.string = s.termAsString;
suggestion.score = s.score;
suggestion.freq = s.docfreq;
suggestions[index--] = suggestion;
}
ArrayUtil.timSort(suggestions, Collections.reverseOrder(comparator));
if (numSug < suggestions.length) {
SuggestWord trimmed[] = new SuggestWord[numSug];
System.arraycopy(suggestions, 0, trimmed, 0, numSug);
suggestions = trimmed;
}
return suggestions;
}
/**
* Provide spelling corrections based on several parameters.
*
* @param term The term to suggest spelling corrections for
* @param numSug The maximum number of spelling corrections
* @param ir The index reader to fetch the candidate spelling corrections from
* @param docfreq The minimum document frequency a potential suggestion need to have in order to be included
* @param editDistance The maximum edit distance candidates are allowed to have
* @param accuracy The minimum accuracy a suggested spelling correction needs to have in order to be included
* @param spare a chars scratch
* @return a collection of spelling corrections sorted by ScoreTerm's natural order.
* @throws IOException If I/O related errors occur
*/
protected Collection suggestSimilar(Term term, int numSug, IndexReader ir, int docfreq, int editDistance,
float accuracy, final CharsRefBuilder spare) throws IOException {
Terms terms = MultiTerms.getTerms(ir, term.field());
if (terms == null) {
return Collections.emptyList();
}
FuzzyTermsEnum e = new FuzzyTermsEnum(terms, term, editDistance, Math.max(minPrefix, editDistance - 1), true);
final PriorityQueue stQueue = new PriorityQueue<>();
BytesRef queryTerm = new BytesRef(term.text());
BytesRef candidateTerm;
ScoreTerm st = new ScoreTerm();
while ((candidateTerm = e.next()) != null) {
// For FuzzyQuery, boost is the score:
float score = e.getBoost();
// ignore uncompetitive hits
if (stQueue.size() >= numSug && score <= stQueue.peek().boost) {
continue;
}
// ignore exact match of the same term
if (queryTerm.bytesEquals(candidateTerm)) {
continue;
}
int df = e.docFreq();
// check docFreq if required
if (df <= docfreq) {
continue;
}
final String termAsString;
if (distance == INTERNAL_LEVENSHTEIN) {
// delay creating strings until the end
termAsString = null;
} else {
spare.copyUTF8Bytes(candidateTerm);
termAsString = spare.toString();
score = distance.getDistance(term.text(), termAsString);
}
if (score < accuracy) {
continue;
}
// add new entry in PQ
st.term = BytesRef.deepCopyOf(candidateTerm);
st.boost = score;
st.docfreq = df;
st.termAsString = termAsString;
st.score = score;
stQueue.offer(st);
// possibly drop entries from queue
st = (stQueue.size() > numSug) ? stQueue.poll() : new ScoreTerm();
e.setMaxNonCompetitiveBoost((stQueue.size() >= numSug) ? stQueue.peek().boost : Float.NEGATIVE_INFINITY);
}
return stQueue;
}
/**
* Holds a spelling correction for internal usage inside {@link DirectSpellChecker}.
*/
protected static class ScoreTerm implements Comparable {
/**
* The actual spellcheck correction.
*/
public BytesRef term;
/**
* The boost representing the similarity from the FuzzyTermsEnum (internal similarity score)
*/
public float boost;
/**
* The df of the spellcheck correction.
*/
public int docfreq;
/**
* The spellcheck correction represented as string, can be null.
*/
public String termAsString;
/**
* The similarity score.
*/
public float score;
/**
* Constructor.
*/
public ScoreTerm() {
}
@Override
public int compareTo(ScoreTerm other) {
if (term.bytesEquals(other.term))
return 0; // consistent with equals
if (this.boost == other.boost)
return other.term.compareTo(this.term);
else
return Float.compare(this.boost, other.boost);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((term == null) ? 0 : term.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
ScoreTerm other = (ScoreTerm) obj;
if (term == null) {
if (other.term != null) return false;
} else if (!term.bytesEquals(other.term)) return false;
return true;
}
}
}