org.languagetool.rules.AbstractSimpleReplaceRule2 Maven / Gradle / Ivy
Show all versions of languagetool-core Show documentation
/* LanguageTool, a natural language style checker
* Copyright (C) 2005 Daniel Naber (http://www.danielnaber.de)
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
* USA
*/
package org.languagetool.rules;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.jetbrains.annotations.NotNull;
import org.languagetool.AnalyzedSentence;
import org.languagetool.AnalyzedTokenReadings;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.tools.StringTools;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
* A rule that matches words which should not be used and suggests correct ones instead.
* Unlike AbstractSimpleReplaceRule, it supports phrases (Ex: "aqua forte" -> "acvaforte").
*
* Note: Merge this into {@link AbstractSimpleReplaceRule}
*
* @author Ionuț Păduraru
*/
public abstract class AbstractSimpleReplaceRule2 extends Rule {
private final Language language;
public abstract String getFileName();
@Override
public abstract String getId();
@Override
public abstract String getDescription();
public abstract String getShort();
/**
* @return A string where {@code $match} will be replaced with the matching word
* and {@code $suggestions} will be replaced with the alternatives. This is the string
* shown to the user.
*/
public abstract String getSuggestion();
/**
* @return the word used to separate multiple suggestions; used only before last suggestion, the rest are comma-separated.
*/
public abstract String getSuggestionsSeparator();
/**
* locale used on case-conversion
*/
public abstract Locale getLocale();
private static final LoadingCache>> cache = CacheBuilder.newBuilder()
.expireAfterWrite(30, TimeUnit.MINUTES)
.build(new CacheLoader>>() {
@Override
public List> load(@NotNull PathAndLanguage lap) throws IOException {
return loadWords(lap.path, lap.lang, lap.caseSensitive);
}
});
public AbstractSimpleReplaceRule2(ResourceBundle messages, Language language) {
super(messages);
this.language = Objects.requireNonNull(language);
super.setCategory(Categories.MISC.getCategory(messages));
}
/**
* use case-sensitive matching.
*/
public boolean isCaseSensitive() {
return false;
}
/**
* @return the list of wrong words for which this rule can suggest correction. The list cannot be modified.
*/
public List> getWrongWords() {
try {
return cache.get(new PathAndLanguage(getFileName(), language, isCaseSensitive()));
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
/**
* Load the list of words.
* Same as {@link AbstractSimpleReplaceRule#loadFromPath} but allows multiple words and a custom message (optional).
* @param filename the file from classpath to load
* @return the list of maps containing the error-corrections pairs. The n-th map contains key strings of (n+1) words.
*/
private static List> loadWords(String filename, Language lang, boolean caseSensitive)
throws IOException {
List> list = new ArrayList<>();
InputStream stream = JLanguageTool.getDataBroker().getFromRulesDirAsStream(filename);
try (
InputStreamReader isr = new InputStreamReader(stream, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(isr))
{
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if (line.isEmpty() || line.charAt(0) == '#') { // ignore comments
continue;
}
String[] parts = line.split("=");
if (parts.length != 2) {
throw new IOException("Format error in file "
+ JLanguageTool.getDataBroker().getFromRulesDirAsUrl(filename)
+ ". Expected exactly 1 '=' character. Line: " + line);
}
String[] wrongForms = parts[0].split("\\|"); // multiple incorrect forms
for (String wrongForm : wrongForms) {
int wordCount = 0;
List tokens = lang.getWordTokenizer().tokenize(wrongForm);
for (String token : tokens) {
if (!StringTools.isWhitespace(token)) {
wordCount++;
}
}
// grow if necessary
for (int i = list.size(); i < wordCount; i++) {
list.add(new HashMap<>());
}
SuggestionWithMessage sugg;
if (parts[1].contains("\t")) {
String[] suggestionParts = parts[1].split("\t");
if (suggestionParts.length != 2) {
throw new IOException("Invalid format - use only one tab character to separate suggestion from the message: " + line);
}
sugg = new SuggestionWithMessage(suggestionParts[0], suggestionParts[1]);
} else {
sugg = new SuggestionWithMessage(parts[1]);
}
list.get(wordCount - 1).put(caseSensitive ? wrongForm : wrongForm.toLowerCase(), sugg);
}
}
}
// seal the result (prevent modification from outside this class)
List> result = new ArrayList<>();
for (Map map : list) {
result.add(Collections.unmodifiableMap(map));
}
return Collections.unmodifiableList(result);
}
private void addToQueue(AnalyzedTokenReadings token,
Queue prevTokens) {
boolean inserted = prevTokens.offer(token);
if (!inserted) {
prevTokens.poll();
prevTokens.offer(token);
}
}
@Override
public RuleMatch[] match(AnalyzedSentence sentence) {
List ruleMatches = new ArrayList<>();
AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
List> wrongWords = getWrongWords();
Queue prevTokens = new ArrayBlockingQueue<>(wrongWords.size());
for (int i = 1; i < tokens.length; i++) {
addToQueue(tokens[i], prevTokens);
StringBuilder sb = new StringBuilder();
List variants = new ArrayList<>();
List prevTokensList =
Arrays.asList(prevTokens.toArray(new AnalyzedTokenReadings[0]));
for (int j = prevTokensList.size() - 1; j >= 0; j--) {
if (j != prevTokensList.size() - 1 && prevTokensList.get(j + 1).isWhitespaceBefore()) {
sb.insert(0, " ");
}
sb.insert(0, prevTokensList.get(j).getToken());
variants.add(0, sb.toString());
}
int len = variants.size(); // prevTokensList and variants have now the same length
for (int j = 0; j < len; j++) { // longest words first
String crt = variants.get(j);
int crtWordCount = len - j;
SuggestionWithMessage crtMatch = isCaseSensitive() ?
wrongWords.get(crtWordCount - 1).get(crt) :
wrongWords.get(crtWordCount - 1).get(crt.toLowerCase(getLocale()));
if (crtMatch != null) {
List replacements = Arrays.asList(crtMatch.getSuggestion().split("\\|"));
String msgSuggestions = "";
for (int k = 0; k < replacements.size(); k++) {
if (k > 0) {
msgSuggestions += (k == replacements.size() - 1 ? getSuggestionsSeparator(): ", ");
}
msgSuggestions += "" + replacements.get(k) + " ";
}
String msg = getSuggestion().replaceFirst("\\$match", crt).replaceFirst("\\$suggestions", msgSuggestions);
if (crtMatch.getMessage() != null) {
msg = crtMatch.getMessage();
}
int startPos = prevTokensList.get(len - crtWordCount).getStartPos();
int endPos = prevTokensList.get(len - 1).getEndPos();
RuleMatch ruleMatch = new RuleMatch(this, sentence, startPos, endPos, msg, getShort());
if (!isCaseSensitive() && StringTools.startsWithUppercase(crt)) {
for (int k = 0; k < replacements.size(); k++) {
replacements.set(k, StringTools.uppercaseFirstChar(replacements.get(k)));
}
}
ruleMatch.setSuggestedReplacements(replacements);
if (!isException(sentence.getText().substring(startPos, endPos))) {
ruleMatches.add(ruleMatch);
}
break;
}
}
}
return toRuleMatchArray(ruleMatches);
}
protected boolean isException(String matchedText) {
return false;
}
static class PathAndLanguage {
final String path;
final Language lang;
final boolean caseSensitive;
PathAndLanguage(String fileName, Language language, boolean caseSensitive) {
this.path = Objects.requireNonNull(fileName);
this.lang = Objects.requireNonNull(language);
this.caseSensitive = caseSensitive;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PathAndLanguage that = (PathAndLanguage) o;
return path.equals(that.path) && lang.equals(that.lang) && caseSensitive == that.caseSensitive;
}
@Override
public int hashCode() {
return Objects.hash(path, lang, caseSensitive);
}
}
}