com.optimaize.langdetect.NgramFrequencyData Maven / Gradle / Ivy
package com.optimaize.langdetect;
import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.profiles.LanguageProfile;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.*;
/**
* Contains frequency information for n-grams coming from multiple {@link LanguageProfile}s.
*
* For each n-gram string it knows the locales (languages) in which it occurs, and how frequent it
* occurs in those languages in relation to other n-grams of the same length in those same languages.
*
* Immutable by definition (can't make Arrays unmodifiable).
*
* @author Fabian Kessler
*/
public final class NgramFrequencyData {
/**
* Key = ngram
* Value = array with probabilities per loaded language, in the same order as {@code langlist}.
*/
@NotNull
private final Map wordLangProbMap;
/**
* All the loaded languages, in exactly the same order as the data is in the double[] in wordLangProbMap.
* Example: if wordLangProbMap has an entry for the n-gram "foo" then for each locale in this langlist here
* it has a value there. Languages that don't know the n-gram have the value 0d.
*/
@NotNull
private final List langlist;
/**
* @param gramLengths for example [1,2,3]
* @throws java.lang.IllegalArgumentException if languageProfiles or gramLengths is empty, or if one of the
* languageProfiles does not have the grams of the required sizes.
*/
@NotNull
public static NgramFrequencyData create(@NotNull Collection languageProfiles, @NotNull Collection gramLengths) throws IllegalArgumentException {
if (languageProfiles.isEmpty()) throw new IllegalArgumentException("No languageProfiles provided!");
if (gramLengths.isEmpty()) throw new IllegalArgumentException("No gramLengths provided!");
Map wordLangProbMap = new HashMap<>();
List langlist = new ArrayList<>();
int langsize = languageProfiles.size();
int index = -1;
for (LanguageProfile profile : languageProfiles) {
index++;
langlist.add( profile.getLocale() );
for (Integer gramLength : gramLengths) {
if (!profile.getGramLengths().contains(gramLength)) {
throw new IllegalArgumentException("The language profile for "+profile.getLocale()+" does not contain "+gramLength+"-grams!");
}
for (Map.Entry ngramEntry : profile.iterateGrams(gramLength)) {
String ngram = ngramEntry.getKey();
Integer frequency = ngramEntry.getValue();
if (!wordLangProbMap.containsKey(ngram)) {
wordLangProbMap.put(ngram, new double[langsize]);
}
double prob = frequency.doubleValue() / profile.getNumGramOccurrences(ngram.length());
wordLangProbMap.get(ngram)[index] = prob;
}
}
}
return new NgramFrequencyData(wordLangProbMap, langlist);
}
private NgramFrequencyData(@NotNull Map wordLangProbMap,
@NotNull List langlist) {
//not making immutable copies because I create them here (optimization).
this.wordLangProbMap = Collections.unmodifiableMap(wordLangProbMap);
this.langlist = Collections.unmodifiableList(langlist);
}
@NotNull
public List getLanguageList() {
return langlist;
}
@NotNull
public LdLocale getLanguage(int pos) {
return langlist.get(pos);
}
/**
* Don't modify this data structure! (Can't make array immutable...)
* @return null if no language profile knows that ngram.
* entries are 0 for languages that don't know that ngram at all.
* The array is in the order of the {@link #getLanguageList()} language list, and has exactly that size.
* impl note: this way the caller can handle it more efficient than returning an empty array.
*/
@Nullable
public double[] getProbabilities(String ngram) {
return wordLangProbMap.get(ngram);
}
}