com.optimaize.langdetect.text.RemoveMinorityScriptsTextFilter Maven / Gradle / Ivy

package com.optimaize.langdetect.text;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 * Removes text written in scripts that are not the dominant script of the text.
 *
 * TODO this does not do special handling for Japanese (3 scripts) and Korean (2 scripts), they should be
 * counted together and kept.
 *
 * @author Fabian Kessler
 */
public class RemoveMinorityScriptsTextFilter implements TextFilter {

    private final double threshold;

    /**
     * If a script has less than this fraction of content compared to the most used one, its text is removed.
     *
     * Example: Latin 10%, Cyrillic 80%, Common 10% (punctuation n'stuff). Now 10 is put in relation to 80.
     *
     * @param threshold 0-1, suggested value is 0.3. If smaller then removed, equal remains.
     */
    public static RemoveMinorityScriptsTextFilter forThreshold(double threshold) {
        return new RemoveMinorityScriptsTextFilter(threshold);
    }

    private RemoveMinorityScriptsTextFilter(double threshold) {
        this.threshold = threshold;
    }

    @Override
    public String filter(CharSequence text) {
        Map counts = countByScript(text);
        if (counts.size()<=1) {
            //nothing to do
            return text.toString();
        } else {
            long most = findMost(counts);
            Set toRemove = new HashSet<>();
            for (Map.Entry entry : counts.entrySet()) {
                if (entry.getValue()==most) continue;
                double ratio = entry.getValue().doubleValue() / most;
                if (ratio <= threshold) {
                    toRemove.add(entry.getKey());
                }
            }
            if (toRemove.isEmpty()) {
                return text.toString();
            } else {
                return remove(text, toRemove);
            }
        }
    }

    private String remove(CharSequence text, Set toRemove) {
        StringBuilder remaining = new StringBuilder();
        Character.UnicodeScript last = null;
        for (int i=0; i counts) {
        long max = 0L;
        for (Long aLong : counts.values()) {
            if (aLong > max) max = aLong;
        }
        return max;
    }

    private Map countByScript(CharSequence text) {
        Map counter = new HashMap<>();
        Character.UnicodeScript last = null;
        for (int i=0; i counter, Character.UnicodeScript unicodeScript) {
        Long number = counter.get(unicodeScript);
        if (number==null) {
            counter.put(unicodeScript, 1L);
        } else {
            counter.put(unicodeScript, number+1);
        }
    }

}