com.optimaize.langdetect.cybozu.util.LangProfile Maven / Gradle / Ivy

/*
 * Copyright 2011 Nakatani Shuyo
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * This file has been modified by François ROLAND.
 */

package com.optimaize.langdetect.cybozu.util;

import org.jetbrains.annotations.NotNull;

import java.io.Serializable;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

/**
 * {@link LangProfile} is a Language Profile Class.
 * Users don't use this class directly.
 *
 * TODO split into builder and immutable class.
 *
 * TODO currently this only makes n-grams with the space before a word included. no n-gram with the space after the word.
 * Example: "foo" creates " fo" as 3gram, but not "oo ". Either this is a bug, or if intended then needs documentation.
 * 
 * @author Nakatani Shuyo
 * @deprecated replaced by LanguageProfile
 */
@Deprecated
public class LangProfile implements Serializable {

	private static final long serialVersionUID = 1L;

    /**
     * n-grams that occur less than this often can be removed using omitLessFreq().
     * This number can change, see LESS_FREQ_RATIO.
     */
	private static final int MINIMUM_FREQ = 2;

    /**
     * Explanation by example:
     *
     * If the most frequent n-gram occurs 1 mio times, then
     * 1'000'000 / this (100'000) = 10.
     * 10 is larger than MINIMUM_FREQ (2), thus MINIMUM_FREQ remains at 2.
     * All n-grams that occur less than 2 times can be removed as noise using omitLessFreq().
     *
     * If the most frequent n-gram occurs 5000 times, then
     * 5'000 / this (100'000) = 0.05.
     * 0.05 is smaller than MINIMUM_FREQ (2), thus MINIMUM_FREQ becomes 0.
     * No n-grams are removed because of insignificance when calling omitLessFreq().
     */
    private static final int LESS_FREQ_RATIO = 100000;

    /**
     * The language name (identifier).
     */
    private String name = null;

    /**
     * Key = ngram, value = count.
     * All n-grams are in here (1-gram, 2-gram, 3-gram).
     */
    private Map freq = new HashMap<>();

    /**
     * Tells how many occurrences of n-grams exist per gram length.
     * When making 1grams, 2grams and 3grams (currently) then this contains 3 entries where
     * element 0 = number occurrences of 1-grams
     * element 1 = number occurrences of 2-grams
     * element 2 = number occurrences of 3-grams
     * Example: if there are 57 1-grams (English language has about that many) and the training text is
     * fairly long, then this number is in the millions.
     */
    private int[] nWords = new int[NGram.N_GRAM];

    /**
     * Constructor for JSONIC 
     */
    public LangProfile() {}

    /**
     * Normal Constructor
     * @param name language name
     */
    public LangProfile(String name) {
        this.setName(name);
    }
    
    /**
     * Add n-gram to profile
     * @param gram
     */
    public void add(@NotNull String gram) {
        if (name == null) throw new IllegalStateException();
        int len = gram.length();
        if (len < 1 || len > NGram.N_GRAM) {
            throw new IllegalArgumentException("ngram length must be 1-3 but was "+len+": >>>"+gram+"<< keys = freq.keySet();
        int roman = 0;
        for(Iterator i = keys.iterator(); i.hasNext(); ){
            String key = i.next();
            int count = freq.get(key);
            if (count <= threshold) {
                nWords[key.length()-1] -= count;
                i.remove();
            } else {
                if (key.matches("^[A-Za-z]$")) {
                    roman += count;
                }
            }
        }

        // roman check
        if (roman < nWords[0] / 3) {
            Set keys2 = freq.keySet();
            for(Iterator i = keys2.iterator(); i.hasNext(); ){
                String key = i.next();
                if (key.matches(".*[A-Za-z].*")) {
                    nWords[key.length()-1] -= freq.get(key);
                    i.remove();
                }
            }
        }
    }

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public Map getFreq() {
		return freq;
	}

	public void setFreq(Map freq) {
		this.freq = freq;
	}

	public int[] getNWords() {
		return nWords;
	}

	public void setNWords(int[] nWords) {
		this.nWords = nWords;
	}
}