pingbu.nlp.NlpLoader Maven / Gradle / Ivy

Go to download
package pingbu.nlp;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

import pingbu.logger.Logger;
import pingbu.storage.FileStorage;
import pingbu.storage.Storage;

/**
 * NLP词典和语法加载器
 * 
 * @author pingbu
 */
public abstract class NlpLoader {
    private static final String TAG = NlpLoader.class.getSimpleName();
    private static final boolean LOG = true;

    private static void log(final String fmt, final Object... args) {
        if (LOG)
            Logger.d(TAG, fmt, args);
    }

    /**
     * 从文本文件加载词典
     *
     * @param name  词典名
     * @param fuzzy 是否模糊匹配
     * @param path  文本文件路径
     * @return 词典对象
     */
    public static Lexicon loadLexicon(final String name, final boolean fuzzy, final String path) {
        try (final FileInputStream f = new FileInputStream(path)) {
            return loadLexicon(name, fuzzy, f);
        } catch (final IOException e) {
            e.printStackTrace();
            return null;
        }
    }

    /**
     * 从文本文件加载词典
     *
     * @param name     词典名
     * @param fuzzy    是否模糊匹配
     * @param storage  文本文件存储
     * @param fileName 文本文件名
     * @return 词典对象
     */
    public static Lexicon loadLexicon(final String name, final boolean fuzzy, final Storage storage, final String fileName) {
        log("==> loadLexicon %s: storage=%sn fileName=%s", name, storage.toString(), fileName);
        try (final InputStream in = storage.open(fileName)) {
            return loadLexicon(name, fuzzy, in);
        } catch (final IOException e) {
            e.printStackTrace();
            return null;
        } finally {
            log("<== loadLexicon %s", name);
        }
    }

    private static Lexicon loadLexicon(final String name, final boolean fuzzy, final InputStream in) {
        try {
            final BufferedReader r = new BufferedReader(new InputStreamReader(in, "UTF-8"));
            final LexiconSimple1 lexicon = new LexiconSimple1(name, fuzzy);
            for (; ; ) {
                final String l = r.readLine();
                if (l == null)
                    break;
                //log(" item '%s'", l);
                lexicon.addItem(l);
            }
            return lexicon;
        } catch (final IOException e) {
            e.printStackTrace();
            return null;
        }
    }

    private static final class GrammarAnalyzeException extends Exception {
        private static final long serialVersionUID = 1L;

        GrammarAnalyzeException(final String msg) {
            super(msg);
        }
    }

    private interface GrammarAnalyzeState {
        boolean analyze(char c) throws GrammarAnalyzeException;
    }

    private static boolean __isTagHeadChar(final char c) {
        return Character.isAlphabetic(c) || c == '_' || c == '$';
    }

    private static boolean __isTagChar(final char c) {
        return __isTagHeadChar(c) || Character.isDigit(c) || c == '-' || c == '.';
    }

    private static final class GrammarAnalyzer {
        private final Parser mParser = new Parser();
        private final Storage mStorage;

        private final class AnalyzeOneGrammar {
            private GrammarAnalyzeState mState = new StateBeforeTag(
                    new StateAfterTag1());
            private String mLexicon = null, mFunction = null;
            private final List