smile.nlp.tokenizer.SimpleTokenizer Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2010-2021 Haifeng Li. All rights reserved.
 *
 * Smile is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Smile is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Smile.  If not, see .
 */

package smile.nlp.tokenizer;

import java.util.ArrayList;
import java.util.regex.Pattern;

/**
 * A word tokenizer that tokenizes English sentences with some differences from
 * TreebankWordTokenizer, notably on handling not-contractions. If a period
 * serves as both the end of sentence and a part of abbreviation, e.g. etc. at
 * the end of sentence, it will generate tokens of "etc." and "." while
 * TreebankWordTokenizer will generate "etc" and ".".
 * 
 * Most punctuation is split from adjoining words. Verb contractions and the
 * Anglo-Saxon genitive of nouns are split into their component morphemes,
 * and each morpheme is tagged separately. Examples
 * 

 *  children's -> children 's
 * 
 parents' -> parents '
 * 
 won't -> will not
 * 
 can't -> can not
 * 
 shan't -> shall not
 * 
 cannot -> can not
 * 
 weren't -> were not
 * 
 'tisn't -> it is not
 * 
 'tis -> it is
 * 
 gonna -> gon na
 * 
 I'm -> I 'm
 * 
 he'll -> he 'll
 * 
 * This tokenizer assumes that the text has already been segmented into
 * sentences. Any periods -- apart from those at the end of a string or before
 * newline -- are assumed to be part of the word they are attached to (e.g. for
 * abbreviations, etc), and are not separately tokenized.
 *
 * @author Haifeng Li
 */
public class SimpleTokenizer implements Tokenizer {
    private static final Pattern WONT_CONTRACTION = Pattern.compile("(?i)\\b(w)(on't)\\b");

    private static final Pattern SHANT_CONTRACTION = Pattern.compile("(?i)\\b(sha)(n't)\\b");

    private static final Pattern AINT_CONTRACTION = Pattern.compile("(?i)\\b(a)(in't)\\b");

    private static final Pattern[] NOT_CONTRACTIONS = {
        Pattern.compile("(?i)\\b(can)('t|not)\\b"),
        Pattern.compile("(?i)(.)(n't)\\b")
    };

    /**
     * List of contractions adapted from Robert MacIntyre's tokenizer.
     */
    private static final Pattern[] CONTRACTIONS2 = {
        Pattern.compile("(?i)(.)('ll|'re|'ve|'s|'m|'d)\\b"),
        Pattern.compile("(?i)\\b(D)('ye)\\b"),
        Pattern.compile("(?i)\\b(Gim)(me)\\b"),
        Pattern.compile("(?i)\\b(Gon)(na)\\b"),
        Pattern.compile("(?i)\\b(Got)(ta)\\b"),
        Pattern.compile("(?i)\\b(Lem)(me)\\b"),
        Pattern.compile("(?i)\\b(Mor)('n)\\b"),
        Pattern.compile("(?i)\\b(T)(is)\\b"),
        Pattern.compile("(?i)\\b(T)(was)\\b"),
        Pattern.compile("(?i)\\b(Wan)(na)\\b")
    };

    private static final Pattern[] CONTRACTIONS3 = {
        Pattern.compile("(?i)\\b(Whad)(dd)(ya)\\b"),
        Pattern.compile("(?i)\\b(Wha)(t)(cha)\\b")
    };

    private static final Pattern[] DELIMITERS = {
        // Separate most punctuation
        Pattern.compile("((?U)[^\\w\\.\\'\\-\\/,&])"),
        // Separate commas if they're followed by space (e.g., don't separate 2,500)
        Pattern.compile("(?U)(,\\s)"),
        // Separate single quotes if they're followed by a space.
        Pattern.compile("(?U)('\\s)"),
        // Separate periods that come before newline or end of string.
        Pattern.compile("(?U)\\. *(\\n|$)"),
        // Separate continuous periods such as ... in ToC.
        Pattern.compile("(?U)(\\.{3,})")

    };

    private static final Pattern WHITESPACE = Pattern.compile("(?U)\\s+");

    private final boolean splitContraction;
    /**
     * Constructor.
     */
    public SimpleTokenizer() {
        this(false);
    }

    /**
     * Constructor.
     * @param splitContraction if true, split adjoining words.
     */
    public SimpleTokenizer(boolean splitContraction) {
        this.splitContraction = splitContraction;
    }

    @Override
    public String[] split(String text) {
        if (splitContraction) {
            text = WONT_CONTRACTION.matcher(text).replaceAll("$1ill not");
            text = SHANT_CONTRACTION.matcher(text).replaceAll("$1ll not");
            text = AINT_CONTRACTION.matcher(text).replaceAll("$1m not");

            for (Pattern regexp : NOT_CONTRACTIONS) {
                text = regexp.matcher(text).replaceAll("$1 not");
            }

            for (Pattern regexp : CONTRACTIONS2) {
                text = regexp.matcher(text).replaceAll("$1 $2");
            }

            for (Pattern regexp : CONTRACTIONS3) {
                text = regexp.matcher(text).replaceAll("$1 $2 $3");
            }
        }

        text = DELIMITERS[0].matcher(text).replaceAll(" $1 ");
        text = DELIMITERS[1].matcher(text).replaceAll(" $1");
        text = DELIMITERS[2].matcher(text).replaceAll(" $1");
        text = DELIMITERS[3].matcher(text).replaceAll(" . ");
        text = DELIMITERS[4].matcher(text).replaceAll(" $1 ");

        String[] words = WHITESPACE.split(text);
        if (words.length > 1 && words[words.length-1].equals(".")) {
            if (EnglishAbbreviations.contains(words[words.length-2])) {
                words[words.length-2] = words[words.length-2] + ".";
            }
        }
        
        ArrayList result = new ArrayList<>();
        for (String token : words) {
            if (!token.isEmpty()) {
                result.add(token);
            }
        }
        
        return result.toArray(new String[0]);
    }
}