smile.nlp.tokenizer.PennTreebankTokenizer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of smile-nlp Show documentation
smile-nlp
There is a newer version: 4.0.0
/*
 * Copyright (c) 2010-2021 Haifeng Li. All rights reserved.
 *
 * Smile is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Smile is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Smile.  If not, see .
 */

package smile.nlp.tokenizer;

import java.util.regex.Pattern;

/**
 * A word tokenizer that tokenizes English sentences using the conventions
 * used by the Penn Treebank. Most punctuation is split from adjoining words.
 * Verb contractions and the Anglo-Saxon genitive of nouns are split into their
 * component morphemes, and each morpheme is tagged separately. Examples
 * 
 *  children's -> children 's
 * 
 parents' -> parents '
 * 
 won't --> wo n't
 * 
 can't -> ca n't
 * 
 weren't -> were n't
 * 
 cannot -> can not
 * 
 'tisn't -> 't is n't
 * 
 'tis -> 't is
 * 
 gonna -> gon na
 * 
 I'm -> I 'm
 * 
 he'll -> he 'll
 * 
 * This tokenizer assumes that the text has already been segmented into
 * sentences. Any periods -- apart from those at the end of a string or before
 * newline -- are assumed to be part of the word they are attached to (e.g. for
 * abbreviations, etc), and are not separately tokenized.
 *
 * @author Haifeng Li
 */
public class PennTreebankTokenizer implements Tokenizer {
    /**
     * List of contractions adapted from Robert MacIntyre's tokenizer.
     */
    private static final Pattern[] CONTRACTIONS2 = {
        Pattern.compile("(?i)(.)('ll|'re|'ve|n't|'s|'m|'d)\\b"),
        Pattern.compile("(?i)\\b(can)(not)\\b"),
        Pattern.compile("(?i)\\b(D)('ye)\\b"),
        Pattern.compile("(?i)\\b(Gim)(me)\\b"),
        Pattern.compile("(?i)\\b(Gon)(na)\\b"),
        Pattern.compile("(?i)\\b(Got)(ta)\\b"),
        Pattern.compile("(?i)\\b(Lem)(me)\\b"),
        Pattern.compile("(?i)\\b(Mor)('n)\\b"),
        Pattern.compile("(?i)\\b(T)(is)\\b"),
        Pattern.compile("(?i)\\b(T)(was)\\b"),
        Pattern.compile("(?i)\\b(Wan)(na)\\b")
    };

    private static final Pattern[] CONTRACTIONS3 = {
        Pattern.compile("(?i)\\b(Whad)(dd)(ya)\\b"),
        Pattern.compile("(?i)\\b(Wha)(t)(cha)\\b")
    };

    private static final Pattern[] DELIMITERS = {
        // Separate most punctuation
        Pattern.compile("(?U)([^\\w\\.\\'\\-\\/,&])"),
        // Separate commas if they're followed by space (e.g., don't separate 2,500)
        Pattern.compile("(?U)(,\\s)"),
        // Separate single quotes if they're followed by a space.
        Pattern.compile("(?U)('\\s)"),
        // Separate periods that come before newline or end of string.
        Pattern.compile("(?U)\\. *(\\n|$)")
    };

    private static final Pattern WHITESPACE = Pattern.compile("(?U)\\s+");

    /**
     * The singleton instance.
     */
    private static final PennTreebankTokenizer singleton = new PennTreebankTokenizer();

    /**
     * Constructor.
     */
    private PennTreebankTokenizer() {

    }

    /**
     * Returns the singleton instance.
     * @return the singleton instance.
     */
    public static PennTreebankTokenizer getInstance() {
        return singleton;
    }

    @Override
    public String[] split(String text) {
        for (Pattern regexp : CONTRACTIONS2)
            text = regexp.matcher(text).replaceAll("$1 $2");

        for (Pattern regexp : CONTRACTIONS3)
            text = regexp.matcher(text).replaceAll("$1 $2 $3");

        text = DELIMITERS[0].matcher(text).replaceAll(" $1 ");
        text = DELIMITERS[1].matcher(text).replaceAll(" $1");
        text = DELIMITERS[2].matcher(text).replaceAll(" $1");
        text = DELIMITERS[3].matcher(text).replaceAll(" . ");

        String[] words = WHITESPACE.split(text);
        if (words.length > 1 && words[words.length-1].equals(".")) {
            if (EnglishAbbreviations.contains(words[words.length-2])) {
                words[words.length-2] = words[words.length-2] + ".";
            }
        }

        return words;
    }
}