• Home
• org.allenai.nlpstack
• nlpstack-tokenize_2.11
• 1.19
• source code
• PennTokenizer.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.allenai.nlpstack.tokenize.PennTokenizer.scala Maven / Gradle / Ivy

package org.allenai.nlpstack.tokenize

import org.allenai.nlpstack.core.Tokenizer

import java.util.regex._

/* The PennTokenizer was used to tokenize the Penn Treebank.
 * The following is a translation from a sed file.  This algorithm
 * is entirely deterministic.  It is composed of regular expression
 * replacements.
 *
 * @author  Michael Schmitz
 */
object PennTokenizer extends Tokenizer {
  val replacements = List(
    // attempt to get correct directional quotes
    ("^\"", "`` "),
    //("""([ (\[{<])""", "$1 `` "),
    ("""\.\.\.""", " ... "),
    ("[,;:@#$%&]", " $0 "),
    ("""([^.]\)\([.])([])}>"']*)[ 	]*$""", "$1 $2$3 "), // scalastyle:ignore
    ("[?!]", " $0 "),
    ("""[](){}<>]""", " $0 "),
    ("--", " $0 "),
    ("$|^", " "),
    ("\"", " '' "),
    (""" ([^'])' """, " '$1 "),
    ("""'([sSmMdD]) """, " '$1 "),
    ("'(ll|re|ve|LL|RE|VE) ", " '$1 "),
    ("(n't|N'T) ", " $1 ")
  ).map {
      case (a, b) =>
        (Pattern.compile(a), b)
    }

  def tokenize(sentence: String) = {
    val split = replacements.foldRight(sentence) {
      case ((t, r), s) =>
        t.matcher(s).replaceAll(r)
    }.trim.split("\\s+")

    Tokenizer.computeOffsets(split, sentence)
  }
}