• Home
• org.allenai.nlpstack
• nlpstack-tokenize_2.11
• 1.19
• source code
• StanfordTokenizer.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.allenai.nlpstack.tokenize.StanfordTokenizer.scala Maven / Gradle / Ivy

package org.allenai.nlpstack.tokenize

import java.io.StringReader
import org.slf4j.bridge.SLF4JBridgeHandler

import scala.collection.{ mutable, JavaConverters }

import edu.stanford.nlp.process.PTBTokenizer
import org.allenai.nlpstack.core.{ Token, Tokenizer }

object StanfordTokenizer extends Tokenizer {
  // redirect stanford's logging
  SLF4JBridgeHandler.removeHandlersForRootLogger()
  SLF4JBridgeHandler.install()

  var averageTokenLength = 6 // low estimates are better
  private val tokenizerFactory = PTBTokenizer.factory()
  tokenizerFactory.setOptions("untokenizable=allKeep")

  def tokenize(sentence: String) = {
    val reader = new StringReader(sentence)
    val tokenizer = tokenizerFactory.getTokenizer(reader)
    val result = new mutable.ArrayBuffer[Token](sentence.length / averageTokenLength)

    while (tokenizer.hasNext) {
      val token = tokenizer.next()
      result += Token(sentence.substring(token.beginPosition(), token.endPosition()), token.beginPosition())
    }
    result
  }
}