• Home
• edu.washington.cs.knowitall.nlptools
• nlptools-tokenize-stanford_2.9.2
• 2.4.1
• source code
• StanfordTokenizer.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

edu.knowitall.tool.tokenize.StanfordTokenizer.scala Maven / Gradle / Ivy

package edu.knowitall
package tool
package tokenize

import scala.collection.JavaConversions._

import edu.stanford.nlp.ling._;
import edu.stanford.nlp.process._;

class StanfordTokenizer extends Tokenizer {
  def tokenize(sentence: String) =
    new PTBTokenizer(
      new java.io.StringReader(sentence),
      new CoreLabelTokenFactory(),
      "").map { stoken =>
    new Token(stoken.word, stoken.beginPosition())
  }.toList
}

object StanfordTokenizerMain extends TokenizerMain {
  val tokenizer = new StanfordTokenizer()
}