• Home
• com.github.halfmatthalfcat
• stringmetric-core_2.12
• 0.28.0
• source code
• StringTokenizer.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.github.halfmatthalfcat.stringmetric.StringTokenizer.scala Maven / Gradle / Ivy

package com.github.halfmatthalfcat.stringmetric

sealed trait StringTokenizer extends Tokenizer[Array[Char]] {
	def tokenize(a: String): Option[Array[String]]
}


final case class NGramTokenizer(n: Int) extends StringTokenizer {
	override def tokenize(a: Array[Char]): Option[Array[Array[Char]]] =
		if (n <= 0 || a.length < n) None
		else Some(sequence(a, Array.empty[Array[Char]], n))

	override def tokenize(a: String): Option[Array[String]] = tokenize(a.toCharArray).map(_.map(_.mkString))

	@annotation.tailrec
	private val sequence: ((Array[Char], Array[Array[Char]], Int) => Array[Array[Char]]) = (i, o, n) =>
		if (i.length <= n) o :+ i
		else sequence(i.tail, o :+ i.take(n), n)
}