commonMain.moe.tlaster.twitter.parser.tokenizer.Tokenizer.kt Maven / Gradle / Ivy
The newest version!
package moe.tlaster.twitter.parser.tokenizer
internal class Tokenizer(
val enableEmoji: Boolean = false,
val enableDomainDetection: Boolean = false,
val enableNonAsciiInUrl: Boolean = true,
val enableEscapeInUrl: Boolean = false,
val validMarkInUserName: List = listOf(),
val validMarkInHashTag: List = listOf(),
) {
private var currentState: State = DataState
private lateinit var tokens: ArrayList
fun parse(reader: Reader): List {
tokens = (0..