• Home
• moe.tlaster
• twitter-parser-jvm
• 0.5.1
• source code
• Tokenizer.kt

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

commonMain.moe.tlaster.twitter.parser.tokenizer.Tokenizer.kt Maven / Gradle / Ivy

package moe.tlaster.twitter.parser.tokenizer


internal class Tokenizer(
    val enableEmoji: Boolean = false,
    val enableDomainDetection: Boolean = false,
    val enableNonAsciiInUrl: Boolean = true,
    val enableEscapeInUrl: Boolean = false,
    val validMarkInUserName: List = listOf(),
) {
    private var currentState: State = DataState
    private lateinit var tokens: ArrayList
    fun parse(reader: Reader): List {
        tokens = (0..