All Downloads are FREE. Search and download functionalities are using the official Maven repository.

commonMain.moe.tlaster.twitter.parser.tokenizer.Tokenizer.kt Maven / Gradle / Ivy

The newest version!
package moe.tlaster.twitter.parser.tokenizer


internal class Tokenizer(
    val enableEmoji: Boolean = false,
    val enableDomainDetection: Boolean = false,
    val enableNonAsciiInUrl: Boolean = true,
    val enableEscapeInUrl: Boolean = false,
    val validMarkInUserName: List = listOf(),
    val validMarkInHashTag: List = listOf(),
) {
    private var currentState: State = DataState
    private lateinit var tokens: ArrayList
    fun parse(reader: Reader): List {
        tokens = (0..




© 2015 - 2024 Weber Informatics LLC | Privacy Policy