commonMain.piacenti.dslmaker.structures.strategies.pretokenizestrategy.Tokenizer.kt Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of dsl-maker-js Show documentation

Kotlin multiplatform library to facilitate creation of DSLs with ANTLR or a simple built in parser

There is a newer version: 1.1.55

package piacenti.dslmaker.structures.strategies.pretokenizestrategy

import piacenti.dslmaker.*
import piacenti.dslmaker.abstraction.ProductionStep
import piacenti.dslmaker.errors.ParserException
import piacenti.dslmaker.getCurrentTimeInMilliSeconds
import piacenti.dslmaker.structures.Grammar

/**
 * @param 
 * @author Piacenti
 */
class Tokenizer(val grammar: Grammar<*>) {
    private val contextStack = mutableListOf()
    private val tokenInfos = grammar.container.tokens
    companion object{
        fun matchTokenRegex(token: ProductionStep, text: String, indexOffset: Int): RegexMatch? {
            val result = regexMatchFromOffset(text, "(?s)^\\s*(?:" + token.regexDefinition + ")", indexOffset)
            return result
        }
    }
    fun tokenize(textInput: String): List {
        val startTime = getCurrentTimeInMilliSeconds()
        val tokens = mutableListOf()
        var text = textInput
        var currentStart = 0
        while (currentStart {
        return tokenInfos.filter { it.context == null || it.context in contextStack }
    }
}