commonMain.pro.felixo.protobuf.schemadocument.SchemaDocumentTokenizer.kt Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of protobuf-kotlin-schemadocument-jvm Show documentation

Protocol Buffers 3 support for Kotlin Multiplatform

The newest version!

package pro.felixo.protobuf.schemadocument

/**
 * Converts .proto syntax into sequences of tokens.
 */
class SchemaDocumentTokenizer {
    fun tokenize(input: String): Sequence = sequence {
        var position = 0

        outer@ while (position < input.length) {
            for (type in TokenType.entries) {
                val matchResult = type.regex.find(input, position)

                if (matchResult != null && matchResult.range.first == position) {
                    val tokenText = matchResult.value
                    type.getToken(tokenText)?.let { yield(it) }
                    position += tokenText.length
                    continue@outer
                }
            }

            throw IllegalArgumentException("Unrecognized token at position $position")
        }
    }
}