
com.exactpro.th2.codec.csv.AbstractDecoder.kt Maven / Gradle / Ivy
/*
* Copyright 2023 Exactpro (Exactpro Systems Limited)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.exactpro.th2.codec.csv
import com.csvreader.CsvReader
import com.exactpro.th2.codec.DecodeException
import mu.KotlinLogging
import java.io.ByteArrayInputStream
import java.io.IOException
import java.nio.charset.Charset
import kotlin.math.min
abstract class AbstractDecoder(
private val charset: Charset,
private val csvDelimiter: Char,
private val defaultHeader: Array?,
private val publishHeader: Boolean,
private val validateLength: Boolean,
private val trimWhitespace: Boolean
) {
protected abstract val RAW_MESSAGE.messageMetadata: Map
protected abstract val RAW_MESSAGE.messageSessionAlias: String
protected abstract val RAW_MESSAGE.rawBody: ByteArray
protected abstract val RAW_MESSAGE.messageProtocol: String
protected abstract val RAW_MESSAGE.logId: String
protected abstract val RAW_MESSAGE.logData: String
protected abstract val ANY_MESSAGE.isParsed: Boolean
protected abstract val ANY_MESSAGE.isRaw: Boolean
protected abstract val ANY_MESSAGE.asRaw: RAW_MESSAGE
protected abstract fun createParsedMessage(sourceMessage: RAW_MESSAGE, outputMessageType: String, body: Map, currentIndex: Int): ANY_MESSAGE
protected abstract fun String.toFieldValue(): BODY_FIELD_VALUE
protected abstract fun Array.toFieldValue(): BODY_FIELD_VALUE
fun decode(messageGroup: List): List {
val groupBuilder = mutableListOf() // ProtoMessageGroup.newBuilder()
val errors: MutableCollection> = mutableListOf()
for (anyMessage in messageGroup) {
if (anyMessage.isParsed) {
groupBuilder += anyMessage
continue
}
if (!anyMessage.isRaw) {
LOGGER.error { "Message should either have a raw or parsed message but has nothing: $anyMessage" }
continue
}
val rawMessage = anyMessage.asRaw
val protocol = rawMessage.messageProtocol
if ("" != protocol && !"csv".equals(protocol, ignoreCase = true)) {
LOGGER.error { "Wrong protocol: message should have empty or 'csv' protocol but has $protocol" }
continue
}
val data = decodeValues(rawMessage.rawBody)
if (data.isEmpty()) {
LOGGER.error { "The raw message does not contains any data: ${rawMessage.logData}" }
errors.add(ErrorHolder("The raw message does not contains any data", rawMessage))
continue
}
decodeCsvData(errors, groupBuilder, rawMessage, data)
}
if (errors.isNotEmpty()) {
throw DecodeException(
"Cannot decode some messages:\n" + errors.joinToString("\n") {
"Message ${it.originalMessage.logId} cannot be decoded because ${it.text}"
}
)
}
return groupBuilder
}
private fun decodeCsvData(
errors: MutableCollection>,
groupBuilder: MutableList,
rawMessage: RAW_MESSAGE,
data: Iterable>
) {
val originalMetadata = rawMessage.messageMetadata
val outputMessageType = originalMetadata.getOrDefault(
OVERRIDE_MESSAGE_TYPE_PROP_NAME_LOWERCASE,
CSV_MESSAGE_TYPE
)
var currentIndex = 0
var header: Array? = defaultHeader
for (strings in data) {
currentIndex++
if (strings.isEmpty()) {
LOGGER.error { "Empty raw at $currentIndex index (starts with 1). Data: $data" }
errors.add(ErrorHolder("Empty raw at $currentIndex index (starts with 1)", rawMessage))
continue
}
if (trimWhitespace) {
trimEachElement(strings)
}
if (header == null) {
LOGGER.debug { "Set header to: ${strings.contentToString()}" }
header = strings
if (publishHeader) {
//groupBuilder += createHeadersMessage(rawMessage, strings, currentIndex)
groupBuilder += createParsedMessage(rawMessage,
HEADER_MSG_TYPE, mapOf(HEADER_FIELD_NAME to strings.toFieldValue()), currentIndex)
}
continue
}
if (strings.size != header.size && validateLength) {
val msg = String.format(
"Wrong fields count in message. Expected count: %d; actual: %d; session alias: %s",
header.size, strings.size, rawMessage.messageSessionAlias
)
LOGGER.error(msg)
LOGGER.debug { rawMessage.toString() }
errors.add(ErrorHolder(msg, rawMessage))
}
val headerLength = header.size
val rowLength = strings.size
var i = 0
val body = mutableMapOf()
while (i < headerLength && i < rowLength) {
val extraLength = getHeaderArrayLength(header, i)
if (extraLength == 1) {
body[header[i]] = strings[i].toFieldValue()
i++
} else {
val values = copyArray(strings, i, i + extraLength)
body[header[i]] = values.toFieldValue()
i += extraLength
}
}
groupBuilder += createParsedMessage(rawMessage, outputMessageType, body, currentIndex)
}
}
private fun copyArray(original: Array, from: Int, to: Int) = original.copyOfRange(from,
min(to, original.size)
)
private fun getHeaderArrayLength(header: Array, index: Int): Int {
var length = 1
var i = index + 1
while (i < header.size && header[i].isEmpty()) {
length++
i++
}
return length
}
private fun decodeValues(body: ByteArray): List> {
try {
ByteArrayInputStream(body).use {
val reader = CsvReader(it, csvDelimiter, charset)
reader.trimWhitespace = trimWhitespace
return try {
val result: MutableList> = ArrayList()
while (reader.readRecord()) {
result.add(reader.values)
}
result
} finally {
reader.close()
}
}
} catch (e: IOException) {
throw RuntimeException("cannot read data from raw bytes", e)
}
}
private class ErrorHolder(
val text: String,
val originalMessage: T
)
private fun trimEachElement(elements: Array) {
for (i in elements.indices) {
elements[i] = elements[i].trim()
}
}
companion object {
private val LOGGER = KotlinLogging.logger {}
private const val HEADER_MSG_TYPE = "Csv_Header"
private const val CSV_MESSAGE_TYPE = "Csv_Message"
private const val HEADER_FIELD_NAME = "Header"
private const val OVERRIDE_MESSAGE_TYPE_PROP_NAME_LOWERCASE = "th2.csv.override_message_type"
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy