
tarlake-spark3_2.12.1.3.0.source-code.reference-kafka.conf Maven / Gradle / Ivy
kafka {
cometOffsetsMode = "STREAM"
customDeserializers {
#"deserialize": "io.confluent.kafka.serializers.KafkaAvroDeserializer"
}
serverOptions {
"bootstrap.servers": "localhost:9092"
}
topics {
logs {
topicName: logs
maxRead = 0
fields = ["key as STRING", "value as STRING", "topic as STRING", "partition as INT", "offset as LONG", "timestamp as TIMESTAMP", "timestampType as INT"]
writeFormat = "parquet"
createOptions {
"cleanup.policy": "compact"
}
accessOptions {
"kafka.bootstrap.servers": "localhost:9092"
"bootstrap.servers": "localhost:9092"
"key.deserializer": "org.apache.kafka.common.serialization.StringDeserializer",
"value.deserializer": "org.apache.kafka.common.serialization.StringDeserializer"
"subscribe": "logs"
}
},
comet_offsets {
topicName: comet_offsets
maxRead = 0
partitions = 1
replication-factor = 1
writeFormat = "parquet"
createOptions {
"cleanup.policy": "compact"
}
accessOptions {
"kafka.bootstrap.servers": "localhost:9092"
"auto.offset.reset": "earliest"
"auto.commit.enable": "false"
"consumer.timeout.ms": "10"
"bootstrap.servers": "localhost:9092"
"key.deserializer": "org.apache.kafka.common.serialization.StringDeserializer",
"value.deserializer": "org.apache.kafka.common.serialization.StringDeserializer"
"subscribe": "comet_offsets"
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy