com.quandoo.lib.reactivekafka.consumer.KafkaConsumer.kt Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of reactive-kafka Show documentation
Show all versions of reactive-kafka Show documentation
A high level kafka consumer which wrapps the low level api of Kafka Reactor and provides a similar usability like Spring Kafka
/**
* Copyright (C) 2019 Quandoo GmbH ([email protected])
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.quandoo.lib.reactivekafka.consumer
import com.google.common.base.MoreObjects
import com.quandoo.lib.reactivekafka.KafkaProperties
import com.quandoo.lib.reactivekafka.consumer.listener.KafkaListenerMeta
import com.quandoo.lib.reactivekafka.util.KafkaConfigHelper
import io.reactivex.Completable
import io.reactivex.Flowable
import io.reactivex.Single
import io.reactivex.subscribers.DisposableSubscriber
import java.time.Duration
import java.util.concurrent.TimeUnit
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.serialization.BytesDeserializer
import org.apache.kafka.common.utils.Bytes
import org.slf4j.LoggerFactory
import reactor.adapter.rxjava.RxJava2Adapter
import reactor.core.publisher.Mono
import reactor.core.scheduler.Scheduler
import reactor.core.scheduler.Schedulers
import reactor.kafka.receiver.KafkaReceiver
import reactor.kafka.receiver.ReceiverOptions
import reactor.kafka.receiver.ReceiverRecord
/**
* @author Emir Dizdarevic
* @since 1.0.0
*/
class KafkaConsumer {
companion object {
private val log = LoggerFactory.getLogger(KafkaConsumer::class.java)
}
private val kafkaProperties: KafkaProperties
private val kafkaListenerMetas: List>
private val schedulers: Map, Scheduler>
constructor(
kafkaProperties: KafkaProperties,
originalKafkaListenerMetas: List>
) {
this.kafkaProperties = kafkaProperties
this.kafkaListenerMetas = mergeConfiguration(kafkaProperties, originalKafkaListenerMetas)
this.schedulers = kafkaListenerMetas.map { it to Schedulers.newParallel("kafka-consumer-${it.topics}", kafkaProperties.consumer!!.parallelism) }.toMap()
checkListenerMetas(kafkaListenerMetas)
}
private fun mergeConfiguration(kafkaProperties: KafkaProperties, listeners: List>): List> {
return listeners.map { kafkaListenerMeta ->
kafkaListenerMeta.copy(
groupId = MoreObjects.firstNonNull(kafkaListenerMeta.groupId, kafkaProperties.consumer?.groupId),
batchSize = MoreObjects.firstNonNull(kafkaListenerMeta.batchSize, kafkaProperties.consumer?.batchSize),
parallelism = MoreObjects.firstNonNull(kafkaListenerMeta.parallelism, kafkaProperties.consumer?.parallelism),
maxPoolIntervalMillis = MoreObjects.firstNonNull(kafkaListenerMeta.maxPoolIntervalMillis, kafkaProperties.consumer?.maxPoolIntervalMillis),
batchWaitMillis = MoreObjects.firstNonNull(kafkaListenerMeta.batchWaitMillis, kafkaProperties.consumer?.batchWaitMillis),
retryBackoffMillis = MoreObjects.firstNonNull(kafkaListenerMeta.retryBackoffMillis, kafkaProperties.consumer?.retryBackoffMillis),
partitionAssignmentStrategy = MoreObjects.firstNonNull(kafkaListenerMeta.partitionAssignmentStrategy, kafkaProperties.consumer?.partitionAssignmentStrategy),
autoOffsetReset = MoreObjects.firstNonNull(kafkaListenerMeta.autoOffsetReset, kafkaProperties.consumer?.autoOffsetReset)
)
}
}
private fun checkListenerMetas(listeners: List>): List> {
check(listeners.isNotEmpty()) { "At least one consumer has to be defined" }
listeners.groupBy { it.groupId }.forEach { entry ->
check(entry.value.map { it.valueClass }.size == entry.value.map { it.valueClass }.toSet().size) { "Only one listener per groupID and Entity can be defined" }
entry.value.forEach {
checkNotNull(it.groupId) { "groupId mandatory" }
checkNotNull(it.batchSize) { "batchSize mandatory" }
checkNotNull(it.parallelism) { "parallelism mandatory" }
checkNotNull(it.maxPoolIntervalMillis) { "maxPoolIntervalMillis mandatory" }
checkNotNull(it.batchWaitMillis) { "batchWaitMillis mandatory" }
checkNotNull(it.retryBackoffMillis) { "retryBackoffMillis mandatory" }
checkNotNull(it.partitionAssignmentStrategy) { "partitionAssignmentStrategy mandatory" }
checkNotNull(it.autoOffsetReset) { "autoOffsetReset mandatory" }
}
}
return listeners
}
private fun createReceiverOptions(kafkaListenerMeta: KafkaListenerMeta): ReceiverOptions {
val consumerProps = HashMap()
.also {
it[ConsumerConfig.GROUP_ID_CONFIG] = kafkaListenerMeta.groupId as String
it[ConsumerConfig.AUTO_OFFSET_RESET_CONFIG] = kafkaListenerMeta.autoOffsetReset as String
it[ConsumerConfig.MAX_POLL_RECORDS_CONFIG] = kafkaListenerMeta.batchSize as Int
it[ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG] = kafkaListenerMeta.maxPoolIntervalMillis as Int
it[ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG] = kafkaListenerMeta.partitionAssignmentStrategy as String
}
.let { KafkaConfigHelper.populateCommonConfig(kafkaProperties, it) }
.let { KafkaConfigHelper.populateSslConfig(kafkaProperties, it) }
.let { KafkaConfigHelper.populateSaslConfig(kafkaProperties, it) }
return ReceiverOptions.create(consumerProps)
// Disable automatic commits
.commitInterval(Duration.ZERO)
.commitBatchSize(0)
.withKeyDeserializer(BytesDeserializer())
.withValueDeserializer(BytesDeserializer())
.subscription(kafkaListenerMeta.topics)
.schedulerSupplier { schedulers[kafkaListenerMeta] }
}
fun start() {
startConsumers()
}
@SuppressWarnings("unchecked")
private fun startConsumers() {
kafkaListenerMetas.forEach {
for (i in 1..it.parallelism!!) {
startConsumer(it)
}
}
}
private fun startConsumer(kafkaListenerMeta: KafkaListenerMeta) {
val receiverOptions = createReceiverOptions(kafkaListenerMeta)
Flowable.defer {
RxJava2Adapter.fluxToFlowable(
KafkaReceiver.create(receiverOptions).receive()
.bufferTimeout(kafkaListenerMeta.batchSize!!, Duration.ofMillis(kafkaListenerMeta.batchWaitMillis!!), schedulers[kafkaListenerMeta]!!)
)
}
.flatMap(
{ receiverRecords ->
when (kafkaListenerMeta.handler) {
is SingleHandler -> {
processSingle(kafkaListenerMeta, receiverRecords).toFlowable().map { receiverRecords }
}
is BatchHandler -> {
processBatch(kafkaListenerMeta, receiverRecords).toFlowable().map { receiverRecords }
}
else -> {
throw IllegalStateException("Unknown handler type: ${kafkaListenerMeta.handler.javaClass}")
}
}
.observeOn(io.reactivex.schedulers.Schedulers.from { r -> schedulers[kafkaListenerMeta]!!.schedule(r) })
.concatMap(
{ receiverRecords ->
if (receiverRecords.isNotEmpty()) {
// All offsets need to be ackd
receiverRecords.forEach { it.receiverOffset().acknowledge() }
val lastReceiverRecordPerTopicPartition = receiverRecords.map { it.receiverOffset().topicPartition() to it }.toMap()
Flowable.fromIterable(lastReceiverRecordPerTopicPartition.values)
.flatMap { receiverRecord ->
RxJava2Adapter.monoToCompletable(receiverRecord.receiverOffset().commit()).toSingle { receiverRecord }.toFlowable()
}
.toList()
.toFlowable()
.map { receiverRecords }
} else {
Flowable.just(receiverRecords)
}
},
1
)
},
1
)
.observeOn(io.reactivex.schedulers.Schedulers.from { r -> schedulers[kafkaListenerMeta]!!.schedule(r) })
.retry()
.subscribeOn(io.reactivex.schedulers.Schedulers.from { r -> schedulers[kafkaListenerMeta]!!.schedule(r) }, true)
.subscribe(
object : DisposableSubscriber>>() {
override fun onStart() {
request(1)
}
override fun onNext(receiverRecord: List>) {
receiverRecord.forEach { logMessage("Message processed", it) }
request(1)
}
override fun onError(ex: Throwable) {
log.error("Consumer terminated", ex)
}
override fun onComplete() {
log.error("Consumer terminated")
}
}
)
}
private fun processSingle(
kafkaListenerMeta: KafkaListenerMeta,
receiverRecords: MutableList>
): Single>> {
return Single.defer {
Flowable.fromIterable(receiverRecords)
.filter { receiverRecord -> preFilterMessage(kafkaListenerMeta, receiverRecord) }
.map { serializeConsumerRecord(kafkaListenerMeta, it) }
.filter { receiverRecord -> filterMessage(kafkaListenerMeta, receiverRecord) }
.concatMapEager { receiverRecord ->
Flowable.defer { Flowable.just((kafkaListenerMeta.handler as SingleHandler).apply(receiverRecord)) }
.concatMapEager { result ->
when (result) {
is Mono<*> -> RxJava2Adapter.monoToCompletable(result).toSingleDefault(receiverRecord).toFlowable()
is Completable -> result.toSingleDefault(1).toFlowable().map { receiverRecord }
else -> Flowable.error>(IllegalStateException("Unknown return type ${result.javaClass}"))
}
}
.doOnError { error -> log.error("Failed to process kafka message", error) }
.retryWhen { receiverRecord -> receiverRecord.delay(kafkaListenerMeta.retryBackoffMillis!!, TimeUnit.MILLISECONDS) }
}
.toList()
}
.doOnError { error -> log.error("Failed to process batch", error) }
.retryWhen { receiverRecord -> receiverRecord.delay(kafkaListenerMeta.retryBackoffMillis!!, TimeUnit.MILLISECONDS) }
}
private fun processBatch(
kafkaListenerMeta: KafkaListenerMeta,
receiverRecords: MutableList>
): Single>> {
return Single.defer {
Flowable.fromIterable(receiverRecords)
.filter { receiverRecord -> preFilterMessage(kafkaListenerMeta, receiverRecord) }
.map { serializeConsumerRecord(kafkaListenerMeta, it) }
.filter { receiverRecord -> filterMessage(kafkaListenerMeta, receiverRecord) }
.toList()
.flatMap { receiverRecords ->
if (receiverRecords.isNotEmpty()) {
Single.defer { Single.just((kafkaListenerMeta.handler as BatchHandler).apply(receiverRecords)) }
.flatMap { result ->
when (result) {
is Mono<*> -> RxJava2Adapter.monoToCompletable(result).toSingleDefault(receiverRecords)
is Completable -> result.toSingleDefault(1).map { receiverRecords }
else -> Single.error>>(IllegalStateException("Unknown return type ${result.javaClass}"))
}
}
.doOnError { error -> log.error("Failed to process kafka message", error) }
.retryWhen { receiverRecords -> receiverRecords.delay(kafkaListenerMeta.retryBackoffMillis!!, TimeUnit.MILLISECONDS) }
} else {
Single.just(emptyList())
}
}
}
.doOnError { error -> log.error("Failed to process batch", error) }
.retryWhen { receiverRecord -> receiverRecord.delay(kafkaListenerMeta.retryBackoffMillis!!, TimeUnit.MILLISECONDS) }
}
private fun preFilterMessage(kafkaListenerMeta: KafkaListenerMeta, receiverRecord: ReceiverRecord): Boolean {
val pass = kafkaListenerMeta.preFilter.apply(receiverRecord)
if (!pass) {
logMessage("Messages pre-filtered out", receiverRecord)
}
return pass
}
private fun filterMessage(kafkaListenerMeta: KafkaListenerMeta, receiverRecord: ReceiverRecord): Boolean {
val pass = kafkaListenerMeta.filter.apply(receiverRecord)
if (!pass) {
logMessage("Messages filtered out", receiverRecord)
}
return pass
}
private fun serializeConsumerRecord(kafkaListenerMeta: KafkaListenerMeta, originalReceiverRecord: ReceiverRecord): ReceiverRecord {
val key = originalReceiverRecord.key()?.let { kafkaListenerMeta.keyDeserializer.deserialize(originalReceiverRecord.topic(), it.get()) }
val value = originalReceiverRecord.value()?.let { kafkaListenerMeta.valueDeserializer.deserialize(originalReceiverRecord.topic(), it.get()) }
return ReceiverRecord(
ConsumerRecord(
originalReceiverRecord.topic(),
originalReceiverRecord.partition(),
originalReceiverRecord.offset(),
originalReceiverRecord.timestamp(),
originalReceiverRecord.timestampType(),
originalReceiverRecord.checksum(),
originalReceiverRecord.serializedKeySize(),
originalReceiverRecord.serializedValueSize(),
key,
value
),
originalReceiverRecord.receiverOffset()
)
}
private fun logMessage(prefix: String, receiverRecord: ReceiverRecord<*, *>) {
log.debug(
"$prefix.\nTopic: {}, Partition: {}, Offset: {}, Headers: {}",
receiverRecord.receiverOffset().topicPartition().topic(),
receiverRecord.receiverOffset().topicPartition().partition(),
receiverRecord.receiverOffset().offset(),
receiverRecord.headers().map { it.key() + ":" + String(it.value()) }
)
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy