no.finn.retriableconsumer.ReliableKafkaConsumerPool Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of retriable-kafka-consumer Show documentation
Retries processing when consuming records from kafka.
There is a newer version: 1.55
Show newest version
package no.finn.retriableconsumer;

import java.io.Closeable;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;

import static java.util.stream.IntStream.rangeClosed;

/**
 * THE class to use from this library.
 *
 * This class will create a set of consumers that subscribes to a given topic and mechanism to
 * resend records that fails to be processed.
 *
 * @param   - the key-type of the kafka topic
 * @param   - the value-type of the kafka-topic
 */
public class ReliableKafkaConsumerPool implements Closeable {


    public final RestartableMonitor monitor;

    /**
     * @param consumerPoolCount   - number of kafka-consumers
     * @param factory             - factory that creates consumer and producer
     * @param topic               - topic to listen to
     * @param processingFunction  - function that will process messages
     * @param pollFunction        - function to poll messages
     * @param retryPeriodInMillis - how long retry-producer should retry the message before giving up
     * @param retryThrottleMillis - how long we should delay before processing the record again
     */
    public ReliableKafkaConsumerPool(
            int consumerPoolCount,
            KafkaClientFactory factory,
            List topic,
            Function, Boolean> processingFunction,
            Function, ConsumerRecords> pollFunction,
            long retryThrottleMillis,
            long retryPeriodInMillis
    ) {
        this(
                consumerPoolCount,
                factory,
                topic,
                processingFunction,
                pollFunction,
                consumer -> null,
                retryThrottleMillis,
                retryPeriodInMillis
        );
    }

    public ReliableKafkaConsumerPool(
            int consumerPoolCount,
            KafkaClientFactory factory,
            List topics,
            Function, Boolean> processingFunction,
            Function, ConsumerRecords> pollFunction,
            Function, Void> afterProcess,
            long retryThrottleMillis,
            long retryDurationInMillis
    ) {

        // queue for safe communication between consumers and retry-producer
        RetryHandler retryHandler = new RetryHandler<>(factory::producer, retryThrottleMillis, factory.groupId());

        // consumers
        List consumers =
                rangeClosed(1, consumerPoolCount)
                        .mapToObj(
                                i ->
                                        new RestartableKafkaConsumer<>(
                                                factory::consumer,
                                                topics,
                                                processingFunction,
                                                pollFunction,
                                                afterProcess,
                                                retryHandler,
                                                retryDurationInMillis))
                        .collect(Collectors.toList());

        //retry-consumers
        consumers.add(
                new RestartableKafkaConsumer<>(
                        factory::consumer,
                        RetryHandler.retryTopicNames(topics, factory.groupId()),
                        processingFunction,
                        pollFunction,
                        ap -> {
                            ap.commitSync();
                            return null;
                        },
                        retryHandler,
                        retryDurationInMillis));

        // monitor all consumers and producer with the same monitor
        monitor = new RestartableMonitor(consumers);
    }

    @Override
    public void close() {
        monitor.close();
    }

    public int size() {
        return monitor.size();
    }
}