no.finn.retriableconsumer.ReliableKafkaConsumerPool Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of retriable-kafka-consumer Show documentation
Retries processing when consuming records from kafka.
There is a newer version: 1.55
Show newest version
package no.finn.retriableconsumer;

import java.io.Closeable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;

import static java.util.stream.IntStream.rangeClosed;

/**
 * THE class to use from this library.
 *
 * This class will create a set of consumers that subscribes to a given topic and mechanism to
 * resend records that fails to be processed.
 *
 * @param  - the key-type of the kafka topic
 * @param  - the value-type of the kafka-topic
 */
public class ReliableKafkaConsumerPool implements Closeable {


    public final RestartableMonitor monitor;

    /**
     * @param consumerPoolCount     - number of kafka-consumers
     * @param factory               - factory that creates consumer and producer
     * @param topicsRetryTopics     - map of topics and associated retry-topics
     * @param processingFunction    - function that will process messages
     * @param pollFunction          - function to poll messages
     * @param expiredHandler        - hook to handle expired messages
     * @param logHandler            - How and what to log
     * @param retryThrottleMillis   - how long we should delay before processing the record again
     * @param retryDurationInMillis - how long retry-producer should retry the message before giving up
     */
    public ReliableKafkaConsumerPool(
            int consumerPoolCount,
            KafkaClientFactory factory,
            Map topicsRetryTopics,
            Function, Boolean> processingFunction,
            Function, ConsumerRecords> pollFunction,
            java.util.function.Consumer> expiredHandler,
            LogHandler logHandler,
            long retryThrottleMillis,
            long retryDurationInMillis) {

        // queue for safe communication between consumers and retry-producer
        RetryHandler retryHandler = new RetryHandler<>(factory::producer,
                                                             retryThrottleMillis,
                                                             topicsRetryTopics,
                                                             logHandler);

        // consumers
        List consumers =
                rangeClosed(1, consumerPoolCount)
                        .mapToObj(
                                i ->
                                        new RestartableKafkaConsumer<>(
                                                factory::consumer,
                                                new ArrayList<>(topicsRetryTopics.keySet()),
                                                processingFunction,
                                                pollFunction,
                                                retryHandler,
                                                expiredHandler,
                                                logHandler,
                                                retryDurationInMillis))
                        .collect(Collectors.toList());

        //retry-consumer
        consumers.add(
                new RestartableKafkaConsumer<>(
                        factory::consumer,
                        new ArrayList<>(topicsRetryTopics.values()),
                        processingFunction,
                        pollFunction,
                        retryHandler,
                        expiredHandler,
                        logHandler,
                        retryDurationInMillis));

        // monitor all consumers and producer with the same monitor
        monitor = new RestartableMonitor(consumers);
    }

    @Override
    public void close() {
        monitor.close();
    }

    public int size() {
        return monitor.size();
    }
}