com.networknt.eventuate.kafka.consumer.KafkaMessageProcessor Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of eventuate-server-kafka Show documentation

A module that is responsible for Kafka communication

There is a newer version: 2.0.11

Show newest version

package com.networknt.eventuate.kafka.consumer;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.BiConsumer;

/**
 * Processes a Kafka message and tracks the message offsets that have been successfully processed and can be committed
 *
 */
public class KafkaMessageProcessor {
  private Logger logger = LoggerFactory.getLogger(getClass());

  private String subscriberId;
  private BiConsumer, BiConsumer> handler;
  private OffsetTracker offsetTracker = new OffsetTracker();

  private BlockingQueue> processedRecords = new LinkedBlockingQueue<>();

  public KafkaMessageProcessor(String subscriberId, BiConsumer, BiConsumer> handler) {
    this.subscriberId = subscriberId;
    this.handler = handler;
  }


  public void process(ConsumerRecord record) {
      offsetTracker.noteUnprocessed(new TopicPartition(record.topic(), record.partition()), record.offset());
      handler.accept(record, (result, t) -> {
        if (t != null) {
          logger.error("Got exception: ", t);
        } else {
          logger.debug("Adding processed record to queue {} {}", subscriberId, record.offset());
          processedRecords.add(record);
        }
      });
  }


  public Map offsetsToCommit() {
    int count = 0;
    while (true) {
      ConsumerRecord record = processedRecords.poll();
      if (record == null)
        break;
      count++;
      offsetTracker.noteProcessed(new TopicPartition(record.topic(), record.partition()), record.offset());
    }
    logger.trace("removed {} {} processed records from queue", subscriberId, count);
    return offsetTracker.offsetsToCommit();
  }

  public void noteOffsetsCommitted(Map offsetsToCommit) {
    offsetTracker.noteOffsetsCommitted(offsetsToCommit);
  }

  public OffsetTracker getPending() {
    return offsetTracker;
  }
}