com.networknt.eventuate.kafka.consumer.EventuateKafkaConsumer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of eventuate-server-kafka Show documentation
A module that is responsible for Kafka communication
There is a newer version: 2.0.11
Show newest version
package com.networknt.eventuate.kafka.consumer;

import com.networknt.config.Config;
import com.networknt.eventuate.kafka.KafkaConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;

/**
 * A Kafka consumer that manually commits offsets and supports asynchronous message processing
 */
public class EventuateKafkaConsumer {

  private static Logger logger = LoggerFactory.getLogger(EventuateKafkaConsumer.class);
  static KafkaConfig config = (KafkaConfig) Config.getInstance().getJsonObjectConfig(KafkaConfig.CONFIG_NAME, KafkaConfig.class);

  private final String subscriberId;
  private final BiConsumer, BiConsumer> handler;
  private final List topics;
  private AtomicBoolean stopFlag = new AtomicBoolean(false);
  private Properties consumerProperties;

  /**
   *
   * @param subscriberId message subscriber id
   * @param handler defined message handler
   * @param topics topics Kafka topic list
   */
  public EventuateKafkaConsumer(String subscriberId, BiConsumer, BiConsumer> handler, List topics) {

    this.subscriberId = subscriberId;
    this.handler = handler;
    this.topics = topics;
    this.consumerProperties = ConsumerPropertiesFactory.makeConsumerProperties(config, subscriberId);
  }

  public static List verifyTopicExistsBeforeSubscribing(KafkaConsumer consumer, String topic) {
    try {
      logger.debug("Verifying Topic {}", topic);
      List partitions = consumer.partitionsFor(topic);
      logger.debug("Got these partitions {} for Topic {}", partitions, topic);
      return partitions;
    } catch (Throwable e) {
      logger.error("Got exception: ", e);
      throw new RuntimeException(e);
    }
  }

  private void maybeCommitOffsets(KafkaConsumer consumer, KafkaMessageProcessor processor) {
    Map offsetsToCommit = processor.offsetsToCommit();
    if (!offsetsToCommit.isEmpty()) {
      logger.debug("Committing offsets {} {}", subscriberId, offsetsToCommit);
      consumer.commitSync(offsetsToCommit);
      logger.debug("Committed offsets {}", subscriberId);
      processor.noteOffsetsCommitted(offsetsToCommit);
    }
  }

  /**
   * start Kafka consumer process
   */
  public void start() {
    try {

      KafkaConsumer consumer = new KafkaConsumer<>(consumerProperties);

      KafkaMessageProcessor processor = new KafkaMessageProcessor(subscriberId, handler);

      for (String topic : topics) {
        verifyTopicExistsBeforeSubscribing(consumer, topic);
      }

      logger.debug("Subscribing to {} {}", subscriberId, topics);

      consumer.subscribe(new ArrayList<>(topics));

      logger.debug("Subscribed to {} {}", subscriberId, topics);

      new Thread(() -> {


        try {
          while (!stopFlag.get()) {
            ConsumerRecords records = consumer.poll(100);
            if (!records.isEmpty())
              logger.debug("Got {} {} records", subscriberId, records.count());

            for (ConsumerRecord record : records) {
              logger.debug("processing record {} {} {}", subscriberId, record.offset(), record.value());
              if (logger.isDebugEnabled())
                logger.debug(String.format("EventuateKafkaAggregateSubscriptions subscriber = %s, offset = %d, key = %s, value = %s", subscriberId, record.offset(), record.key(), record.value()));
              if ( record !=null &&  record.key() !=null &&  record.value()!=null) {
                processor.process(record);
              }
            }
            if (!records.isEmpty())
              logger.debug("Processed {} {} records", subscriberId, records.count());

            maybeCommitOffsets(consumer, processor);

            if (!records.isEmpty())
              logger.debug("To commit {} {}", subscriberId, processor.getPending());

          }

          maybeCommitOffsets(consumer, processor);

        } catch (Throwable e) {
          logger.error("Got exception: ", e);
          throw new RuntimeException(e);
        }

      }, "Eventuate-subscriber-" + subscriberId).start();

    } catch (Exception e) {
      logger.error("Error subscribing", e);
      throw new RuntimeException(e);
    }
  }

  public void stop() {
    stopFlag.set(true);
  }

}