com.mageddo.kafka.client.ConsumerFactory Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of kafka-client Show documentation
null
There is a newer version: 1.0.2
package com.mageddo.kafka.client;

import java.time.Duration;
import java.util.ArrayList;
import java.util.List;

import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;

import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG;
import static org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG;

@Slf4j
public class ConsumerFactory implements AutoCloseable {

  private List> consumers = new ArrayList<>();
  private boolean closed;

  public static  ConsumerSupplier defaultConsumerSupplier() {
    return config -> new KafkaConsumer<>(config.props());
  }

  public void consume(Consumers consumerConfig) {
    if (consumerConfig.consumers() == Integer.MIN_VALUE) {
      log.info(
          "status=disabled-consumer, groupId={}, topics={}",
          consumerConfig.props()
              .get(GROUP_ID_CONFIG),
          consumerConfig.topics()
      );
      return;
    }
    this.checkReasonablePollInterval(consumerConfig);

    for (int i = 0; i < consumerConfig.consumers() - 1; i++) {
      final ThreadConsumer consumer = getInstance(create(consumerConfig), consumerConfig);
      consumer.start();
    }
    log.info("status={} consumers started", consumerConfig.consumers());
  }

  ThreadConsumer getInstance(Consumer consumer, Consumers consumerConfig) {
    if (consumerConfig.batchCallback() != null) {
      return bindInstance(new BatchConsumer<>(consumer, consumerConfig));
    }
    return bindInstance(new RecordConsumer<>(consumer, consumerConfig));
  }

  private ThreadConsumer bindInstance(ThreadConsumer consumer) {
    this.consumers.add(consumer);
    return consumer;
  }

  Consumer create(Consumers consumerConfig) {
    return consumerConfig.consumerSupplier()
        .get(consumerConfig);
  }

  private void checkReasonablePollInterval(Consumers consumerConfig) {
    final int defaultPollInterval = (int) Duration
        .ofMinutes(5)
        .toMillis();

    final int currentPollInterval = (int) consumerConfig
        .props()
        .getOrDefault(MAX_POLL_INTERVAL_MS_CONFIG, defaultPollInterval);

    final RetryPolicy retryPolicy = consumerConfig.retryPolicy();

    final long retryMaxWaitTime = retryPolicy
        .calcMaxTotalWaitTime()
        .toMillis();

    if (currentPollInterval < retryMaxWaitTime) {
      notifyNotRecommendedRetryPolicy(currentPollInterval, retryPolicy, retryMaxWaitTime);
    }
  }

  void notifyNotRecommendedRetryPolicy(int currentPollInterval, RetryPolicy retryPolicy, long retryMaxWaitTime) {
    log.warn(
        "msg=your 'max.poll.interval.ms' is set to a value less than the retry policy, it will cause consumer "
            + "rebalancing, increase 'max.poll.interval.ms' or decrease the retry policy delay or retries, "
            + "max.poll.interval.ms={}, retryMaxWaitTime={} (retries={}, delay={})",
        Duration.ofMillis(currentPollInterval),
        Duration.ofMillis(retryMaxWaitTime),
        retryPolicy.getMaxTries(),
        retryPolicy.getDelay()
    );
  }

  @Override
  public void close() throws Exception {
    if(closed){
      return ;
    }
    for (ThreadConsumer consumer : this.consumers) {
      consumer.close();
    }
    this.closed = true;
  }

  /**
   * wait until all consumers threads termination
   */
  @SneakyThrows
  public ConsumerFactory waitFor() {
    Thread.currentThread().join();
    return this;
  }
}