com.mageddo.kafka.client.BatchConsumer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of kafka-client Show documentation
null
There is a newer version: 1.0.2
package com.mageddo.kafka.client;

import java.util.List;

import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.TopicPartition;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@RequiredArgsConstructor
public class BatchConsumer extends DefaultConsumer {

  private final Consumer consumer;
  private final Consumers consumers;

  @Override
  protected void consume(ConsumerRecords records) {
    final Retrier retrier = Retrier
        .builder()
        .retryPolicy(this.consumers.retryPolicy())
        .onRetry(() -> {
          log.info("failed to consume");
          for (final TopicPartition partition : records.partitions()) {
            commitFirstRecord(this.consumer, records, partition);
          }
        })
        .onExhausted((lastFailure) -> {
          log.info("status=exhausted-tries, records={}", records.count());
          records.forEach(record -> this.doRecoverWhenAvailable(
              DefaultRecoverContext
              .builder()
              .consumer(this.consumer)
              .lastFailure(lastFailure)
              .record(record)
              .build(),
              this.consumers.recoverCallback()
          ));
        })
        .build();

    retrier.run(() -> {
      if (log.isTraceEnabled()) {
        log.trace("status=consuming, records={}", records);
      }
      try {
        this.consumers
            .batchCallback()
            .accept(
                DefaultCallbackContext
                    .builder()
                    .consumer(this.consumer)
                    .records(records)
                    .build(),
                records
            );
      } catch (Exception e) {
        Exceptions.throwException(e);
      }
    });
    this.consumer.commitSync();
  }

  @Override
  protected Consumer consumer() {
    return this.consumer;
  }

  @Override
  protected Consumers consumerConfig() {
    return this.consumers;
  }

  private void commitFirstRecord(Consumer consumer, ConsumerRecords records, TopicPartition partition) {
    final ConsumerRecord firstRecord = getFirstRecord(records, partition);
    if (firstRecord != null) {
      commitSyncRecord(consumer, firstRecord);
    }
  }

  private ConsumerRecord getFirstRecord(ConsumerRecords records, TopicPartition partition) {
    final List> partitionRecords = records.records(partition);
    return partitionRecords.isEmpty() ? null : partitionRecords.get(0);
  }

}