net.kut3.messaging.kafka.KafkaConsumer Maven / Gradle / Ivy
/*
* Copyright 2019 Kut3Net.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.kut3.messaging.kafka;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import net.kut3.messaging.BatchConsumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import net.kut3.messaging.Message;
/**
*
*/
public final class KafkaConsumer {
private final ConsumerBuilder builder;
private KafkaConsumer(ConsumerBuilder builder) {
this.builder = builder;
}
/**
*
* @return A new {@link ConsumerBuilder}
*/
public static ConsumerBuilder newBuilder() {
return new ConsumerBuilder();
}
/**
*
*/
public static final class ConsumerBuilder {
private final Map props;
private final List topics;
private OffsetMode offsetMode = OffsetMode.LATEST;
private ConsumerBuilder() {
this.props = new HashMap<>();
this.topics = new ArrayList<>();
}
/**
*
* @return A comma-separated list of host and port pairs. A host and
* port pair uses : as the separator.
*/
public String servers() {
return (String) this.props.get("bootstrap.servers");
}
/**
*
* @param value A comma-separated list of host and port pairs. A host
* and port pair uses : as the separator.
* @return the current {@link ConsumerBuilder} instance
*/
public ConsumerBuilder servers(String value) {
this.props.put("bootstrap.servers", value);
return this;
}
/**
*
* @return Id of the consumer group belongs to
*/
public String groupId() {
return (String) this.props.get("group.id");
}
/**
*
* @param value id of the group this consumer belongs
* @return The current {@link ConsumerBuilder} instance
*/
public ConsumerBuilder groupId(String value) {
this.props.put("group.id", value);
return this;
}
/**
*
* @return An id string to pass to the server when making requests. The
* purpose of this is to be able to track the source of requests beyond
* just ip/port by allowing a logical application name to be included in
* server-side request logging.
*/
public String clientId() {
return (String) this.props.get("client.id");
}
/**
*
* @param value An id string to pass to the server when making requests.
* The purpose of this is to be able to track the source of requests
* beyond just ip/port by allowing a logical application name to be
* included in server-side request logging.
* @return The current {@link ConsumerBuilder} instance
*/
public ConsumerBuilder clientId(String value) {
this.props.put("group.id", value);
return this;
}
/**
*
* @param value One of {@link OffsetMode} value. Default value is
* {@link OffsetMode#LATEST}.
* @return The current {@link ConsumerBuilder} instance
*/
public ConsumerBuilder offsetMode(OffsetMode value) {
this.offsetMode = value;
return this;
}
/**
* Default value is {@link OffsetMode#LATEST}.
*
* @return Indicate this consumer in which offset mode
*/
public OffsetMode offsetMode() {
return this.offsetMode;
}
/**
*
* @param topics The Kafka topics will be consumed
* @return the current {@link ConsumerBuilder}
*/
public ConsumerBuilder addTopics(List topics) {
topics.stream().filter((topic) -> !(this.topics.contains(topic)))
.forEachOrdered((topic) -> {
this.topics.add(topic);
});
return this;
}
/**
*
* @return The properties container
*/
Map properties() {
return this.props;
}
/**
*
* @return a new the current {@link KafkaConsumer}
*/
public KafkaConsumer build() {
return new KafkaConsumer(this);
}
}
/**
*
* @return List of consuming topics
*/
public List topics() {
return this.builder.topics;
}
/**
*
* @param messageProcessor The processor will process delivered message
*/
public void start(Consumer messageProcessor) {
Map props = this.builder.properties();
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "3000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("auto.offset.reset", this.builder.offsetMode().asString());
org.apache.kafka.clients.consumer.KafkaConsumer consumer
= new org.apache.kafka.clients.consumer.KafkaConsumer<>(props);
consumer.subscribe(this.builder.topics);
while (true) {
ConsumerRecords records
= consumer.poll(Duration.ofMillis(10000));
for (ConsumerRecord record : records) {
messageProcessor.accept(new KafkaMessage(record));
}
}
}
/**
*
* @param messageProcessor The processor will process delivered message
*/
public void start(BatchConsumer messageProcessor) {
Map props = this.builder.properties();
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "3000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("auto.offset.reset", this.builder.offsetMode.asString());
org.apache.kafka.clients.consumer.KafkaConsumer consumer
= new org.apache.kafka.clients.consumer.KafkaConsumer<>(props);
consumer.subscribe(this.builder.topics);
while (true) {
ConsumerRecords records
= consumer.poll(Duration.ofMillis(10000));
if (records.isEmpty()) {
System.out.println("No records pooled after 10000ms");
continue;
}
for (ConsumerRecord record : records) {
messageProcessor.addToBatch(new KafkaMessage(record));
}
messageProcessor.processBatch();
}
}
@Override
public String toString() {
return "{topics=" + this.builder.topics
+ "}";
}
}