com.github.ddth.kafka.internal.KafkaConsumer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of ddth-kafka Show documentation
Show all versions of ddth-kafka Show documentation
DDTH's Kafka Libraries and Utilities
package com.github.ddth.kafka.internal;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.ddth.kafka.AbstractKafkaMessagelistener;
import com.github.ddth.kafka.IKafkaMessageListener;
import com.github.ddth.kafka.KafkaClient;
import com.github.ddth.kafka.KafkaMessage;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
/**
* A simple Kafka consumer client.
*
*
* Each {@link KafkaConsumer} is associated with a unique consumer-group-id.
*
*
*
* One single {@link KafkaConsumer} is used to consume messages from multiple
* topics.
*
*
* @author Thanh Ba Nguyen
* @since 1.0.0
*/
public class KafkaConsumer {
private final Logger LOGGER = LoggerFactory.getLogger(KafkaConsumer.class);
private String consumerGroupId;
private boolean consumeFromBeginning = false;
/* Mapping {topic -> consumer-connector} */
private ConcurrentMap topicConsumerConnectors = new ConcurrentHashMap();
/* Mapping {topic -> [kafka-message-listerners]} */
private Multimap topicMessageListeners = HashMultimap.create();
/* Mapping {topic -> [kafka-consumer-workers]} */
private Multimap topicConsumerWorkers = HashMultimap.create();
private KafkaClient kafkaClient;
/**
* Constructs an new {@link KafkaConsumer} object.
*/
public KafkaConsumer(KafkaClient kafkaClient, String consumerGroupId) {
this.kafkaClient = kafkaClient;
this.consumerGroupId = consumerGroupId;
}
/**
* Constructs an new {@link KafkaConsumer} object.
*/
public KafkaConsumer(KafkaClient kafkaClient, String consumerGroupId,
boolean consumeFromBeginning) {
this.kafkaClient = kafkaClient;
this.consumerGroupId = consumerGroupId;
this.consumeFromBeginning = consumeFromBeginning;
}
/**
* Each Kafka consumer is associated with a consumer group id.
*
*
* If two or more consumers have a same group-id, and consume messages from
* a same topic: messages will be consumed just like a queue: no message is
* consumed by more than one consumer. Which consumer consumes which message
* is undetermined.
*
*
*
* If two or more consumers with different group-ids, and consume messages
* from a same topic: messages will be consumed just like publish-subscribe
* pattern: one message is consumed by all consumers.
*
*
* @return
*/
public String getConsumerGroupId() {
return consumerGroupId;
}
/**
* See {@link #getConsumerGroupId()}.
*
* @param consumerGroupId
* @return
*/
public KafkaConsumer setConsumerGroupId(String consumerGroupId) {
this.consumerGroupId = consumerGroupId;
return this;
}
/**
* Consume messages from the beginning? See {@code auto.offset.reset} option
* at http://kafka.apache.org/08/configuration.html.
*
* @return
*/
public boolean isConsumeFromBeginning() {
return consumeFromBeginning;
}
/**
* Alias of {@link #isConsumeFromBeginning()}.
*
* @return
*/
public boolean getConsumeFromBeginning() {
return consumeFromBeginning;
}
/**
* Consume messages from the beginning? See {@code auto.offset.reset} option
* at http://kafka.apache.org/08/configuration.html.
*
* @param consumeFromBeginning
* @return
*/
public KafkaConsumer setConsumeFromBeginning(boolean consumeFromBeginning) {
this.consumeFromBeginning = consumeFromBeginning;
return this;
}
/**
* Initializing method.
*/
public void init() {
}
/**
* Destroying method.
*/
public void destroy() {
Set topicNames = new HashSet(topicConsumerConnectors.keySet());
for (String topic : topicNames) {
try {
_removeConsumer(topic);
} catch (Exception e) {
LOGGER.warn(e.getMessage(), e);
}
}
}
/**
* Builds consumer configurations.
*
* @param consumerGroupId
* @param consumeFromBeginning
* @param autoCommitOffset
* @return
*/
private ConsumerConfig _buildConsumerConfig(String consumerGroupId,
boolean consumeFromBeginning, boolean autoCommitOffset, boolean leaderAutoRebalance) {
Properties props = new Properties();
props.put("zookeeper.connect", kafkaClient.getZookeeperConnectString());
props.put("group.id", consumerGroupId);
props.put("zookeeper.session.timeout.ms", "600000");
props.put("zookeeper.connection.timeout.ms", "10000");
props.put("zookeeper.sync.time.ms", "2000");
props.put("socket.timeout.ms", "5000");
props.put("fetch.wait.max.ms", "2000");
props.put("auto.offset.reset", consumeFromBeginning ? "smallest" : "largest");
/*
* New in v1.1.1
*/
if (leaderAutoRebalance) {
props.put("auto.leader.rebalance.enable", "true");
props.put("rebalance.backoff.ms", "10000");
props.put("refresh.leader.backoff.ms", "1000");
} else {
props.put("auto.leader.rebalance.enable", "false");
}
props.put("controlled.shutdown.enable", "true");
props.put("controlled.shutdown.max.retries", "5");
props.put("controlled.shutdown.retry.backoff.ms", "10000");
if (autoCommitOffset) {
props.put("auto.commit.enable", "true");
props.put("auto.commit.interval.ms", "1000");
} else {
props.put("auto.commit.enable", "false");
}
return new ConsumerConfig(props);
}
/**
* Creates a consumer for a topic.
*
* @param topic
* @param autoCommitOffset
* @return
*/
private ConsumerConnector _createConsumer(String topic, boolean autoCommitOffset,
boolean leaderAutoRebalance) {
ConsumerConfig consumerConfig = _buildConsumerConfig(consumerGroupId, consumeFromBeginning,
autoCommitOffset, leaderAutoRebalance);
ConsumerConnector consumer = Consumer.createJavaConsumerConnector(consumerConfig);
return consumer;
}
/**
* Prepares worker(s) to consume messages from a topic.
*
* @param topic
* @param singleThread
* if {@code true}, always create one thread to consume message,
* {@code false} will create number of threads equals to number
* of topic's partitions
* @param consumer
* @param autoCommitOffset
*/
private void _initConsumerWorkers(String topic, boolean singleThread,
ConsumerConnector consumer, boolean autoCommitOffset) {
Map topicCountMap = new HashMap();
int numThreads = kafkaClient.getTopicNumPartitions(topic);
if (numThreads < 1 || singleThread) {
numThreads = 1;
}
topicCountMap.put(topic, new Integer(numThreads));
Map>> consumerMap = consumer
.createMessageStreams(topicCountMap);
List> streams = consumerMap.get(topic);
for (KafkaStream stream : streams) {
/* Note: Multimap.get() never returns null */
/*
* Note: Changes make to the returned collection will update the
* underlying multimap, and vice versa.
*/
Collection messageListeners = topicMessageListeners.get(topic);
KafkaConsumerWorker worker = new KafkaConsumerWorker(consumer, autoCommitOffset,
stream, messageListeners);
topicConsumerWorkers.put(topic, worker);
kafkaClient.submitTask(worker);
}
}
private ConsumerConnector _initConsumer(String topic, boolean singleThread,
boolean leaderAutoRebalance) {
ConsumerConnector consumer = topicConsumerConnectors.get(topic);
if (consumer == null) {
consumer = _createConsumer(topic, !singleThread, leaderAutoRebalance);
ConsumerConnector existingConsumer = topicConsumerConnectors.putIfAbsent(topic,
consumer);
if (existingConsumer != null) {
consumer.shutdown();
consumer = existingConsumer;
} else {
_initConsumerWorkers(topic, singleThread, consumer, !singleThread);
}
}
return consumer;
}
private void _removeConsumer(String topic) {
// cleanup workers for a topic-consumer
Collection workers = topicConsumerWorkers.removeAll(topic);
if (workers != null) {
for (KafkaConsumerWorker worker : workers) {
try {
worker.stop();
} catch (Exception e) {
LOGGER.warn(e.getMessage(), e);
}
}
}
// cleanup message-listeners
@SuppressWarnings("unused")
Collection listeners = topicMessageListeners.removeAll(topic);
// finally, cleanup the consumer-connector
try {
ConsumerConnector consumer = topicConsumerConnectors.remove(topic);
if (consumer != null) {
consumer.shutdown();
}
} catch (Exception e) {
LOGGER.warn(e.getMessage(), e);
}
}
/**
* Adds a message listener for a topic.
*
* @param topic
* @param messageListener
* @return {@code true} if successful, {@code false} otherwise (the listener
* may have been added already)
*/
public boolean addMessageListener(String topic, IKafkaMessageListener messageListener) {
synchronized (topicMessageListeners) {
if (!topicMessageListeners.put(topic, messageListener)) {
return false;
}
_initConsumer(topic, false, true);
return true;
}
}
/**
* Adds a message listener for a topic.
*
* @param topic
* @param messageListener
* @param singleThread
* @param leaderAutoRebalance
* allow leadership rebalance (see
* http://kafka.apache.org/documentation.html)
* @return {@code true} if successful, {@code false} otherwise (the listener
* may have been added already)
*/
public boolean addMessageListener(String topic, IKafkaMessageListener messageListener,
boolean singleThread, boolean leaderAutoRebalance) {
synchronized (topicMessageListeners) {
if (!topicMessageListeners.put(topic, messageListener)) {
return false;
}
_initConsumer(topic, singleThread, leaderAutoRebalance);
return true;
}
}
/**
* Removes a topic message listener.
*
* @param topic
* @param messageListener
* @return {@code true} if successful, {@code false} otherwise (the topic
* may have no such listener added before)
*/
public boolean removeMessageListener(String topic, IKafkaMessageListener messageListener) {
synchronized (topicMessageListeners) {
if (!topicMessageListeners.remove(topic, messageListener)) {
return false;
}
// Collection listeners =
// topicListeners.get(topic);
// if (listeners == null || listeners.size() == 0) {
// // no more listeners, remove the consumer
// removeConsumer(topic);
// }
}
return true;
}
/**
* Consumes one message from a topic.
*
*
* This method blocks until message is available.
*
*
* @param topic
* @return
* @throws InterruptedException
*/
synchronized public KafkaMessage consume(String topic) throws InterruptedException {
final BlockingQueue buffer = new LinkedBlockingQueue();
final IKafkaMessageListener listener = new AbstractKafkaMessagelistener(topic, this) {
@Override
public void onMessage(KafkaMessage message) {
removeMessageListener(message.topic(), this);
buffer.add(message);
}
};
addMessageListener(topic, listener, true, false);
KafkaMessage result = buffer.take();
removeMessageListener(topic, listener);
return result;
}
/**
* Consumes one message from a topic, wait up to specified wait time.
*
* @param topic
* @param waitTime
* @param waitTimeUnit
* @return {@code null} if there is no message available
* @throws InterruptedException
*/
synchronized public KafkaMessage consume(String topic, long waitTime, TimeUnit waitTimeUnit)
throws InterruptedException {
final BlockingQueue buffer = new LinkedBlockingQueue();
final IKafkaMessageListener listener = new AbstractKafkaMessagelistener(topic, this) {
@Override
public void onMessage(KafkaMessage message) {
removeMessageListener(message.topic(), this);
buffer.add(message);
}
};
addMessageListener(topic, listener, true, false);
KafkaMessage result = buffer.poll(waitTime, waitTimeUnit);
removeMessageListener(topic, listener);
if (result == null) {
result = buffer.poll();
}
return result;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy