com.github.ddth.kafka.internal.KafkaConsumerWorker Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ddth-kafka Show documentation
DDTH's Kafka Libraries and Utilities
There is a newer version: 2.0.0
package com.github.ddth.kafka.internal;

import java.util.Collection;
import java.util.HashSet;
import java.util.concurrent.CountDownLatch;

import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.ddth.kafka.IKafkaMessageListener;
import com.github.ddth.kafka.KafkaMessage;

public class KafkaConsumerWorker implements Runnable {

    private Logger LOGGER = LoggerFactory.getLogger(KafkaConsumerWorker.class);

    private KafkaStream kafkaStream;
    private ConsumerConnector consumer;
    private boolean autoCommitOffset;

    private boolean stop = false;
    private Collection messageListerners;

    public KafkaConsumerWorker(ConsumerConnector consumer, boolean autoCommitOffset,
            KafkaStream kafkaStream,
            Collection messageListerners) {
        this.consumer = consumer;
        this.autoCommitOffset = autoCommitOffset;
        this.kafkaStream = kafkaStream;
        this.messageListerners = messageListerners;
    }

    /**
     * Stops the worker.
     */
    public void stop() {
        stop = true;
    }

    @Override
    public void run() {
        ConsumerIterator it = kafkaStream.iterator();
        while (!stop) {
            Collection msgListeners = new HashSet();
            MessageAndMetadata mm = null;
            synchronized (messageListerners) {
                if (messageListerners.size() > 0 && it.hasNext()) {
                    msgListeners.addAll(messageListerners);
                    mm = it.next();
                }
            }

            if (msgListeners.size() > 0 && mm != null) {
                /**
                 * Got a message, convert it to {@link KafkaMessage} object.
                 */
                final KafkaMessage message = new KafkaMessage(mm);

                final CountDownLatch countDownLatch = new CountDownLatch(msgListeners.size());
                for (final IKafkaMessageListener listerner : msgListeners) {
                    // Delivery the consumed message to n-listeners
                    // asynchronously
                    Thread t = new Thread("Kafka-Consumer-Delivery") {
                        public void run() {
                            try {
                                listerner.onMessage(message);
                            } catch (Exception e) {
                                LOGGER.warn(e.getMessage(), e);
                            } finally {
                                countDownLatch.countDown();
                            }
                        }
                    };
                    t.start();
                }
                try {
                    countDownLatch.await();
                    if (!autoCommitOffset) {
                        consumer.commitOffsets(true);
                    }
                } catch (InterruptedException e) {
                    LOGGER.warn(e.getMessage(), e);
                }
            } else {
                Thread.yield();
            }
        }
    }
}