org.graylog2.radio.transports.kafka.KafkaProducer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of graylog2-radio Show documentation
Graylog radio
There is a newer version: 1.3.4
/**
 * This file is part of Graylog.
 *
 * Graylog is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Graylog is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Graylog.  If not, see .
 */
package org.graylog2.radio.transports.kafka;

import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.RadioMessage;
import org.graylog2.plugin.ServerStatus;
import org.graylog2.radio.Configuration;
import org.graylog2.radio.transports.RadioTransport;
import org.msgpack.MessagePack;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Properties;

import static com.codahale.metrics.MetricRegistry.name;

public class KafkaProducer implements RadioTransport {
    private static final Logger LOG = LoggerFactory.getLogger(KafkaProducer.class);
    public static final String KAFKA_TOPIC = "graylog2-radio-messages";

    private final Producer producer;
    private final MessagePack pack;
    private final Meter incomingMessages;
    private final Meter rejectedMessages;
    private final Timer processTime;

    @Inject
    public KafkaProducer(ServerStatus serverStatus, Configuration configuration, MetricRegistry metricRegistry) {
        pack = new MessagePack();

        // Use a separate class loader for msgpack to avoid generation of duplicate class names.
        // The JavaassistTemplateBuilder used by MessagePack uses a sequence number for class naming
        // and is not thread-safe.
        pack.setClassLoader(new ClassLoader(Thread.currentThread().getContextClassLoader()) {});

        Properties props = new Properties();
        props.put("metadata.broker.list", configuration.getKafkaBrokers());
        props.put("partitioner.class", "kafka.producer.DefaultPartitioner");
        props.put("serializer.class", "kafka.serializer.DefaultEncoder");
        props.put("request.required.acks", String.valueOf(configuration.getKafkaRequiredAcks()));
        props.put("client.id", "graylog2-radio-" + serverStatus.getNodeId().toString());
        props.put("producer.type", configuration.getKafkaProducerType());
        props.put("queue.buffering.max.ms", String.valueOf(configuration.getKafkaBatchMaxWaitMs()));
        props.put("batch.num.messages", String.valueOf(configuration.getKafkaBatchSize()));

        ProducerConfig config = new ProducerConfig(props);
        producer = new Producer<>(config);

        incomingMessages = metricRegistry.meter(name(KafkaProducer.class, "incomingMessages"));
        rejectedMessages = metricRegistry.meter(name(KafkaProducer.class, "rejectedMessages"));
        processTime = metricRegistry.timer(name(KafkaProducer.class, "processTime"));
    }

    @Override
    public void send(Message msg) {
        KeyedMessage data;

        try(Timer.Context context = processTime.time()) {
            incomingMessages.mark();
            data = new KeyedMessage<>(
                    KAFKA_TOPIC, msg.getId().getBytes(StandardCharsets.UTF_8), RadioMessage.serialize(pack, msg));

            producer.send(data);
        } catch(IOException e) {
            LOG.error("Could not serialize message.", e);
            rejectedMessages.mark();
        }
    }
}