com.pinterest.doctorkafka.tools.BrokerStatsReader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of kafkastats Show documentation
Collectoring the kafka stats for DoctorKafka
There is a newer version: 0.3.0-rc.2
package com.pinterest.doctorkafka.tools;

import com.pinterest.doctorkafka.BrokerStats;
import com.pinterest.doctorkafka.util.OperatorUtil;
import com.pinterest.doctorkafka.util.KafkaUtils;

import org.apache.avro.Schema;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Arrays;
import java.util.Properties;

public class BrokerStatsReader {

  private static final Logger LOG = LogManager.getLogger(BrokerStatsReader.class);
  private static final String ZOOKEEPER = "zookeeper";
  private static final String STATS_TOPIC = "topic";
  private static final String SECURITY_PROTOCOL = "security.protocol";
  private static final DecoderFactory avroDecoderFactory = DecoderFactory.get();

  private static final Options options = new Options();

  private static CommandLine parseCommandLine(String[] args) {
    if (args.length < 4) {
      printUsageAndExit();
    }

    Option zookeeper = new Option(ZOOKEEPER, true, "zookeeper connection string");
    Option statsTopic = new Option(STATS_TOPIC, true, "kafka topic for broker stats");
    Option securityProtcol = new Option(SECURITY_PROTOCOL, true, "security protocol for reading from kafka");
    options.addOption(zookeeper).addOption(statsTopic).addOption(securityProtcol);

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = null;
    try {
      cmd = parser.parse(options, args);
    } catch (ParseException | NumberFormatException e) {
      printUsageAndExit();
    }
    return cmd;
  }

  private static void printUsageAndExit() {
    HelpFormatter formatter = new HelpFormatter();
    formatter.printHelp("BrokerStatsReader", options);
    System.exit(1);
  }

  public static void main(String[] args) throws Exception {
    CommandLine commandLine = parseCommandLine(args);
    String zkUrl = commandLine.getOptionValue(ZOOKEEPER);
    String statsTopic = commandLine.getOptionValue(STATS_TOPIC);

    String bootstrapBrokers = OperatorUtil.getBrokers(zkUrl, SecurityProtocol.PLAINTEXT);
    Properties props = new Properties();
    props.put(KafkaUtils.BOOTSTRAP_SERVERS, bootstrapBrokers);
    props.put("group.id", "broker_statsreader_group");
    props.put("enable.auto.commit", "false");
    props.put("auto.commit.interval.ms", "1000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");

    Schema schema = BrokerStats.getClassSchema();
    KafkaConsumer consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(statsTopic));
    while (true) {
      ConsumerRecords records = consumer.poll(100);
      for (ConsumerRecord record : records) {
        System.out.printf("offset = %d, key.size = %d, value.size = %s%n",
            record.offset(), record.key().length, record.value().length);
        try {
          BinaryDecoder binaryDecoder = avroDecoderFactory.binaryDecoder(record.value(), null);
          SpecificDatumReader reader = new SpecificDatumReader<>(schema);
          BrokerStats result = new BrokerStats();
          reader.read(result, binaryDecoder);
          System.out.println(result);
        } catch (Exception e) {
          LOG.error("Fail to decode an message", e);
        }
      }
    }
  }
}