All Downloads are FREE. Search and download functionalities are using the official Maven repository.

net.opentsdb.tsd.KafkaRpcPlugin Maven / Gradle / Ivy

Go to download

A consumer and publisher for OpenTSDB Kafka messages

There is a newer version: 2.4.0
Show newest version
// This file is part of OpenTSDB.
// Copyright (C) 2018  The OpenTSDB Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package net.opentsdb.tsd;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.jboss.netty.util.Timeout;
import org.jboss.netty.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Maps;
import com.google.common.util.concurrent.AtomicDouble;
import com.stumbleupon.async.Deferred;

import net.opentsdb.core.TSDB;
import net.opentsdb.stats.StatsCollector;
import net.opentsdb.tsd.KafkaRpcPluginThread.CounterType;

public class KafkaRpcPlugin extends RpcPlugin implements TimerTask {
  private static final Logger LOG = LoggerFactory.getLogger(
      KafkaRpcPlugin.class);

  public static KafkaRpcPlugin KAFKA_RPC_REFERENCE = null;
  
  private final String host;
  
  private ExecutorService thread_pool;
  
  private List consumer_groups;
  
  private KafkaRpcPluginConfig config;
  
  private boolean track_metric_prefix;
  
  private TSDB tsdb;
  
  private final ConcurrentHashMap> namespace_counters;
  
  private final Map totals_counters;
  private final AtomicLong messages_received = new AtomicLong();
  private final AtomicLong datapoints_received = new AtomicLong();
  private final AtomicLong deserialization_errors = new AtomicLong();
  private final AtomicLong restarts = new AtomicLong();
  private final AtomicLong rebalance_failures = new AtomicLong();
  private final AtomicDouble cumulative_rate_delay = new AtomicDouble();
  private final AtomicDouble kafka_wait_time = new AtomicDouble();
  
  public KafkaRpcPlugin() {
    try {
      host = InetAddress.getLocalHost().getHostName();
    } catch (UnknownHostException e) {
      LOG.error("Unable to get hostname!", e);
      throw new RuntimeException("WTF? Shouldn't be here", e);
    }
    thread_pool = Executors.newCachedThreadPool();
    namespace_counters = new ConcurrentHashMap>(
        CounterType.values().length);
    totals_counters = Maps.newHashMapWithExpectedSize(CounterType.values().length);
  }
  
  @Override
  public void initialize(final TSDB tsdb) {
    this.tsdb = tsdb;
    config = new KafkaRpcPluginConfig(tsdb.getConfig());
    consumer_groups = createConsumerGroups();
    LOG.info("Launching " + consumer_groups.size() + " Kafka consumer groups...");
    for (final KafkaRpcPluginGroup group : consumer_groups) {
      group.start();
    }
    LOG.info("Launched " + consumer_groups.size() + " Kafka consumer groups");
    tsdb.getTimer().newTimeout(this, 100, TimeUnit.MILLISECONDS);
    // Sync just in case the HTTP plugin loads or tries to fetch stats before 
    // we finish initializing.
    synchronized (tsdb) {
      KAFKA_RPC_REFERENCE = this;
    }
    LOG.info("Initialized KafkaRpcPlugin.");
  }

  @Override
  public Deferred shutdown() {
    LOG.info("Shutting down KafkaRpcPlugin.");
    try {
      if (consumer_groups != null) {
        for (final KafkaRpcPluginGroup group : consumer_groups) {
          try {
            group.shutdown();
          } catch (Throwable e) {
            LOG.error("Failed shutting down Kafka Consumer: " + group 
                + ". Continuing with shutdown.", e);
          }
        }
      }
    } catch (Exception e) {
      LOG.error("Uncaught exception during shutdown", e);
      return Deferred.fromError(e);
    } catch (Throwable e) {
      LOG.error("Fatal exception in Kafka thread: " + this, e);
      return Deferred.fromError(new RuntimeException("WTF? Unexpected exception "
          + "shutting down Kafka consumer.", e));
    }
    LOG.info("Successfully shutdown KafkaRpcPlugin.");
    return Deferred.fromResult(null);
  }

  @Override
  public String version() {
    return "2.4.0";
  }

  @Override
  public void collectStats(final StatsCollector collector) {
    for (final Entry entry : totals_counters.entrySet()) {
      collector.record("KafkaRpcPlugin." + entry.getKey(), entry.getValue().get());
    }
  }

  /**
   * Returns the reference for the shared namespace counters map for updating.
   * @return The master counters map.
   */
  public ConcurrentMap> getNamespaceCounters() {
    return namespace_counters;
  }
  
  public boolean trackMetricPrefix() {
    return track_metric_prefix;
  }
  
  KafkaRpcPluginConfig getConfig() {
    return config;
  }
    
  ExecutorService getKafkaPool() {
    return thread_pool;
  }
  
  TSDB getTSDB() {
    return tsdb;
  }
  
  String getHost() {
    return host;
  }

  /**
   * Creates a set of kafka consumer groups
   * @throws IllegalArgumentException if a group couldn't be set up due to
   * missing configuration parameters.
   */
  List createConsumerGroups() {
    final String group_list = config.getString(
        KafkaRpcPluginConfig.PLUGIN_PROPERTY_BASE + "groups");
    if (group_list == null || group_list.isEmpty()) {
        throw new IllegalArgumentException("Missing " + 
            KafkaRpcPluginConfig.PLUGIN_PROPERTY_BASE + "groups");
    }
    final String[] groups = group_list.split(",");
    
    final List consumerGroups = 
        new ArrayList(groups.length);
    for (final String group : groups) {
        final KafkaRpcPluginGroup consumerGroup = 
            new KafkaRpcPluginGroup(this, group);
        consumerGroups.add(consumerGroup);
    }
    return consumerGroups;
  }

  /**
   * This is the TimerTask that runs periodically to perform metrics
   * aggregation into the totals map.
   */
  @Override
  public void run(final Timeout timeout) throws Exception {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Running metric aggregation in Kafka RPC plugin.");
    }
    
    try {
      final Map temp_totals_counters = 
          new HashMap(CounterType.values().length);  
      
      long tempMessageReceived = 0;
      long tempDatapointsReceived = 0;
      long tempDeserializationError = 0;
      long tempRestarts = 0;
      long tempRebalanceFailures = 0;
      double tempCumulativeRateDelay = 0;
      double tempKafkaWaitTime = 0;
      
      for (final KafkaRpcPluginGroup group : consumer_groups) {
        //group.getNamespaceCounters(tempNamespaceCounters);
        tempMessageReceived += group.getMessagesReceived();
        tempDatapointsReceived += group.getDatapointsReceived();
        tempDeserializationError += group.getDeserializationErrors();
        tempRestarts += group.getRestarts();
        tempRebalanceFailures += group.getRebalanceFailures();
        tempCumulativeRateDelay += group.getCumulativeRateDelay();
        tempKafkaWaitTime += group.getKafkaWaitTime();
      }
      
      messages_received.set(tempMessageReceived);
      datapoints_received.set(tempDatapointsReceived);
      deserialization_errors.set(tempDeserializationError);
      restarts.set(tempRestarts);
      rebalance_failures.set(tempRebalanceFailures);
      cumulative_rate_delay.set(tempCumulativeRateDelay);
      kafka_wait_time.set(tempKafkaWaitTime);
      
      // agg the namespace counters into total
      for (final Entry> namespaceCountersMap : 
          namespace_counters.entrySet()) {
        for (final Entry counter : 
            namespaceCountersMap.getValue().entrySet()) {
          Long val = temp_totals_counters.get(namespaceCountersMap.getKey());
          if (val == null) {
            temp_totals_counters.put(namespaceCountersMap.getKey(), counter.getValue().get());
          } else {
            temp_totals_counters.put(namespaceCountersMap.getKey(), counter.getValue().get() + val);
          }
        }
      }
      
      // replace the master totals now
      for (Entry total : temp_totals_counters.entrySet()) {
        AtomicLong value = totals_counters.get(total.getKey());
        if (value == null) {
          value = new AtomicLong();
          totals_counters.put(total.getKey(), value);
        }
        value.set(total.getValue());
      }
    } catch (Exception e) {
      LOG.error("Unexpected exception in the Metrics thread", e);
    } catch (Throwable e) {
      LOG.error("Fatal exception in the Metrics thread", e);
      System.exit(1);
    }
    
    // always remember to reschedule!
    tsdb.getTimer().newTimeout(this, config.getLong(
        KafkaRpcPluginConfig.METRIC_AGG_FREQUENCY), 
        TimeUnit.MILLISECONDS);
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("Completed metric aggregation in Kafka RPC plugin.");
    }
  }

  public Map getRates() {
    final Map map = 
        new HashMap(consumer_groups.size());
    for (final KafkaRpcPluginGroup group : consumer_groups) {
      map.put(group.getGroupID(), group.getRate());
    }
    return map;
  }
  
  public void setRate(final String group_id, final double rate) {
    if (group_id == null || group_id.isEmpty()) {
      throw new IllegalArgumentException("GroupID cannot be null or empty");
    }
    if (rate < 0) {
      throw new IllegalArgumentException("Rate cannot be less than zero: " + rate);
    }
    
    for (final KafkaRpcPluginGroup group : consumer_groups) {
      if (group.getGroupID().toLowerCase().equals(group_id.toLowerCase())) {
        group.setRate(rate);
        return;
      }
    }
    
    throw new IllegalArgumentException("GroupID " + group_id + " was not found");
  }
  
  public final Map>> getPerThreadStats() {
    final Map>> map = 
        new HashMap>>();
    for (final KafkaRpcPluginGroup group : consumer_groups) {
      map.put(group.getGroupID(), group.getPerThreadMetrics());
    }
    return map;
  }
}
    

    

    

            

    

            



    
    






    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy