gobblin.metrics.influxdb.InfluxDBReporter Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package gobblin.metrics.influxdb;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import org.influxdb.dto.Point;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Counting;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metered;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.typesafe.config.Config;
import gobblin.configuration.ConfigurationKeys;
import gobblin.metrics.Measurements;
import gobblin.metrics.reporter.ConfiguredScheduledReporter;
import gobblin.util.ConfigUtils;
import static gobblin.metrics.Measurements.*;
/**
* InfluxDB reporter for metrics
*
* @author Lorand Bendig
*
*/
public class InfluxDBReporter extends ConfiguredScheduledReporter {
private final InfluxDBPusher influxDBPusher;
private static final Logger LOGGER = LoggerFactory.getLogger(InfluxDBReporter.class);
public InfluxDBReporter(Builder builder, Config config) {
super(builder, config);
if (builder.influxDBPusher.isPresent()) {
this.influxDBPusher = builder.influxDBPusher.get();
} else {
this.influxDBPusher =
new InfluxDBPusher.Builder(builder.url, builder.username, builder.password, builder.database,
builder.connectionType).build();
}
}
/**
* A static factory class for obtaining new {@link gobblin.metrics.influxdb.InfluxDBReporter.Builder}s
*
* @see gobblin.metrics.influxdb.InfluxDBReporter.Builder
*/
public static class Factory {
public static BuilderImpl newBuilder() {
return new BuilderImpl();
}
}
public static class BuilderImpl extends Builder {
@Override
protected BuilderImpl self() {
return this;
}
}
/**
* Builder for {@link InfluxDBReporter}. Defaults to no filter, reporting rates in seconds and times in
* milliseconds using TCP sending type
*/
public static abstract class Builder> extends
ConfiguredScheduledReporter.Builder {
protected MetricFilter filter;
protected String url;
protected String username;
protected String password;
protected String database;
protected InfluxDBConnectionType connectionType;
protected Optional influxDBPusher;
protected Builder() {
super();
this.name = "InfluxDBReporter";
this.influxDBPusher = Optional.absent();
this.filter = MetricFilter.ALL;
this.connectionType = InfluxDBConnectionType.TCP;
}
/**
* Set {@link gobblin.metrics.influxdb.InfluxDBPusher} to use.
*/
public T withInfluxDBPusher(InfluxDBPusher pusher) {
this.influxDBPusher = Optional.of(pusher);
return self();
}
/**
* Set connection parameters for the {@link gobblin.metrics.influxdb.InfluxDBPusher} creation
*/
public T withConnection(String url, String username, String password, String database) {
this.url = url;
this.username = username;
this.password = password;
this.database = database;
return self();
}
/**
* Set {@link gobblin.metrics.influxdb.InfluxDBConnectionType} to use.
*/
public T withConnectionType(InfluxDBConnectionType connectionType) {
this.connectionType = connectionType;
return self();
}
/**
* Only report metrics which match the given filter.
*
* @param filter a {@link MetricFilter}
* @return {@code this}
*/
public T filter(MetricFilter filter) {
this.filter = filter;
return self();
}
/**
* Builds and returns {@link InfluxDBReporter}.
*
* @return InfluxDBReporter
*/
public InfluxDBReporter build(Properties props) throws IOException {
return new InfluxDBReporter(this, ConfigUtils.propertiesToConfig(props,
Optional.of(ConfigurationKeys.METRICS_CONFIGURATIONS_PREFIX)));
}
}
@Override
protected void report(SortedMap gauges, SortedMap counters,
SortedMap histograms, SortedMap meters, SortedMap timers,
Map tags) {
String prefix = getMetricNamePrefix(tags);
long timestamp = System.currentTimeMillis();
List points = Lists.newArrayList();
try {
for (Map.Entry gauge : gauges.entrySet()) {
reportGauge(points, prefix, gauge.getKey(), gauge.getValue(), timestamp);
}
for (Map.Entry counter : counters.entrySet()) {
reportCounter(points, prefix, counter.getKey(), counter.getValue(), timestamp);
}
for (Map.Entry histogram : histograms.entrySet()) {
reportHistogram(points, prefix, histogram.getKey(), histogram.getValue(), timestamp);
}
for (Map.Entry meter : meters.entrySet()) {
reportMetered(points, prefix, meter.getKey(), meter.getValue(), timestamp);
}
for (Map.Entry timer : timers.entrySet()) {
reportTimer(points, prefix, timer.getKey(), timer.getValue(), timestamp);
}
influxDBPusher.push(points);
} catch (IOException ioe) {
LOGGER.error("Error sending metrics to InfluxDB", ioe);
}
}
private void reportGauge(List points, String prefix, String name, Gauge gauge, long timestamp)
throws IOException {
String metricName = getKey(prefix, name);
points.add(buildMetricAsPoint(metricName, gauge.getValue(), timestamp));
}
private void reportCounter(List points, String prefix, String name, Counting counter, long timestamp)
throws IOException {
String metricName = getKey(prefix, name, COUNT.getName());
points.add(buildMetricAsPoint(metricName, counter.getCount(), false, timestamp));
}
private void reportHistogram(List points, String prefix, String name, Histogram histogram, long timestamp)
throws IOException {
reportCounter(points, prefix, name, histogram, timestamp);
reportSnapshot(points, prefix, name, histogram.getSnapshot(), timestamp, false);
}
private void reportTimer(List points, String prefix, String name, Timer timer, long timestamp)
throws IOException {
reportSnapshot(points, prefix, name, timer.getSnapshot(), timestamp, true);
reportMetered(points, prefix, name, timer, timestamp);
}
private void reportSnapshot(List points, String prefix, String name, Snapshot snapshot, long timestamp,
boolean convertDuration) throws IOException {
String baseMetricName = getKey(prefix, name);
points.add(buildMetricAsPoint(getKey(baseMetricName, MIN), snapshot.getMin(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, MAX), snapshot.getMax(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, MEAN), snapshot.getMean(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, STDDEV), snapshot.getStdDev(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, MEDIAN), snapshot.getMedian(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, PERCENTILE_75TH), snapshot.get75thPercentile(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, PERCENTILE_95TH), snapshot.get95thPercentile(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, PERCENTILE_98TH), snapshot.get98thPercentile(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, PERCENTILE_99TH), snapshot.get99thPercentile(), convertDuration, timestamp));
points.add(buildMetricAsPoint(getKey(baseMetricName, PERCENTILE_999TH), snapshot.get999thPercentile(), convertDuration, timestamp));
}
private void reportMetered(List points, String prefix, String name, Metered metered, long timestamp)
throws IOException {
reportCounter(points,prefix, name, metered, timestamp);
String baseMetricName = getKey(prefix, name);
points.add(buildRateAsPoint(getKey(baseMetricName, RATE_1MIN), metered.getOneMinuteRate(), timestamp));
points.add(buildRateAsPoint(getKey(baseMetricName, RATE_5MIN), metered.getFiveMinuteRate(), timestamp));
points.add(buildRateAsPoint(getKey(baseMetricName, RATE_15MIN), metered.getFifteenMinuteRate(), timestamp));
points.add(buildRateAsPoint(getKey(baseMetricName, MEAN_RATE), metered.getMeanRate(), timestamp));
}
private Point buildMetricAsPoint(String metricName, Number value, boolean toDuration, long timestamp)
throws IOException {
Number metricValue = toDuration ? convertDuration(value.doubleValue()) : value;
return buildMetricAsPoint(metricName, metricValue, timestamp);
}
private Point buildRateAsPoint(String metricName, double value, long timestamp)
throws IOException {
return buildMetricAsPoint(metricName, convertRate(value), timestamp);
}
private Point buildMetricAsPoint(String name, Object value, long timestamp) throws IOException {
return Point.measurement(name).field("value", value).time(timestamp, TimeUnit.MILLISECONDS).build();
}
private String getKey(String baseName, Measurements measurements) {
return getKey(baseName, measurements.getName());
}
private String getKey(String... keys) {
return JOINER.join(keys);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy