com.arpnetworking.metrics.mad.Aggregator Maven / Gradle / Ivy
/**
* Copyright 2015 Groupon.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.arpnetworking.metrics.mad;
import com.arpnetworking.commons.builder.OvalBuilder;
import com.arpnetworking.commons.observer.Observable;
import com.arpnetworking.commons.observer.Observer;
import com.arpnetworking.logback.annotations.LogValue;
import com.arpnetworking.metrics.mad.model.Record;
import com.arpnetworking.steno.LogValueMapFactory;
import com.arpnetworking.steno.Logger;
import com.arpnetworking.steno.LoggerFactory;
import com.arpnetworking.tsdcore.model.DefaultKey;
import com.arpnetworking.tsdcore.model.Key;
import com.arpnetworking.tsdcore.sinks.Sink;
import com.arpnetworking.tsdcore.statistics.Statistic;
import com.arpnetworking.utility.Launchable;
import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import net.sf.oval.constraint.NotNull;
import org.joda.time.Period;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
/**
* Performs aggregation of Record instances per Period.
* This class is thread safe.
*
* @author Ville Koskela (ville dot koskela at inscopemetrics dot com)
*/
// NOTE: The _periodWorkerExecutor is accessed both in synchronized lifecycle methods like launch() and shutdown() but
// also non-synchronized methods like notify(). Access to _periodWorkerExecutor does not need to be synchronized.
@SuppressFBWarnings("IS2_INCONSISTENT_SYNC")
public final class Aggregator implements Observer, Launchable {
/**
* {@inheritDoc}
*/
@Override
public synchronized void launch() {
LOGGER.debug()
.setMessage("Launching aggregator")
.addData("aggregator", this)
.log();
_periodWorkers.clear();
if (!_periods.isEmpty()) {
_periodWorkerExecutor = Executors.newCachedThreadPool(r -> new Thread(r, "PeriodWorker"));
}
}
/**
* {@inheritDoc}
*/
@Override
public synchronized void shutdown() {
LOGGER.debug()
.setMessage("Stopping aggregator")
.addData("aggregator", this)
.log();
for (final List periodCloserList : _periodWorkers.values()) {
periodCloserList.forEach(com.arpnetworking.metrics.mad.PeriodWorker::shutdown);
}
_periodWorkers.clear();
if (_periodWorkerExecutor != null) {
_periodWorkerExecutor.shutdown();
try {
_periodWorkerExecutor.awaitTermination(10, TimeUnit.SECONDS);
} catch (final InterruptedException e) {
LOGGER.warn("Unable to shutdown period worker executor", e);
}
_periodWorkerExecutor = null;
}
}
/**
* {@inheritDoc}
*/
@Override
public void notify(final Observable observable, final Object event) {
if (!(event instanceof Record)) {
LOGGER.error()
.setMessage("Observed unsupported event")
.addData("event", event)
.log();
return;
}
final Record record = (Record) event;
final Key key = new DefaultKey(createDimensions(record));
LOGGER.trace()
.setMessage("Processing record")
.addData("record", record)
.addData("key", key)
.log();
for (final PeriodWorker periodWorker : _periodWorkers.computeIfAbsent(key, this::createPeriodWorkers)) {
periodWorker.record(record);
}
}
/**
* Generate a Steno log compatible representation.
*
* @return Steno log compatible representation.
*/
@LogValue
public Object toLogValue() {
return LogValueMapFactory.builder(this)
.put("sink", _sink)
.put("timerStatistics", _specifiedTimerStatistics)
.put("counterStatistics", _specifiedCounterStatistics)
.put("gaugeStatistics", _specifiedGaugeStatistics)
.put("periodWorkers", _periodWorkers)
.build();
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return toLogValue().toString();
}
private ImmutableMap createDimensions(final Record record) {
// TODO(ville): Promote user specified annotations to dimensions.
final ImmutableMap.Builder dimensionBuilder = ImmutableMap.builder();
dimensionBuilder.put(Key.HOST_DIMENSION_KEY, record.getAnnotations().get(Key.HOST_DIMENSION_KEY));
dimensionBuilder.put(Key.SERVICE_DIMENSION_KEY, record.getAnnotations().get(Key.SERVICE_DIMENSION_KEY));
dimensionBuilder.put(Key.CLUSTER_DIMENSION_KEY, record.getAnnotations().get(Key.CLUSTER_DIMENSION_KEY));
return dimensionBuilder.build();
}
private List createPeriodWorkers(final Key key) {
final List periodWorkerList = Lists.newArrayListWithExpectedSize(_periods.size());
for (final Period period : _periods) {
final PeriodWorker periodWorker = new PeriodWorker.Builder()
.setPeriod(period)
.setBucketBuilder(
new Bucket.Builder()
.setKey(key)
.setSpecifiedCounterStatistics(_specifiedCounterStatistics)
.setSpecifiedGaugeStatistics(_specifiedGaugeStatistics)
.setSpecifiedTimerStatistics(_specifiedTimerStatistics)
.setDependentCounterStatistics(_dependentCounterStatistics)
.setDependentGaugeStatistics(_dependentGaugeStatistics)
.setDependentTimerStatistics(_dependentTimerStatistics)
.setSpecifiedStatistics(_cachedSpecifiedStatistics)
.setDependentStatistics(_cachedDependentStatistics)
.setPeriod(period)
.setSink(_sink))
.build();
periodWorkerList.add(periodWorker);
_periodWorkerExecutor.execute(periodWorker);
}
LOGGER.info()
.setMessage("Created period workers")
.addData("key", key)
.addData("periodWorkersSize", periodWorkerList.size())
.log();
return periodWorkerList;
}
private ImmutableSet computeDependentStatistics(final ImmutableSet statistics) {
final ImmutableSet.Builder builder = ImmutableSet.builder();
for (final Statistic statistic : statistics) {
statistic.getDependencies().stream().filter(dependency -> !statistics.contains(dependency)).forEach(builder::add);
}
return builder.build();
}
private Aggregator(final Builder builder) {
_periods = ImmutableSet.copyOf(builder._periods);
_sink = builder._sink;
_specifiedCounterStatistics = ImmutableSet.copyOf(builder._counterStatistics);
_specifiedGaugeStatistics = ImmutableSet.copyOf(builder._gaugeStatistics);
_specifiedTimerStatistics = ImmutableSet.copyOf(builder._timerStatistics);
_dependentCounterStatistics = computeDependentStatistics(_specifiedCounterStatistics);
_dependentGaugeStatistics = computeDependentStatistics(_specifiedGaugeStatistics);
_dependentTimerStatistics = computeDependentStatistics(_specifiedTimerStatistics);
final ImmutableMap.Builder> statisticsBuilder = ImmutableMap.builder();
for (final Map.Entry> entry : builder._statistics.entrySet()) {
final Pattern pattern = Pattern.compile(entry.getKey());
final ImmutableSet statistics = ImmutableSet.copyOf(entry.getValue());
statisticsBuilder.put(pattern, statistics);
}
_statistics = statisticsBuilder.build();
_cachedSpecifiedStatistics = CacheBuilder
.newBuilder()
.concurrencyLevel(1)
.build(
new CacheLoader>>() {
// TODO(vkoskela): Add @NonNull annotation to metric. [ISSUE-?]
@Override
public Optional> load(final String metric) throws Exception {
for (final Map.Entry> entry : _statistics.entrySet()) {
final Pattern pattern = entry.getKey();
final ImmutableSet statistics = entry.getValue();
if (pattern.matcher(metric).matches()) {
return Optional.of(statistics);
}
}
return Optional.absent();
}
});
_cachedDependentStatistics = CacheBuilder
.newBuilder()
.concurrencyLevel(1)
.build(new CacheLoader>>() {
// TODO(vkoskela): Add @NonNull annotation to metric. [ISSUE-?]
@Override
public Optional> load(final String metric) throws Exception {
final Optional> statistics = _cachedSpecifiedStatistics.get(metric);
if (statistics.isPresent()) {
return Optional.of(computeDependentStatistics(statistics.get()));
} else {
return Optional.absent();
}
}
});
}
private final ImmutableSet _periods;
private final Sink _sink;
private final ImmutableSet _specifiedTimerStatistics;
private final ImmutableSet _specifiedCounterStatistics;
private final ImmutableSet _specifiedGaugeStatistics;
private final ImmutableSet _dependentTimerStatistics;
private final ImmutableSet _dependentCounterStatistics;
private final ImmutableSet _dependentGaugeStatistics;
private final ImmutableMap> _statistics;
private final LoadingCache>> _cachedSpecifiedStatistics;
private final LoadingCache>> _cachedDependentStatistics;
private final Map> _periodWorkers = Maps.newConcurrentMap();
private ExecutorService _periodWorkerExecutor = null;
private static final Logger LOGGER = LoggerFactory.getLogger(Aggregator.class);
/**
* Builder implementation for Aggregator.
*/
public static final class Builder extends OvalBuilder {
/**
* Public constructor.
*/
public Builder() {
super(Aggregator::new);
}
/**
* Set the sink. Cannot be null or empty.
*
* @param value The sink.
* @return This Builder instance.
*/
public Builder setSink(final Sink value) {
_sink = value;
return this;
}
/**
* Set the periods. Cannot be null or empty.
*
* @param value The periods.
* @return This Builder instance.
*/
public Builder setPeriods(final Set value) {
_periods = value;
return this;
}
/**
* Set the timer statistics. Cannot be null or empty.
*
* @param value The timer statistics.
* @return This Builder instance.
*/
public Builder setTimerStatistics(final Set value) {
_timerStatistics = value;
return this;
}
/**
* Set the counter statistics. Cannot be null or empty.
*
* @param value The counter statistics.
* @return This Builder instance.
*/
public Builder setCounterStatistics(final Set value) {
_counterStatistics = value;
return this;
}
/**
* Set the gauge statistics. Cannot be null or empty.
*
* @param value The gauge statistics.
* @return This Builder instance.
*/
public Builder setGaugeStatistics(final Set value) {
_gaugeStatistics = value;
return this;
}
/**
* The statistics to compute for a metric pattern. Optional. Cannot be null.
* Default is empty.
*
* @param value The gauge statistics.
* @return This instance of Builder.
*/
public Builder setStatistics(final Map> value) {
_statistics = value;
return this;
}
@NotNull
private Sink _sink;
@NotNull
private Set _periods;
@NotNull
private Set _timerStatistics;
@NotNull
private Set _counterStatistics;
@NotNull
private Set _gaugeStatistics;
@NotNull
private Map> _statistics = Collections.emptyMap();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy