Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* Copyright 2015 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.servo.publish.atlas;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.netflix.archaius.config.EmptyConfig;
import com.netflix.servo.Metric;
import com.netflix.servo.annotations.DataSourceType;
import com.netflix.servo.monitor.BasicCounter;
import com.netflix.servo.monitor.BasicGauge;
import com.netflix.servo.monitor.Counter;
import com.netflix.servo.monitor.Gauge;
import com.netflix.servo.monitor.MonitorConfig;
import com.netflix.servo.monitor.Monitors;
import com.netflix.servo.monitor.Pollers;
import com.netflix.servo.monitor.Stopwatch;
import com.netflix.servo.monitor.Timer;
import com.netflix.servo.publish.MetricObserver;
import com.netflix.servo.tag.BasicTag;
import com.netflix.servo.tag.BasicTagList;
import com.netflix.servo.tag.Tag;
import com.netflix.servo.tag.TagList;
import iep.com.netflix.iep.http.BasicServerRegistry;
import iep.com.netflix.iep.http.RxHttp;
import iep.io.reactivex.netty.protocol.http.client.HttpClientResponse;
import io.netty.buffer.ByteBuf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.functions.Func1;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
/**
* Observer that forwards metrics to atlas. In addition to being MetricObserver, it also supports
* a push model that sends metrics as soon as possible (asynchronously).
*/
public class AtlasMetricObserver implements MetricObserver {
private static final Logger LOGGER = LoggerFactory.getLogger(AtlasMetricObserver.class);
private static final Tag ATLAS_COUNTER_TAG = new BasicTag("atlas.dstype", "counter");
private static final Tag ATLAS_GAUGE_TAG = new BasicTag("atlas.dstype", "gauge");
private static final UpdateTasks NO_TASKS = new UpdateTasks(0, null, -1L);
private static final String FILE_DATE_FORMAT = "yyyy_MM_dd_HH_mm_ss_SSS";
private final JsonFactory jsonFactory = new JsonFactory();
protected final HttpHelper httpHelper;
protected final ServoAtlasConfig config;
protected final long sendTimeoutMs; // in milliseconds
protected final long stepMs; // in milliseconds
private final Counter numMetricsTotal = Monitors.newCounter("numMetricsTotal");
private final Timer updateTimer = Monitors.newTimer("update");
private final Counter numMetricsDroppedSendTimeout = newErrCounter("numMetricsDropped",
"sendTimeout");
private final Counter numMetricsDroppedQueueFull = newErrCounter("numMetricsDropped",
"sendQueueFull");
private final Counter numMetricsDroppedHttpErr = newErrCounter("numMetricsDropped",
"httpError");
private final Counter numMetricsSent = Monitors.newCounter("numMetricsSent");
private final TagList commonTags;
private final BlockingQueue pushQueue;
@SuppressWarnings("unused")
private final Gauge pushQueueSize = new BasicGauge<>(
MonitorConfig.builder("pushQueue").build(), new Callable() {
@Override
public Integer call() throws Exception {
return pushQueue.size();
}
});
protected boolean shouldDumpPayload() {
return false;
}
protected String getPayloadDirectory() {
String tmp = System.getProperty("java.io.tmpdir");
return tmp != null ? tmp : "/tmp";
}
private final Thread pushThread;
private final AtomicBoolean shouldPushMetrics = new AtomicBoolean(true);
/**
* Create an observer that can send metrics to atlas with a given
* config and list of common tags.
* This method will use the default poller index of 0.
*/
public AtlasMetricObserver(ServoAtlasConfig config, TagList commonTags) {
this(config, commonTags, 0);
}
/**
* Create an observer that can send metrics to atlas with a given config, list of common tags,
* and poller index.
*/
public AtlasMetricObserver(ServoAtlasConfig config, TagList commonTags, int pollerIdx) {
this(config, commonTags, pollerIdx,
new HttpHelper(new RxHttp(EmptyConfig.INSTANCE, new BasicServerRegistry())));
}
/**
* Create an atlas observer. For internal use of servo only.
*/
public AtlasMetricObserver(ServoAtlasConfig config,
TagList commonTags,
int pollerIdx,
HttpHelper httpHelper) {
this.httpHelper = httpHelper;
this.config = config;
this.stepMs = Pollers.getPollingIntervals().get(pollerIdx);
this.sendTimeoutMs = stepMs * 9 / 10;
this.commonTags = commonTags;
pushQueue = new LinkedBlockingQueue<>(config.getPushQueueSize());
pushThread = new Thread(new PushProcessor(), "BaseAtlasMetricObserver-Push");
pushThread.setDaemon(true);
pushThread.start();
}
/**
* Stop attempting to send metrics to atlas.
* Cleans up the thread that is created by this observer.
*/
public void stop() {
shouldPushMetrics.set(false);
// since we're probably blocking on getting an element from the
// push queue, interrupt the thread now
pushThread.interrupt();
}
protected static Counter newErrCounter(String name, String err) {
return new BasicCounter(MonitorConfig.builder(name).withTag("error", err).build());
}
protected static Metric asGauge(Metric m) {
return new Metric(m.getConfig().withAdditionalTag(ATLAS_GAUGE_TAG),
m.getTimestamp(), m.getValue());
}
protected static Metric asCounter(Metric m) {
return new Metric(m.getConfig().withAdditionalTag(ATLAS_COUNTER_TAG),
m.getTimestamp(), m.getValue());
}
protected static boolean isCounter(Metric m) {
final TagList tags = m.getConfig().getTags();
final String value = tags.getValue(DataSourceType.KEY);
return value != null && value.equals(DataSourceType.COUNTER.name());
}
protected static boolean isGauge(Metric m) {
final TagList tags = m.getConfig().getTags();
final String value = tags.getValue(DataSourceType.KEY);
return value != null && value.equals(DataSourceType.GAUGE.name());
}
protected static boolean isRate(Metric m) {
final TagList tags = m.getConfig().getTags();
final String value = tags.getValue(DataSourceType.KEY);
return DataSourceType.RATE.name().equals(value)
|| DataSourceType.NORMALIZED.name().equals(value);
}
protected static List identifyDsTypes(List metrics) {
// since we never generate atlas.dstype = counter we can do the following:
return metrics.stream().map(m -> isRate(m) ? m : asGauge(m)).collect(Collectors.toList());
}
@Override
public String getName() {
return "atlas";
}
private List identifyCountersForPush(List metrics) {
List transformed = new ArrayList<>(metrics.size());
for (Metric m : metrics) {
Metric toAdd = m;
if (isCounter(m)) {
toAdd = asCounter(m);
} else if (isGauge(m)) {
toAdd = asGauge(m);
}
transformed.add(toAdd);
}
return transformed;
}
/**
* Immediately send metrics to the backend.
*
* @param rawMetrics Metrics to be sent. Names and tags will be sanitized.
*/
public void push(List rawMetrics) {
List validMetrics = ValidCharacters.toValidValues(filter(rawMetrics));
List metrics = transformMetrics(validMetrics);
LOGGER.debug("Scheduling push of {} metrics", metrics.size());
final UpdateTasks tasks = getUpdateTasks(BasicTagList.EMPTY,
identifyCountersForPush(metrics));
final int maxAttempts = 5;
int attempts = 1;
while (!pushQueue.offer(tasks) && attempts <= maxAttempts) {
++attempts;
final UpdateTasks droppedTasks = pushQueue.remove();
LOGGER.warn("Removing old push task due to queue full. Dropping {} metrics.",
droppedTasks.numMetrics);
numMetricsDroppedQueueFull.increment(droppedTasks.numMetrics);
}
if (attempts >= maxAttempts) {
LOGGER.error("Unable to push update of {}", tasks);
numMetricsDroppedQueueFull.increment(tasks.numMetrics);
} else {
LOGGER.debug("Queued push of {}", tasks);
}
}
protected void sendNow(UpdateTasks updateTasks) {
if (updateTasks.numMetrics == 0) {
return;
}
final Stopwatch s = updateTimer.start();
int totalSent = 0;
try {
totalSent = httpHelper.sendAll(updateTasks.tasks,
updateTasks.numMetrics, sendTimeoutMs);
LOGGER.debug("Sent {}/{} metrics to atlas", totalSent, updateTasks.numMetrics);
} finally {
s.stop();
int dropped = updateTasks.numMetrics - totalSent;
numMetricsDroppedSendTimeout.increment(dropped);
}
}
protected boolean shouldIncludeMetric(Metric metric) {
return true;
}
/**
* Return metrics to be sent to the main atlas deployment.
* Metrics will be sent if their publishing policy matches atlas and if they
* will *not* be sent to the aggregation cluster.
*/
protected List filter(List metrics) {
final List filtered = metrics.stream().filter(this::shouldIncludeMetric)
.collect(Collectors.toList());
LOGGER.debug("Filter: input {} metrics, output {} metrics",
metrics.size(), filtered.size());
return filtered;
}
protected List transformMetrics(List metrics) {
return metrics;
}
@Override
public void update(List rawMetrics) {
List valid = ValidCharacters.toValidValues(rawMetrics);
List metrics = identifyDsTypes(filter(valid));
List transformed = transformMetrics(metrics);
sendNow(getUpdateTasks(getCommonTags(), transformed));
}
private UpdateTasks getUpdateTasks(TagList tags, List metrics) {
if (!config.shouldSendMetrics()) {
LOGGER.debug("Plugin disabled or running on a dev environment. Not sending metrics.");
return NO_TASKS;
}
if (metrics.isEmpty()) {
LOGGER.debug("metrics list is empty, no data being sent to server");
return NO_TASKS;
}
final int numMetrics = metrics.size();
final Metric[] atlasMetrics = new Metric[metrics.size()];
metrics.toArray(atlasMetrics);
numMetricsTotal.increment(numMetrics);
final List> tasks = new ArrayList<>();
final String uri = config.getAtlasUri();
LOGGER.debug("writing {} metrics to atlas ({})", numMetrics, uri);
int i = 0;
while (i < numMetrics) {
final int remaining = numMetrics - i;
final int batchSize = Math.min(remaining, config.batchSize());
final Metric[] batch = new Metric[batchSize];
System.arraycopy(atlasMetrics, i, batch, 0, batchSize);
final Observable sender = getSenderObservable(tags, batch);
tasks.add(sender);
i += batchSize;
}
assert i == numMetrics;
LOGGER.debug("succeeded in creating {} observable(s) to send metrics with total size {}",
tasks.size(), numMetrics);
return new UpdateTasks(numMetrics * getNumberOfCopies(), tasks,
System.currentTimeMillis());
}
protected int getNumberOfCopies() {
return 1;
}
private String getPayloadPrefix() {
SimpleDateFormat fmt = new SimpleDateFormat(FILE_DATE_FORMAT);
fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
return fmt.format(new Date());
}
protected void dumpPayload(File out, JsonPayload payload) throws IOException {
JsonGenerator generator = jsonFactory.createGenerator(out, JsonEncoding.UTF8);
generator.setPrettyPrinter(new AtlasPrettyPrinter());
payload.toJson(generator);
}
protected Observable getSenderObservable(TagList tags, Metric[] batch) {
JsonPayload payload = new UpdateRequest(tags, batch, batch.length);
if (shouldDumpPayload()) {
String prefix = getPayloadPrefix();
try {
Path tempFile = Files.createTempFile(Paths.get(getPayloadDirectory()), prefix, ".json");
dumpPayload(tempFile.toFile(), payload);
} catch (IOException ex) {
LOGGER.debug("Ignoring error writing payload sent to atlas: {}", ex.getMessage());
}
}
return httpHelper.postSmile(config.getAtlasUri(), payload)
.map(withBookkeeping(batch.length));
}
/**
* Get the list of common tags that will be added to all metrics sent by this Observer.
*/
protected TagList getCommonTags() {
return commonTags;
}
/**
* Utility function to map an Observable<ByteBuf> to an Observable<Integer> while also
* updating our counters for metrics sent and errors.
*/
protected Func1, Integer> withBookkeeping(final int batchSize) {
return response -> {
boolean ok = response.getStatus().code() == 200;
if (ok) {
numMetricsSent.increment(batchSize);
} else {
LOGGER.info("Status code: {} - Lost {} metrics",
response.getStatus().code(), batchSize);
numMetricsDroppedHttpErr.increment(batchSize);
}
return batchSize;
};
}
private static class UpdateTasks {
private final int numMetrics;
private final List> tasks;
private final long timestamp;
UpdateTasks(int numMetrics, List> tasks, long timestamp) {
this.numMetrics = numMetrics;
this.tasks = tasks;
this.timestamp = timestamp;
}
@Override
public String toString() {
return "UpdateTasks{numMetrics=" + numMetrics + ", tasks="
+ tasks + ", timestamp=" + timestamp + '}';
}
}
private class PushProcessor implements Runnable {
@Override
public void run() {
while (shouldPushMetrics.get()) {
try {
sendNow(pushQueue.take());
} catch (InterruptedException e) {
LOGGER.debug("Interrupted trying to get next UpdateTask to push");
break;
} catch (Exception t) {
LOGGER.info("Caught unexpected exception pushing metrics", t);
}
}
}
}
}
