com.wavefront.agent.data.AbstractDataSubmissionTask Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of proxy Show documentation
Show all versions of proxy Show documentation
Service for batching and relaying metric traffic to Wavefront
package com.wavefront.agent.data;
import static com.wavefront.common.Utils.isWavefrontResponse;
import static java.lang.Boolean.TRUE;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
import com.google.common.base.Throwables;
import com.wavefront.agent.queueing.TaskQueue;
import com.wavefront.common.TaggedMetricName;
import com.wavefront.common.logger.MessageDedupingLogger;
import com.wavefront.data.ReportableEntityType;
import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.Histogram;
import com.yammer.metrics.core.MetricName;
import com.yammer.metrics.core.TimerContext;
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;
import javax.net.ssl.SSLHandshakeException;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.core.Response;
/**
* A base class for data submission tasks.
*
* @param task type
* @author [email protected].
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
abstract class AbstractDataSubmissionTask>
implements DataSubmissionTask {
private static final int MAX_RETRIES = 15;
private static final Logger log =
new MessageDedupingLogger(
Logger.getLogger(AbstractDataSubmissionTask.class.getCanonicalName()), 1000, 1);
@JsonProperty protected long enqueuedTimeMillis = Long.MAX_VALUE;
@JsonProperty protected int attempts = 0;
@JsonProperty protected int serverErrors = 0;
@JsonProperty protected String handle;
@JsonProperty protected ReportableEntityType entityType;
@JsonProperty protected Boolean limitRetries = null;
protected transient Histogram timeSpentInQueue;
protected transient Supplier timeProvider;
protected transient EntityProperties properties;
protected transient TaskQueue backlog;
AbstractDataSubmissionTask() {}
/**
* @param properties entity-specific wrapper for runtime properties.
* @param backlog backing queue.
* @param handle port/handle
* @param entityType entity type
* @param timeProvider time provider (in millis)
*/
AbstractDataSubmissionTask(
EntityProperties properties,
TaskQueue backlog,
String handle,
ReportableEntityType entityType,
@Nullable Supplier timeProvider) {
this.properties = properties;
this.backlog = backlog;
this.handle = handle;
this.entityType = entityType;
this.timeProvider = MoreObjects.firstNonNull(timeProvider, System::currentTimeMillis);
}
@Override
public long getEnqueuedMillis() {
return enqueuedTimeMillis;
}
@Override
public ReportableEntityType getEntityType() {
return entityType;
}
abstract Response doExecute() throws DataSubmissionException;
public TaskResult execute() {
if (enqueuedTimeMillis < Long.MAX_VALUE) {
if (timeSpentInQueue == null) {
timeSpentInQueue =
Metrics.newHistogram(
new TaggedMetricName(
"buffer", "queue-time", "port", handle, "content", entityType.toString()));
}
timeSpentInQueue.update(timeProvider.get() - enqueuedTimeMillis);
}
attempts += 1;
TimerContext timer =
Metrics.newTimer(
new MetricName("push." + handle, "", "duration"),
TimeUnit.MILLISECONDS,
TimeUnit.MINUTES)
.time();
try (Response response = doExecute()) {
Metrics.newCounter(
new TaggedMetricName("push", handle + ".http." + response.getStatus() + ".count"))
.inc();
if (response.getStatus() >= 200 && response.getStatus() < 300) {
Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered"))
.inc(this.weight());
return TaskResult.DELIVERED;
}
switch (response.getStatus()) {
case 406:
case 429:
return handleStatus429();
case 401:
case 403:
log.warning(
"["
+ handle
+ "] HTTP "
+ response.getStatus()
+ ": "
+ "Please verify that \""
+ entityType
+ "\" is enabled for your account!");
return checkStatusAndQueue(QueueingReason.AUTH, false);
case 407:
case 408:
if (isWavefrontResponse(response)) {
log.warning(
"["
+ handle
+ "] HTTP "
+ response.getStatus()
+ " (Unregistered proxy) "
+ "received while sending data to Wavefront - please verify that your token is "
+ "valid and has Proxy Management permissions!");
} else {
log.warning(
"["
+ handle
+ "] HTTP "
+ response.getStatus()
+ " "
+ "received while sending data to Wavefront - please verify your network/HTTP proxy"
+ " settings!");
}
return checkStatusAndQueue(QueueingReason.RETRY, false);
case 413:
splitTask(1, properties.getDataPerBatch())
.forEach(
x ->
x.enqueue(
enqueuedTimeMillis == Long.MAX_VALUE ? QueueingReason.SPLIT : null));
return TaskResult.PERSISTED_RETRY;
default:
serverErrors += 1;
if (serverErrors > MAX_RETRIES && TRUE.equals(limitRetries)) {
log.info(
"["
+ handle
+ "] HTTP "
+ response.getStatus()
+ " received while sending "
+ "data to Wavefront, max retries reached");
return TaskResult.DELIVERED;
} else {
log.info(
"["
+ handle
+ "] HTTP "
+ response.getStatus()
+ " received while sending "
+ "data to Wavefront, retrying");
return checkStatusAndQueue(QueueingReason.RETRY, true);
}
}
} catch (DataSubmissionException ex) {
if (ex instanceof IgnoreStatusCodeException) {
Metrics.newCounter(new TaggedMetricName("push", handle + ".http.404.count")).inc();
Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered"))
.inc(this.weight());
return TaskResult.DELIVERED;
}
throw new RuntimeException("Unhandled DataSubmissionException", ex);
} catch (ProcessingException ex) {
Throwable rootCause = Throwables.getRootCause(ex);
if (rootCause instanceof UnknownHostException) {
log.warning(
"["
+ handle
+ "] Error sending data to Wavefront: Unknown host "
+ rootCause.getMessage()
+ ", please check your network!");
} else if (rootCause instanceof ConnectException
|| rootCause instanceof SocketTimeoutException) {
log.warning(
"["
+ handle
+ "] Error sending data to Wavefront: "
+ rootCause.getMessage()
+ ", please verify your network/HTTP proxy settings!");
} else if (ex.getCause() instanceof SSLHandshakeException) {
log.warning(
"["
+ handle
+ "] Error sending data to Wavefront: "
+ ex.getCause()
+ ", please verify that your environment has up-to-date root certificates!");
} else {
log.warning("[" + handle + "] Error sending data to Wavefront: " + rootCause);
}
if (log.isLoggable(Level.FINE)) {
log.log(Level.FINE, "Full stacktrace: ", ex);
}
return checkStatusAndQueue(QueueingReason.RETRY, false);
} catch (Exception ex) {
log.warning(
"[" + handle + "] Error sending data to Wavefront: " + Throwables.getRootCause(ex));
if (log.isLoggable(Level.FINE)) {
log.log(Level.FINE, "Full stacktrace: ", ex);
}
return checkStatusAndQueue(QueueingReason.RETRY, true);
} finally {
timer.stop();
}
}
@SuppressWarnings("unchecked")
@Override
public void enqueue(@Nullable QueueingReason reason) {
enqueuedTimeMillis = timeProvider.get();
try {
backlog.add((T) this);
if (reason != null) {
Metrics.newCounter(
new TaggedMetricName(
entityType + "." + handle, "queued", "reason", reason.toString()))
.inc(this.weight());
}
} catch (IOException e) {
Metrics.newCounter(new TaggedMetricName("buffer", "failures", "port", handle)).inc();
log.severe(
"["
+ handle
+ "] CRITICAL (Losing data): WF-1: Error adding task to the queue: "
+ e.getMessage());
}
}
private TaskResult checkStatusAndQueue(QueueingReason reason, boolean requeue) {
if (reason == QueueingReason.AUTH) return TaskResult.REMOVED;
if (enqueuedTimeMillis == Long.MAX_VALUE) {
if (properties.getTaskQueueLevel().isLessThan(TaskQueueLevel.ANY_ERROR)) {
return TaskResult.RETRY_LATER;
}
enqueue(reason);
return TaskResult.PERSISTED;
}
if (requeue) {
enqueue(null);
return TaskResult.PERSISTED_RETRY;
} else {
return TaskResult.RETRY_LATER;
}
}
protected TaskResult handleStatus429() {
if (enqueuedTimeMillis == Long.MAX_VALUE) {
if (properties.getTaskQueueLevel().isLessThan(TaskQueueLevel.PUSHBACK)) {
return TaskResult.RETRY_LATER;
}
enqueue(QueueingReason.PUSHBACK);
return TaskResult.PERSISTED;
}
if (properties.isSplitPushWhenRateLimited()) {
List splitTasks =
splitTask(properties.getMinBatchSplitSize(), properties.getDataPerBatch());
if (splitTasks.size() == 1) return TaskResult.RETRY_LATER;
splitTasks.forEach(x -> x.enqueue(null));
return TaskResult.PERSISTED;
}
return TaskResult.RETRY_LATER;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy