com.amazonaws.services.kinesis.metrics.impl.CWPublisherRunnable Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of amazon-kinesis-client Show documentation
Show all versions of amazon-kinesis-client Show documentation
The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data
from Amazon Kinesis.
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.amazonaws.services.kinesis.metrics.impl;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* A CWPublisherRunnable contains the logic of when to publish metrics.
*
* @param
*/
public class CWPublisherRunnable implements Runnable {
private static final Log LOG = LogFactory.getLog(CWPublisherRunnable.class);
private final ICWMetricsPublisher metricsPublisher;
private final MetricAccumulatingQueue queue;
private final long bufferTimeMillis;
/*
* Number of metrics that will cause us to flush.
*/
private int flushSize;
private boolean shuttingDown = false;
private boolean shutdown = false;
private long lastFlushTime = Long.MAX_VALUE;
private int maxJitter;
private Random rand = new Random();
private int nextJitterValueToUse = 0;
/**
* Constructor.
*
* @param metricsPublisher publishes metrics
* @param bufferTimeMillis time between publishing metrics
* @param maxQueueSize max size of metrics to publish
* @param batchSize size of batch that can be published at a time
*/
public CWPublisherRunnable(ICWMetricsPublisher metricsPublisher,
long bufferTimeMillis,
int maxQueueSize,
int batchSize) {
this(metricsPublisher, bufferTimeMillis, maxQueueSize, batchSize, 0);
}
public CWPublisherRunnable(ICWMetricsPublisher metricsPublisher,
long bufferTimeMillis,
int maxQueueSize,
int batchSize,
int maxJitter) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Constructing CWPublisherRunnable with maxBufferTimeMillis %d maxQueueSize %d batchSize %d maxJitter %d",
bufferTimeMillis,
maxQueueSize,
batchSize,
maxJitter));
}
this.metricsPublisher = metricsPublisher;
this.bufferTimeMillis = bufferTimeMillis;
this.queue = new MetricAccumulatingQueue(maxQueueSize);
this.flushSize = batchSize;
this.maxJitter = maxJitter;
}
@Override
public void run() {
while (!shutdown) {
try {
runOnce();
} catch (Throwable t) {
LOG.error("Encountered throwable in CWPublisherRunable", t);
}
}
LOG.info("CWPublication thread finished.");
}
/**
* Exposed for testing purposes.
*/
public void runOnce() {
List> dataToPublish = null;
synchronized (queue) {
/*
* We should send if:
*
* it's been maxBufferTimeMillis since our last send
* or if the queue contains > batchSize elements
* or if we're shutting down
*/
long timeSinceFlush = Math.max(0, getTime() - lastFlushTime);
if (timeSinceFlush >= bufferTimeMillis || queue.size() >= flushSize || shuttingDown) {
dataToPublish = queue.drain(flushSize);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Drained %d datums from queue", dataToPublish.size()));
}
if (shuttingDown) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Shutting down with %d datums left on the queue", queue.size()));
}
// If we're shutting down, we successfully shut down only when the queue is empty.
shutdown = queue.isEmpty();
}
} else {
long waitTime = bufferTimeMillis - timeSinceFlush;
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Waiting up to %dms for %d more datums to appear.", waitTime, flushSize
- queue.size()));
}
try {
// Wait for enqueues for up to maxBufferTimeMillis.
queue.wait(waitTime);
} catch (InterruptedException e) {
}
}
}
if (dataToPublish != null) {
try {
metricsPublisher.publishMetrics(dataToPublish);
} catch (Throwable t) {
LOG.error("Caught exception thrown by metrics Publisher in CWPublisherRunnable", t);
}
// Changing the value of lastFlushTime will change the time when metrics are flushed next.
lastFlushTime = getTime() + nextJitterValueToUse;
if (maxJitter != 0) {
// nextJittervalueToUse will be a value between (-maxJitter,+maxJitter)
nextJitterValueToUse = maxJitter - rand.nextInt(2 * maxJitter);
}
}
}
/**
* Overrideable for testing purposes.
*/
protected long getTime() {
return System.currentTimeMillis();
}
public void shutdown() {
LOG.info("Shutting down CWPublication thread.");
synchronized (queue) {
shuttingDown = true;
queue.notify();
}
}
public boolean isShutdown() {
return shutdown;
}
/**
* Enqueues metric data for publication.
*
* @param data collection of MetricDatum to enqueue
*/
public void enqueue(Collection> data) {
synchronized (queue) {
if (shuttingDown) {
LOG.warn(String.format("Dropping metrics %s because CWPublisherRunnable is shutting down.", data));
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Enqueueing %d datums for publication", data.size()));
}
for (MetricDatumWithKey datumWithKey : data) {
if (!queue.offer(datumWithKey.key, datumWithKey.datum)) {
LOG.warn("Metrics queue full - dropping metric " + datumWithKey.datum);
}
}
// If this is the first enqueue, start buffering from now.
if (lastFlushTime == Long.MAX_VALUE) {
lastFlushTime = getTime();
}
queue.notify();
}
}
}