com.amazonaws.services.kinesis.producer.KinesisProducer Maven / Gradle / Ivy
Show all versions of amazon-kinesis-producer Show documentation
/*
* Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Amazon Software License (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/asl/
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazonaws.services.kinesis.producer;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.channels.FileLock;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import javax.xml.bind.DatatypeConverter;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.services.kinesis.producer.protobuf.Messages;
import com.amazonaws.services.kinesis.producer.protobuf.Messages.Flush;
import com.amazonaws.services.kinesis.producer.protobuf.Messages.Message;
import com.amazonaws.services.kinesis.producer.protobuf.Messages.MetricsRequest;
import com.amazonaws.services.kinesis.producer.protobuf.Messages.MetricsResponse;
import com.amazonaws.services.kinesis.producer.protobuf.Messages.PutRecord;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
/**
* An interface to the native KPL daemon. This class handles the creation,
* destruction and use of the child process.
*
*
* Use a single instance within the application whenever possible:
*
*
* - One child process is spawned per instance of KinesisProducer. Additional
* instances introduce overhead and reduce aggregation efficiency.
* - All streams within a region that can be accessed with the same
* credentials can share the same KinesisProducer instance.
* - The {@link #addUserRecord} methods are thread safe, and be called
* concurrently.
* - Therefore, unless you need to put to multiple regions, or need to use
* different credentials for different streams, you should avoid creating
* multiple instances of KinesisProducer.
*
*
*
* @author chaodeng
*
*/
public class KinesisProducer {
private static final Logger log = LoggerFactory.getLogger(KinesisProducer.class);
private static final BigInteger UINT_128_MAX = new BigInteger(StringUtils.repeat("FF", 16), 16);
private static final Object EXTRACT_BIN_MUTEX = new Object();
private static final AtomicInteger callbackCompletionPoolNumber = new AtomicInteger(0);
private final KinesisProducerConfiguration config;
private final Map env;
private final AtomicLong messageNumber = new AtomicLong(1);
private final Map> futures = new ConcurrentHashMap<>();
private final ExecutorService callbackCompletionExecutor = new ThreadPoolExecutor(
1,
Runtime.getRuntime().availableProcessors() * 4,
5,
TimeUnit.MINUTES,
new LinkedBlockingQueue(),
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("kpl-callback-pool-" + callbackCompletionPoolNumber.getAndIncrement() + "-thread-%d")
.build(),
new RejectedExecutionHandler() {
/**
* Execute the runnable inline if we can't submit it to the
* executor. This shouldn't happen since we're using a linked
* queue which doesn't have a bound; but it's here just in case.
*/
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
r.run();
}
});
private String pathToExecutable;
private String pathToLibDir;
private String pathToTmpDir;
private volatile Daemon child;
private volatile long lastChild = System.nanoTime();
private volatile boolean destroyed = false;
private class MessageHandler implements Daemon.MessageHandler {
@Override
public void onMessage(final Message m) {
callbackCompletionExecutor.submit(new Runnable() {
@Override
public void run() {
if (m.hasPutRecordResult()) {
onPutRecordResult(m);
} else if (m.hasMetricsResponse()) {
onMetricsResponse(m);
} else {
log.error("Unexpected message type from child process");
}
}
});
}
@Override
public void onError(final Throwable t) {
// Don't log error if the user called destroy
if (!destroyed) {
log.error("Error in child process", t);
}
// Fail all outstanding futures
for (final Map.Entry> entry : futures.entrySet()) {
callbackCompletionExecutor.submit(new Runnable() {
@Override
public void run() {
entry.getValue().setException(t);
}
});
}
futures.clear();
// Only restart child if it's not an irrecoverable error, and if
// there has been some time (3 seconds) between the last child
// creation. If the child process crashes almost immediately, we're
// going to abort to avoid going into a loop.
if (!(t instanceof IrrecoverableError) && System.nanoTime() - lastChild > 3e9) {
lastChild = System.nanoTime();
child = new Daemon(pathToExecutable, new MessageHandler(), pathToTmpDir, config, env);
}
}
/**
* Matches up the incoming PutRecordResult to an outstanding future, and
* completes that future with the appropriate data.
*
*
* We adapt the protobuf PutRecordResult to another simpler
* PutRecordResult class to hide the protobuf stuff from the user.
*
* @param msg
*/
private void onPutRecordResult(Message msg) {
SettableFuture f = getFuture(msg);
UserRecordResult result = UserRecordResult.fromProtobufMessage(msg.getPutRecordResult());
if (result.isSuccessful()) {
f.set(result);
} else {
f.setException(new UserRecordFailedException(result));
}
}
private void onMetricsResponse(Message msg) {
SettableFuture> f = getFuture(msg);
List userMetrics = new ArrayList<>();
MetricsResponse res = msg.getMetricsResponse();
for (Messages.Metric metric : res.getMetricsList()) {
userMetrics.add(new Metric(metric));
}
f.set(userMetrics);
}
private SettableFuture getFuture(Message msg) {
long id = msg.getSourceId();
@SuppressWarnings("unchecked")
SettableFuture f = (SettableFuture) futures.remove(id);
if (f == null) {
throw new RuntimeException("Future for message id " + id + " not found");
}
return f;
}
}
/**
* Start up a KinesisProducer instance.
*
*
* Since this creates a child process, it is fairly expensive. Avoid
* creating more than one per application, unless putting to multiple
* regions at the same time. All streams in the same region can share the
* same instance.
*
*
* All methods in KinesisProducer are thread-safe.
*
* @param config
* Configuration for the KinesisProducer. See the docs for that
* class for details.
*
* @see KinesisProducerConfiguration
*/
public KinesisProducer(KinesisProducerConfiguration config) {
this.config = config;
extractBinaries();
env = new ImmutableMap.Builder()
.put("LD_LIBRARY_PATH", pathToLibDir)
.put("DYLD_LIBRARY_PATH", pathToLibDir)
.build();
child = new Daemon(pathToExecutable, new MessageHandler(), pathToTmpDir, config, env);
}
/**
* Start up a KinesisProducer instance.
*
*
* Since this creates a child process, it is fairly expensive. Avoid
* creating more than one per application, unless putting to multiple
* regions at the same time. All streams in the same region can share the
* same instance.
*
*
* The KPL will use a set of default configurations. You can set custom
* configuration using the constructor that takes a {@link KinesisProducerConfiguration}
* object.
*
*
* All methods in KinesisProducer are thread-safe.
*/
public KinesisProducer() {
this(new KinesisProducerConfiguration());
}
/**
* Connect to a running daemon. Does not start a child process. Used for
* testing.
*
* @param inPipe
* @param outPipe
*/
protected KinesisProducer(File inPipe, File outPipe) {
this.config = null;
this.env = null;
child = new Daemon(inPipe, outPipe, new MessageHandler());
}
/**
* Put a record asynchronously. A {@link ListenableFuture} is returned that
* can be used to retrieve the result, either by polling or by registering a
* callback.
*
*
* The return value can be disregarded if you do not wish to process the
* result. Under the covers, the KPL will automatically re-attempt puts in
* case of transient errors (including throttling). A failed result is
* generally returned only if an irrecoverable error is detected (e.g.
* trying to put to a stream that doesn't exist), or if the record expires.
*
*
* Thread safe.
*
*
* To add a listener to the future:
*
*
* ListenableFuture<PutRecordResult> f = myKinesisProducer.addUserRecord(...);
* com.google.common.util.concurrent.Futures.addCallback(f, callback, executor);
*
*
* where callback is an instance of
* {@link com.google.common.util.concurrent.FutureCallback} and
* executor is an instance of
* {@link java.util.concurrent.Executor}.
*
* Important:
*
* If long-running tasks are performed in the callbacks, it is recommended
* that a custom executor be provided when registering callbacks to ensure
* that there are enough threads to achieve the desired level of
* parallelism. By default, the KPL will use an internal thread pool to
* execute callbacks, but this pool may not have a sufficient number of
* threads if a large number is desired.
*
* Another option would be to hand the result off to a different component
* for processing and keep the callback routine fast.
*
* @param stream
* Stream to put to.
* @param partitionKey
* Partition key. Length must be at least one, and at most 256
* (inclusive).
* @param data
* Binary data of the record. Maximum size 1MiB.
* @return A future for the result of the put.
* @throws IllegalArgumentException
* if input does not meet stated constraints
* @throws DaemonException
* if the child process is dead
* @see ListenableFuture
* @see UserRecordResult
* @see KinesisProducerConfiguration#setRecordTtl(long)
* @see UserRecordFailedException
*/
public ListenableFuture addUserRecord(String stream, String partitionKey, ByteBuffer data) {
return addUserRecord(stream, partitionKey, null, data);
}
/**
* Put a record asynchronously. A {@link ListenableFuture} is returned that
* can be used to retrieve the result, either by polling or by registering a
* callback.
*
*
* The return value can be disregarded if you do not wish to process the
* result. Under the covers, the KPL will automatically reattempt puts in
* case of transient errors (including throttling). A failed result is
* generally returned only if an irrecoverable error is detected (e.g.
* trying to put to a stream that doesn't exist), or if the record expires.
*
*
* Thread safe.
*
*
* To add a listener to the future:
*
*
* ListenableFuture<PutRecordResult> f = myKinesisProducer.addUserRecord(...);
* com.google.common.util.concurrent.Futures.addCallback(f, callback, executor);
*
*
* where callback is an instance of
* {@link com.google.common.util.concurrent.FutureCallback} and
* executor is an instance of
* {@link java.util.concurrent.Executor}.
*
* Important:
*
* If long-running tasks are performed in the callbacks, it is recommended
* that a custom executor be provided when registering callbacks to ensure
* that there are enough threads to achieve the desired level of
* parallelism. By default, the KPL will use an internal thread pool to
* execute callbacks, but this pool may not have a sufficient number of
* threads if a large number is desired.
*
* Another option would be to hand the result off to a different component
* for processing and keep the callback routine fast.
*
* @param stream
* Stream to put to.
* @param partitionKey
* Partition key. Length must be at least one, and at most 256
* (inclusive).
* @param explicitHashKey
* The hash value used to explicitly determine the shard the data
* record is assigned to by overriding the partition key hash.
* Must be a valid string representation of a positive integer
* with value between 0 and 2^128 - 1 (inclusive).
* @param data
* Binary data of the record. Maximum size 1MiB.
* @return A future for the result of the put.
* @throws IllegalArgumentException
* if input does not meet stated constraints
* @throws DaemonException
* if the child process is dead
* @see ListenableFuture
* @see UserRecordResult
* @see KinesisProducerConfiguration#setRecordTtl(long)
* @see UserRecordFailedException
*/
public ListenableFuture addUserRecord(String stream, String partitionKey, String explicitHashKey, ByteBuffer data) {
if (stream == null) {
throw new IllegalArgumentException("Stream name cannot be null");
}
stream = stream.trim();
if (stream.length() == 0) {
throw new IllegalArgumentException("Stream name cannot be empty");
}
if (partitionKey == null) {
throw new IllegalArgumentException("partitionKey cannot be null");
}
if (partitionKey.length() < 1 || partitionKey.length() > 256) {
throw new IllegalArgumentException(
"Invalid parition key. Length must be at least 1 and at most 256, got " + partitionKey.length());
}
try {
partitionKey.getBytes("UTF-8");
} catch (Exception e) {
throw new IllegalArgumentException("Partition key must be valid UTF-8");
}
BigInteger b = null;
if (explicitHashKey != null) {
explicitHashKey = explicitHashKey.trim();
try {
b = new BigInteger(explicitHashKey);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid explicitHashKey, must be an integer, got " + explicitHashKey);
}
if (b != null) {
if (b.compareTo(UINT_128_MAX) > 0 || b.compareTo(BigInteger.ZERO) < 0) {
throw new IllegalArgumentException(
"Invalid explicitHashKey, must be greater or equal to zero and less than or equal to (2^128 - 1), got " +
explicitHashKey);
}
}
}
if (data != null && data.remaining() > 1024 * 1024) {
throw new IllegalArgumentException(
"Data must be less than or equal to 1MB in size, got " + data.remaining() + " bytes");
}
long id = messageNumber.getAndIncrement();
SettableFuture f = SettableFuture.create();
futures.put(id, f);
PutRecord.Builder pr = PutRecord.newBuilder()
.setStreamName(stream)
.setPartitionKey(partitionKey)
.setData(data != null ? ByteString.copyFrom(data) : ByteString.EMPTY);
if (b != null) {
pr.setExplicitHashKey(b.toString(10));
}
Message m = Message.newBuilder()
.setId(id)
.setPutRecord(pr.build())
.build();
child.add(m);
return f;
}
/**
* Get the number of unfinished records currently being processed. The
* records could either be waiting to be sent to the child process, or have
* reached the child process and are being worked on.
*
*
* This is equal to the number of futures returned from {@link #addUserRecord}
* that have not finished.
*
* @return The number of unfinished records currently being processed.
*/
public int getOutstandingRecordsCount() {
return futures.size();
}
/**
* Get metrics from the KPL.
*
*
* The KPL computes and buffers useful metrics. Use this method to retrieve
* them. The same metrics are also uploaded to CloudWatch (unless disabled).
*
*
* Multiple metrics exist for the same name, each with a different list of
* dimensions (e.g. stream name). This method will fetch all metrics with
* the provided name.
*
*
* See the stand-alone metrics documentation for details about individual
* metrics.
*
*
* This method is synchronous and will block while the data is being
* retrieved.
*
* @param metricName
* Name of the metrics to fetch.
* @param windowSeconds
* Fetch data from the last N seconds. The KPL maintains data at
* per second granularity for the past minute. To get total
* cumulative data since the start of the program, use the
* overloads that do not take this argument.
* @return A list of metrics with the provided name.
* @throws ExecutionException
* If an error occurred while fetching metrics from the child
* process.
* @throws InterruptedException
* If the thread is interrupted while waiting for the response
* from the child process.
* @see Metric
*/
public List getMetrics(String metricName, int windowSeconds) throws InterruptedException, ExecutionException {
MetricsRequest.Builder mrb = MetricsRequest.newBuilder();
if (metricName != null) {
mrb.setName(metricName);
}
if (windowSeconds > 0) {
mrb.setSeconds(windowSeconds);
}
long id = messageNumber.getAndIncrement();
SettableFuture> f = SettableFuture.create();
futures.put(id, f);
child.add(Message.newBuilder()
.setId(id)
.setMetricsRequest(mrb.build())
.build());
return f.get();
}
/**
* Get metrics from the KPL.
*
*
* The KPL computes and buffers useful metrics. Use this method to retrieve
* them. The same metrics are also uploaded to CloudWatch (unless disabled).
*
*
* Multiple metrics exist for the same name, each with a different list of
* dimensions (e.g. stream name). This method will fetch all metrics with
* the provided name.
*
*
* The retrieved data represents cumulative statistics since the start of
* the program. To get data from a smaller window, use
* {@link #getMetrics(String, int)}.
*
*
* See the stand-alone metrics documentation for details about individual
* metrics.
*
*
* This method is synchronous and will block while the data is being
* retrieved.
*
* @param metricName
* Name of the metrics to fetch.
* @return A list of metrics with the provided name.
* @throws ExecutionException
* If an error occurred while fetching metrics from the child
* process.
* @throws InterruptedException
* If the thread is interrupted while waiting for the response
* from the child process.
* @see Metric
*/
public List getMetrics(String metricName) throws InterruptedException, ExecutionException {
return getMetrics(metricName, -1);
}
/**
* Get metrics from the KPL.
*
*
* The KPL computes and buffers useful metrics. Use this method to retrieve
* them. The same metrics are also uploaded to CloudWatch (unless disabled).
*
*
* This method fetches all metrics available. To fetch only metrics with a
* given name, use {@link #getMetrics(String)}.
*
*
* The retrieved data represents cumulative statistics since the start of
* the program. To get data from a smaller window, use
* {@link #getMetrics(int)}.
*
*
* See the stand-alone metrics documentation for details about individual
* metrics.
*
*
* This method is synchronous and will block while the data is being
* retrieved.
*
* @return A list of all of the metrics maintained by the KPL.
* @throws ExecutionException
* If an error occurred while fetching metrics from the child
* process.
* @throws InterruptedException
* If the thread is interrupted while waiting for the response
* from the child process.
* @see Metric
*/
public List getMetrics() throws InterruptedException, ExecutionException {
return getMetrics(null);
}
/**
* Get metrics from the KPL.
*
*
* The KPL computes and buffers useful metrics. Use this method to retrieve
* them. The same metrics are also uploaded to CloudWatch (unless disabled).
*
*
* This method fetches all metrics available. To fetch only metrics with a
* given name, use {@link #getMetrics(String, int)}.
*
*
* See the stand-alone metrics documentation for details about individual
* metrics.
*
*
* This method is synchronous and will block while the data is being
* retrieved.
*
* @param windowSeconds
* Fetch data from the last N seconds. The KPL maintains data at
* per second granularity for the past minute. To get total
* cumulative data since the start of the program, use the
* overloads that do not take this argument.
* @return A list of metrics with the provided name.
* @throws ExecutionException
* If an error occurred while fetching metrics from the child
* process.
* @throws InterruptedException
* If the thread is interrupted while waiting for the response
* from the child process.
* @see Metric
*/
public List getMetrics(int windowSeconds) throws InterruptedException, ExecutionException {
return getMetrics(null, windowSeconds);
}
/**
* Immediately kill the child process. This will cause all outstanding
* futures to fail immediately.
*
*
* To perform a graceful shutdown instead, there are several options:
*
*
* - Call {@link #flush()} and wait (perhaps with a time limit) for all
* futures. If you were sending a very high volume of data you may need to
* call flush multiple times to clear all buffers.
* - Poll {@link #getOutstandingRecordsCount()} until it returns 0.
* - Call {@link #flushSync()}, which blocks until completion.
*
*
* Once all records are confirmed with one of the above, call destroy to
* terminate the child process. If you are terminating the JVM then calling
* destroy is unnecessary since it will be done automatically.
*/
public void destroy() {
destroyed = true;
child.destroy();
}
/**
* Instruct the child process to perform a flush, sending some of the
* records it has buffered for the specified stream.
*
*
* This does not guarantee that all buffered records will be sent, only that
* most of them will; to flush all records and wait for completion, use
* {@link #flushSync}.
*
*
* This method returns immediately without blocking.
*
* @param stream
* Stream to flush
* @throws DaemonException
* if the child process is dead
*/
public void flush(String stream) {
Flush.Builder f = Flush.newBuilder();
if (stream != null) {
f.setStreamName(stream);
}
Message m = Message.newBuilder()
.setId(messageNumber.getAndIncrement())
.setFlush(f.build())
.build();
child.add(m);
}
/**
* Instruct the child process to perform a flush, sending some of the
* records it has buffered. Applies to all streams.
*
*
* This does not guarantee that all buffered records will be sent, only that
* most of them will; to flush all records and wait for completion, use
* {@link #flushSync}.
*
*
* This method returns immediately without blocking.
*
* @throws DaemonException
* if the child process is dead
*/
public void flush() {
flush(null);
}
/**
* Instructs the child process to flush all records and waits until all
* records are complete (either succeeding or failing).
*
*
* The wait includes any retries that need to be performed. Depending on
* your configuration of record TTL and request timeout, this can
* potentially take a long time if the library is having trouble delivering
* records to the backend, for example due to network problems.
*
*
* This is useful if you need to shutdown your application and want to make
* sure all records are delivered before doing so.
*
* @throws DaemonException
* if the child process is dead
*
* @see KinesisProducerConfiguration#setRecordTtl(long)
* @see KinesisProducerConfiguration#setRequestTimeout(long)
*/
public void flushSync() {
while (getOutstandingRecordsCount() > 0) {
flush();
try {
Thread.sleep(500);
} catch (InterruptedException e) { }
}
}
private void extractBinaries() {
synchronized (EXTRACT_BIN_MUTEX) {
String os = SystemUtils.OS_NAME;
if (SystemUtils.IS_OS_WINDOWS) {
os = "windows";
} else if (SystemUtils.IS_OS_LINUX) {
os = "linux";
} else if (SystemUtils.IS_OS_MAC_OSX) {
os = "osx";
} else {
throw new RuntimeException("Your operation system is not supported (" +
os + "), the KPL only supports Linux, OSX and Windows");
}
String root = "amazon-kinesis-producer-native-binaries";
String tmpDir = config.getTempDirectory();
if (tmpDir.trim().length() == 0) {
tmpDir = System.getProperty("java.io.tmpdir");
}
tmpDir = Paths.get(tmpDir, root).toString();
pathToTmpDir = tmpDir;
String binPath = config.getNativeExecutable();
if (binPath != null && !binPath.trim().isEmpty()) {
pathToExecutable = binPath.trim();
log.warn("Using non-default native binary at " + pathToExecutable);
pathToLibDir = "";
} else {
log.info("Extracting binaries to " + tmpDir);
try {
File tmpDirFile = new File(tmpDir);
if (!tmpDirFile.exists() && !tmpDirFile.mkdirs()) {
throw new IOException("Could not create tmp dir " + tmpDir);
}
String extension = os.equals("windows") ? ".exe" : "";
String executableName = "kinesis_producer" + extension;
byte[] bin = IOUtils.toByteArray(
this.getClass().getClassLoader().getResourceAsStream(root + "/" + os + "/" + executableName));
MessageDigest md = MessageDigest.getInstance("SHA1");
String mdHex = DatatypeConverter.printHexBinary(md.digest(bin)).toLowerCase();
pathToExecutable = Paths.get(pathToTmpDir, "kinesis_producer_" + mdHex + extension).toString();
File extracted = new File(pathToExecutable);
if (extracted.exists()) {
try (FileInputStream fis = new FileInputStream(extracted);
FileLock lock = fis.getChannel().lock(0, Long.MAX_VALUE, true)) {
boolean contentEqual = false;
if (extracted.length() == bin.length) {
byte[] existingBin = IOUtils.toByteArray(new FileInputStream(extracted));
contentEqual = Arrays.equals(bin, existingBin);
}
if (!contentEqual) {
throw new SecurityException("The contents of the binary " + extracted.getAbsolutePath()
+ " is not what it's expected to be.");
}
}
} else {
try (FileOutputStream fos = new FileOutputStream(extracted);
FileLock lock = fos.getChannel().lock()) {
IOUtils.write(bin, fos);
}
extracted.setExecutable(true);
}
pathToLibDir = pathToTmpDir;
} catch (Exception e) {
throw new RuntimeException("Could not copy native binaries to temp directory " + tmpDir, e);
}
}
}
}
}