ch.qos.logback.more.appenders.KinesisStreamLogbackAppender Maven / Gradle / Ivy
/**
* Copyright (c) 2018 sndyuk
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package ch.qos.logback.more.appenders;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import com.amazonaws.services.kinesis.model.PutRecordsRequest;
import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry;
import com.amazonaws.services.kinesis.model.PutRecordsResult;
import com.amazonaws.services.kinesis.model.PutRecordsResultEntry;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.encoder.Encoder;
import ch.qos.logback.more.appenders.IntervalEmitter.EventMapper;
import ch.qos.logback.more.appenders.IntervalEmitter.IntervalAppender;
/**
* Appender for Kinesis Stream. It appends entries for every emitInterval.
*
* @author sndyuk
*/
public class KinesisStreamLogbackAppender extends KinesisStreamAppenderBase {
private IntervalEmitter emitter;
private PartitionKey partitionKey = new RandomPartitionKey();
private long emitInterval = 10000;
public void setAwsConfig(AwsConfig config) {
this.config = config;
}
public void setStreamName(String streamName) {
this.streamName = streamName;
}
public void setShardCount(int shardCount) {
this.shardCount = shardCount;
}
public void setCreateStreamDestination(boolean createStreamDestination) {
this.createStreamDestination = createStreamDestination;
}
public void setPartitionKey(PartitionKey partitionKey) {
this.partitionKey = partitionKey;
}
public void setEmitInterval(long emitInterval) {
this.emitInterval = emitInterval;
}
public void setEncoder(Encoder encoder) {
this.encoder = encoder;
}
@Override
public void start() {
this.emitter = new IntervalEmitter(emitInterval,
new KinesisEventMapper(), new KinesisIntervalAppender());
super.start();
}
@Override
public void stop() {
try {
emitter.emitForShutdown(10000, 10);
} catch (Exception e) {
// Ignore
}
try {
super.stop();
} catch (Exception e) {
// Ignore
}
}
@Override
protected void append(ILoggingEvent eventObject) {
emitter.append(eventObject);
}
private final class KinesisEventMapper implements EventMapper {
@Override
public PutRecordsRequestEntry map(ILoggingEvent event) {
PutRecordsRequestEntry entry = new PutRecordsRequestEntry();
entry.setData(ByteBuffer.wrap(encoder.encode(event)));
entry.setPartitionKey(partitionKey.get(event));
return entry;
}
}
private final class KinesisIntervalAppender implements IntervalAppender {
@Override
public boolean append(List entries) {
if (!active) {
ensureKinesisStream();
}
return append(entries, 0);
}
private boolean append(List entries, int retryCount) {
if (retryCount > 3) {
StringBuilder sb = new StringBuilder("Could not append the Kinesis stream entry. Failed entries:");
for (PutRecordsRequestEntry entry : entries) {
sb.append(System.lineSeparator()).append(entry.getData());
}
addError(sb.toString());
return true; // true to forcibly remove unexpected failed entries.
}
try {
if (retryCount > 0) {
Thread.sleep(1000 * retryCount);
}
PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
putRecordsRequest.setStreamName(streamName);
putRecordsRequest.setRecords(entries);
PutRecordsResult putRecordsResult = kinesis.putRecords(putRecordsRequest);
if (putRecordsResult.getFailedRecordCount() == 0) {
return true;
}
if (putRecordsResult.getFailedRecordCount() == entries.size()) {
return false; // Leave it to logback retry mechanism.
}
// Retry
List failedEntries = new ArrayList(putRecordsResult.getFailedRecordCount());
for (int i = 0; i < entries.size(); i++) {
PutRecordsResultEntry resultEntry = putRecordsResult.getRecords().get(i);
if (resultEntry.getErrorCode() != null) {
failedEntries.add(entries.get(i));
}
append(failedEntries, retryCount + 1);
}
} catch (RuntimeException e) {
addError("Unexpected runtime error while appending kinesis entries.", e);
} catch (InterruptedException e) {
// pass
}
return true;
}
}
public static class RandomPartitionKey implements PartitionKey {
private Random rand;
private int maxPartition;
public RandomPartitionKey() {
this.rand = new Random();
this.maxPartition = 10000;
}
@Override
public String get(ILoggingEvent event) {
return String.valueOf(rand.nextInt(maxPartition));
}
}
public static class SinglePartitionKey implements PartitionKey {
@Override
public String get(ILoggingEvent event) {
return "0";
}
}
}