org.komamitsu.fluency.buffer.PackedForwardBuffer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of fluency Show documentation
Show all versions of fluency Show documentation
Yet another fluent logger
package org.komamitsu.fluency.buffer;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.komamitsu.fluency.BufferFullException;
import org.komamitsu.fluency.sender.Sender;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessagePacker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
public class PackedForwardBuffer
extends Buffer
{
public static final String FORMAT_TYPE = "packed_forward";
private static final Logger LOG = LoggerFactory.getLogger(PackedForwardBuffer.class);
private final Map retentionBuffers = new HashMap();
private final LinkedBlockingQueue flushableBuffers = new LinkedBlockingQueue();
private final Queue backupBuffers = new ConcurrentLinkedQueue();
private final BufferPool bufferPool;
private PackedForwardBuffer(PackedForwardBuffer.Config bufferConfig)
{
super(bufferConfig);
bufferPool = new BufferPool(bufferConfig.getInitialBufferSize(), bufferConfig.getMaxBufferSize());
}
protected PackedForwardBuffer.Config getConfig()
{
return (PackedForwardBuffer.Config) bufferConfig;
}
private RetentionBuffer prepareBuffer(String tag, int writeSize)
throws BufferFullException
{
RetentionBuffer retentionBuffer = retentionBuffers.get(tag);
if (retentionBuffer != null && retentionBuffer.getByteBuffer().remaining() > writeSize) {
return retentionBuffer;
}
int existingDataSize = 0;
int newRetentionBufferSize;
if (retentionBuffer == null) {
newRetentionBufferSize = getConfig().getInitialBufferSize();
}
else{
existingDataSize = retentionBuffer.getByteBuffer().position();
newRetentionBufferSize = (int) (retentionBuffer.getByteBuffer().capacity() * getConfig().getBufferExpandRatio());
}
while (newRetentionBufferSize < (writeSize + existingDataSize)) {
newRetentionBufferSize *= getConfig().getBufferExpandRatio();
}
ByteBuffer acquiredBuffer = bufferPool.acquireBuffer(newRetentionBufferSize);
if (acquiredBuffer == null) {
throw new BufferFullException("Buffer is full. bufferConfig=" + getConfig() + ", bufferPool=" + bufferPool);
}
RetentionBuffer newBuffer = new RetentionBuffer(acquiredBuffer);
if (retentionBuffer != null) {
retentionBuffer.getByteBuffer().flip();
newBuffer.getByteBuffer().put(retentionBuffer.getByteBuffer());
bufferPool.returnBuffer(retentionBuffer.getByteBuffer());
}
LOG.trace("prepareBuffer(): allocate a new buffer. tag={}, buffer={}", tag, newBuffer);
retentionBuffers.put(tag, newBuffer);
return newBuffer;
}
private void loadDataToRetentionBuffers(String tag, ByteBuffer src)
throws IOException
{
synchronized (retentionBuffers) {
RetentionBuffer buffer = prepareBuffer(tag, src.remaining());
buffer.getByteBuffer().put(src);
buffer.getLastUpdatedTimeMillis().set(System.currentTimeMillis());
moveRetentionBufferIfNeeded(tag, buffer);
}
}
@Override
protected void loadBufferFromFile(List params, FileChannel channel)
{
if (params.size() != 1) {
throw new IllegalArgumentException("The number of params should be 1: params=" + params);
}
String tag = params.get(0);
try {
MappedByteBuffer src = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size());
loadDataToRetentionBuffers(tag, src);
}
catch (Exception e) {
LOG.error("Failed to load data to flushableBuffers: params={}, channel={}", params, channel);
}
}
private void saveBuffer(TaggableBuffer buffer)
{
saveBuffer(Collections.singletonList(buffer.getTag()), buffer.getByteBuffer());
}
@Override
protected void saveAllBuffersToFile()
throws IOException
{
moveRetentionBuffersToFlushable(true); // Just in case
TaggableBuffer flushableBuffer;
while ((flushableBuffer = flushableBuffers.poll()) != null) {
saveBuffer(flushableBuffer);
}
while ((flushableBuffer = backupBuffers.poll()) != null) {
saveBuffer(flushableBuffer);
}
}
@Override
public void append(String tag, long timestamp, Map data)
throws IOException
{
ObjectMapper objectMapper = objectMapperHolder.get();
ByteArrayOutputStream outputStream = outputStreamHolder.get();
outputStream.reset();
objectMapper.writeValue(outputStream, Arrays.asList(timestamp, data));
outputStream.close();
loadDataToRetentionBuffers(tag, ByteBuffer.wrap(outputStream.toByteArray()));
}
private void moveRetentionBufferIfNeeded(String tag, RetentionBuffer buffer)
throws IOException
{
if (buffer.getByteBuffer().position() > getConfig().getBufferRetentionSize()) {
moveRetentionBufferToFlushable(tag, buffer);
}
}
private void moveRetentionBuffersToFlushable(boolean force)
throws IOException
{
long expiredThreshold = System.currentTimeMillis() - getConfig().getBufferRetentionTimeMillis();
synchronized (retentionBuffers) {
for (Map.Entry entry : retentionBuffers.entrySet()) {
// it can be null because moveRetentionBufferToFlushable() can set null
if (entry.getValue() != null) {
if (force || entry.getValue().getLastUpdatedTimeMillis().get() < expiredThreshold) {
moveRetentionBufferToFlushable(entry.getKey(), entry.getValue());
}
}
}
}
}
private void moveRetentionBufferToFlushable(String tag, RetentionBuffer buffer)
throws IOException
{
try {
LOG.trace("moveRetentionBufferToFlushable(): tag={}, buffer={}", tag, buffer);
buffer.getByteBuffer().flip();
flushableBuffers.put(new TaggableBuffer(tag, buffer.getByteBuffer()));
retentionBuffers.put(tag, null);
}
catch (InterruptedException e) {
throw new IOException("Failed to move retention buffer due to interruption", e);
}
}
@Override
public String bufferFormatType()
{
return FORMAT_TYPE;
}
@Override
public void flushInternal(Sender sender, boolean force)
throws IOException
{
moveRetentionBuffersToFlushable(force);
TaggableBuffer flushableBuffer;
ByteArrayOutputStream header = new ByteArrayOutputStream();
MessagePacker messagePacker = MessagePack.newDefaultPacker(header);
while ((flushableBuffer = flushableBuffers.poll()) != null) {
boolean keepBuffer = false;
try {
LOG.trace("flushInternal(): bufferUsage={}, flushableBuffer={}", getBufferUsage(), flushableBuffer);
String tag = flushableBuffer.getTag();
ByteBuffer byteBuffer = flushableBuffer.getByteBuffer();
if (bufferConfig.isAckResponseMode()) {
messagePacker.packArrayHeader(3);
}
else {
messagePacker.packArrayHeader(2);
}
messagePacker.packString(tag);
messagePacker.packRawStringHeader(byteBuffer.limit());
messagePacker.flush();
try {
ByteBuffer headerBuffer = ByteBuffer.wrap(header.toByteArray());
List dataList = Arrays.asList(headerBuffer, byteBuffer);
if (bufferConfig.isAckResponseMode()) {
String uuid = UUID.randomUUID().toString();
byte[] uuidBytes = uuid.getBytes(CHARSET);
synchronized (sender) {
sender.sendWithAck(dataList, uuidBytes);
}
} else {
synchronized (sender) {
sender.send(dataList);
}
}
}
catch (IOException e) {
LOG.warn("Failed to send data. The data is going to be saved into the buffer again: data={}", flushableBuffer);
keepBuffer = true;
throw e;
}
}
finally {
header.reset();
if (keepBuffer) {
try {
flushableBuffers.put(flushableBuffer);
}
catch (InterruptedException e1) {
LOG.warn("Failed to save the data into the buffer. Trying to save it in extra buffer: chunk={}", flushableBuffer);
backupBuffers.add(flushableBuffer);
}
}
else {
bufferPool.returnBuffer(flushableBuffer.getByteBuffer());
}
}
}
}
@Override
protected synchronized void closeInternal()
{
retentionBuffers.clear();
bufferPool.releaseBuffers();
}
@Override
public long getAllocatedSize()
{
return bufferPool.getAllocatedSize();
}
@Override
public long getBufferedDataSize()
{
long size = 0;
synchronized (retentionBuffers) {
for (Map.Entry buffer : retentionBuffers.entrySet()) {
if (buffer.getValue() != null && buffer.getValue().getByteBuffer() != null) {
size += buffer.getValue().getByteBuffer().position();
}
}
}
for (TaggableBuffer buffer : flushableBuffers) {
if (buffer.getByteBuffer() != null) {
size += buffer.getByteBuffer().remaining();
}
}
return size;
}
private static class RetentionBuffer
{
private final AtomicLong lastUpdatedTimeMillis = new AtomicLong();
private final ByteBuffer byteBuffer;
public RetentionBuffer(ByteBuffer byteBuffer)
{
this.byteBuffer = byteBuffer;
}
public AtomicLong getLastUpdatedTimeMillis()
{
return lastUpdatedTimeMillis;
}
public ByteBuffer getByteBuffer()
{
return byteBuffer;
}
@Override
public String toString()
{
return "RetentionBuffer{" +
"lastUpdatedTimeMillis=" + lastUpdatedTimeMillis +
", byteBuffer=" + byteBuffer +
'}';
}
}
private static class TaggableBuffer
{
private final String tag;
private final ByteBuffer byteBuffer;
public TaggableBuffer(String tag, ByteBuffer byteBuffer)
{
this.tag = tag;
this.byteBuffer = byteBuffer;
}
public String getTag()
{
return tag;
}
public ByteBuffer getByteBuffer()
{
return byteBuffer;
}
@Override
public String toString()
{
return "TaggableBuffer{" +
"tag='" + tag + '\'' +
", byteBuffer=" + byteBuffer +
'}';
}
}
public static class Config extends Buffer.Config
{
private int initialBufferSize = 1024 * 1024;
private float bufferExpandRatio = 2.0f;
private int bufferRetentionSize = 4 * 1024 * 1024;
private int bufferRetentionTimeMillis = 400;
@Override
protected PackedForwardBuffer.Config self() {
return this;
}
public int getInitialBufferSize()
{
return initialBufferSize;
}
public PackedForwardBuffer.Config setInitialBufferSize(int initialBufferSize)
{
this.initialBufferSize = initialBufferSize;
return this;
}
public float getBufferExpandRatio()
{
return bufferExpandRatio;
}
public PackedForwardBuffer.Config setBufferExpandRatio(float bufferExpandRatio)
{
this.bufferExpandRatio = bufferExpandRatio;
return this;
}
public int getBufferRetentionSize()
{
return bufferRetentionSize;
}
public PackedForwardBuffer.Config setBufferRetentionSize(int bufferRetentionSize)
{
this.bufferRetentionSize = bufferRetentionSize;
return this;
}
public int getBufferRetentionTimeMillis()
{
return bufferRetentionTimeMillis;
}
public PackedForwardBuffer.Config setBufferRetentionTimeMillis(int bufferRetentionTimeMillis)
{
this.bufferRetentionTimeMillis = bufferRetentionTimeMillis;
return this;
}
@Override
public String toString()
{
return "Config{" +
"initialBufferSize=" + initialBufferSize +
", bufferExpandRatio=" + bufferExpandRatio +
", bufferRetentionSize=" + bufferRetentionSize +
", bufferRetentionTimeMillis=" + bufferRetentionTimeMillis +
"} " + super.toString();
}
@Override
protected PackedForwardBuffer createInstanceInternal()
{
return new PackedForwardBuffer(this);
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy