com.ebay.jetstream.batcher.AutoFlushBatcher Maven / Gradle / Ivy
The newest version!
/*******************************************************************************
* Copyright © 2012-2015 eBay Software Foundation
* This program is dual licensed under the MIT and Apache 2.0 licenses.
* Please see LICENSE for more information.
*******************************************************************************/
package com.ebay.jetstream.batcher;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author [email protected] - This is a generic batcher that batches data items of the specified type and flushes the batch of data items. The flush is triggered when
* the batch queue contains items equal to the specified threshold or the flush timer fires. The threshold and flush timer interval are both
* configurable.
*/
public abstract class AutoFlushBatcher implements Runnable {
private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.messaging");
private AtomicInteger m_flushInterval = new AtomicInteger(100);
private AtomicInteger m_maxFlushSz = new AtomicInteger(40);
private AutoFlushWriterQueue m_queue;
private Thread m_batchAssembler;
private AtomicBoolean m_shutdown = new AtomicBoolean(false);
private int m_maxQueueSz = 100000;
private int m_waitBetweenFlush = 30; // in millis - default = 30 millis
public int getWaitBetweenFlush() {
return m_waitBetweenFlush;
}
public void setWaitBetweenFlush(int waitBetweenFlush) {
m_waitBetweenFlush = waitBetweenFlush;
}
public int getMaxQueueSz() {
return m_maxQueueSz;
}
public void setMaxQueueSz(int maxQueueSz) {
m_maxQueueSz = maxQueueSz;
}
public int getFlushInterval() {
return m_flushInterval.get();
}
public void setFlushInterval(int flushInterval) {
m_flushInterval.set(flushInterval);
}
public void start() {
m_batchAssembler = new Thread(this, "BatchAssemblerThread");
m_batchAssembler.start();
}
/**
* @return
*/
public int getMaxFlushSz() {
return m_maxFlushSz.get();
}
/**
* @param maxFlushSz
*/
public void setMaxFlushSz(int maxFlushSz) {
this.m_maxFlushSz.set(maxFlushSz);
}
/**
*
*/
public AutoFlushBatcher() {
m_queue = new AutoFlushWriterQueue();
m_queue.setMaxFlushSz(m_maxFlushSz.get());
start();
}
/**
* @param autoFlushSz
* @param flushInterval
* in millis
*/
public AutoFlushBatcher(int autoFlushSz, int flushInterval) {
m_maxFlushSz.set(autoFlushSz);
m_flushInterval.set(flushInterval);
m_queue = new AutoFlushWriterQueue();
m_queue.setMaxFlushSz(m_maxFlushSz.get());
start();
}
/**
* @param e
*/
public void write(T e) throws Exception {
if (m_shutdown.get())
throw new Exception("Batcher has been shutdown");
m_queue.add(e);
}
/**
* @param items
* - derived class to implement this method
*/
public abstract void flush(List items);
/**
*
*/
public void shutdown() {
m_shutdown.set(true);
LOGGER.info( "Shutting down AutoFlushBufferWriter");
while (m_queue.size() > 0) {
List events = m_queue.get(getMaxFlushSz());
if (events != null)
flush(events);
}
}
@Override
public void run() {
while (!m_shutdown.get()) {
long elapsedTime = System.currentTimeMillis()
- m_queue.getLastFlushTime();
if (m_queue.isTimeToFlush()
|| (elapsedTime > m_flushInterval.get())) {
m_queue.setLastFlushTime(System.currentTimeMillis());
List events = m_queue.get(getMaxFlushSz());
if (events == null)
continue;
flush(events);
} else {
try {
Thread.sleep(getWaitBetweenFlush()); // wait 30 millis
} catch (InterruptedException e) {
}
}
}
}
}