com.addc.commons.queue.PersistingQueue Maven / Gradle / Ivy
Show all versions of addc-queues Show documentation
package com.addc.commons.queue;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.addc.commons.Constants;
import com.addc.commons.database.Database;
import com.addc.commons.database.DatabaseException;
import com.addc.commons.database.derby.DerbyDatabase;
import com.addc.commons.i18n.I18nTextFactory;
import com.addc.commons.jmx.MBeanServerHelper;
import com.addc.commons.queue.persistence.DefaultElementSerializer;
import com.addc.commons.queue.persistence.PersistentQueue;
import com.addc.commons.queue.persistence.PersistentQueueException;
/**
* The PersistingQueue supplies a bounded queue that will optionally persist any
* overflow and save any queue elements that are left in the buffer on shutdown.
*
* When the queue is created/started it will fill itself with any elements in
* persistence allowing recovery of any unread elements from the previos session.
*
* The order of the elements added is guaranteed for any queue reader.
*
* If persistence is not enabled, the queue will throttle when its buffer is
* full to try to give the reader time to get elements from the queue and
* liberate space in the buffer. If the throttling fails to free any space, the
* element will be dropped and any {@link PersistingQueueTransitionListener}s
* will be notified.
*
* The class is thread-safe.
*
*/
public class PersistingQueue {
private static final Long[] RETRY_DELAYS= { 10L, 13L, 18L, 25L };
private static final Logger LOGGER= LoggerFactory.getLogger(PersistingQueue.class);
private final PersistentQueue persistentQueue;
private final LinkedBlockingQueue payloadBuffer;
private final Object bufferLock= new Object();
private final Object takeLock= new Object();
private final List throttleDelays;
private final PersistingQueueStatistics statistics;
private final List listeners= Collections
.synchronizedList(new LinkedList());
private final List> transitionListeners= Collections
.synchronizedList(new LinkedList>());
private Iterator throttleIterator;
private boolean shutdown;
private boolean interruptTake;
private final boolean dumpStatisticsOnExit;
/**
* Create new persistingQueue that will use the
* {@link DefaultElementSerializer} if persistence is enabled in the
* configuration and will not dump statistics on shutdown.
*
* @param config
* The configuration for underlying persistence
* @param statistics
* The {@link PersistingQueueStatistics} to use. This will be
* registered as an MBean if its getObjectName() does not return
* null.
* @param maxBufferSize
* The maximum number of messages in the queue before they are
* persisted or dropped
* @throws DatabaseException
* If the {@link Database} cannot be initialized
* @throws PersistentQueueException
* If the internal {@link PersistentQueue} cannot be
* initialized.
*/
public PersistingQueue(PersistenceConfig config, PersistingQueueStatistics statistics, int maxBufferSize)
throws DatabaseException, PersistentQueueException {
this(config, statistics, false, maxBufferSize, null);
}
/**
* Create new persistingQueue
*
* @param config
* The configuration for underlying persistence
* @param statistics
* The {@link PersistingQueueStatistics} to use. This will be
* registered as an MBean if its getObjectName() does not return
* null.
* @param dumpStatisticsOnExit
* true to dump statistics on exit
* @param maxBufferSize
* The maximum number of messages in the queue before they are
* persisted or dropped
* @param serializer
* The {@link ElementSerializer} to use if persistence is
* enabled, if null a
* {@link DefaultElementSerializer} will be used
* @throws DatabaseException
* If the {@link Database} cannot be initialized
* @throws PersistentQueueException
* If the internal {@link PersistentQueue} cannot be
* initialized.
*/
public PersistingQueue(PersistenceConfig config, PersistingQueueStatistics statistics,
boolean dumpStatisticsOnExit, int maxBufferSize, ElementSerializer serializer)
throws DatabaseException, PersistentQueueException {
this.dumpStatisticsOnExit= dumpStatisticsOnExit;
this.payloadBuffer= new LinkedBlockingQueue<>(maxBufferSize);
this.statistics= statistics;
this.throttleDelays= Arrays.asList(RETRY_DELAYS);
if (config.isPersistent()) {
Database database= new DerbyDatabase(config.getDerbyDbDir(), config.isEncrypted());
if (serializer == null) {
this.persistentQueue= new PersistentQueue<>(database, new DefaultElementSerializer());
} else {
this.persistentQueue= new PersistentQueue<>(database, serializer);
}
LOGGER.info("Using persistent queue");
fillBufferFromPersistentQueue();
} else {
this.persistentQueue= null;
}
MBeanServerHelper.getInstance().registerStandardMBean(statistics, statistics.getObjectName());
LOGGER.info("Created Persisting Queue");
}
/**
* This method is synchronized internally as several threads can put elements in the queue.
*
* @param element
* element to enqueue
* @throws IllegalStateException
* If the queue has been shut down
*/
public void put(T element) {
if (shutdown) {
throw new IllegalStateException(
I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.QUEUE_SHUTDOWN));
}
synchronized (bufferLock) {
LOGGER.debug("{}", element);
statistics.itemCreated(element);
if (payloadBuffer.offer(element)) {
checkStateAndNotify();
} else {
if (isPersistent()) {
saveToPersistentQueue(element);
} else {
// We try to give the put another chance just in case
// the buffer isn't being read fast enough
throttleIterator= throttleDelays.iterator();
boolean success= false;
while (throttleIterator.hasNext() && !success) {
throttle();
success= payloadBuffer.offer(element);
}
noPersistencePostProcessing(element, success);
}
}
}
LOGGER.trace("Signal take()");
synchronized (takeLock) {
takeLock.notify();
}
LOGGER.trace("Signalled take()");
}
/**
* Waits until there is something in the buffer or untill the take is interrupted.
*
* @return the next available element. Can be null if the take was
* interrupted.
* @throws IllegalStateException
* If the queue has been shut down
*/
@SuppressWarnings("PMD.AssignmentInOperand")
public T take() {
if (shutdown) {
throw new IllegalStateException(
I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.QUEUE_SHUTDOWN));
}
T element= null;
synchronized (takeLock) {
// interruptTake= false;
while (!interruptTake && (element= poll()) == null) {
try {
takeLock.wait(500L);
} catch (@SuppressWarnings("unused") InterruptedException e) {
LOGGER.info("The take has been interrupted");
break;
}
}
interruptTake= false;
}
return element;
}
/**
* This will interrupt an ongoing take method if there is one; otherwise it
* does nothing.
*
*/
public void interruptTake() {
synchronized (takeLock) {
interruptTake= true;
takeLock.notifyAll();
}
}
/**
* Returns the first element in the buffer which may be null if the buffer
* is empty. After removing the first element from the buffer and if the
* queue is persistent, the buffer will be filled from the persistent queue
* if there are any elements available there.
*
* @return the next available element which may be null
* @throws IllegalStateException
* If the queue has been shut down
*/
public T poll() {
if (shutdown) {
throw new IllegalStateException(
I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.QUEUE_SHUTDOWN));
}
T element= null;
synchronized (bufferLock) {
if (payloadBuffer.isEmpty()) {
fillBufferFromPersistentQueue();
}
if (!payloadBuffer.isEmpty()) {
element= payloadBuffer.remove();
checkStateAndNotify();
fillBufferFromPersistentQueue();
}
}
LOGGER.debug("{}", element);
return element;
}
/**
* Saves the element passed to it (the last element polled) and any elements
* stored in the buffer to the persistent queue
*
* @param element
* A pending element that should be stored before the buffer
*/
public void shutdown(T element) {
if (shutdown) {
throw new IllegalStateException(
I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.QUEUE_SHUTDOWN));
}
this.shutdown= true;
if (isPersistent()) {
if (element != null) {
LOGGER.info("Saving first outstanding element to persistent queue...");
saveToPersistentQueue(element);
}
for (T payload : payloadBuffer) {
LOGGER.info("Pushing queue to persistence...");
saveToPersistentQueue(payload);
}
persistentQueue.shutdown();
} else {
if (element != null) {
statistics.itemDropped(element);
}
for (T payload : payloadBuffer) {
statistics.itemDropped(payload);
}
payloadBuffer.clear();
}
for (PersistingQueueTransitionListener listener : transitionListeners) {
listener.onShutdown();
}
if (dumpStatisticsOnExit) {
LOGGER.info(statistics.getQueueStatistics());
}
LOGGER.info("Unregistering Statistics MBean ...");
if (!MBeanServerHelper.getInstance().unregisterMBean(statistics.getObjectName())) {
LOGGER.error("Failed to unregister MBean {}", statistics.getObjectName());
}
LOGGER.info("Terminated.");
}
/**
* Query whether the queue is shut down
*
* @return whether the queue is shut down
*/
public boolean isShutdown() {
return shutdown;
}
private void noPersistencePostProcessing(T element, boolean success) {
if (success) {
for (PersistingQueueTransitionListener listener : transitionListeners) {
listener.onPut();
}
} else {
LOGGER.debug("Dropping {}", element);
statistics.itemDropped(element);
for (PersistingQueueTransitionListener listener : transitionListeners) {
listener.onDrop(element);
}
}
}
/**
* Gets all the events it can from the persistent queue and puts them in the
* buffer
*/
@SuppressWarnings("PMD.AssignmentInOperand")
private void fillBufferFromPersistentQueue() {
if (isPersistent()) {
LOGGER.debug("Fill the buffer from database");
T element;
boolean done= false;
do {
element= persistentQueue.poll();
if (element == null) {
done= true;
} else {
statistics.itemReadFromPersistence(element);
LOGGER.debug("Read {} from persistence", element);
done= payloadBuffer.offer(element);
}
} while (!done);
}
}
private void saveToPersistentQueue(T element) {
try {
persistentQueue.put(element);
statistics.itemWrittenToPersistence(element);
LOGGER.debug("Wrote {} to persistence", element);
} catch (Exception e) {
LOGGER.error(I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.QUEUE_DROP_ELEMENT), e);
statistics.itemDropped(element);
}
}
/**
* Sleep for a time before trying to put the batch again in the buffer
*
* @param incomingBatch
* @throws InterruptedException
*/
private void throttle() {
long nextTimeToSleep= throttleIterator.next();
LOGGER.debug("Sleeping for {}", nextTimeToSleep);
try {
Thread.sleep(nextTimeToSleep);
} catch (InterruptedException e) {
LOGGER.debug("Error while throttling", e);
}
}
/**
* Adds a {@link PersistingQueueStateListener}
*
* @param listener
* the listener to be added
*/
public void addListener(PersistingQueueStateListener listener) {
this.listeners.add(listener);
}
/**
* Adds a {@link PersistingQueueTransitionListener}
*
* @param listener
* The listener to add
*/
public void addTransitionListener(PersistingQueueTransitionListener listener) {
this.transitionListeners.add(listener);
}
/**
* Checks the state of this batch queue and notifies all the listeners if it
* is needed
*
*/
private void checkStateAndNotify() {
for (PersistingQueueStateListener listener : listeners) {
if (payloadBuffer.remainingCapacity() < 2) {
listener.onGettingFull();
} else if (payloadBuffer.size() == 1) {
listener.onGettingEmpty();
}
}
for (PersistingQueueTransitionListener listener : transitionListeners) {
listener.onPut();
}
}
/**
* For testing
*/
boolean isPersistent() {
return persistentQueue != null;
}
/**
* For testing
*/
PersistentQueue getPersistentQueue() {
return persistentQueue;
}
/**
* For testing
*/
void clear() {
payloadBuffer.clear();
if (isPersistent()) {
try {
persistentQueue.clear();
} catch (PersistentQueueException e) {
LOGGER.error("FATAL: Error clearing persistent queue.", e);
}
}
}
/**
* for testing
*/
LinkedBlockingQueue getPayloadBuffer() {
return payloadBuffer;
}
}