com.addc.commons.queue.persistence.PersistentQueue Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of addc-queues Show documentation
The addc-queues library supplies support for internal persistent queues using an optional DERBY database for storage.
The newest version!
package com.addc.commons.queue.persistence;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.addc.commons.Constants;
import com.addc.commons.Mutex;
import com.addc.commons.database.Database;
import com.addc.commons.i18n.I18nTextFactory;
import com.addc.commons.queue.ElementSerializer;

/**
 * PersistingQueue stores and retrieves objects to and from persistent storage
 * in a database.
 * 
 */
public class PersistentQueue {

    private static final String TABLE= "QUEUE";
    private static final String CREATE_TABLE= "CREATE TABLE " + TABLE
            + " (index BIGINT, deleted SMALLINT DEFAULT 0, item BLOB, PRIMARY KEY (index))";
    private static final String SELECT_MAX_SQL= "SELECT MAX(index) FROM " + TABLE;
    private static final String SELECT_MIN_SQL= "SELECT MIN(index) FROM " + TABLE;
    private static final String SELECT_COUNT_SQL= "SELECT COUNT(index) FROM " + TABLE;
    private static final String INSERT_ELEMENT_SQL= "INSERT INTO " + TABLE + "(index,item) VALUES(?,?)";
    private static final String SELECT_NEXT_BATCH_SQL= "SELECT index, deleted, item FROM " + TABLE + " WHERE index=?";
    private static final String UPDATE_ELEMENT_FOR_DELETE_SQL= "UPDATE " + TABLE + " SET deleted=1 WHERE index=?";
    private static final String REMOVE_DELETED_ELEMENTS_SQL= "DELETE FROM " + TABLE + " WHERE deleted=1";
    private static final String DELETE_ALL_RECORDS_SQL= "DELETE FROM " + TABLE;

    private static final long NO_INDEX= -1L;

    /**
     * Clear deleted items from the queue every DELETE_THRESHOLD polls
     */
    private static final long DELETE_THRESHOLD= 250;

    private static final Logger LOGGER= LoggerFactory.getLogger(PersistentQueue.class);

    private final Mutex idxMutex= new Mutex();
    private Database persistentQueueDb;
    private PreparedStatement insertElement;
    private PreparedStatement selectNextElement;
    private PreparedStatement setElementDeleted;
    private PreparedStatement removeDeletedElements;

    private ElementSerializer serializerEngine;

    private Connection putConn;
    private Connection pollConn;

    private long firstIndex= NO_INDEX;
    private long lastIndex= NO_INDEX;
    private long batchDeleteCounter;

    private boolean shutdown;

    /**
     * Create a PersistentQueueDerby instance that uses the
     * {@link DefaultElementSerializer} and is not confidential.
     * 
     * @param database
     *            The {@link Database} to use
     * @throws PersistentQueueException
     *             If initialization fails
     */
    public PersistentQueue(Database database) throws PersistentQueueException {
        this(database, new DefaultElementSerializer());
    }

    /**
     * Create a PersistentQueueImpl instance
     * 
     * @param database
     *            The {@link Database} to use
     * @param serializerEngine
     *            The {@link ElementSerializer} to use
     * @throws PersistentQueueException
     *             If initialization fails
     */
    public PersistentQueue(Database database, ElementSerializer serializerEngine) throws PersistentQueueException {
        try {
            persistentQueueDb= database;
            createDatatableIfNeeded();
            createPutConnection();
            createPollConnection();
            deleteTaggedElements();
            initFirstIndex();
            initLastIndex();
        } catch (SQLException e) {
            throw new PersistentQueueException(e);
        }

        this.serializerEngine= serializerEngine;
    }

    /**
     * Put an element onto the queue
     * 
     * @param element
     *            The element to put
     * @throws PersistentQueueException
     *             if there was a problem accessing the underlying database
     */
    public void put(T element) throws PersistentQueueException {
        LOGGER.debug("Persist {}", element);

        long index= getIndex();
        try {
            insertElement.setLong(1, index);
            insertElement.setBytes(2, serializerEngine.serializeItem(element));
            insertElement.executeUpdate();
            updateIndices(index);
            LOGGER.debug("{} has been persisted at index {}", element, index);

        } catch (SQLException | IOException e) {
            LOGGER.error("Failed to put element", e);
            throw new PersistentQueueException(e);
        }
    }

    /**
     * Check to see if there is an element available in the queue
     * 
     * @return The element if available or null
     */
    @SuppressWarnings("PMD.IdenticalCatchBranches")
    public T poll() {
        T element= null;
        ResultSet rs= null;
        long index= getFirstIndex();
        LOGGER.debug("First index in dB is {}", index);
        if (index != NO_INDEX) {
            try {
                selectNextElement.clearParameters();
                selectNextElement.setLong(1, index);
                rs= selectNextElement.executeQuery();
                if (rs.next()) {
                    // create the EventBatch object
                    byte[] payloadBytes= rs.getBytes(3);
                    element= serializerEngine.deserializeItem(payloadBytes);

                    // Tag the row for deletion
                    setElementDeleted.clearParameters();
                    setElementDeleted.setLong(1, index);
                    setElementDeleted.execute();
                    incFirstIndex();
                    batchDeleteCounter++;
                }
            } catch (SQLException e) {
                LOGGER.error(I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.PQ_READ_ERROR), e);
            } catch (IOException | ClassNotFoundException e) {
                LOGGER.error(I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.PQ_CORRUPTED), e);
            } finally {
                closeResultSet(rs);
                if (batchDeleteCounter > DELETE_THRESHOLD) {
                    deleteTaggedElements();
                }
            }
            LOGGER.debug("Returns from queue {}", element);
        }
        return element;
    }

    private long getIndex() {
        synchronized (idxMutex) {
            return ++lastIndex;
        }
    }

    private void initLastIndex() throws SQLException {
        synchronized (idxMutex) {
            ResultSet rs= null;
            PreparedStatement selectMaxStatement= null;
            try {
                selectMaxStatement= putConn.prepareStatement(SELECT_MAX_SQL);
                rs= selectMaxStatement.executeQuery();
                if (rs.next()) {
                    lastIndex= rs.getLong(1);
                } else {
                    lastIndex= 0L;
                }

            } finally {
                closeResultSet(rs);
                closeStatement(selectMaxStatement);
            }
            LOGGER.debug("Last index in dB is {}", lastIndex);
        }
    }

    /**
     * Clear the queue deleting all the data in the underlying database
     * 
     * @throws PersistentQueueException
     *             if there was a problem accessing the underlying database
     */
    public void clear() throws PersistentQueueException {
        try (PreparedStatement deleteAllRecords= putConn.prepareStatement(DELETE_ALL_RECORDS_SQL)) {
            deleteAllRecords.execute();
        } catch (SQLException e) {
            throw new PersistentQueueException(e);
        }
    }

    /**
     * @return The location of the underlying database
     */
    public String getQueuePath() {
        return persistentQueueDb.getLocation();
    }

    /**
     * Query whether the queue is shut dowm
     * 
     * @return true if the queue is shut dowm
     */
    public boolean isShutdown() {
        return shutdown;
    }

    /**
     * Shut down the queue
     */
    public void shutdown() {
        try {
            deleteTaggedElements();
        } finally {
            closeDatabase();
            shutdown= true;
        }

    }

    private void initFirstIndex() throws PersistentQueueException {
        synchronized (idxMutex) {
            if (getRowCount() == 0L) {
                firstIndex= NO_INDEX;
            } else {
                try (PreparedStatement statement= putConn.prepareStatement(SELECT_MIN_SQL);
                        ResultSet rs= statement.executeQuery()) {
                    if (rs.next()) {
                        firstIndex= rs.getLong(1);
                    }
                } catch (SQLException e) {
                    throw new PersistentQueueException(e);
                }
            }
            LOGGER.debug("First index in table is {}", firstIndex);
        }
    }

    private long getRowCount() throws PersistentQueueException {
        long rowCount= 0;
        try (PreparedStatement statement= putConn.prepareStatement(SELECT_COUNT_SQL);
                ResultSet rs= statement.executeQuery()) {
            if (rs.next()) {
                rowCount= rs.getLong(1);
            }
        } catch (SQLException e) {
            throw new PersistentQueueException(e);
        }
        return rowCount;
    }

    private void createPollConnection() throws SQLException {
        pollConn= persistentQueueDb.getConnection();
        selectNextElement= pollConn.prepareStatement(SELECT_NEXT_BATCH_SQL);
        setElementDeleted= pollConn.prepareStatement(UPDATE_ELEMENT_FOR_DELETE_SQL);
        removeDeletedElements= pollConn.prepareStatement(REMOVE_DELETED_ELEMENTS_SQL);
    }

    private void createPutConnection() throws SQLException {
        putConn= persistentQueueDb.getConnection();
        insertElement= putConn.prepareStatement(INSERT_ELEMENT_SQL);
    }

    /**
     * So we can test it made package protected
     */
    @SuppressWarnings("PMD.PreserveStackTrace")
    void deleteTaggedElements() {
        try {
            LOGGER.info("Cleanup queue...");
            removeDeletedElements.execute();
            LOGGER.debug("deleted {} rows", removeDeletedElements.getUpdateCount());
            LOGGER.info("Cleanup completed.");
        } catch (SQLException e) {
            LOGGER.error(I18nTextFactory.getTranslator(Constants.BASENAME).translate(Constants.PQ_DELETE_ERROR), e);
        } finally {
            batchDeleteCounter= 0;
        }
    }

    private void closeResultSet(ResultSet rs) {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                LOGGER.error("Could not close result set", e);
            }
        }
    }

    private void closeDatabase() {
        LOGGER.info("Close queue connections");
        closePollConn();
        closePutConn();
        persistentQueueDb.shutdown();
    }

    private void closePollConn() {
        closeStatement(removeDeletedElements);
        closeStatement(selectNextElement);
        closeStatement(setElementDeleted);
        closeConnection(pollConn);
        pollConn= null;
    }

    private void closePutConn() {
        closeStatement(insertElement);
        closeConnection(putConn);
        putConn= null;
    }

    /**
     * Checks if the table already exists in the database and creates it if not
     * 
     * @throws SQLException
     */
    private void createDatatableIfNeeded() throws SQLException {
        PreparedStatement createTableStmt= null;
        Connection conn= null;
        try {
            if (persistentQueueDb.hasTable(TABLE)) {
                LOGGER.debug("Data table {} already exists.", TABLE);
            } else {
                conn= persistentQueueDb.getConnection();
                createTableStmt= conn.prepareStatement(CREATE_TABLE);
                createTableStmt.execute();
                LOGGER.debug("Data table {} has been created.", TABLE);
            }
        } finally {
            closeStatement(createTableStmt);
            closeConnection(conn);
        }
    }

    private void closeStatement(PreparedStatement statement) {
        if (statement != null) {
            try {
                statement.close();
            } catch (SQLException e) {
                LOGGER.debug("Error closing statement", e);
            }
        }
    }

    private void closeConnection(Connection connection) {
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
                LOGGER.debug("Error when closing the connection", e);
            }
        }
    }

    private void updateIndices(long index) {
        synchronized (idxMutex) {
            if (firstIndex == NO_INDEX) {
                firstIndex= index;
            }
            LOGGER.debug("Indices: first={}, last={}", firstIndex, lastIndex);
        }
    }

    private long getFirstIndex() {
        synchronized (idxMutex) {
            return firstIndex;
        }
    }

    private void incFirstIndex() {
        synchronized (idxMutex) {
            if (firstIndex < lastIndex) {
                firstIndex++;
            } else {
                firstIndex= NO_INDEX;
            }
        }
    }

    Connection getConnection() throws SQLException {
        return persistentQueueDb.getConnection();
    }

}