com.ning.billing.queue.DBBackedQueue Maven / Gradle / Ivy
/*
* Copyright 2010-2013 Ning, Inc.
*
* Ning licenses this file to you under the Apache License, version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.ning.billing.queue;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import org.skife.jdbi.v2.Transaction;
import org.skife.jdbi.v2.TransactionStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.ning.billing.Hostname;
import com.ning.billing.queue.api.PersistentQueueConfig;
import com.ning.billing.queue.api.PersistentQueueEntryLifecycleState;
import com.ning.billing.queue.dao.EventEntryModelDao;
import com.ning.billing.queue.dao.QueueSqlDao;
import com.ning.billing.clock.Clock;
import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
/**
* This class abstract the interaction with the database tables which store the persistent entries for the bus events or
* notification events.
*
* This can be configured to either cache the recordId for the entries that are ready be fetched so that we avoid expansive
* queries to the database. Alternatively, the inflight queue is not used and the search query is always run when we need to retrieve
* new entries.
*
* @param
*/
public class DBBackedQueue {
private static final Logger log = LoggerFactory.getLogger(DBBackedQueue.class);
//
// Exponential backup retry logic, where we retyr up to 10 times for a maxium of about 10 sec ((521 + 256 + 128 + ... ) * 10)
//
private final static long INFLIGHT_ENTRIES_INITIAL_POLL_SLEEP_MS = 10;
private final static int INFLIGHT_ENTRIES_POLL_MAX_RETRY = 10;
private final String DB_QUEUE_LOG_ID;
private final AtomicBoolean isQueueOpenForWrite;
private final AtomicBoolean isQueueOpenForRead;
private final QueueSqlDao sqlDao;
private final Clock clock;
private final Queue inflightEvents;
private final PersistentQueueConfig config;
private final boolean useInflightQueue;
private final Counter totalInflightProcessed;
private final Counter totalProcessed;
private final Counter totalInflightWritten;
private final Counter totalWritten;
public DBBackedQueue(final Clock clock,
final QueueSqlDao sqlDao,
final PersistentQueueConfig config,
final String dbBackedQId,
final MetricRegistry metricRegistry) {
this.useInflightQueue = config.isUsingInflightQueue();
this.sqlDao = sqlDao;
this.config = config;
this.inflightEvents = useInflightQueue ? new LinkedBlockingQueue(config.getQueueCapacity()) : null;
this.isQueueOpenForWrite = new AtomicBoolean(false);
this.isQueueOpenForRead = new AtomicBoolean(false);
this.clock = clock;
this.totalInflightProcessed = metricRegistry.counter(MetricRegistry.name(DBBackedQueue.class, dbBackedQId + "-totalInflightProcessed"));
this.totalProcessed = metricRegistry.counter(MetricRegistry.name(DBBackedQueue.class, dbBackedQId + "-totalProcessed"));
this.totalInflightWritten = metricRegistry.counter(MetricRegistry.name(DBBackedQueue.class, dbBackedQId + "-totalInflightWritten"));
this.totalWritten = metricRegistry.counter(MetricRegistry.name(DBBackedQueue.class, dbBackedQId + "-totalWritten"));
DB_QUEUE_LOG_ID = "DBBackedQueue-" + dbBackedQId + ": ";
}
public void initialize() {
final List entries = fetchReadyEntries(config.getPrefetchEntries());
if (entries.size() == 0) {
isQueueOpenForWrite.set(true);
isQueueOpenForRead.set(true);
} else if (entries.size() < config.getPrefetchEntries()) {
isQueueOpenForWrite.set(true);
isQueueOpenForRead.set(false);
} else {
isQueueOpenForWrite.set(false);
isQueueOpenForRead.set(false);
}
if (useInflightQueue) {
inflightEvents.clear();
}
// Lame, no more clear API
totalInflightProcessed.dec(totalInflightProcessed.getCount());
totalProcessed.dec(totalProcessed.getCount());
totalInflightWritten.dec(totalInflightWritten.getCount());
totalWritten.dec(totalWritten.getCount());
log.info(DB_QUEUE_LOG_ID + "Initialized with isQueueOpenForWrite = " + isQueueOpenForWrite.get() + ", isQueueOpenForRead" + isQueueOpenForRead.get());
}
public void insertEntry(final T entry) {
sqlDao.inTransaction(new Transaction>() {
@Override
public Void inTransaction(final QueueSqlDao transactional, final TransactionStatus status) throws Exception {
insertEntryFromTransaction(transactional, entry);
return null;
}
});
}
public void insertEntryFromTransaction(final QueueSqlDao transactional, final T entry) {
transactional.insertEntry(entry, config.getTableName());
totalWritten.inc();
if (useInflightQueue && isQueueOpenForWrite.get()) {
Long lastInsertId = transactional.getLastInsertId();
boolean success = inflightEvents.offer(lastInsertId);
if (log.isDebugEnabled()) {
log.debug(DB_QUEUE_LOG_ID + "Inserting entry " + lastInsertId +
(success ? " into inflightQ" : " into disk") +
" [" + entry.getEventJson() + "]" );
}
// Q overflowed
if (!success) {
final boolean q = isQueueOpenForWrite.compareAndSet(true, false);
if (q) {
log.info(DB_QUEUE_LOG_ID + "Closing Q for write: Overflowed with recordId = " + lastInsertId);
}
} else {
totalInflightWritten.inc();
}
}
}
public List getReadyEntries() {
List candidates = ImmutableList.of();
// If we are not configured to use inflightQ then run expensive query
if (!useInflightQueue) {
final List entriesToClaim = fetchReadyEntries(config.getMaxEntriesClaimed());
if (entriesToClaim.size() > 0) {
candidates = claimEntries(entriesToClaim);
}
return candidates;
}
if (isQueueOpenForRead.get()) {
candidates = fetchReadyEntriesFromIds();
// There are entries in the Q, we just return those
if (candidates.size() > 0) {
return claimEntries(candidates);
}
// There are no more entries in the Q but the Q is not open for write so either there is nothing to be read, or
// the Q overflowed previously so we disable reading from the Q and continue below.
if (!isQueueOpenForWrite.get()) {
final boolean q = isQueueOpenForRead.compareAndSet(true, false);
if (q) {
log.info(DB_QUEUE_LOG_ID + " Closing Q for read");
}
}
}
if (!isQueueOpenForRead.get()) {
List prefetchedEntries = fetchReadyEntries(config.getPrefetchEntries());
// There is a small number so we re-enable adding entries in the Q
if (prefetchedEntries.size() < config.getPrefetchEntries()) {
log.info(DB_QUEUE_LOG_ID + " Opening Q for write");
isQueueOpenForWrite.compareAndSet(false, true);
}
// Only keep as many candidates as we are allowed to
final int candidateSize = prefetchedEntries.size() > config.getMaxEntriesClaimed() ? config.getMaxEntriesClaimed() : prefetchedEntries.size();
candidates = prefetchedEntries.subList(0, candidateSize);
totalProcessed.inc(candidates.size());
//
// If we see that we catch up with entries in the inflightQ, we need to switch mode and remove entries we are processing
// Failure to remove the entries would NOT trigger a bug, but might waste cycles where getReadyEntries() would return less
// elements as expected, because entries have already been processed.
//
if (removeInflightEventsWhenSwitchingToQueueOpenForRead(candidates)) {
log.info(DB_QUEUE_LOG_ID + " Opening Q for read");
final boolean q = isQueueOpenForRead.compareAndSet(false, true);
if (q) {
log.info(DB_QUEUE_LOG_ID + " Opening Q for read");
}
}
return claimEntries(candidates);
}
return candidates;
}
private boolean removeInflightEventsWhenSwitchingToQueueOpenForRead(final List candidates) {
boolean foundEntryInInflightEvents = false;
for (T entry : candidates) {
foundEntryInInflightEvents = inflightEvents.remove(entry.getRecordId());
}
return foundEntryInInflightEvents;
}
public void updateOnError(final T entry) {
sqlDao.inTransaction(new Transaction>() {
@Override
public Void inTransaction(final QueueSqlDao transactional, final TransactionStatus status) throws Exception {
transactional.updateOnError(entry.getRecordId(), clock.getUTCNow().toDate(), entry.getErrorCount(), config.getTableName());
return null;
}
});
}
public void moveEntryToHistory(final T entry) {
sqlDao.inTransaction(new Transaction>() {
@Override
public Void inTransaction(final QueueSqlDao transactional, final TransactionStatus status) throws Exception {
moveEntryToHistoryFromTransaction(transactional, entry);
return null;
}
});
}
public void moveEntryToHistoryFromTransaction(final QueueSqlDao transactional, final T entry) {
transactional.insertEntry(entry, config.getHistoryTableName());
transactional.removeEntry(entry.getRecordId(), config.getTableName());
}
private List fetchReadyEntriesFromIds() {
final int size = config.getMaxEntriesClaimed() < inflightEvents.size() ? config.getMaxEntriesClaimed() : inflightEvents.size();
final List recordIds = new ArrayList(size);
for (int i = 0; i < size; i++) {
final Long entryId = inflightEvents.poll();
if (entryId != null) {
totalInflightProcessed.inc();
totalProcessed.inc();
recordIds.add(entryId);
}
}
// Before we return we filter on AVAILABLE entries for precaution; the case could potentially happen
// at the time when we switch from !isQueueOpenForRead -> isQueueOpenForRead with two thread in parallel.
//
List result = ImmutableList.of();
if (recordIds.size() > 0) {
if (log.isDebugEnabled()) {
log.debug(DB_QUEUE_LOG_ID + "fetchReadyEntriesFromIds, size = " + size + ", ids = " + Joiner.on(", ").join(recordIds));
}
final List entriesFromIds = getEntriesFromIds(recordIds);
result = ImmutableList.copyOf(Collections2.filter(entriesFromIds, new Predicate() {
@Override
public boolean apply(final T input) {
return (input.getProcessingState() == PersistentQueueEntryLifecycleState.AVAILABLE);
}
}));
}
return result;
}
//
// When there are some entries in the inflightQ, 3 cases may have occured:
// 1. The thread that posted the entry already committed his transaction and in which case the entry
// should be found on disk
// 2. The thread that posted the entry rolled back and therefore that entry will never make it on disk, it should
// be ignored.
// 3. The thread that posted the entry did not complete its transaction and so we don't know whether or not the entry
// will make it on disk.
//
// The code below looks for all entries by retrying the lookup on disk, and if eventually returns the one that have been found.
// Note that:
// - It is is OK for that thread to sleep and retry as this is its nature -- it sleeps and polls
// - If for some reason the entry is not found but the transaction eventually commits, we will end up in a situation
// where we have entries AVALAIBLE on disk; those would be cleared as we restart the service. If this ends up being an issue
// we could had some additional logics to catch them.
//
private List getEntriesFromIds(final List recordIds) {
int originalSize = recordIds.size();
List result = new ArrayList(recordIds.size());
int nbTries = 0;
do {
final List tmp = sqlDao.getEntriesFromIds(recordIds, config.getTableName());
if (tmp.size() > 0) {
for (T cur : tmp) {
recordIds.remove(cur.getRecordId());
}
result.addAll(tmp);
}
if (result.size() < originalSize) {
try {
long sleepTime = INFLIGHT_ENTRIES_INITIAL_POLL_SLEEP_MS * (int) Math.pow(2, nbTries);
Thread.sleep(sleepTime);
log.info(DB_QUEUE_LOG_ID + "Sleeping " + sleepTime + " for IDS = " + Joiner.on(",").join(recordIds));
} catch (InterruptedException e) {
log.warn(DB_QUEUE_LOG_ID + "Thread " + Thread.currentThread() + " got interrupted");
Thread.currentThread().interrupt();
return result;
}
}
nbTries++;
} while (result.size() < originalSize && nbTries < INFLIGHT_ENTRIES_POLL_MAX_RETRY);
if (recordIds.size() > 0) {
log.warn(DB_QUEUE_LOG_ID + " Missing inflight entries from disk, recordIds = [" + Joiner.on(",").join(recordIds) + " ]");
}
return result;
}
private List fetchReadyEntries(int size) {
final Date now = clock.getUTCNow().toDate();
final List entries = sqlDao.getReadyEntries(now, Hostname.get(), size, config.getTableName());
return entries;
}
private List claimEntries(List candidates) {
return ImmutableList.copyOf(Collections2.filter(candidates, new Predicate() {
@Override
public boolean apply(final T input) {
return claimEntry(input);
}
}));
}
private boolean claimEntry(T entry) {
final Date nextAvailable = clock.getUTCNow().plus(config.getClaimedTime().getMillis()).toDate();
final boolean claimed = (sqlDao.claimEntry(entry.getRecordId(), clock.getUTCNow().toDate(), Hostname.get(), nextAvailable, config.getTableName()) == 1);
if (claimed && log.isDebugEnabled()) {
log.debug(DB_QUEUE_LOG_ID + "Claiming entry " + entry.getRecordId());
}
return claimed;
}
public QueueSqlDao getSqlDao() {
return sqlDao;
}
public boolean isQueueOpenForWrite() {
return isQueueOpenForWrite.get();
}
public boolean isQueueOpenForRead() {
return isQueueOpenForRead.get();
}
public long getTotalInflightProcessed() {
return totalInflightProcessed.getCount();
}
public long getTotalProcessed() {
return totalProcessed.getCount();
}
public long getTotalInflightWritten() {
return totalInflightWritten.getCount();
}
public long getTotalWritten() {
return totalWritten.getCount();
}
}