net.sf.jabb.txsdp.DefaultTransactionalStreamDataBatchProcessing Maven / Gradle / Ivy
/**
*
*/
package net.sf.jabb.txsdp;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import net.sf.jabb.dstream.ReceiveStatus;
import net.sf.jabb.dstream.StreamDataSupplier;
import net.sf.jabb.dstream.StreamDataSupplierWithIdAndPositionRange;
import net.sf.jabb.dstream.StreamDataSupplierWithIdAndRange;
import net.sf.jabb.dstream.ex.DataStreamInfrastructureException;
import net.sf.jabb.seqtx.ReadOnlySequentialTransaction;
import net.sf.jabb.seqtx.SequentialTransaction;
import net.sf.jabb.seqtx.SequentialTransactionsCoordinator;
import net.sf.jabb.seqtx.SequentialTransactionsCoordinator.TransactionCounts;
import net.sf.jabb.seqtx.ex.DuplicatedTransactionIdException;
import net.sf.jabb.seqtx.ex.TransactionStorageInfrastructureException;
import net.sf.jabb.util.parallel.WaitStrategy;
import net.sf.jabb.util.text.DurationFormatter;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Transactional batch processing of stream data.
*
* @author James Hu
*
* @param type of the data/message/event
*
*/
public class DefaultTransactionalStreamDataBatchProcessing implements TransactionalStreamDataBatchProcessing {
static final Logger logger = LoggerFactory.getLogger(DefaultTransactionalStreamDataBatchProcessing.class);
protected String id;
protected FlexibleBatchProcessor batchProcessor;
protected List> suppliers;
protected SequentialTransactionsCoordinator txCoordinator;
protected Options processorOptions;
protected Map processors = new ConcurrentHashMap<>();
public List> getSuppliers() {
return suppliers;
}
@Override
public void setSuppliers(List> suppliers) {
this.suppliers = suppliers;
}
/**
* Get the transaction coordinator
* @return the txCoordinator
*/
public SequentialTransactionsCoordinator getTransactionCoordinator() {
return txCoordinator;
}
/**
* Constructor
* @param id ID of this processing
* @param processorOptions options
* @param txCoordinator transactions coordinator
* @param processor batch processor
* @param suppliers stream data suppliers
*/
public DefaultTransactionalStreamDataBatchProcessing(String id, Options processorOptions, SequentialTransactionsCoordinator txCoordinator,
FlexibleBatchProcessor processor, List> suppliers){
this.id = id;
this.processorOptions = new Options(processorOptions);
this.txCoordinator = txCoordinator;
this.batchProcessor = processor;
this.suppliers = new ArrayList<>();
this.suppliers.addAll(suppliers);
}
/**
* Constructor
* @param id ID of this processing
* @param processorOptions options
* @param txCoordinator transactions coordinator
* @param processor batch processor
* @param suppliers stream data suppliers
*/
@SafeVarargs
public DefaultTransactionalStreamDataBatchProcessing(String id, Options processorOptions, SequentialTransactionsCoordinator txCoordinator,
FlexibleBatchProcessor processor, StreamDataSupplierWithIdAndRange... suppliers){
this(id, processorOptions, txCoordinator, processor, Arrays.asList(suppliers));
}
/**
* Constructor
* @param id ID of this processing
* @param processorOptions options
* @param txCoordinator transactions coordinator
* @param processor simple batch processor
* @param maxBatchSize the maximum number of data items to be fetched and processed in a batch.
* The actual numbers of data items in a batch will be smaller than or equal to this number.
* @param receiveTimeout total duration allowed for receiving all the data items in a closed range batch/transaction.
* When fetching data for retrying a previously failed transaction which has a closed data range,
* this argument decides how long the processors wait for all the data to be fetched.
* If the full range of data cannot be fetched within this duration,
* the batch processing transaction will abort. Normally this duration should be long enough to make
* sure that data items as many as maxBatchSize can always be successfully fetched.
* @param receiveTimeoutForOpenRange total duration allowed for receiving all the data items in a open range batch/transaction.
* When fetching data for a new transaction which has an open data range (from a specific time to the infinity),
* this argument decides how long the processors wait for a batch of data to be fetched.
* The longer this duration is, the more data will probably be fetched for a batch, and the larger the
* batch is and the longer the end-to-end processing delay is.
* Normally it should be shorter than receiveTimeout
* @param suppliers stream data suppliers
*/
public DefaultTransactionalStreamDataBatchProcessing(String id, Options processorOptions, SequentialTransactionsCoordinator txCoordinator,
SimpleBatchProcessor processor, int maxBatchSize, Duration receiveTimeout, Duration receiveTimeoutForOpenRange,
List> suppliers){
this(id, processorOptions, txCoordinator,
new SimpleFlexibleBatchProcessor(processor, maxBatchSize, receiveTimeout, receiveTimeoutForOpenRange),
suppliers);
}
/**
* Constructor
* @param id ID of this processing
* @param processorOptions options
* @param txCoordinator transactions coordinator
* @param processor simple batch processor
* @param maxBatchSize the maximum number of data items to be fetched and processed in a batch.
* The actual numbers of data items in a batch will be smaller than or equal to this number.
* @param receiveTimeout total duration allowed for receiving all the data items in a closed range batch/transaction.
* When fetching data for retrying a previously failed transaction which has a closed data range,
* this argument decides how long the processors wait for all the data to be fetched.
* If the full range of data cannot be fetched within this duration,
* the batch processing transaction will abort. Normally this duration should be long enough to make
* sure that data items as many as maxBatchSize can always be successfully fetched.
* @param receiveTimeoutForOpenRange total duration allowed for receiving all the data items in a open range batch/transaction.
* When fetching data for a new transaction which has an open data range (from a specific time to the infinity),
* this argument decides how long the processors wait for a batch of data to be fetched.
* The longer this duration is, the more data will probably be fetched for a batch, and the larger the
* batch is and the longer the end-to-end processing delay is.
* Normally it should be shorter than receiveTimeout
* @param suppliers stream data suppliers
*/
@SafeVarargs
public DefaultTransactionalStreamDataBatchProcessing(String id, Options processorOptions, SequentialTransactionsCoordinator txCoordinator,
SimpleBatchProcessor processor, int maxBatchSize, Duration receiveTimeout, Duration receiveTimeoutForOpenRange,
StreamDataSupplierWithIdAndPositionRange... suppliers){
this(id, processorOptions, txCoordinator, processor, maxBatchSize, receiveTimeout, receiveTimeoutForOpenRange, Arrays.asList(suppliers));
}
@Override
public Runnable createProcessor(String processorId){
Validate.notNull(processorId, "Processor id cannot be null");
Processor runnable;
if (processors.containsKey(processorId)){
throw new IllegalArgumentException("Another runnable with the same processor ID already exists: " + processorId);
}
runnable = new Processor(processorId);
processors.put(processorId, runnable);
return runnable;
}
/**
* Start processing. Once started, the processing can later be paused or stopped.
* @param runnable the runnable to be started
*/
protected void start(Processor runnable){
if (runnable.state.compareAndSet(State.READY, State.RUNNING)){
return;
}
if (runnable.state.compareAndSet(State.PAUSED, State.RUNNING)){
return;
}
throw new IllegalStateException("Cannot start when in " + runnable.state.get() + " state: " + runnable.processorId);
}
/**
* Pause processing. Once paused, the processing can later be started or stopped.
* @param runnable the runnable to be paused
*/
protected void pause(Processor runnable){
if (runnable.state.compareAndSet(State.RUNNING, State.PAUSING)){
return;
}
throw new IllegalStateException("Cannot pause when not in running state: " + runnable.processorId);
}
/**
* Stop processing. Once stopped, the processing cannot be restarted.
* @param runnable the runnable to be stopped
*/
protected void stop(Processor runnable){
if (runnable.state.compareAndSet(State.RUNNING, State.STOPPING)){
return;
}
if (runnable.state.compareAndSet(State.PAUSED, State.STOPPING)){
return;
}
if (runnable.state.compareAndSet(State.PAUSING, State.STOPPING)){
return;
}
}
@Override
public void start(String processorId){
Processor runnable = processors.get(processorId);
Validate.notNull(runnable, "There is no processor with the id: " + processorId);
start(runnable);
}
@Override
public void pause(String processorId){
Processor runnable = processors.get(processorId);
Validate.notNull(runnable, "There is no processor with the id: " + processorId);
pause(runnable);
}
@Override
public void stop(String processorId){
Processor runnable = processors.get(processorId);
Validate.notNull(runnable, "There is no processor with the id: " + processorId);
stop(runnable);
}
@Override
public void remove(String processorId) {
processors.remove(processorId);
}
@Override
public void removeUnused() {
List stopped = getProcessorStatus().entrySet().stream()
.filter(entry->entry.getValue().getState().isUnused())
.map(entry->entry.getKey())
.collect(Collectors.toList());
processors.keySet().removeAll(stopped);
}
/**
* Start processing. Once started, the processing can later be paused or stopped.
*/
@Override
public void startAll(){
for (Processor runnable: processors.values()){
start(runnable);
}
}
/**
* Pause processing. Once paused, the processing can later be started or stopped.
*/
@Override
public void pauseAll(){
for (Processor runnable: processors.values()){
pause(runnable);
}
}
/**
* Stop processing. Once stopped, the processing cannot be restarted.
*/
@Override
public void stopAll(){
for (Processor runnable: processors.values()){
stop(runnable);
}
}
protected String seriesId(StreamDataSupplierWithIdAndRange supplierWithId){
if (id == null || id.length() == 0){
return supplierWithId.getId().replace('/', '_');
}else{
return id + "_" + supplierWithId.getId().replace('/', '_');
}
}
class Processor implements Runnable{
protected AtomicReference state = new AtomicReference<>(State.READY);
private String processorId;
Processor(String processorId){
this.processorId = processorId;
}
private void await(){
long millis = processorOptions.getTransactionAcquisitionDelay().toMillis();
if (millis > 0){
WaitStrategy waitStrategy = processorOptions.getWaitStrategy();
try{
waitStrategy.await(millis);
}catch(InterruptedException ie){
waitStrategy.handleInterruptedException(ie);
}
}
}
private boolean allProcessed(boolean[] outOfRangeReached){
for (boolean b: outOfRangeReached){
if (!b){
return false;
}
}
return true;
}
@Override
public void run() {
logger.debug("[{}] Start running: {}", processorId, state);
List> localSuppliers = new ArrayList<>(suppliers.size());
localSuppliers.addAll(suppliers);
boolean[] outOfRangeReached = new boolean[localSuppliers.size()];
// make sure we start from a random partition, and then do a round robin afterwards
Random random = new Random();
int partition = random.nextInt(outOfRangeReached.length);
// reuse these data structures in the thread
ProcessingContextImpl context = new ProcessingContextImpl(txCoordinator);
boolean sticky = false;
while(!state.compareAndSet(State.STOPPING, State.STOPPED)){
if (!localSuppliers.equals(suppliers)){ // if suppliers changed
localSuppliers.clear();
localSuppliers.addAll(suppliers);
outOfRangeReached = new boolean[localSuppliers.size()];
partition = partition % outOfRangeReached.length;
}
while(state.get() == State.RUNNING){
if (allProcessed(outOfRangeReached)){
state.set(State.FINISHED);
break;
}
long startTime = System.currentTimeMillis();
int attempts = 0;
SequentialTransaction transaction = null;
String seriesId = null;
StreamDataSupplierWithIdAndRange supplierWithIdAndRange = null;
StreamDataSupplier supplier = null;
// if sticky is true, we don't need to get a transaction skeleton from the coordinator. and we don't change to another partition
boolean newSticky = processorOptions.stickyMode == Options.STICKY_WHEN_OPEN_RANGE_SUCCEEDED && context.isOpenRangeSuccessfullyClosed ||
processorOptions.stickyMode == Options.STICKY_WHEN_OPEN_RANGE_SUCCEEDED_OR_NO_DATA && (context.isOpenRangeSuccessfullyClosed || context.isOpenRangeAbortedBecauseNothingReceived);
if (sticky != newSticky){
sticky = newSticky;
logger.debug("Processor '{}' {}stick on '{}'", processorId, sticky ? "" : "no longer ", seriesId(localSuppliers.get(partition)));
}
if (!sticky){
partition = (partition+1) % outOfRangeReached.length;
}
try{
while (state.get() == State.RUNNING && !outOfRangeReached[partition]){
supplierWithIdAndRange = localSuppliers.get(partition);
supplier = supplierWithIdAndRange.getSupplier();
seriesId = seriesId(supplierWithIdAndRange);
if (sticky){
transaction = context.transaction; // assume that we can get start position directly from the last transaction that had just finished
break;
}
attempts++;
try {
transaction = txCoordinator.startTransaction(seriesId, processorId,
processorOptions.getInitialTransactionTimeoutDuration(),
processorOptions.getMaxInProgressTransactions(), processorOptions.getMaxRetringTransactions());
} catch (Exception e) {
logger.warn("[{}] Processor {} startTransaction(...) failed", seriesId, processorId, e);
}
if (transaction != null){
break;
}
await();
partition = (partition+1) % outOfRangeReached.length;
}
// got a skeleton, with matching seriesId
while (transaction != null && (!transaction.hasStarted() || sticky && transaction == context.transaction) && state.get() == State.RUNNING){
String previousTransactionId;
String previousEndPosition;
String startPosition = null;
if (sticky && transaction == context.transaction && context.isOpenRangeAbortedBecauseNothingReceived){
previousTransactionId = context.previousTransactionPreviousTransactionId;
previousEndPosition = context.previousTransactionEndPosition;
startPosition = transaction.getStartPosition(); // just retry last one, the transaction object is from context
}else{
previousTransactionId = transaction.getTransactionId();
if (sticky && transaction == context.transaction && context.isOpenRangeSuccessfullyClosed){
previousEndPosition = transaction.getEndPosition(); // the transaction object is from context
}else{ // non sticky or another skeleton was returned from startTransaction(...)
previousEndPosition = transaction.getStartPosition(); // the transaction object is returned by the coordinator
}
if (previousEndPosition == null){
startPosition = ""; // the beginning of the range
}else{
startPosition = supplier.nextStartPosition(previousEndPosition);
}
}
transaction.setTransactionId(null);
transaction.setStartPosition(startPosition);
transaction.setEndPositionNull(); // for an open range transaction
transaction.setTimeout(processorOptions.getInitialTransactionTimeoutDuration());
attempts++;
context.previousTransactionPreviousTransactionId = previousTransactionId;
context.previousTransactionEndPosition = previousEndPosition;
transaction = txCoordinator.startTransaction(seriesId, previousTransactionId, previousEndPosition, transaction,
processorOptions.getMaxInProgressTransactions(), processorOptions.getMaxRetringTransactions());
if (logger.isDebugEnabled()){
logger.debug("[{}] Processor {} tried to start transaction: sticky={}, previousTransactionId={}, previousEndPosition={}, startPosition={}, returnedTransactionId={}, returnedTransactionHasStarted={}",
seriesId, processorId, sticky, previousTransactionId, previousEndPosition, startPosition, transaction == null ? null : transaction.getTransactionId(), transaction == null ? null : transaction.hasStarted());
}
}
}catch(TransactionStorageInfrastructureException e){
logger.debug("[{}] In transaction storage infrastructure error happened", seriesId, e);
transaction = null;
}catch(DuplicatedTransactionIdException e){
logger.warn("[{}] Transaction ID is duplicated: " + transaction.getTransactionId(), seriesId, e);
transaction = null;
}catch(Exception e){
logger.error("[{}] Error happened", seriesId, e);
transaction = null;
}finally{
// clear those flags so that the next round will start from a new state
// otherwise sometimes the processor may stick to the same partition forever
context.isOpenRangeAbortedBecauseNothingReceived = false;
context.isOpenRangeSuccessfullyClosed = false;
context.isOutOfRangeMessageReached = false;
}
if (transaction != null && transaction.hasStarted() && state.get() == State.RUNNING){
if (logger.isDebugEnabled()){
logger.debug("[{}] Processor {} got a {} transaction {} ({}-{}] after {} attempts: {}",
seriesId,
processorId,
(transaction.getAttempts() == 1 ? "new" : "failed"),
transaction.getTransactionId(),
transaction.getStartPosition(), transaction.getEndPosition(),
attempts,
DurationFormatter.formatSince(startTime));
}
doTransaction(context.withSeriesId(seriesId).withTransaction(transaction), supplierWithIdAndRange);
if (context.isOutOfRangeMessageReached){
String finishedPosition;
try {
finishedPosition = SequentialTransactionsCoordinator.getFinishedPosition(txCoordinator.getRecentTransactions(seriesId));
if (finishedPosition != null &&
(finishedPosition.equals(transaction.getStartPosition()) || finishedPosition.equals(transaction.getEndPosition()))){
outOfRangeReached[partition] = true;
}
} catch (Exception e) {
logger.warn("[{}] Processor {} failed to get recent transactions", seriesId, processorId, e);
}
}
}else{ // can't get a transaction
if (state.get() == State.RUNNING && !outOfRangeReached[partition]){
await();
}
}
} // state.get() == State.RUNNING
state.compareAndSet(State.PAUSING, State.PAUSED);
if (allProcessed(outOfRangeReached)){
state.set(State.FINISHED);
break;
}
} // state.compareAndSet(State.STOPPING, State.STOPPED)
logger.debug("[{}] Finish running: {}", processorId, state);
}
/**
* perform a batch processing transaction
* @param context the processing context which will be updated in this method
* @param supplierWithIdAndRange the stream data supplier
*/
protected void doTransaction(ProcessingContextImpl context, StreamDataSupplierWithIdAndRange supplierWithIdAndRange) {
String seriesId = context.seriesId;
SequentialTransaction transaction = context.transaction;
Boolean succeeded = false;
String fetchedLastPosition = null;
ReceiveStatus receiveStatus = null;
boolean isInitiallyOpenRange = transaction.getEndPosition() == null;
boolean isOpenRangeClosed = false;
boolean isProcessingFailed = false;
try{
if (!batchProcessor.initialize(context)){
throw new Exception("Unable to initilize processor");
}
long receiveTimeoutMillis = batchProcessor.receive(context, null); // keep it for logging
receiveStatus = supplierWithIdAndRange.receiveInRange(msg->{
long remaining = batchProcessor.receive(context, msg);
return state.get() == State.RUNNING ? remaining : 0;
}, transaction.getStartPosition(), transaction.getEndPosition());
fetchedLastPosition = receiveStatus.getLastPosition();
if (fetchedLastPosition != null){
if (isInitiallyOpenRange){ // we need to close the open range
try{
txCoordinator.updateTransactionEndPosition(seriesId, processorId, transaction.getTransactionId(), fetchedLastPosition);
transaction.setEndPosition(fetchedLastPosition);
isOpenRangeClosed = true;
}catch(Exception e){
throw new Exception("Unable to update end position in open range transaction", e);
}
}else{ // we need to make sure that all items in the range had been fetched
if (!fetchedLastPosition.equals(transaction.getEndPosition())){
throw new Exception("Unable to fetch all the data in range within duration " + DurationFormatter.format(receiveTimeoutMillis));
}
}
succeeded = batchProcessor.finish(context);
}else{
if (logger.isDebugEnabled()){
logDebugInTransaction("Fetched nothing within " + DurationFormatter.format(receiveTimeoutMillis), context, fetchedLastPosition, receiveStatus.isOutOfRangeReached());
}
if (receiveStatus.isOutOfRangeReached()){ // all data in range had been purged
succeeded = batchProcessor.finish(context);
}
}
}catch(Exception e){
if (logger.isDebugEnabled()){
logDebugInTransaction("Processing is not successful", context, fetchedLastPosition, e);
}
}
if (succeeded == null){ // the batchProcessor will handle transaction by itself
// do nothing because the batchProcessor will do it later
}else if (succeeded){ // succeeded
try{
//txCoordinator.finishTransaction(seriesId, processorId, transaction.getTransactionId(), fetchedLastPosition);
txCoordinator.finishTransaction(seriesId, processorId, transaction.getTransactionId());
}catch(Exception e){
if (logger.isDebugEnabled()){
logDebugInTransaction("Unable to finish transaction", context, fetchedLastPosition, e);
}
}
}else{ // failed
isProcessingFailed = true; // may also because that nothing had been received for the open range
try{
txCoordinator.abortTransaction(seriesId, processorId, transaction.getTransactionId());
if (logger.isDebugEnabled()){
logDebugInTransaction("Aborted transaction", context, fetchedLastPosition);
}
}catch(Exception e){
if (logger.isDebugEnabled()){
logDebugInTransaction("Unable to abort transaction", context, fetchedLastPosition, e);
}
}
}
context.isOutOfRangeMessageReached = receiveStatus == null ? false : receiveStatus.isOutOfRangeReached();
context.isOpenRangeSuccessfullyClosed = isInitiallyOpenRange && isOpenRangeClosed && !isProcessingFailed;
context.isOpenRangeAbortedBecauseNothingReceived = isInitiallyOpenRange && fetchedLastPosition == null;
}
protected void logDebugInTransaction(String message, ProcessingContextImpl context, String fetchedLastPosition, Exception e){
SequentialTransaction transaction = context.transaction;
logger.debug("[{} - {}] " + message + ": transactionId={}, startPosition={}, endPosition={}, fetchedLastPosition={}. Exception: {}",
context.seriesId, processorId, transaction.getTransactionId(), transaction.getStartPosition(), transaction.getEndPosition(),
fetchedLastPosition, exceptionSummary(e));
}
protected void logDebugInTransaction(String message, ProcessingContextImpl context, String fetchedLastPosition){
SequentialTransaction transaction = context.transaction;
logger.debug("[{} - {}] " + message + ": transactionId={}, startPosition={}, endPosition={}, fetchedLastPosition={}",
context.seriesId, processorId, transaction.getTransactionId(), transaction.getStartPosition(), transaction.getEndPosition(),
fetchedLastPosition);
}
protected void logDebugInTransaction(String message, ProcessingContextImpl context, String fetchedLastPosition, boolean isOutOfRangeReached){
SequentialTransaction transaction = context.transaction;
logger.debug("[{} - {}] " + message + ": transactionId={}, startPosition={}, endPosition={}, fetchedLastPosition={}, isOutOfRangeReached={}",
context.seriesId, processorId, transaction.getTransactionId(), transaction.getStartPosition(), transaction.getEndPosition(),
fetchedLastPosition, isOutOfRangeReached);
}
}
static protected String exceptionSummary(final Throwable ex){
Throwable e = ex;
StringBuilder sb = new StringBuilder();
sb.append('[');
sb.append(e.getClass().getName());
sb.append(":").append(e.getMessage());
sb.append(']');
for(int i = 0; i < 5 && e.getCause() != null && e.getCause() != e; i ++){
e = e.getCause();
sb.append(" caused by [");
sb.append(e.getClass().getName());
sb.append(":").append(e.getMessage());
sb.append(']');
}
return sb.toString();
}
@Override
public ProcessorStatus getProcessorStatus(String processorId) {
Processor processor = processors.get(processorId);
if (processor == null){
return null;
}else{
return new ProcessorStatus(processor.state.get());
}
}
@Override
public SortedMap getProcessorStatus(){
SortedMap result = new TreeMap<>();
for (Processor runnable: processors.values()){
result.put(runnable.processorId, new ProcessorStatus(runnable.state.get()));
}
return result;
}
@Override
public LinkedHashMap getStreamStatus() throws TransactionStorageInfrastructureException, DataStreamInfrastructureException{
List> localSuppliers = new ArrayList<>(suppliers.size());
localSuppliers.addAll(suppliers);
LinkedHashMap result = new LinkedHashMap<>(localSuppliers.size());
for (StreamDataSupplierWithIdAndRange supplier: localSuppliers){
String seriesId = seriesId(supplier);
List transactions = txCoordinator.getRecentTransactions(seriesId);
TransactionCounts transactionCounts = SequentialTransactionsCoordinator.getTransactionCounts(transactions);
String finishedPosition = SequentialTransactionsCoordinator.getFinishedPosition(transactions);
String lastUnfinishedStartPosition = null;
String lastUnfinishedEndPosition = null;
Instant finishedEnqueuedTime = null;
Instant lastUnfinishedStartEnqueuedTime = null;
Instant lastUnfinishedEndEnqueuedTime = null;
if (transactions != null && transactions.size() > 0){
ReadOnlySequentialTransaction tx = transactions.get(transactions.size() - 1);
if (tx.isInProgress()){
lastUnfinishedStartPosition = tx.getStartPosition();
lastUnfinishedEndPosition = tx.getEndPosition();
if (StringUtils.isNotBlank(lastUnfinishedStartPosition)){
lastUnfinishedStartEnqueuedTime = supplier.getSupplier().enqueuedTime(lastUnfinishedStartPosition);
}
if (StringUtils.isNotBlank(lastUnfinishedEndPosition)){
lastUnfinishedEndEnqueuedTime = supplier.getSupplier().enqueuedTime(lastUnfinishedEndPosition);
}
}
}
if (finishedPosition != null){
finishedEnqueuedTime = supplier.getSupplier().enqueuedTime(finishedPosition);
}
StreamStatus status = new StreamStatus();
status.transactionCounts = transactionCounts;
status.finishedPosition = finishedPosition;
status.lastUnfinishedStartPosition = lastUnfinishedStartPosition;
status.lastUnfinishedEndPosition = lastUnfinishedEndPosition;
status.finishedEnqueuedTime = finishedEnqueuedTime;
status.lastUnfinishedStartEnqueuedTime = lastUnfinishedStartEnqueuedTime;
status.lastUnfinishedEndEnqueuedTime = lastUnfinishedEndEnqueuedTime;
result.put(supplier.getId(), status);
}
return result;
}
/**
* Options for the processing.
*
* - initialTransactionTimeoutDuration - the initial timeout duration for the transactions. It should be long enough
* to allow data items for a batch to be fetched, Otherwise the processors may not have a chance to renew the transaction
* timeout before the transaction times out.
* - maxInProgressTransactions - maximum number of transactions allowed to be in progress at the same time
* - maxRetringTransactions - among in progress transactions, the maximum number of retrying transactions allowed at the same time
* - transactionAcquisitionDelay - time to wait before next try to get a batch of data items for processing when
* previously there was no data available for processing
* - waitStrategy - the {@link WaitStrategy} specifying how to wait for a specific time duration
* - noStick/stickyWhenOpenRangeSucceeded/stickyWhenOpenRangeSucceededOrNoData - how processors stick to suppliers
*
* @author James Hu
*
*/
static public class Options{
static public final int STICKY_NEVER = 0;
static public final int STICKY_WHEN_OPEN_RANGE_SUCCEEDED = 1;
static public final int STICKY_WHEN_OPEN_RANGE_SUCCEEDED_OR_NO_DATA = 2;
private Duration initialTransactionTimeoutDuration;
private int maxInProgressTransactions;
private int maxRetringTransactions;
private Duration transactionAcquisitionDelay;
private WaitStrategy waitStrategy;
private int stickyMode = STICKY_NEVER;
public Options(){
}
public Options(Options that){
this.initialTransactionTimeoutDuration = that.initialTransactionTimeoutDuration;
this.maxInProgressTransactions = that.maxInProgressTransactions;
this.maxRetringTransactions = that.maxRetringTransactions;
this.transactionAcquisitionDelay = that.transactionAcquisitionDelay;
this.waitStrategy = that.waitStrategy;
this.stickyMode = that.stickyMode;
}
public Duration getInitialTransactionTimeoutDuration() {
return initialTransactionTimeoutDuration;
}
public void setInitialTransactionTimeoutDuration(Duration initialTransactionTimeoutDuration) {
this.initialTransactionTimeoutDuration = initialTransactionTimeoutDuration;
}
public Options withInitialTransactionTimeoutDuration(Duration initialTransactionTimeoutDuration) {
this.initialTransactionTimeoutDuration = initialTransactionTimeoutDuration;
return this;
}
public int getMaxInProgressTransactions() {
return maxInProgressTransactions;
}
public void setMaxInProgressTransactions(int maxInProgressTransactions) {
this.maxInProgressTransactions = maxInProgressTransactions;
}
public Options withMaxInProgressTransactions(int maxInProgressTransactions) {
this.maxInProgressTransactions = maxInProgressTransactions;
return this;
}
public int getMaxRetringTransactions() {
return maxRetringTransactions;
}
public void setMaxRetringTransactions(int maxRetringTransactions) {
this.maxRetringTransactions = maxRetringTransactions;
}
public Options withMaxRetringTransactions(int maxRetringTransactions) {
this.maxRetringTransactions = maxRetringTransactions;
return this;
}
public Duration getTransactionAcquisitionDelay() {
return transactionAcquisitionDelay;
}
public void setTransactionAcquisitionDelay(Duration transactionAcquisitionDelay) {
this.transactionAcquisitionDelay = transactionAcquisitionDelay;
}
public Options withTransactionAcquisitionDelay(Duration transactionAcquisitionDelay) {
this.transactionAcquisitionDelay = transactionAcquisitionDelay;
return this;
}
public WaitStrategy getWaitStrategy() {
return waitStrategy;
}
public void setWaitStrategy(WaitStrategy waitStrategy) {
this.waitStrategy = waitStrategy;
}
public Options withWaitStrategy(WaitStrategy waitStrategy) {
this.waitStrategy = waitStrategy;
return this;
}
public Options withStickyMode(int stickyMode){
this.stickyMode = stickyMode;
return this;
}
/**
* Processors will always try to get and process a transaction from another supplier
* after processed (successful or not) a transaction from a supplier.
* @return the same Options object
*/
public Options withNoSticky(){
this.stickyMode = STICKY_NEVER;
return this;
}
/**
* Processors will keep trying to process transactions from the same supplier
* only if previous transaction was a successfully closed open range one.
* @return the same Options object
*/
public Options withStickyWhenOpenRangeSucceeded(){
this.stickyMode = STICKY_WHEN_OPEN_RANGE_SUCCEEDED;
return this;
}
/**
* Processors will keep trying to process transactions from the same supplier
* if previous transaction was an open range one and the transaction succeeded
* or there was no data received in the transaction.
* @return the same Options object
*/
public Options withStickyWhenOpenRangeSucceededOrNoData(){
this.stickyMode = STICKY_WHEN_OPEN_RANGE_SUCCEEDED_OR_NO_DATA;
return this;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy