com.transferwise.tasks.TasksService Maven / Gradle / Ivy
Show all versions of tw-tasks-core Show documentation
package com.transferwise.tasks;
import com.transferwise.common.context.TwContextClockHolder;
import com.transferwise.common.context.UnitOfWorkManager;
import com.transferwise.common.gracefulshutdown.GracefulShutdownStrategy;
import com.transferwise.tasks.dao.ITaskDao;
import com.transferwise.tasks.domain.BaseTask;
import com.transferwise.tasks.domain.BaseTask1;
import com.transferwise.tasks.domain.FullTaskRecord;
import com.transferwise.tasks.domain.TaskStatus;
import com.transferwise.tasks.entrypoints.EntryPoint;
import com.transferwise.tasks.entrypoints.EntryPointsGroups;
import com.transferwise.tasks.entrypoints.EntryPointsNames;
import com.transferwise.tasks.entrypoints.IEntryPointsService;
import com.transferwise.tasks.entrypoints.IMdcService;
import com.transferwise.tasks.handler.interfaces.ITaskHandlerRegistry;
import com.transferwise.tasks.helpers.ICoreMetricsTemplate;
import com.transferwise.tasks.helpers.executors.IExecutorsHelper;
import com.transferwise.tasks.processing.ITaskRegistrationDecorator;
import com.transferwise.tasks.triggering.ITasksExecutionTriggerer;
import com.transferwise.tasks.utils.LogUtils;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.Ordered;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
@Slf4j
public class TasksService implements ITasksService, GracefulShutdownStrategy, InitializingBean {
@Autowired
private ITaskDao taskDao;
@Autowired
private ITasksExecutionTriggerer tasksExecutionTriggerer;
@Autowired
private TasksProperties tasksProperties;
@Autowired
private IExecutorsHelper executorsHelper;
@Autowired
private IPriorityManager priorityManager;
@Autowired
private ITaskHandlerRegistry taskHandlerRegistry;
@Autowired
private IMdcService mdcService;
@Autowired
private UnitOfWorkManager unitOfWorkManager;
@Autowired
private IEntryPointsService entryPointsHelper;
@Autowired
private IEnvironmentValidator environmentValidator;
@Autowired
private ICoreMetricsTemplate coreMetricsTemplate;
@Autowired(required = false)
private List taskRegistrationDecorators = new ArrayList<>();
private ExecutorService afterCommitExecutorService;
private TxSyncAdapterFactory txSyncAdapterFactory;
private final AtomicInteger inProgressAfterCommitTasks = new AtomicInteger();
private final AtomicInteger activeAfterCommitTasks = new AtomicInteger();
@Override
public void afterPropertiesSet() {
environmentValidator.validate();
if (tasksProperties.isAsyncTaskTriggering()) {
afterCommitExecutorService = executorsHelper.newScheduledExecutorService("tsac", tasksProperties.getAsyncTaskTriggeringsConcurrency());
txSyncAdapterFactory = AsynchronouslyTriggerTaskTxSyncAdapter::new;
} else {
txSyncAdapterFactory = (mdcService, unitOfWorkManager, task) -> new SynchronouslyTriggerTaskTxSyncAdapter(task);
}
log.info("Tasks service initialized for client '" + tasksProperties.getClientId() + "'.");
coreMetricsTemplate.registerInProgressTriggeringsCount(inProgressAfterCommitTasks);
coreMetricsTemplate.registerActiveTriggeringsCount(activeAfterCommitTasks);
}
@Override
@EntryPoint(usesExisting = true)
@Transactional(rollbackFor = Exception.class)
public AddTaskResponse addTask(AddTaskRequest requestParam) {
return entryPointsHelper.continueOrCreate(EntryPointsGroups.TW_TASKS_ENGINE, EntryPointsNames.ADD_TASK,
() -> {
AddTaskRequest request = requestParam;
mdcService.put(request.getTaskId(), 0L);
mdcService.putType(request.getType());
mdcService.putSubType(request.getSubType());
for (ITaskRegistrationDecorator decorator : taskRegistrationDecorators) {
request = decorator.decorate(request);
}
ZonedDateTime now = ZonedDateTime.now(TwContextClockHolder.getClock());
final TaskStatus status =
request.getRunAfterTime() == null || !request.getRunAfterTime().isAfter(now) ? TaskStatus.SUBMITTED : TaskStatus.WAITING;
final int priority = priorityManager.normalize(request.getPriority());
if (StringUtils.isEmpty(StringUtils.trim(request.getType()))) {
throw new IllegalStateException("Task type is mandatory, but '" + request.getType() + "' was provided.");
}
ZonedDateTime maxStuckTime =
request.getExpectedQueueTime() == null ? now.plus(tasksProperties.getTaskStuckTimeout()) : now.plus(request.getExpectedQueueTime());
ITaskDao.InsertTaskResponse insertTaskResponse = taskDao.insertTask(
new ITaskDao.InsertTaskRequest().setData(request.getData()).setKey(request.getUniqueKey())
.setRunAfterTime(request.getRunAfterTime())
.setSubType(request.getSubType())
.setType(request.getType()).setTaskId(request.getTaskId())
.setMaxStuckTime(maxStuckTime).setStatus(status).setPriority(priority)
.setCompression(request.getCompression())
.setTaskContext(request.getTaskContext())
);
coreMetricsTemplate.registerTaskAdding(request.getType(), request.getUniqueKey(),
insertTaskResponse.isInserted(), request.getRunAfterTime(), request.getData());
if (!insertTaskResponse.isInserted()) {
coreMetricsTemplate.registerDuplicateTask(request.getType(), !request.isWarnWhenTaskExists());
if (request.isWarnWhenTaskExists()) {
log.warn("Task with uuid '" + request.getTaskId() + "'"
+ (request.getUniqueKey() == null ? "" : " and key '" + request.getUniqueKey() + "'")
+ " already exists (type " + request.getType() + ", subType " + request.getSubType() + ").");
}
return new AddTaskResponse().setResult(AddTaskResponse.Result.ALREADY_EXISTS);
}
final UUID taskId = insertTaskResponse.getTaskId();
mdcService.put(taskId, 0L);
log.debug("Task '{}' created with status {}.", taskId, status);
if (status == TaskStatus.SUBMITTED) {
triggerTask(new BaseTask().setId(taskId).setType(request.getType()).setPriority(priority));
}
return new AddTaskResponse().setResult(AddTaskResponse.Result.OK).setTaskId(taskId);
});
}
@Override
@EntryPoint(usesExisting = true)
@Transactional(rollbackFor = Exception.class)
public boolean resumeTask(ResumeTaskRequest request) {
return entryPointsHelper.continueOrCreate(EntryPointsGroups.TW_TASKS_ENGINE, EntryPointsNames.RESUME_TASK,
() -> {
UUID taskId = request.getTaskId();
mdcService.put(request.getTaskId(), request.getVersion());
BaseTask1 task = taskDao.getTask(taskId, BaseTask1.class);
if (task == null) {
log.debug("Cannot resume task '" + taskId + "' as it was not found.");
return false;
}
mdcService.put(task);
long version = task.getVersion();
if (version != request.getVersion()) {
coreMetricsTemplate.registerFailedStatusChange(task.getType(), task.getStatus(), TaskStatus.SUBMITTED);
log.debug("Expected version " + request.getVersion() + " does not match " + version + ".");
return false;
}
if (task.getStatus().equals(TaskStatus.WAITING.name()) || task.getStatus().equals(TaskStatus.NEW.name())) {
if (!taskDao.markAsSubmitted(taskId, version++, taskHandlerRegistry.getExpectedProcessingMoment(task))) {
coreMetricsTemplate.registerFailedStatusChange(task.getType(), task.getStatus(), TaskStatus.SUBMITTED);
if (log.isDebugEnabled()) {
log.debug("Can not resume task '" + taskId + "', expected version " + request.getVersion()
+ " does not match " + task.getVersion() + ".");
}
return false;
}
} else if (!task.getStatus().equals(TaskStatus.SUBMITTED.name())) {
if (!request.isForce()) {
if (log.isDebugEnabled()) {
log.debug("Can not resume task {}, it has wrong state '{}'.", LogUtils.asParameter(task.getVersionId()), task.getStatus());
}
return false;
} else {
log.warn("Task '" + taskId + "' will be force resumed. Status will change from '" + task.getStatus() + "' to 'SUBMITTED'.");
}
if (!taskDao.markAsSubmitted(taskId, version++, taskHandlerRegistry.getExpectedProcessingMoment(task))) {
coreMetricsTemplate.registerFailedStatusChange(task.getType(), task.getStatus(), TaskStatus.SUBMITTED);
log.debug("Can not resume task {}, it has wrong version '{}'.", LogUtils.asParameter(task.getVersionId()), task.getStatus());
return false;
}
}
triggerTask(task.toBaseTask().setVersion(version));
return true;
});
}
@Override
@EntryPoint(usesExisting = true)
@Transactional(rollbackFor = Exception.class)
public RescheduleTaskResponse rescheduleTask(RescheduleTaskRequest request) {
return entryPointsHelper.continueOrCreate(EntryPointsGroups.TW_TASKS_ENGINE, EntryPointsNames.RESCHEDULE_TASK,
() -> {
UUID taskId = request.getTaskId();
mdcService.put(request.getTaskId(), request.getVersion());
FullTaskRecord task = taskDao.getTask(taskId, FullTaskRecord.class);
if (task == null) {
log.debug("Cannot reschedule task '" + taskId + "' as it was not found.");
return new RescheduleTaskResponse().setResult(RescheduleTaskResponse.Result.NOT_FOUND).setTaskId(taskId);
}
mdcService.put(task);
long version = task.getVersion();
if (version != request.getVersion()) {
coreMetricsTemplate.registerFailedNextEventTimeChange(task.getType(), task.getNextEventTime(), request.getRunAfterTime());
log.debug("Expected version " + request.getVersion() + " does not match " + version + ".");
return new RescheduleTaskResponse().setResult(RescheduleTaskResponse.Result.NOT_FOUND).setTaskId(taskId);
}
if (task.getStatus().equals(TaskStatus.WAITING.name())) {
if (!taskDao.setNextEventTime(taskId, request.getRunAfterTime(), version, TaskStatus.WAITING.name())) {
coreMetricsTemplate.registerFailedNextEventTimeChange(task.getType(), task.getNextEventTime(), request.getRunAfterTime());
return new RescheduleTaskResponse().setResult(RescheduleTaskResponse.Result.FAILED).setTaskId(taskId);
} else {
coreMetricsTemplate.registerTaskRescheduled(null, task.getType());
return new RescheduleTaskResponse().setResult(RescheduleTaskResponse.Result.OK).setTaskId(taskId);
}
}
coreMetricsTemplate.registerFailedNextEventTimeChange(task.getType(), task.getNextEventTime(), request.getRunAfterTime());
return new RescheduleTaskResponse().setResult(RescheduleTaskResponse.Result.NOT_ALLOWED).setTaskId(taskId);
});
}
@Override
@EntryPoint(usesExisting = true)
@Transactional(rollbackFor = Exception.class)
public GetTaskResponse getTask(GetTaskRequest request) {
return entryPointsHelper.continueOrCreate(EntryPointsGroups.TW_TASKS_ENGINE, EntryPointsNames.GET_TASK,
() -> {
UUID taskId = request.getTaskId();
mdcService.put(request.getTaskId());
FullTaskRecord task = taskDao.getTask(taskId, FullTaskRecord.class);
if (task == null) {
log.debug("Cannot get task '" + taskId + "' as it was not found.");
return new GetTaskResponse()
.setResult(GetTaskResponse.Result.NOT_FOUND)
.setTaskId(taskId);
}
mdcService.put(task);
return new GetTaskResponse().setResult(GetTaskResponse.Result.OK)
.setTaskId(taskId)
.setType(task.getType())
.setVersion(task.getVersion())
.setPriority(task.getPriority())
.setStatus(task.getStatus())
.setNextEventTime(task.getNextEventTime());
});
}
@Override
public void startTasksProcessing(String bucketId) {
tasksExecutionTriggerer.startTasksProcessing(bucketId);
}
@Override
public Future stopTasksProcessing(String bucketId) {
return tasksExecutionTriggerer.stopTasksProcessing(bucketId);
}
@Override
public ITasksService.TasksProcessingState getTasksProcessingState(String bucketId) {
return tasksExecutionTriggerer.getTasksProcessingState(bucketId);
}
/**
* We register an after commit hook here, so as soon as transaction has finished, the task will be triggered.
*
* The problem with many Spring provided transactions manager, is that this hook is called, after commit has happened,
* but before the resources (db connections) are released.
*
*
If we would do it in the same thread, and because triggering needs its own database connection for marking as submit,
* we would create a deadlock on pool exhaustion situations (easiest to test with db pool of 1).
*
*
So we do it asynchronously in different threads. In this case, if triggering will be slower than creating tasks,
* we may exhaust memory, so a maximum limit of waiting triggerings has to be enforced.
*
*
Notice, that with proper JTA transaction manager (TW uses Gaffer in some services),
* this deadlock problem does not exist and we may trigger in the same thread.
*/
protected void triggerTask(BaseTask task) {
TransactionSynchronizationManager.registerSynchronization(txSyncAdapterFactory.create(mdcService, unitOfWorkManager, task));
}
@Override
public boolean canShutdown() {
return inProgressAfterCommitTasks.get() == 0;
}
private void doTriggerTask(BaseTask task) {
activeAfterCommitTasks.incrementAndGet();
try {
tasksExecutionTriggerer.trigger(task);
if (log.isDebugEnabled()) {
log.debug("Task {} triggered. AfterCommit queue size is {}.", LogUtils.asParameter(task.getVersionId()), inProgressAfterCommitTasks.get());
}
} catch (Throwable t) {
log.error("Triggering task '{}' failed.", task.getVersionId(), t);
} finally {
activeAfterCommitTasks.decrementAndGet();
}
}
@FunctionalInterface
private interface TxSyncAdapterFactory {
TransactionSynchronization create(IMdcService mdcService, UnitOfWorkManager unitOfWorkManager, BaseTask task);
}
@RequiredArgsConstructor
private class SynchronouslyTriggerTaskTxSyncAdapter extends TransactionSynchronizationAdapter {
private final BaseTask task;
@Override
public void afterCommit() {
inProgressAfterCommitTasks.incrementAndGet();
try {
doTriggerTask(task);
} finally {
inProgressAfterCommitTasks.decrementAndGet();
}
}
}
/**
* This system is not required for happy flows anymore, because we do not change database then, task is already in SUBMITTED state.
*
*
However, for unhappy case, the task can move to ERROR and we need to modify database for that.
*/
@RequiredArgsConstructor
private class AsynchronouslyTriggerTaskTxSyncAdapter extends TransactionSynchronizationAdapter {
private final IMdcService mdcService;
private final UnitOfWorkManager unitOfWorkManager;
private final BaseTask task;
@Override
public void afterCommit() {
if (inProgressAfterCommitTasks.incrementAndGet() >= tasksProperties.getMaxAsyncTaskTriggerings()) {
// Need to ignore, if we wait for empty space, we can create deadlock with database resources.
log.warn("Task {} was not triggered, because resources have been exhausted.", LogUtils.asParameter(task.getVersionId()));
inProgressAfterCommitTasks.decrementAndGet();
return;
}
afterCommitExecutorService.submit(() -> {
try {
afterCommit0();
} catch (Throwable t) {
log.error("Triggering task '{}' failed.", task.getVersionId(), t);
}
});
}
@EntryPoint
private void afterCommit0() {
unitOfWorkManager.createEntryPoint(EntryPointsGroups.TW_TASKS_ENGINE, EntryPointsNames.TRIGGER_TASK).toContext().execute(
() -> {
try {
mdcService.put(task);
doTriggerTask(task);
} finally {
inProgressAfterCommitTasks.decrementAndGet();
}
});
}
}
/**
* This allows to work on Spring 4.x as well.
*
*
Spring 4.x does not have default methods in `TransactionSynchronization` interface.
*/
private static class TransactionSynchronizationAdapter implements TransactionSynchronization {
@Override
public int getOrder() {
return Ordered.LOWEST_PRECEDENCE;
}
@Override
public void suspend() {
}
@Override
public void resume() {
}
@Override
public void flush() {
}
@Override
public void beforeCommit(boolean readOnly) {
}
@Override
public void beforeCompletion() {
}
@Override
public void afterCommit() {
}
@Override
public void afterCompletion(int status) {
}
}
}