org.shoulder.batch.service.impl.BatchProcessor Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of shoulder-batch Show documentation
Shoulder 扩展-批处理模块，提供批量数据导入、导出、异步校验、导入历史记录管理等能力。
There is a newer version: 0.8.1
package org.shoulder.batch.service.impl;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.shoulder.batch.enums.BatchErrorCodeEnum;
import org.shoulder.batch.enums.ProcessStatusEnum;
import org.shoulder.batch.model.BatchDataSlice;
import org.shoulder.batch.model.BatchRecordDetail;
import org.shoulder.batch.model.DataItem;
import org.shoulder.batch.service.ext.BatchTaskSliceHandler;
import org.shoulder.core.exception.CommonErrorCodeEnum;
import org.shoulder.core.i18.Translator;
import org.shoulder.core.log.Logger;
import org.shoulder.core.log.LoggerFactory;
import org.shoulder.core.util.ContextUtils;

import javax.annotation.Nonnull;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.BlockingQueue;

/**
 * 批处理工人
 * 实现了返回结果，具体操作还需要使用者实现 {@link BatchTaskSliceHandler}
 *
 * @author lym
 */
public class BatchProcessor implements Runnable {

    private final static Logger log = LoggerFactory.getLogger(BatchProcessor.class);

    /**
     * 任务队列
     */
    private final BlockingQueue taskQueue;
    /**
     * 产生结果队列
     */
    private final BlockingQueue resultQueue;

    /**
     * 数据处理器
     */
    private Collection batchTaskSliceHandlers =
        ContextUtils.getBeansOfType(BatchTaskSliceHandler.class).values();

    protected Translator translator;

    protected String name;

    public BatchProcessor(String name, BlockingQueue taskQueue,
                          BlockingQueue resultQueue) {
        this.name = name;
        this.taskQueue = taskQueue;
        this.resultQueue = resultQueue;
        this.translator = ContextUtils.getBean(Translator.class);
    }

    @Override
    public void run() {
        BatchDataSlice task;
        // 不断尝试从任务队列取，直至取不到，结束
        int taskProcessed = 0;
        for (; (task = taskQueue.poll()) != null; taskProcessed++) {
            List results = doWork(task);
            putResult(results);
        }
        log.info("{} stop, processed {}", getName(), taskProcessed);
    }

    /**
     * 放入结果队列
     *
     * @param results 处理完毕的结果
     */
    private void putResult(List results) {
        int put = 0;
        try {
            for (; put < results.size(); put++) {
                resultQueue.put(results.get(put));
            }
        } catch (InterruptedException e) {
            log.error("put result into queue FAIL, size=" + results.size() + " put=" + put, e);
        }
    }

    public String getName() {
        return name;
    }

    /**
     * 执行批处理任务
     * 数据类型 dataType, 操作类型 operationType 全部确定，且所有数据相同
     */
    public List doWork(@Nonnull BatchDataSlice task) {
        log.info("task start. {}", task.toString());
        if (CollectionUtils.isEmpty(task.getBatchList())) {
            return Collections.emptyList();
        }
        // 这里委派给使用者提供的数据处理扩展点
        String dataType = task.getDataType();
        String operation = task.getOperationType();
        BatchTaskSliceHandler taskHandler = batchTaskSliceHandlers.stream()
            .filter(handler -> handler.support(dataType, operation))
            .findFirst()
            // 若不存在则肯定是代码写错了，直接抛出异常
            .orElseThrow(() -> BatchErrorCodeEnum.DATA_TYPE_OR_OPERATION_NOT_SUPPORT
                .toException(dataType, operation));
        List taskResult = null;
        try {
            taskResult = taskHandler.handle(task);
        } catch (Exception e) {
            log.error("worker " + getName() + " processed failed", e);
        }
        log.info("task {}-{} finished", task.getTaskId(), task.getSequence());
        return recheckResultList(task, ListUtils.emptyIfNull(taskResult));
    }

    /**
     * 检查结果数，是否和数据数目一致，否则自动补充失败
     */
    private List recheckResultList(@Nonnull BatchDataSlice task,
                                                      @Nonnull List resultDetailList) {
        int exceptNum = task.getBatchList().size();
        int actuallyNum = resultDetailList.size();
        if (exceptNum == actuallyNum) {
            return resultDetailList;
        }
        log.warnWithErrorCode(BatchErrorCodeEnum.TASK_SLICE_RESULT_INVALID.getCode(),
            BatchErrorCodeEnum.TASK_SLICE_RESULT_INVALID.getMessage(), exceptNum, actuallyNum);
        // 为没有返回结果的任务进行补偿填充，认为失败了
        for (DataItem dataItem : task.getBatchList()) {
            int index = dataItem.getIndex();
            BatchRecordDetail except =
                resultDetailList.stream()
                    .filter(result -> index == result.getIndex())
                    .findFirst().orElse(null);
            if (except != null) {
                continue;
            }
            except = new BatchRecordDetail(index, ProcessStatusEnum.IMPORT_FAILED.getCode(),
                    CommonErrorCodeEnum.UNKNOWN.getCode());
            resultDetailList.add(except);
        }
        return resultDetailList;
    }

}