com.qiniu.process.Base Maven / Gradle / Ivy

Go to download
package com.qiniu.process;

import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.qiniu.common.QiniuException;
import com.qiniu.interfaces.IFileChecker;
import com.qiniu.interfaces.ILineProcess;
import com.qiniu.persistence.FileSaveMapper;
import com.qiniu.util.*;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

/**
 * process 基类，实现 ILineProcess 的方法，并实现 Cloneable 接口支持 clone。该类为抽象类，且子类需要实现一些抽象方法（子类实现抽象方法时不需
 * 要同步代码），包内提供的子类均为非线程安全，且都调用该基类的 close 方法，多线程时需使用 clone 的对象，否则容易引发线程安全问题，同时，即使子类
 * 使用同步方法，多线程时调用了一次 close 会导致其他线程跑异常。
 * @param  line 数据范型
 */
public abstract class Base implements ILineProcess, Cloneable {

    protected String processName;
    protected String accessId;
    protected String secretKey;
    protected String bucket;
    protected int batchSize;
    protected int retryTimes = 5;
    protected boolean autoIncrease;
    protected int index;
    protected AtomicInteger saveIndex;
    protected String savePath;
    protected FileSaveMapper fileSaveMapper;
    protected IFileChecker iFileChecker;
    protected String checkType;
    protected boolean canceled;

    public Base(String processName, String accessId, String secretKey, String bucket) {
        this.processName = processName;
        this.accessId = accessId;
        this.secretKey = secretKey;
        this.bucket = bucket;
        this.iFileChecker = key -> null;
    }

    public Base(String processName, String accessKey, String secretKey, String bucket, String savePath, int saveIndex)
            throws IOException {
        this(processName, accessKey, secretKey, bucket);
        this.index = saveIndex;
        this.saveIndex = new AtomicInteger(saveIndex);
        this.savePath = savePath;
        this.fileSaveMapper = new FileSaveMapper(savePath, processName, String.valueOf(saveIndex));
        this.fileSaveMapper.preAddWriter("need_retry");
    }

    public String getProcessName() {
        return this.processName;
    }

    public void setAutoIncrease(boolean autoIncrease) {
        this.autoIncrease = autoIncrease;
    }

    public void setBatchSize(int batchSize) throws IOException {
        if (!ProcessUtils.canBatch(processName)) {
            throw new IOException(processName + " is not support batch operation.");
        } else if (batchSize > 1000 || batchSize < 1) {
            throw new IOException("batch size must less than 1000 and more than 1.");
        } else {
            this.batchSize = batchSize;
        }
    }

    public void setRetryTimes(int retryTimes) {
        this.retryTimes = retryTimes < 1 ? 5 : retryTimes;
    }

    @SuppressWarnings("unchecked")
    public Base clone() throws CloneNotSupportedException {
        Base base = (Base)super.clone();
        if (checkType != null && !"".equals(checkType)) base.iFileChecker = fileCheckerInstance();
        if (fileSaveMapper == null) return base;
        try {
            base.fileSaveMapper = autoIncrease ?
                    new FileSaveMapper(savePath, processName, String.valueOf(saveIndex.addAndGet(1))) :
                    new FileSaveMapper(savePath);
            base.fileSaveMapper.preAddWriter("need_retry");
        } catch (IOException e) {
            throw new CloneNotSupportedException(e.getMessage() + ", init writer failed.");
        }
        return base;
    }

    public void changeSaveOrder(String order) throws IOException {
        try {
            this.fileSaveMapper.changePrefixAndSuffix(processName, order);
        } catch (NullPointerException e) {
            throw new IOException("instance without savePath can not call changeSaveOrder method.");
        }
    }

    protected IFileChecker fileCheckerInstance() {
        return key -> null;
    }

    public void setCheckType(String checkType) {
        this.checkType = checkType;
        this.iFileChecker = fileCheckerInstance();
    }

    /**
     * 当处理结束应该从 line 中记录哪些关键信息
     * @param line 输入的 line
     * @return 返回需要记录的信息字符串
     */
    protected abstract String resultInfo(T line);

    protected List putBatchOperations(List processList) throws IOException {
        return processList;
    }

    /**
     * 对 lineList 执行 batch 的操作，因为默认是实现单个资源请求的操作，部分操作不支持 batch，因此需要 batch 操作时子类需要重写该方法。
     * @param lineList 代执行的文件信息列表
     * @return 返回执行响应信息的字符串
     * @throws Exception 执行失败抛出的异常
     */
    protected String batchResult(List lineList) throws Exception {
        throw new IOException("no default batch operation, please implements batch processing by yourself.");
    }

    /**
     * 处理 batchOperations 执行的结果，将输入的文件信息和结果对应地记录下来
     * @param processList batch 操作的资源列表
     * @param result batch 操作之后的响应结果
     * @return 返回需要进行重试的记录列表
     * @throws Exception 处理结果失败抛出的异常
     */
    protected List parseBatchResult(List processList, String result) throws Exception {
        if (result == null || "".equals(result)) throw new IOException("not valid json.");
        List retryList = null;
        JsonArray jsonArray = JsonUtils.fromJson(result, JsonArray.class);
        // 正常情况下 jsonArray 和 processList 的长度是相同的，将输入行信息和执行结果一一对应记录
        if (processList.size() != jsonArray.size()) throw new IOException("not matched process and return size.");
        JsonObject jsonObject;
        for (int j = 0; j < processList.size(); j++) {
            jsonObject = jsonArray.get(j).getAsJsonObject();
            switch (HttpRespUtils.checkStatusCode(jsonObject.get("code").getAsInt())) {
                case 1:
                    fileSaveMapper.writeSuccess(resultInfo(processList.get(j)) + "\t" + jsonObject, false);
                    break;
                case 0:
                    if (retryList == null) retryList = new ArrayList<>();
                    retryList.add(processList.get(j)); // 放回重试列表
                    break;
                case -1:
                    fileSaveMapper.writeError(resultInfo(processList.get(j)) + "\t" + jsonObject, false);
                    break;
            }
        }
        return retryList;
    }

    /**
     * 批量处理输入行，具体执行的操作取决于 batchResult 方法的实现。
     * @param lineList 输入列表
     * @param batchSize 一次批量处理请求处理的文件个数
     * @param retryTimes 每一行信息处理时需要的重试次数
     * @throws IOException 处理失败可能抛出的异常
     */
    private void batchProcess(List lineList, int batchSize, int retryTimes) throws IOException {
        int times = (lineList.size() + batchSize - 1) / batchSize;
        List processList;
        int retry;
        String message = null;
        QiniuException exception = null;
        for (int i = 0; i < times; i++) {
            if (canceled) break;
            retry = retryTimes;
            processList = lineList.subList(batchSize * i, i == times - 1 ? lineList.size() : batchSize * (i + 1));
            // 加上 processList.size() > 0 的选择原因是会在每一次处理 batch 操作的结果时将需要重试的记录加入重试列表进行返回，并且在
            // 没有异常的情况下当前的 processList 会执行到没有重试记录返回时才结束，parseBatchResult 会返回可以重试的列表，无重试记录则返回
            // 空，重试次数小于 0 时设置 processList = null
            while (processList != null && processList.size() > 0) {
                try {
                    processList = putBatchOperations(processList);
                    if (processList.size() > 0) {
                        processList = parseBatchResult(processList, batchResult(processList));
                    }
                } catch (QiniuException e) {
                    retry = HttpRespUtils.checkException(e, retry);
                    message = HttpRespUtils.getMessage(e);
                    exception = e;
                } catch (Exception e) {
                    retry = 0;
                    message = e.getMessage();
                    exception = null;
                }
                switch (retry) {
                    case 0: fileSaveMapper.writeError(String.join("\n", processList.stream()
                            .map(this::resultInfo).collect(Collectors.toList())) + "\t" + message, false);
                        processList = null; break;
                    case -1: fileSaveMapper.writeToKey("need_retry", String.join("\n", processList
                            .stream().map(this::resultInfo).collect(Collectors.toList())) + "\t" + message, false);
                        processList = null; break;
                    case -2: fileSaveMapper.writeError(String.join("\n", lineList
                            .subList(batchSize * i, lineList.size()).stream()
                            .map(this::resultInfo).collect(Collectors.toList())) + "\t" + message, false);
                        if (exception != null) throw exception;
                }
                if (exception != null && exception.response != null) exception.response.close();
            }
        }
    }

    /**
     * 单个文件进行操作的方法，返回操作的结果字符串，要求子类必须实现该方法，支持单个资源依次请求操作
     * @param line 输入 line
     * @return 操作结果的字符串
     * @throws Exception 操作失败时的返回
     */
    abstract protected String singleResult(T line) throws Exception;

    /**
     * 处理 singleProcess 执行的结果，默认情况下直接使用 resultInfo 拼接 result 成一行执行持久化写入，部分 process 可能对结果做进一步判断
     * 需要重写该方法
     * @param line 输入的 map 数据
     * @param result singleResult 的结果字符串
     * @throws Exception 处理结果失败抛出异常
     */
    protected void parseSingleResult(T line, String result) throws Exception {
        fileSaveMapper.writeSuccess(result, false);
    }

    /**
     * 对输入的文件信息列表单个进行操作，具体的操作方法取决于 singleResult 方法
     * @param lineList 输入列表
     * @param retryTimes 每一行信息处理时需要的重试次数
     * @throws IOException 处理失败可能抛出的异常
     */
    private void singleProcess(List lineList, int retryTimes) throws IOException {
        int retry;
        T line;
        String message;
        QiniuException exception;
        for (int i = 0; i < lineList.size(); i++) {
            line = lineList.get(i);
            retry = retryTimes;
            while (retry > 0) {
                try {
                    parseSingleResult(line, singleResult(line));
                    break;
                } catch (QiniuException e) {
                    retry = HttpRespUtils.checkException(e, retry);
                    message = HttpRespUtils.getMessage(e);
                    exception = e;
                } catch (Exception e) {
                    retry = 0;
                    message = e.getMessage();
                    exception = null;
                }
                switch (retry) {
                    case 0: fileSaveMapper.writeError(resultInfo(line) + "\t" + message, false); break;
                    case -1: fileSaveMapper.writeToKey("need_retry", resultInfo(line) + "\t" + message, false);
                        break;
                    case -2: fileSaveMapper.writeError(String.join("\n", lineList.subList(i, lineList.size())
                            .stream().map(this::resultInfo).collect(Collectors.toList())) + "\t" + message, false);
                        throw exception;
                }
                if (exception != null && exception.response != null) exception.response.close();
            }
        }
    }

    public String processLine(T line) throws IOException {
        try {
            return singleResult(line);
        } catch (NullPointerException e) {
            if (canceled) {
                throw new IOException("processor in canceled state.");
            } else if (batchSize < 0) { // 如果是关闭了那么 batchSize 应该小于 0
                throw new IOException("input is empty or the processor may be already closed.");
            } else {
                throw new IOException("instance without savePath can not call this batch process method.");
            }
        } catch (IOException e) {
            throw e;
        } catch (Exception e) {
            throw new IOException(e.getMessage(), e);
        }
    }

    /**
     * 公开的操作调用方法入口，通过判断 batch size 来决定调用哪个方法
     * @param lineList 输入的文件信息列表
     * @throws IOException 处理过程中出现的异常
     */
    public void processLine(List lineList) throws IOException {
        try {
            if (batchSize > 1) batchProcess(lineList, batchSize, retryTimes);
            else singleProcess(lineList, retryTimes);
        } catch (NullPointerException e) {
            if (canceled) {
////            // nothing to do
            } else if (batchSize < 0) { // 如果是关闭了那么 batchSize 应该小于 0
                throw new IOException("input is empty or the processor may be already closed.");
            } else {
                throw new IOException("instance without savePath can not call this batch process method.");
            }
        } catch (IOException e) {
            throw e;
        } catch (Exception e) {
            throw new IOException(e.getMessage());
        }
    }

    public void closeResource() {
        accessId = null;
        secretKey = null;
        bucket = null;
        batchSize = -1;
        saveIndex = null;
        savePath = null;
        if (fileSaveMapper != null) fileSaveMapper.closeWriters();
        fileSaveMapper = null;
        iFileChecker = null;
    }

    public void cancel() {
        canceled = true;
        closeResource();
    }
}