com.qiniu.service.datasource.FileInput Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of qsuits Show documentation
qiniu-suits is a efficient tools for qiniu api implemented by java8.
There is a newer version: 8.4.8
package com.qiniu.service.datasource;

import com.qiniu.common.QiniuException;
import com.qiniu.persistence.FileMap;
import com.qiniu.service.convert.MapToString;
import com.qiniu.service.convert.LineToMap;
import com.qiniu.service.interfaces.ILineProcess;
import com.qiniu.service.interfaces.ITypeConvert;
import com.qiniu.util.HttpResponseUtils;
import com.qiniu.util.SystemUtils;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;

public class FileInput implements IDataSource {

    final private String filePath;
    final private String parseType;
    final private String separator;
    final private Map indexMap;
    final private int threads;
    final private int unitLen;
    final private String savePath;
    private boolean saveTotal;
    private String saveFormat;
    private String saveSeparator;
    private List rmFields;
    private ExecutorService executorPool; // 线程池
    private AtomicBoolean exitBool; // 多线程的原子操作 bool 值
    private ILineProcess> processor; // 定义的资源处理器

    public FileInput(String filePath, String parseType, String separator, Map indexMap, int threads,
                     int unitLen, String savePath) {
        this.filePath = filePath;
        this.parseType = parseType;
        this.separator = separator;
        this.indexMap = indexMap;
        this.threads = threads;
        this.unitLen = unitLen;
        this.savePath = savePath;
        this.saveTotal = false;
    }

    public void setResultOptions(boolean saveTotal, String format, String separator, List rmFields) {
        this.saveTotal = saveTotal;
        this.saveFormat = format;
        this.saveSeparator = separator;
        this.rmFields = rmFields;
    }

    public void setProcessor(ILineProcess> processor) {
        this.processor = processor;
    }

    private void export(BufferedReader reader, FileMap fileMap, ILineProcess> processor)
            throws IOException {
        ITypeConvert> typeConverter = new LineToMap(parseType, separator, indexMap);
        ITypeConvert, String> writeTypeConverter = new MapToString(saveFormat, saveSeparator, rmFields);
        List srcList = new ArrayList<>();
        List> infoMapList;
        List writeList;
        String line = null;
        boolean goon = true;
        while (goon) {
            // 避免文件过大，行数过多，使用 lines() 的 stream 方式直接转换可能会导致内存泄漏，故使用 readLine() 的方式
            try { line = reader.readLine(); } catch (IOException e) { e.printStackTrace(); }
            if (line == null) goon = false;
            else srcList.add(line);
            if (srcList.size() >= unitLen || line == null) {
                infoMapList = typeConverter.convertToVList(srcList);
                if (typeConverter.getErrorList().size() > 0)
                    fileMap.writeError(String.join("\n", typeConverter.consumeErrorList()), false);
                if (saveTotal) {
                    writeList = writeTypeConverter.convertToVList(infoMapList);
                    if (writeList.size() > 0) fileMap.writeSuccess(String.join("\n", writeList), false);
                }
                // 如果抛出异常需要检测下异常是否是可继续的异常，如果是程序可继续的异常，忽略当前异常保持数据源读取过程继续进行
                try {
                    if (processor != null) processor.processLine(infoMapList);
                } catch (QiniuException e) {
                    HttpResponseUtils.processException(e, 1, null, null);
                }
                srcList = new ArrayList<>();
            }
        }
    }

    private void execInThreads(FileMap initFileMap) throws Exception {
        HashMap readersMap = initFileMap.getReaderMap();
        List keys = new ArrayList<>(readersMap.keySet());
        for (int i = 0; i < keys.size(); i++) {
            // 如果是第一个线程直接使用初始的 processor 对象，否则使用 clone 的 processor 对象
            ILineProcess> lineProcessor = processor == null ? null :
                    i == 0 ? processor : processor.clone();
            String order = String.valueOf(i);
            String key = keys.get(i);
            executorPool.execute(() -> {
                try {
                    BufferedReader reader = readersMap.get(key);
                    FileMap fileMap = new FileMap(savePath, "fileinput", order);
                    fileMap.initDefaultWriters();
                    String record = "order " + order + ": " + key;
                    String next;
                    try {
                        initFileMap.writeKeyFile("result", record + "\treading...", true);
                        export(reader, fileMap, lineProcessor);
                        next = reader.readLine();
                        if (next == null) record += "\tsuccessfully done";
                        else record += "\tnextLine:" + next;
                        System.out.println(record);
                    } catch (IOException e) {
                        try { next = reader.readLine(); } catch (IOException ex) { next = ex.getMessage(); }
                        record += "\tnextLine:" + next + "\t" + e.getMessage().replaceAll("\n", "\t");
                        e.printStackTrace();
                        throw e;
                    } finally {
                        initFileMap.writeKeyFile("result", record, true);
                        fileMap.closeWriters();
                        if (lineProcessor != null) lineProcessor.closeResource();
                    }
                } catch (Exception e) {
                    SystemUtils.exit(exitBool, e);
                }
            });
        }
    }

    public void export() throws Exception {
        FileMap inputFiles = new FileMap(savePath);
        File sourceFile = new File(filePath);
        if (sourceFile.isDirectory()) {
            inputFiles.initReaders(filePath);
        } else {
            inputFiles.initReader(filePath);
        }

        int filesCount = inputFiles.getReaderMap().size();
        int runningThreads = filesCount < threads ? filesCount : threads;
        String info = "read files" + (processor == null ? "" : " and " + processor.getProcessName());
        System.out.println(info + " running...");
        executorPool = Executors.newFixedThreadPool(runningThreads);
        exitBool = new AtomicBoolean(false);
        execInThreads(inputFiles);
        executorPool.shutdown();
        while (!executorPool.isTerminated()) Thread.sleep(1000);
        inputFiles.closeReaders();
        System.out.println(info + " finished");
    }
}