com.qiniu.datasource.LocalFileContainer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of qsuits Show documentation
qiniu-suits is a efficient tools for qiniu api implemented by java8.
There is a newer version: 8.4.8
package com.qiniu.datasource;

import com.qiniu.convert.LineToMap;
import com.qiniu.convert.MapToString;
import com.qiniu.interfaces.IReader;
import com.qiniu.interfaces.ITypeConvert;
import com.qiniu.persistence.FileSaveMapper;
import com.qiniu.interfaces.IResultOutput;
import com.qiniu.util.FileUtils;

import java.io.*;
import java.util.*;

public class LocalFileContainer extends FileContainer> {

    public LocalFileContainer(String filePath, String parseFormat, String separator, String addKeyPrefix,
                              String rmKeyPrefix, Map linesMap, Map indexMap,
                              List fields, int unitLen, int threads) {
        super(filePath, parseFormat, separator, addKeyPrefix, rmKeyPrefix, linesMap, indexMap, fields, unitLen, threads);
    }

    @Override
    protected ITypeConvert> getNewConverter() throws IOException {
        return new LineToMap(parse, separator, addKeyPrefix, rmKeyPrefix, indexMap);
    }

    @Override
    protected ITypeConvert, String> getNewStringConverter() throws IOException {
        return new MapToString(saveFormat, saveSeparator, fields);
    }

    @Override
    public String getSourceName() {
        return "local";
    }

    @Override
    protected IResultOutput getNewResultSaver(String order) throws IOException {
        return order != null ? new FileSaveMapper(savePath, getSourceName(), order) : new FileSaveMapper(savePath);
    }

    private List getFiles(File directory) throws IOException {
        File[] fs = directory.listFiles();
        if (fs == null) throw new IOException("The current path you gave may be incorrect: " + directory);
        List files = new ArrayList<>();
//        Objects.requireNonNull(directory.listFiles());
        for(File f : fs) {
            if (f.isDirectory()) {
                files.addAll(getFiles(f));
            } else {
                String type = FileUtils.contentType(f);
                if (type.startsWith("text") || type.equals("application/octet-stream")) {
                    files.add(f);
                }
            }
        }
        return files;
    }

    @Override
    protected List> getFileReaders(String path) throws IOException {
        List> fileReaders = new ArrayList<>();
        if (linesMap != null && linesMap.size() > 0) {
            try { path = FileUtils.realPathWithUserHome(path); } catch (IOException ignored) {}
            String type;
            for (Map.Entry entry : linesMap.entrySet()) {
                File file = new File(path, entry.getKey());
                if (!file.exists()) file = new File(entry.getKey());
                if (file.isDirectory()) throw new IOException("the filename defined in lines map can not be directory.");
                else {
                    type = FileUtils.contentType(file);
                    if (type.startsWith("text") || type.equals("application/octet-stream")) {
                        fileReaders.add(new LocalFileReader(file, linesMap.get(file.getPath()), unitLen));
                    } else {
                        throw new IOException("please provide the \'text\' file. The current path you gave is: " + path);
                    }
                }
            }
        } else {
            path = FileUtils.realPathWithUserHome(path);
            if (path.equals(FileUtils.realPathWithUserHome(savePath))) {
                throw new IOException("the save-path can not be same as path.");
            }
            File sourceFile = new File(path);
            if (sourceFile.isDirectory()) {
                List files = getFiles(sourceFile);
                for (File file : files) {
                    fileReaders.add(new LocalFileReader(file, linesMap.get(file.getPath()), unitLen));
                }
            } else {
                String type = FileUtils.contentType(sourceFile);
                if (type.startsWith("text") || type.equals("application/octet-stream")) {
                    fileReaders.add(new LocalFileReader(sourceFile, linesMap.get(sourceFile.getPath()), unitLen));
                } else {
                    throw new IOException("please provide the \'text\' file. The current path you gave is: " + path);
                }
            }
        }
        if (fileReaders.size() == 0) throw new IOException("please provide the \'text\' file in the directory. " +
                "The current path you gave is: " + path);
        return fileReaders;
    }
}