com.qiniu.datasource.FileContainer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of qsuits Show documentation
Show all versions of qsuits Show documentation
qiniu-suits is a efficient tools for qiniu api implemented by java8.
package com.qiniu.datasource;
import com.qiniu.common.JsonRecorder;
import com.qiniu.common.QiniuException;
import com.qiniu.interfaces.IDataSource;
import com.qiniu.interfaces.ILineProcess;
import com.qiniu.interfaces.IReader;
import com.qiniu.interfaces.ITypeConvert;
import com.qiniu.persistence.FileSaveMapper;
import com.qiniu.interfaces.IResultOutput;
import com.qiniu.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Signal;
import sun.misc.SignalHandler;
import java.io.File;
import java.io.IOException;
import java.time.Clock;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static com.qiniu.entry.CommonParams.lineFormats;
public abstract class FileContainer implements IDataSource, IResultOutput, T> {
private static final File errorLogFile = new File(String.join(".", LogUtils.getLogPath(LogUtils.QSUITS), LogUtils.ERROR));
private static final File infoLogFile = new File(String.join(".", LogUtils.getLogPath(LogUtils.QSUITS), LogUtils.INFO));
private static final File procedureLogFile = new File(String.join(".", LogUtils.getLogPath(LogUtils.PROCEDURE), LogUtils.LOG_EXT));
private static final Logger rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
private static final Logger errorLogger = LoggerFactory.getLogger(LogUtils.ERROR);
private static final Logger infoLogger = LoggerFactory.getLogger(LogUtils.INFO);
private static final Logger procedureLogger = LoggerFactory.getLogger(LogUtils.PROCEDURE);
private String filePath;
protected String parse;
protected String separator;
protected String addKeyPrefix;
protected String rmKeyPrefix;
protected Map linesMap;
protected Map indexMap;
protected int unitLen;
protected int threads;
protected int retryTimes = 5;
protected String savePath;
protected boolean saveTotal;
protected String saveFormat;
protected String saveSeparator;
protected List rmFields;
protected List fields;
private ILineProcess processor; // 定义的资源处理器
private ConcurrentMap> saverMap = new ConcurrentHashMap<>(threads);
private ConcurrentMap> processorMap = new ConcurrentHashMap<>(threads);
public FileContainer(String filePath, String parse, String separator, String addKeyPrefix, String rmKeyPrefix,
Map linesMap, Map indexMap, List fields, int unitLen,
int threads) throws IOException {
this.filePath = filePath;
this.parse = parse;
this.separator = separator;
this.addKeyPrefix = addKeyPrefix;
this.rmKeyPrefix = rmKeyPrefix;
this.linesMap = linesMap == null ? new HashMap<>() : linesMap;
this.indexMap = indexMap;
this.unitLen = unitLen;
this.threads = threads;
// default save parameters
this.saveTotal = false; // 默认全记录不保存
this.savePath = "result";
this.saveFormat = "tab";
this.saveSeparator = "\t";
if (fields == null || fields.size() == 0) {
this.fields = ConvertingUtils.getOrderedFields(this.indexMap, rmFields);
}
else this.fields = fields;
}
// 不调用则各参数使用默认值
public void setSaveOptions(boolean saveTotal, String savePath, String format, String separator, List rmFields)
throws IOException {
this.saveTotal = saveTotal;
this.savePath = savePath;
this.saveFormat = format;
if (!lineFormats.contains(saveFormat)) throw new IOException("please check your format for map to string.");
this.saveSeparator = separator;
this.rmFields = rmFields;
if (rmFields != null && rmFields.size() > 0) {
this.fields = ConvertingUtils.getFields(fields, rmFields);
}
}
public void setRetryTimes(int retryTimes) {
this.retryTimes = retryTimes;
}
public void setProcessor(ILineProcess processor) {
this.processor = processor;
}
protected abstract ITypeConvert getNewConverter() throws IOException;
protected abstract ITypeConvert getNewStringConverter() throws IOException;
private JsonRecorder recorder = new JsonRecorder();
public void export(IReader reader, IResultOutput saver, ILineProcess processor) throws Exception {
ITypeConvert converter = getNewConverter();
ITypeConvert stringConverter = null;
if (saveTotal) {
stringConverter = getNewStringConverter();
saver.preAddWriter("failed");
}
String lastLine = reader.lastLine();
List srcList = null;
List convertedList;
List writeList;
int retry;
while (lastLine != null) {
if (LocalDateTime.now(DatetimeUtils.clock_Default).isAfter(pauseDateTime)) {
synchronized (object) {
object.wait();
}
}
retry = retryTimes + 1;
while (retry > 0) {
try {
srcList = reader.readLines();
retry = 0;
} catch (IOException e) {
retry--;
if (retry == 0) throw e;
}
}
convertedList = converter.convertToVList(srcList);
if (converter.errorSize() > 0) saver.writeError(converter.errorLines(), false);
if (stringConverter != null) {
writeList = stringConverter.convertToVList(convertedList);
if (writeList.size() > 0) saver.writeSuccess(String.join("\n", writeList), false);
if (stringConverter.errorSize() > 0)
saver.writeToKey("failed", stringConverter.errorLines(), false);
}
// 如果抛出异常需要检测下异常是否是可继续的异常，如果是程序可继续的异常，忽略当前异常保持数据源读取过程继续进行
try {
if (processor != null) processor.processLine(convertedList);
} catch (QiniuException e) {
// 这里其实逻辑上没有做重试次数的限制，因为返回的 retry 始终大于等于 -1，所以不是必须抛出的异常则会跳过，process 本身会
// 保存失败的记录，除非是 process 出现 599 状态码才会抛出异常
if (HttpRespUtils.checkException(e, 2) < -1) throw e;
if (e.response != null) e.response.close();
}
try { FileUtils.createIfNotExists(procedureLogFile); } catch (IOException ignored) {}
procedureLogger.info(recorder.put(reader.getName(), lastLine));
lastLine = reader.lastLine();
}
}
protected abstract IResultOutput getNewResultSaver(String order) throws IOException;
void reading(IReader reader) {
int order = UniOrderUtils.getOrder();
String orderStr = String.valueOf(order);
ILineProcess lineProcessor = null;
IResultOutput saver = null;
try {
saver = getNewResultSaver(orderStr);
saverMap.put(orderStr, saver);
if (processor != null) {
lineProcessor = processor.clone();
processorMap.put(orderStr, lineProcessor);
}
export(reader, saver, lineProcessor);
recorder.remove(reader.getName());
} catch (QiniuException e) {
try { FileUtils.createIfNotExists(errorLogFile); } catch (IOException ignored) {}
errorLogger.error("{}: {}, {}", reader.getName(), recorder.getString(reader.getName()), e.error(), e);
if (e.response != null) e.response.close();
} catch (Throwable e) {
try { FileUtils.createIfNotExists(errorLogFile); } catch (IOException ignored) {}
errorLogger.error("{}: {}", reader.getName(), recorder.getString(reader.getName()), e);
} finally {
try { FileUtils.createIfNotExists(infoLogFile); } catch (IOException ignored) {}
infoLogger.info("{}\t{}\t{}", orderStr, reader.getName(), reader.count());
if (saver != null) {
saver.closeWriters();
saver = null; // let gc work
}
saverMap.remove(orderStr);
if (lineProcessor != null) {
lineProcessor.closeResource();
lineProcessor = null;
}
UniOrderUtils.returnOrder(order);
reader.close();
}
}
void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException ignored) {
int i = 0;
while (i < millis) i++;
}
}
private void endAction() throws IOException {
ILineProcess processor;
for (Map.Entry> saverEntry : saverMap.entrySet()) {
saverEntry.getValue().closeWriters();
processor = processorMap.get(saverEntry.getKey());
if (processor != null) processor.closeResource();
}
String record = recorder.toString();
if (recorder.size() > 0) {
String path = new File(savePath).getCanonicalPath();
FileSaveMapper saveMapper = new FileSaveMapper(new File(path).getParent());
saveMapper.setAppend(false);
saveMapper.setFileExt(".json");
String fileName = path.substring(path.lastIndexOf(FileUtils.pathSeparator) + 1) + "-lines";
saveMapper.addWriter(fileName);
saveMapper.writeToKey(fileName, record, true);
saveMapper.closeWriters();
rootLogger.info("please check the lines breakpoint in {}.json, " +
"it can be used for one more time reading remained lines", fileName);
}
procedureLogger.info(record);
}
private void showdownHook() {
SignalHandler handler = signal -> {
try {
pauseDateTime = LocalDateTime.MIN;
endAction();
} catch (IOException e) {
rootLogger.error("showdown error", e);
}
System.exit(0);
};
try { // 设置 INT 信号 (Ctrl + C 中断执行) 交给指定的信号处理器处理，废掉系统自带的功能
Signal.handle(new Signal("INT"), handler); } catch (Exception ignored) {}
try { Signal.handle(new Signal("TERM"), handler); } catch (Exception ignored) {}
try { Signal.handle(new Signal("USR1"), handler); } catch (Exception ignored) {}
try { Signal.handle(new Signal("USR2"), handler); } catch (Exception ignored) {}
}
protected abstract List> getFileReaders(String path) throws IOException;
public void export() throws Exception {
List> fileReaders = getFileReaders(filePath);
int filesCount = fileReaders.size();
int runningThreads = filesCount < threads ? filesCount : threads;
String info = processor == null ?
String.join(" ", "read objects from file(s):", filePath) :
String.join(" ", "read objects from file(s):", filePath, "and", processor.getProcessName());
rootLogger.info("{} running...", info);
rootLogger.info("order\tpath\tquantity");
ExecutorService executorPool = Executors.newFixedThreadPool(runningThreads);
showdownHook();
try {
String start = null;
for (IReader fileReader : fileReaders) {
recorder.put(fileReader.getName(), start);
executorPool.execute(() -> reading(fileReader));
}
executorPool.shutdown();
while (!executorPool.isTerminated()) {
sleep(2000);
}
rootLogger.info("{} finished.", info);
endAction();
} catch (Throwable e) {
executorPool.shutdownNow();
rootLogger.error("export failed", e);
endAction();
System.exit(-1);
}
}
private final Object object = new Object();
private LocalDateTime pauseDateTime = LocalDateTime.MAX;
public void export(LocalDateTime startTime, long pauseDelay, long duration) throws Exception {
if (startTime != null) {
Clock clock = Clock.systemDefaultZone();
LocalDateTime now = LocalDateTime.now(clock);
if (startTime.minusWeeks(1).isAfter(now)) {
throw new Exception("startTime is not allowed to exceed next week");
}
while (now.isBefore(startTime)) {
System.out.printf("\r%s", LocalDateTime.now(clock).toString().substring(0, 19));
sleep(1000);
now = LocalDateTime.now(clock);
}
}
if (duration <= 0 || pauseDelay < 0) {
export();
} else if (duration > 84600 || duration < 1800) {
throw new Exception("duration can not be bigger than 23.5 hours or smaller than 0.5 hours.");
} else {
pauseDateTime = LocalDateTime.now().plusSeconds(pauseDelay);
Timer timer = new Timer();
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
synchronized (object) {
object.notifyAll();
}
pauseDateTime = LocalDateTime.now().plusSeconds(86400 - duration);
// pauseDateTime = LocalDateTime.now().plusSeconds(20 - duration);
}
}, (pauseDelay + duration) * 1000, 86400000);
// }, (pauseDelay + duration) * 1000, 20000);
export();
timer.cancel();
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy