dev.fitko.fitconnect.core.io.FileChunker Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of client Show documentation
Show all versions of client Show documentation
Library that provides client access to the FIT-Connect api-endpoints for sending, subscribing and
routing
package dev.fitko.fitconnect.core.io;
import dev.fitko.fitconnect.core.utils.Preconditions;
import dev.fitko.fitconnect.core.utils.StopWatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;
import static java.util.Comparator.comparing;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
public class FileChunker {
private static final Logger LOGGER = LoggerFactory.getLogger(FileChunker.class);
private static final String CHUNK_INDEX_FORMAT = "%06d"; // 6 digits -> max 999.999 named chunks
private static final String CHUNK_FILE_ENDING = ".chunk";
private static final String PART_SUFFIX = "_part_";
public static final String FRAGMENT_PREFIX = "fragment_";
/**
* Splits a file path into chunks and writes them into a given directory.
*
* @param filePath path to a file that should be chunked
* @param chunkSize max. size of each chunk
* @param outputDir output directory the chunks will be written to
* @return list of chunked files
* @throws IOException if an error occurred whilst reading the file or writing the chunk
*/
public List chunkFile(final Path filePath, final int chunkSize, final Path outputDir) throws IOException {
try (InputStream is = Files.newInputStream(filePath)) {
return chunkStream(is, chunkSize, filePath.getFileName().toString(), outputDir);
}
}
/**
* Splits a stream into chunked files and writes them into a given directory.
*
* @param stream input-stream of the data
* @param chunkSize max. size of each chunk
* @param fileName name of the original file
* @param outputDir output directory the chunks will be written to
* @return list of chunked files
* @throws IOException if an error occurred whilst reading the file or writing the chunk
*/
public List chunkStream(final InputStream stream, final int chunkSize, final String fileName, final Path outputDir) throws IOException {
Preconditions.checkArgumentAndThrow(chunkSize <= 0, "chunk size must be larger than zero, but was " + chunkSize);
final var buffer = new byte[chunkSize];
final var fileChunks = new ArrayList();
final var name = fileName == null ? UUID.randomUUID().toString() : fileName;
int chunkCount = 0;
int readBytes = stream.read(buffer);
final var start = StopWatch.start();
while (readBytes > -1) {
final File fileChunk = writeFileChunk(generateFileName(name, chunkCount), buffer, readBytes, outputDir);
LOGGER.debug("Wrote {} bytes of chunk #{}", readBytes, ++chunkCount);
fileChunks.add(fileChunk);
readBytes = stream.read(buffer);
}
LOGGER.info("Chunking to {} files took {}", chunkCount, StopWatch.stop(start));
return fileChunks;
}
/**
* Concatenates all file parts of a given directory to a single file and deletes the file chunks afterward.
*
* @param path path that contains the file chunks with ".chunk" extension
* @param fileName filename the merged file is written to
* @return optional of the merged file, is empty if no chunks were found
* @throws IOException if an error occurred whilst reading the directory path or merging the chunks
*/
public Optional concatChunks(final Path path, final String fileName) throws IOException {
final List chunkedFiles = Stream.of(requireNonNull(path.toFile().listFiles()))
.filter(file -> !file.isDirectory())
.filter(file -> file.getName().contains(CHUNK_FILE_ENDING))
.sorted(comparing(File::getName))
.collect(toList());
if (chunkedFiles.isEmpty()) {
return Optional.empty();
}
final File mergedFile = path.resolve(fileName).toFile();
LOGGER.info("Merging {} chunks into file {}", chunkedFiles.size(), mergedFile.getPath());
final var start = StopWatch.start();
try (FileOutputStream fos = new FileOutputStream(mergedFile)) {
for (final File chunk : chunkedFiles) {
try (InputStream is = Files.newInputStream(chunk.toPath())) {
is.transferTo(fos);
fos.flush();
chunk.delete();
}
}
}
LOGGER.info("Merging chunks took {}", StopWatch.stop(start));
return Optional.of(mergedFile);
}
/**
* Writes a file chunk with an index to the filesystem.
*
* @param data as byte[] that should be written
* @param chunkIndex index of the current chunk that is used in the filename
* @param outputDir the target path dir the chunk is written to
* @return that file tha was written
* @throws IOException if there was an error whilst writing the file
*/
public File writeFileChunk(final byte[] data, final int chunkIndex, final Path outputDir) throws IOException {
final String fileName = generateFileName(chunkIndex);
return writeFileChunk(fileName, data, data.length, outputDir);
}
private File writeFileChunk(final String fileName, final byte[] buffer, final int length, final Path outputDir) throws IOException {
final File chunk = outputDir.resolve(fileName).toFile();
try (FileOutputStream fos = new FileOutputStream(chunk)) {
fos.write(buffer, 0, length);
}
return chunk;
}
private static String generateFileName(final String fileName, final int chunkCount) {
return fileName + PART_SUFFIX + String.format(CHUNK_INDEX_FORMAT, chunkCount) + CHUNK_FILE_ENDING;
}
private static String generateFileName(final int chunkCount) {
return FRAGMENT_PREFIX + String.format(CHUNK_INDEX_FORMAT, chunkCount) + CHUNK_FILE_ENDING;
}
}