org.apache.jackrabbit.oak.commons.FileIOUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of org.apache.sling.testing.sling-mock-oak Show documentation
Show all versions of org.apache.sling.testing.sling-mock-oak Show documentation
Implements a resource resolver type for Jackrabbit Oak that can be used in unit tests based on Sling Mocks.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jackrabbit.oak.commons;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Strings;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.io.Closeables.close;
import static com.google.common.io.FileWriteMode.APPEND;
import static com.google.common.io.Files.asByteSink;
import static com.google.common.io.Files.move;
import static com.google.common.io.Files.newWriter;
import static java.io.File.createTempFile;
import static org.apache.commons.io.FileUtils.forceDelete;
import static org.apache.commons.io.IOUtils.closeQuietly;
import static org.apache.commons.io.IOUtils.copyLarge;
import static org.apache.commons.io.LineIterator.closeQuietly;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.escapeLineBreak;
import static org.apache.jackrabbit.oak.commons.sort.EscapeUtils.unescapeLineBreaks;
import static org.apache.jackrabbit.oak.commons.sort.ExternalSort.mergeSortedFiles;
import static org.apache.jackrabbit.oak.commons.sort.ExternalSort.sortInBatch;
/**
* Simple File utils
*/
public final class FileIOUtils {
private FileIOUtils() {
}
public final static Comparator lexComparator = new Comparator() {
@Override public int compare(String s1, String s2) {
return s1.compareTo(s2);
}
};
public final static Function passThruTransformer = new Function() {
@Nullable @Override public String apply(@Nullable String input) {
return input;
}
};
/**
* Sorts the given file externally using the {@link #lexComparator} and removes duplicates.
*
* @param file file whose contents needs to be sorted
*/
public static void sort(File file) throws IOException {
File sorted = createTempFile("fleioutilssort", null);
merge(sortInBatch(file, lexComparator, true), sorted);
move(sorted, file);
}
/**
* Sorts the given file externally with the given comparator and removes duplicates.
*
* @param file file whose contents needs to be sorted
* @param comparator to compare
* @throws IOException
*/
public static void sort(File file, Comparator comparator) throws IOException {
File sorted = createTempFile("fleioutilssort", null);
merge(sortInBatch(file, comparator, true), sorted, comparator);
move(sorted, file);
}
/**
* Merges a list of files after sorting with the {@link #lexComparator}.
*
* @param files files to merge
* @param output merge output file
* @throws IOException
*/
public static void merge(List files, File output) throws IOException {
mergeSortedFiles(
files,
output, lexComparator, true);
}
/**
* Merges a list of files after sorting with the given comparator.
*
* @param files files to merge
* @param output merge output file
* @throws IOException
*/
public static void merge(List files, File output, Comparator comparator) throws IOException {
mergeSortedFiles(
files,
output, comparator, true);
}
/**
* Copies an input stream to a file.
*
* @param stream steam to copy
* @return
* @throws IOException
*/
public static File copy(InputStream stream) throws IOException {
File file = createTempFile("fleioutilscopy", null);
copyInputStreamToFile(stream, file);
return file;
}
/**
* Appends the contents of the list of files to the given file and deletes the files
* if the delete flag is enabled.
*
* If there is a scope for lines in the files containing line break characters it should be
* ensured that the files are written with {@link #writeAsLine(BufferedWriter, String, boolean)}
* with true to escape line break characters.
* @param files
* @param appendTo
* @throws IOException
*/
public static void append(List files, File appendTo, boolean delete) throws IOException {
OutputStream appendStream = null;
boolean threw = true;
try {
appendStream = asByteSink(appendTo, APPEND).openBufferedStream();
for (File f : files) {
InputStream iStream = new FileInputStream(f);
try {
copyLarge(iStream, appendStream);
} finally {
closeQuietly(iStream);
}
}
threw = false;
} finally {
if (delete) {
for (File f : files) {
f.delete();
}
}
close(appendStream, threw);
}
}
/**
* Writes a string as a new line into the given buffered writer and optionally
* escapes the line for line breaks.
*
* @param writer to write the string
* @param str the string to write
* @param escape whether to escape string for line breaks
* @throws IOException
*/
public static void writeAsLine(BufferedWriter writer, String str, boolean escape) throws IOException {
if (escape) {
writer.write(escapeLineBreak(str));
} else {
writer.write(str);
}
writer.newLine();
}
/**
* Writes string from the given iterator to the given file and optionally
* escape the written strings for line breaks.
*
* @param iterator the source of the strings
* @param f file to write to
* @param escape whether to escape for line breaks
* @return count
* @throws IOException
*/
public static int writeStrings(Iterator iterator, File f, boolean escape)
throws IOException {
return writeStrings(iterator, f, escape, null, "");
}
/**
* Writes string from the given iterator to the given file and optionally
* escape the written strings for line breaks.
*
* @param iterator the source of the strings
* @param f file to write to
* @param escape escape whether to escape for line breaks
* @param logger logger to log progress
* @param message message to log
* @return
* @throws IOException
*/
public static int writeStrings(Iterator iterator, File f, boolean escape,
@Nullable Logger logger, @Nullable String message) throws IOException {
return writeStrings(iterator, f, escape, passThruTransformer, logger, message);
}
/**
* Writes string from the given iterator to the given file and optionally
* escape the written strings for line breaks.
*
* @param iterator the source of the strings
* @param f file to write to
* @param escape escape whether to escape for line breaks
* @param transformer any transformation on the input
* @param logger logger to log progress
* @param message message to log
* @return
* @throws IOException
*/
public static int writeStrings(Iterator iterator, File f, boolean escape,
@NotNull Function transformer, @Nullable Logger logger, @Nullable String message) throws IOException {
BufferedWriter writer = newWriter(f, UTF_8);
boolean threw = true;
int count = 0;
try {
while (iterator.hasNext()) {
writeAsLine(writer, transformer.apply(iterator.next()), escape);
count++;
if (logger != null) {
if (count % 1000 == 0) {
logger.info(Strings.nullToEmpty(message) + count);
}
}
}
threw = false;
} finally {
close(writer, threw);
}
return count;
}
/**
* Reads strings from the given stream into a set and optionally unescaping for line breaks.
*
* @param stream the source of the strings
* @param unescape whether to unescape for line breaks
* @return set
* @throws IOException
*/
public static Set readStringsAsSet(InputStream stream, boolean unescape) throws IOException {
BufferedReader reader = null;
Set set = newHashSet();
boolean threw = true;
try {
reader = new BufferedReader(new InputStreamReader(stream, Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
if (unescape) {
set.add(unescapeLineBreaks(line));
} else {
set.add(line);
}
}
threw = false;
} finally {
close(reader, threw);
}
return set;
}
/**
* Composing iterator which unescapes for line breaks and delegates to the given comparator.
* When using this it should be ensured that the data source has been correspondingly escaped.
*
* @param delegate the actual comparison iterator
* @return comparator aware of line breaks
*/
public static Comparator lineBreakAwareComparator (Comparator delegate) {
return new FileIOUtils.TransformingComparator(delegate, new Function() {
@Nullable
@Override
public String apply(@Nullable String input) {
return unescapeLineBreaks(input);
}
});
}
/**
*
* Copy the input stream to the given file. Delete the file in case of exception.
*
* @param source the input stream source
* @param destination the file to write to
* @throws IOException
*/
public static void copyInputStreamToFile(final InputStream source, final File destination) throws IOException {
boolean success = false;
try {
FileUtils.copyInputStreamToFile(source, destination);
success = true;
} finally {
if (!success) {
forceDelete(destination);
}
}
}
/**
* Decorates the given comparator and applies the function before delegating to the decorated
* comparator.
*/
public static class TransformingComparator implements Comparator {
private Comparator delegate;
private Function func;
public TransformingComparator(Comparator delegate, Function func) {
this.delegate = delegate;
this.func = func;
}
@Override
public int compare(String s1, String s2) {
return delegate.compare(func.apply(s1), func.apply(s2));
}
}
/**
* FileLineDifferenceIterator class which iterates over the difference of 2 files line by line.
*
* If there is a scope for lines in the files containing line break characters it should be
* ensured that both the files are written with
* {@link #writeAsLine(BufferedWriter, String, boolean)} with true to escape line break
* characters.
*/
public static class FileLineDifferenceIterator extends AbstractIterator implements Closeable {
private final PeekingIterator peekMarked;
private final LineIterator marked;
private final LineIterator all;
private Function transformer = new Function() {
@Override
public String apply(String input) {
return input;
}
};
public FileLineDifferenceIterator(LineIterator marked, LineIterator available) throws IOException {
this(marked, available, null);
}
public FileLineDifferenceIterator(File marked, File available,
@Nullable Function transformer) throws IOException {
this(FileUtils.lineIterator(marked, UTF_8.toString()),
FileUtils.lineIterator(available, UTF_8.toString()), transformer);
}
public FileLineDifferenceIterator(LineIterator marked, LineIterator available,
@Nullable Function transformer) throws IOException {
this.marked = marked;
this.peekMarked = Iterators.peekingIterator(marked);
this.all = available;
if (transformer != null) {
this.transformer = transformer;
}
}
@Override
protected String computeNext() {
String diff = computeNextDiff();
if (diff == null) {
close();
return endOfData();
}
return diff;
}
@Override
public void close() {
LineIterator.closeQuietly(marked);
LineIterator.closeQuietly(all);
}
private String computeNextDiff() {
if (!all.hasNext()) {
return null;
}
//Marked finish the rest of all are part of diff
if (!peekMarked.hasNext()) {
return all.next();
}
String diff = null;
while (all.hasNext() && diff == null) {
diff = all.next();
while (peekMarked.hasNext()) {
String marked = peekMarked.peek();
int comparisonResult = transformer.apply(diff).compareTo(transformer.apply((marked)));
if (comparisonResult > 0) {
//Extra entries in marked. Ignore them and move on
peekMarked.next();
} else if (comparisonResult == 0) {
//Matching entry found in marked move past it. Not a
//dif candidate
peekMarked.next();
diff = null;
break;
} else {
//This entry is not found in marked entries
//hence part of diff
return diff;
}
}
}
return diff;
}
}
/**
* Implements a {@link java.io.Closeable} wrapper over a {@link LineIterator}.
* Also has a transformer to transform the output. If the underlying file is
* provide then it deletes the file on {@link #close()}.
*
* If there is a scope for lines in the file containing line break characters it should be
* ensured that the files is written with
* {@link #writeAsLine(BufferedWriter, String, boolean)} with true to escape line break
* characters and should be properly unescaped on read.
* A custom transformer can also be provided to unescape.
*
* @param the type of elements in the iterator
*/
public static class BurnOnCloseFileIterator extends AbstractIterator implements Closeable {
private final Logger log = LoggerFactory.getLogger(getClass());
private final LineIterator iterator;
private final Function transformer;
private File backingFile;
public BurnOnCloseFileIterator(LineIterator iterator, Function transformer) {
this.iterator = iterator;
this.transformer = transformer;
}
public BurnOnCloseFileIterator(LineIterator iterator, File backingFile,
Function transformer) {
this.iterator = iterator;
this.transformer = transformer;
this.backingFile = backingFile;
}
@Override
protected T computeNext() {
if (iterator.hasNext()) {
return transformer.apply(iterator.next());
}
try {
close();
} catch (IOException e) {
log.warn("Error closing iterator", e);
}
return endOfData();
}
@Override
public void close() throws IOException {
closeQuietly(iterator);
if (backingFile != null && backingFile.exists()) {
forceDelete(backingFile);
}
}
public static BurnOnCloseFileIterator wrap(LineIterator iter) {
return new BurnOnCloseFileIterator(iter, new Function() {
public String apply(String s) {
return s;
}
});
}
public static BurnOnCloseFileIterator wrap(LineIterator iter, File backingFile) {
return new BurnOnCloseFileIterator(iter, backingFile,
new Function() {
public String apply(String s) {
return s;
}
});
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy