org.kuali.common.util.FileSystemUtils Maven / Gradle / Ivy
/**
* Copyright 2010-2014 The Kuali Foundation
*
* Licensed under the Educational Community License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.opensource.org/licenses/ecl2.php
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.kuali.common.util;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.kuali.common.util.base.Threads;
import org.kuali.common.util.execute.CopyFileRequest;
import org.kuali.common.util.execute.CopyFileResult;
import org.kuali.common.util.file.DirDiff;
import org.kuali.common.util.file.DirRequest;
import org.kuali.common.util.file.MD5Result;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FileSystemUtils {
private static final Logger logger = LoggerFactory.getLogger(FileSystemUtils.class);
public static final String RECURSIVE_FILE_INCLUDE_PATTERN = "**/**";
public static final List DEFAULT_RECURSIVE_INCLUDES = Arrays.asList(RECURSIVE_FILE_INCLUDE_PATTERN);
private static final String SVN_PATTERN = "**/.svn/**";
private static final String GIT_PATTERN = "**/.git/**";
public static final List DEFAULT_SCM_IGNORE_PATTERNS = Arrays.asList(SVN_PATTERN, GIT_PATTERN);
/**
* Return a recursive listing of all files in the directory ignoring ++/.svn/++ and ++/.git/++
*/
public static List getAllNonScmFiles(File dir) {
return getAllNonScmFiles(dir, DEFAULT_SCM_IGNORE_PATTERNS);
}
/**
* Return a recursive listing of all files in the directory ignoring files that match scmIgnorePatterns
*/
public static List getAllNonScmFiles(File dir, List scmIgnorePatterns) {
SimpleScanner scanner = new SimpleScanner(dir, DEFAULT_RECURSIVE_INCLUDES, scmIgnorePatterns);
return scanner.getFiles();
}
/**
* This method recursively copies one file system directory to another directory under the control of SCM. Before doing so, it records 3 types of files:
*
*
* 1 - both - files that exist in both directories
* 2 - dir1Only - files that exist in the source directory but not the SCM directory
* 3 - dir2Only - files that exist in the SCM directory but not the source directory
*
*
* This provides enough information for SCM tooling to then complete the work of making the SCM directory exactly match the file system directory and commit any changes to the
* SCM system.
*/
@Deprecated
public static DirectoryDiff prepareScmDir(PrepareScmDirRequest request) {
return prepareScmDir(request, null, false);
}
/**
* This method recursively copies one file system directory to another directory under the control of SCM. Before doing so, it records 3 types of files:
*
*
* 1 - both - files that exist in both directories
* 2 - dir1Only - files that exist in the source directory but not the SCM directory
* 3 - dir2Only - files that exist in the SCM directory but not the source directory
*
*
* This provides enough information for SCM tooling to then complete the work of making the SCM directory exactly match the file system directory and commit any changes to the
* SCM system.
*
* @deprecated
*/
@Deprecated
public static DirectoryDiff prepareScmDir(PrepareScmDirRequest request, File relativeDir, boolean diffOnly) {
// Make sure we are configured correctly
Assert.notNull(request, "request is null");
Assert.notNull(request.getSrcDir(), "srcDir is null");
Assert.notNull(request.getScmDir(), "scmDir is null");
// Both must already exist and must be directories (can't be a regular file)
Assert.isExistingDir(request.getSrcDir(), "srcDir is not an existing directory");
Assert.isExistingDir(request.getScmDir(), "scmDir is not an existing directory");
// Setup a diff request
DirectoryDiffRequest diffRequest = new DirectoryDiffRequest();
diffRequest.setDir1(request.getSrcDir());
diffRequest.setDir2(request.getScmDir());
diffRequest.setExcludes(request.getScmIgnorePatterns());
// Record the differences between the two directories
DirectoryDiff diff = getDiff(diffRequest);
// Copy files from the source directory to the SCM directory
if (!diffOnly) {
org.kuali.common.util.execute.CopyFilePatternsExecutable exec = new org.kuali.common.util.execute.CopyFilePatternsExecutable();
exec.setSrcDir(request.getSrcDir());
exec.setDstDir(request.getScmDir());
exec.setExcludes(request.getScmIgnorePatterns());
exec.setRelativeDir(relativeDir);
exec.execute();
}
// Return the diff so we'll know what SCM needs to add/delete from its directory
return diff;
}
public static List getFiles(File dir, List includes, List excludes) {
SimpleScanner scanner = new SimpleScanner(dir, includes, excludes);
return scanner.getFiles();
}
@Deprecated
public static DirectoryDiff getDiff(File dir1, File dir2, List includes, List excludes) {
DirectoryDiffRequest request = new DirectoryDiffRequest();
request.setDir1(dir1);
request.setDir2(dir2);
request.setIncludes(includes);
request.setExcludes(excludes);
return getDiff(request);
}
/**
* Compare 2 directories on the file system and return an object containing the results. All of the files contained in either of the 2 directories get aggregated into 5
* categories.
*
*
* 1 - Both - Files that exist in both directories
* 2 - Different - Files that exist in both directories but who's MD5 checksums do not match
* 3 - Identical - Files that exist in both directories with matching MD5 checksums
* 4 - Source Dir Only - Files that exist only in the source directory
* 5 - Target Dir Only - Files that exist only in the target directory
*
*
* The 5 lists in DirDiff contain the relative paths to files for each category.
*/
public static DirDiff getMD5Diff(DirRequest request) {
// Do a quick diff (just figures out what files are unique to each directory vs files that are in both)
DirDiff diff = getQuickDiff(request);
// Do a deep diff
// This computes MD5 checksums for any files present in both directories
fillInMD5Results(diff);
// return the diff result
return diff;
}
public static List getMD5Results(List sources, List targets) {
Assert.isTrue(sources.size() == targets.size(), "lists are not the same size");
List results = new ArrayList();
for (int i = 0; i < sources.size(); i++) {
File source = sources.get(i);
File target = targets.get(i);
MD5Result md5Result = getMD5Result(source, target);
results.add(md5Result);
}
return results;
}
protected static void fillInMD5Results(DirDiff diff) {
List sources = getFullPaths(diff.getSourceDir(), diff.getBoth());
List targets = getFullPaths(diff.getTargetDir(), diff.getBoth());
List results = getMD5Results(sources, targets);
List different = new ArrayList();
List identical = new ArrayList();
for (MD5Result md5Result : results) {
String sourceChecksum = md5Result.getSourceChecksum();
String targetChecksum = md5Result.getTargetChecksum();
Assert.notNull(sourceChecksum, "sourceChecksum is null");
Assert.notNull(targetChecksum, "targetChecksum is null");
if (StringUtils.equals(sourceChecksum, targetChecksum)) {
identical.add(md5Result);
} else {
different.add(md5Result);
}
}
//
diff.setDifferent(different);
diff.setIdentical(identical);
}
public static MD5Result getMD5Result(File source, File target) {
String sourceChecksum = LocationUtils.getMD5Checksum(source);
String targetChecksum = LocationUtils.getMD5Checksum(target);
return new MD5Result(source, sourceChecksum, target, targetChecksum);
}
/**
* Compare 2 directories on the file system and return an object containing the results. All of the files contained in either of the 2 directories get placed into one of 3
* categories.
*
*
* 1 - Both - Files that exist in both directories
* 2 - Dir 1 Only - Files that exist only in directory 1
* 3 - Dir 2 Only - Files that exist only in directory 2
*
*
* The 3 lists in DirectoryDiff contain the relative paths to files for each category.
*/
@Deprecated
public static DirectoryDiff getDiff(DirectoryDiffRequest request) {
DirRequest newRequest = new DirRequest();
newRequest.setExcludes(request.getExcludes());
newRequest.setIncludes(request.getIncludes());
newRequest.setSourceDir(request.getDir1());
newRequest.setTargetDir(request.getDir2());
DirDiff diff = getQuickDiff(newRequest);
DirectoryDiff dd = new DirectoryDiff(diff.getSourceDir(), diff.getTargetDir());
dd.setBoth(diff.getBoth());
dd.setDir1Only(diff.getSourceDirOnly());
dd.setDir2Only(diff.getTargetDirOnly());
return dd;
}
public static DirDiff getQuickDiff(DirRequest request) {
// Get a listing of files from both directories using the exact same includes/excludes
List sourceFiles = getFiles(request.getSourceDir(), request.getIncludes(), request.getExcludes());
List targetFiles = getFiles(request.getTargetDir(), request.getIncludes(), request.getExcludes());
// Get the unique set of paths for each file relative to their parent directory
Set sourcePaths = new HashSet(getRelativePaths(request.getSourceDir(), sourceFiles));
Set targetPaths = new HashSet(getRelativePaths(request.getTargetDir(), targetFiles));
// Paths that exist in both directories
Set both = SetUtils.intersection(sourcePaths, targetPaths);
// Paths that exist in source but not target
Set sourceOnly = SetUtils.difference(sourcePaths, targetPaths);
// Paths that exist in target but not source
Set targetOnly = SetUtils.difference(targetPaths, sourcePaths);
logger.debug("source={}, sourceOnly.size()={}", request.getSourceDir(), sourceOnly.size());
logger.debug("target={}, targetOnly.size()={}", request.getTargetDir(), targetOnly.size());
// Store the information we've collected into a result object
DirDiff result = new DirDiff(request.getSourceDir(), request.getTargetDir());
// Store the relative paths on the diff object
result.setBoth(new ArrayList(both));
result.setSourceDirOnly(new ArrayList(sourceOnly));
result.setTargetDirOnly(new ArrayList(targetOnly));
// Sort the relative paths
Collections.sort(result.getBoth());
Collections.sort(result.getSourceDirOnly());
Collections.sort(result.getTargetDirOnly());
// return the diff
return result;
}
/**
* Examine the contents of a text file, stopping as soon as it contains token, or timeout is exceeded, whichever comes first.
*/
public static MonitorTextFileResult monitorTextFile(File file, String token, int intervalMillis, int timeoutMillis, String encoding) {
// Make sure we are configured correctly
Assert.notNull(file, "file is null");
Assert.hasText(token, "token has no text");
Assert.hasText(encoding, "encoding has no text");
Assert.isTrue(intervalMillis > 0, "interval must be a positive integer");
Assert.isTrue(timeoutMillis > 0, "timeout must be a positive integer");
// Setup some member variables to record what happens
long start = System.currentTimeMillis();
long stop = start + timeoutMillis;
boolean exists = false;
boolean contains = false;
boolean timeoutExceeded = false;
long now = -1;
String content = null;
// loop until timeout is exceeded or we find the token inside the file
for (;;) {
// Always pause (unless this is the first iteration)
if (now != -1) {
Threads.sleep(intervalMillis);
}
// Check to make sure we haven't exceeded our timeout limit
now = System.currentTimeMillis();
if (now > stop) {
timeoutExceeded = true;
break;
}
// If the file does not exist, no point in going any further
exists = LocationUtils.exists(file);
if (!exists) {
continue;
}
// The file exists, check to see if the token we are looking for is present in the file
content = LocationUtils.toString(file, encoding);
contains = StringUtils.contains(content, token);
if (contains) {
// We found what we are looking for, we are done
break;
}
}
// Record how long the overall process took
long elapsed = now - start;
// Fill in a pojo detailing what happened
MonitorTextFileResult mtfr = new MonitorTextFileResult(exists, contains, timeoutExceeded, elapsed);
mtfr.setAbsolutePath(LocationUtils.getCanonicalPath(file));
mtfr.setContent(content);
return mtfr;
}
public static List syncFiles(List requests) throws IOException {
List results = new ArrayList();
for (SyncRequest request : requests) {
SyncResult result = syncFiles(request);
results.add(result);
}
return results;
}
public static SyncResult syncFilesQuietly(SyncRequest request) {
try {
return syncFiles(request);
} catch (IOException e) {
throw new IllegalStateException("Unexpected IO error");
}
}
public static SyncResult syncFiles(SyncRequest request) throws IOException {
logger.info("Sync [{}] -> [{}]", request.getSrcDir(), request.getDstDir());
List dstFiles = getAllNonScmFiles(request.getDstDir());
List srcFiles = request.getSrcFiles();
List dstPaths = getRelativePaths(request.getDstDir(), dstFiles);
List srcPaths = getRelativePaths(request.getSrcDir(), srcFiles);
List adds = new ArrayList();
List updates = new ArrayList();
List deletes = new ArrayList();
for (String srcPath : srcPaths) {
boolean existing = dstPaths.contains(srcPath);
if (existing) {
updates.add(srcPath);
} else {
adds.add(srcPath);
}
}
for (String dstPath : dstPaths) {
boolean extra = !srcPaths.contains(dstPath);
if (extra) {
deletes.add(dstPath);
}
}
copyFiles(request.getSrcDir(), request.getSrcFiles(), request.getDstDir());
SyncResult result = new SyncResult();
result.setAdds(getFullPaths(request.getDstDir(), adds));
result.setUpdates(getFullPaths(request.getDstDir(), updates));
result.setDeletes(getFullPaths(request.getDstDir(), deletes));
return result;
}
protected static void copyFiles(File srcDir, List files, File dstDir) throws IOException {
for (File file : files) {
String relativePath = getRelativePath(srcDir, file);
File dstFile = new File(dstDir, relativePath);
FileUtils.copyFile(file, dstFile);
}
}
public static List getFullPaths(File dir, Set relativePaths) {
return getFullPaths(dir, new ArrayList(relativePaths));
}
public static List getSortedFullPaths(File dir, List relativePaths) {
List files = getFullPaths(dir, relativePaths);
Collections.sort(files);
return files;
}
public static List getFullPaths(File dir, List relativePaths) {
List files = new ArrayList();
for (String relativePath : relativePaths) {
File file = new File(dir, relativePath);
File canonical = new File(LocationUtils.getCanonicalPath(file));
files.add(canonical);
}
return files;
}
protected static List getRelativePaths(File dir, List files) {
List relativePaths = new ArrayList();
for (File file : files) {
String relativePath = getRelativePath(dir, file);
relativePaths.add(relativePath);
}
return relativePaths;
}
/**
* Return true if child lives on the file system somewhere underneath parent, false otherwise.
*/
public static boolean isParent(File parent, File child) {
if (parent == null || child == null) {
return false;
}
String parentPath = LocationUtils.getCanonicalPath(parent);
String childPath = LocationUtils.getCanonicalPath(child);
if (StringUtils.equals(parentPath, childPath)) {
return false;
} else {
return StringUtils.contains(childPath, parentPath);
}
}
/**
* Return the relative path to file from parentDir. parentDir is optional and can be null. If parentDir is not
* supplied (or is not a parent directory to file the canonical path to file is returned.
*/
public static String getRelativePathQuietly(File parentDir, File file) {
Assert.notNull(file, "file is null");
if (isParent(parentDir, file)) {
return getRelativePath(parentDir, file);
} else {
return LocationUtils.getCanonicalPath(file);
}
}
public static String getRelativePath(File dir, File file) {
String dirPath = LocationUtils.getCanonicalPath(dir);
String filePath = LocationUtils.getCanonicalPath(file);
if (!StringUtils.contains(filePath, dirPath)) {
throw new IllegalArgumentException(file + " does not reside under " + dir);
}
return StringUtils.remove(filePath, dirPath);
}
public static List getCopyFileRequests(File srcDir, List includes, List excludes, File dstDir) {
SimpleScanner scanner = new SimpleScanner(srcDir, includes, excludes);
List srcFiles = scanner.getFiles();
List requests = new ArrayList();
for (File srcFile : srcFiles) {
String relativePath = FileSystemUtils.getRelativePath(srcDir, srcFile);
File dstFile = new File(dstDir, relativePath);
CopyFileRequest request = new CopyFileRequest(srcFile, dstFile);
requests.add(request);
}
return requests;
}
public static CopyFileResult copyFile(File src, File dst) {
try {
long start = System.currentTimeMillis();
boolean overwritten = dst.exists();
FileUtils.copyFile(src, dst);
return new CopyFileResult(src, dst, overwritten, System.currentTimeMillis() - start);
} catch (IOException e) {
throw new IllegalStateException("Unexpected IO error", e);
}
}
public static List copyFiles(List requests) {
List results = new ArrayList();
for (CopyFileRequest request : requests) {
CopyFileResult result = copyFile(request.getSource(), request.getDestination());
results.add(result);
}
return results;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy