com.metaeffekt.artifact.analysis.utils.FileUtils Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.utils;
import org.apache.commons.io.IOUtils;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.taskdefs.Checksum;
import org.apache.tools.ant.taskdefs.Delete;
import org.apache.tools.ant.taskdefs.Tar;
import org.apache.tools.ant.taskdefs.Zip;
import org.metaeffekt.core.inventory.processor.model.Constants;
import org.mozilla.universalchardet.UniversalDetector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.StreamUtils;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
public class FileUtils extends org.metaeffekt.core.util.FileUtils {
private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class);
public static final AntPathMatcher ANT_PATH_MATCHER = new AntPathMatcher();
public static final void deleteDir(File dir) {
if (dir.exists()) {
Delete delete = new Delete();
Project project = new Project();
project.setBaseDir(dir);
delete.setProject(project);
delete.setDir(dir);
delete.execute();
}
}
public static String[] scanDirectoryForFiles(File targetDir, String... includes) {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(targetDir);
scanner.setIncludes(includes);
scanner.setCaseSensitive(false);
scanner.scan();
return scanner.getIncludedFiles();
}
public static String[] scanDirectoryForFiles(File targetDir, String[] includes, String[] excludes) {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(targetDir);
scanner.setIncludes(includes);
scanner.setExcludes(excludes);
scanner.setCaseSensitive(false);
scanner.scan();
return scanner.getIncludedFiles();
}
public static String[] scanDirectoryForFiles(File targetDir, boolean addDefaultExcludes, String... includes) {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(targetDir);
scanner.setIncludes(includes);
scanner.setCaseSensitive(false);
if (addDefaultExcludes) {
scanner.addDefaultExcludes();
}
scanner.scan();
return scanner.getIncludedFiles();
}
public static String[] scanDirectoryForFolders(File targetDir, String... includes) {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(targetDir);
scanner.setIncludes(includes);
scanner.setCaseSensitive(false);
scanner.scan();
return scanner.getIncludedDirectories();
}
public static String computeChecksum(File file) {
Checksum checksum = new Checksum();
Project p = new Project();
checksum.setProject(p);
checksum.setFile(file);
String checksumVariable = "checksum";
checksum.setProperty(checksumVariable);
checksum.execute();
return p.getProperty(checksumVariable);
}
public static String computeSha1Checksum(File file) {
Checksum checksum = new Checksum();
Project p = new Project();
checksum.setProject(p);
checksum.setFile(file);
String checksumVariable = "checksum";
checksum.setProperty(checksumVariable);
checksum.setAlgorithm("SHA-1");
checksum.execute();
return p.getProperty(checksumVariable);
}
public static void validateExists(File file) {
if (!file.exists()) {
throw new IllegalStateException(String.format("File '%s' does not exist.", file.getAbsolutePath()));
}
}
public static boolean compareFolders(File sourceDir, File targetDir) {
validateExists(sourceDir);
validateExists(targetDir);
HashSet sourceFileSet = fileToSet(sourceDir);
HashSet targetFileSet = fileToSet(targetDir);
List messages = new ArrayList<>();
// compare from source to target
for (String file : sourceFileSet) {
if (!targetFileSet.contains(file)) {
messages.add(String.format("Target folder does not contain file %s", file));
}
}
// compare from target to source
for (String file : targetFileSet) {
if (!sourceFileSet.contains(file)) {
messages.add(String.format("Source folder does not contain file %s", file));
}
}
// compare file content by comparing checksums
for (String file : sourceFileSet) {
String sourceCS = computeChecksum(new File(sourceDir, file));
String targetCS = computeChecksum(new File(targetDir, file));
if (!sourceCS.equals(targetCS)) {
messages.add(String.format("Content of file %s differs.", file));
}
}
// print report (full diff)
for (String message : messages) {
LOG.info(message);
}
return !messages.isEmpty();
}
public static HashSet fileToSet(File dir) {
String[] files = scanDirectoryForFiles(dir, true, "**/*");
HashSet fileSet = new HashSet<>();
for (String file : files) {
fileSet.add(file);
}
return fileSet;
}
/**
* Uses the native zip command to zip the file. Uses the -X attribute to create files with deterministic checksum.
*
* @param sourceDir The directory to zip (recursively).
* @param targetZipFile The target zip file name.
* @throws IOException If the commands could not be executed.
*/
public static void zipNative(File sourceDir, File targetZipFile) throws IOException {
// FIXME: touch all files in source dir to normalize timestamp
String[] touchArray = {"find", ".", "-type", "f", "-exec", "touch", "-t", "202001010000", "{}", "+"};
Process touch = Runtime.getRuntime().exec(touchArray, new String[]{}, sourceDir);
// block until touch is completed
waitForProcess(touch);
// run native zip process
String[] zip = {"zip", "-X", "-r", targetZipFile.getAbsolutePath(), "."};
Process p = Runtime.getRuntime().exec(zip, new String[]{}, sourceDir);
// block until zip is completed
waitForProcess(p);
// check exit value and write error to System.err in case exit code is not 0
if (p.exitValue() != 0) {
StreamUtils.copy(p.getErrorStream(), System.err);
}
}
public static void waitForProcess(Process p) {
try {
while (p.isAlive()) {
p.waitFor(1000, TimeUnit.MILLISECONDS);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
IOUtils.copy(p.getInputStream(), baos);
} catch (IOException e) {
LOG.error("Unable to copy input stream into output stream.", e);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
/**
* Uses the native zip command to zip the file. Uses the -X attribute to create files with deterministic checksum.
*
* @param sourceDir The directory to zip (recursively).
* @param targetZipFile The target zip file name.
*/
public static void zipAnt(File sourceDir, File targetZipFile) {
Zip zip = new Zip();
Project project = new Project();
project.setBaseDir(sourceDir);
zip.setProject(project);
zip.setBasedir(sourceDir);
zip.setCompress(true);
zip.setDestFile(targetZipFile);
zip.setFollowSymlinks(false);
zip.execute();
}
public static void gzip(File sourceDir, File targetZipFile) throws IOException {
Tar zip = new Tar();
Project project = new Project();
project.setBaseDir(sourceDir);
zip.setBasedir(sourceDir);
zip.setProject(project);
zip.setDestFile(targetZipFile);
zip.setDescription("");
zip.execute();
}
public static boolean matches(String absolutePath, String[] patterns) {
for (String pattern : patterns) {
pattern = pattern.trim();
// Springs AntPatternMatcher requires a trailing "/" to match an absolute path
pattern = "/" + pattern;
if (ANT_PATH_MATCHER.match(pattern, absolutePath)) {
return true;
}
}
// none of the patterns matched
return false;
}
public static void createDirectoryContentChecksumFile(File baseDir, File targetContentChecksumFile) throws IOException {
final StringBuilder checksumSequence = new StringBuilder();
final String[] files = FileUtils.scanDirectoryForFiles(baseDir, new String[]{"**/*"}, new String[]{"**/.DS_Store*"});
Arrays.stream(files).map(fileName -> normalizePathToLinux(fileName)).sorted(String::compareTo).forEach(fileName -> {
final File file = new File(baseDir, fileName);
try {
final String fileChecksum = FileUtils.computeChecksum(file);
if (checksumSequence.length() > 0) {
checksumSequence.append(0);
}
checksumSequence.append(fileChecksum);
} catch (Exception e) {
if (FileUtils.isSymlink(file)) {
LOG.warn("Cannot compute checksum for symbolic link file [{}].", file);
} else {
LOG.warn("Cannot compute checksum for file [{}].", file);
}
}
});
FileUtils.writeStringToFile(targetContentChecksumFile, checksumSequence.toString(), FileUtils.ENCODING_UTF_8);
}
public static File findSingleFile(File baseFile, String pattern) {
final String[] files = scanDirectoryForFiles(baseFile, pattern);
if (files.length == 1) {
return new File(baseFile, files[0]);
}
return null;
}
public static File findSingleDir(File baseFile, String pattern) {
final String[] dirs = FileUtils.scanDirectoryForFolders(baseFile, pattern);
if (dirs.length == 1) {
return new File(baseFile, dirs[0]);
}
return null;
}
public static String computeContentChecksum(File file, File tmpFolder) throws IOException {
final File contentChecksumFile = new File(tmpFolder, file.getName() + ".content.md5");
createDirectoryContentChecksumFile(file, contentChecksumFile);
try {
return FileUtils.computeChecksum(contentChecksumFile);
} finally {
deleteQuietly(contentChecksumFile);
}
}
public static void forceMkDirQuietly(File file) {
try {
FileUtils.forceMkdir(file);
} catch (IOException exception) {
throw new IllegalStateException("Cannot create folder [" + file + "].");
}
}
public static boolean exists(String filePath) {
if (filePath == null) return false;
return new File(filePath).exists();
}
public static String detectEncoding(File file) throws IOException {
String encoding = UniversalDetector.detectCharset(file);
if (encoding == null) {
// fallback to UTF-8
encoding = FileUtils.ENCODING_UTF_8;
}
return encoding;
}
} © 2015 - 2025 Weber Informatics LLC | Privacy Policy