com.metaeffekt.artifact.analysis.utils.ArchiveUtils Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.utils;
import com.github.katjahahn.parser.FileFormatException;
import com.github.katjahahn.parser.PEData;
import com.github.katjahahn.parser.PELoader;
import com.github.katjahahn.tools.PEFileDumper;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.tools.ant.Project;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class ArchiveUtils extends org.metaeffekt.core.util.ArchiveUtils {
private static final Logger LOG = LoggerFactory.getLogger(ArchiveUtils.class);
private static final Set peExtensions = new HashSet<>();
static {
// FIXME: how can we extend the definition in configuration from a consumer perspective?
org.metaeffekt.core.util.ArchiveUtils.registerZipExtension("mbizip");
// erlang support; see https://www.erlang.org/doc/man/code.html
org.metaeffekt.core.util.ArchiveUtils.registerZipExtension("ez");
peExtensions.add("dll");
peExtensions.add("exe");
peExtensions.add("cab");
peExtensions.add("msi");
peExtensions.add("pyd");
}
/**
* Recursively unpacks all archives in the given directory.
*
* @param baseDir The baseDir to start in.
* @param issues The list of issues to add to.
* @throws IOException If the file could not be unpacked.
*/
public static void recursiveUnpack(File baseDir, List issues) throws IOException {
final String[] files = FileUtils.scanDirectoryForFiles(baseDir, "**/*.tar.gz", "**/*.tar.bz2", "**/*.tgz",
"**/*.tar.xz", "**/*.tar", "**/*.deb", "**/*.rpm", "**/*.apk", "**/*.zip", "**/*.war", "**/*.jar",
"**/*.mbizip", "**/*.nupkg", "**/*.nupack", "**/*.aar", "**/*.dll", "**/*.pyd", "**/*.exe", "**/*.jmod", "**/modules, **/*.ez");
for (String file : files) {
File archiveFile = new File(baseDir, file);
File targetPath = new File(archiveFile.getParentFile(), "[" + archiveFile.getName() + "]");
if (!targetPath.exists()) {
boolean created = targetPath.mkdirs();
if (unpackIfPossible(archiveFile, targetPath, new ArrayList<>())) {
// delete the file after successful expansion
FileUtils.forceDelete(archiveFile);
// recurse into just unpacked folder
recursiveUnpack(targetPath, issues);
} else {
FileUtils.deleteDir(targetPath);
issues.add("Cannot unpack " + archiveFile);
}
}
}
}
public static boolean unpackIfPossible(File archiveFile, File targetDir, List issues) {
final Project project = new Project();
project.setBaseDir(archiveFile.getParentFile());
String extension = FilenameUtils.getExtension(archiveFile.getName()).toLowerCase();
if (extension.isEmpty()) {
extension = archiveFile.getName().toLowerCase();
}
// FIXME: perhaps xz is the better option here
// try pe unwrap
try {
// this code should be moved to the scan report adapter; we may face embedded dlls
if (peExtensions.contains(extension)) {
final PEData peData = PELoader.loadPE(archiveFile);
final PEFileDumper peFileDumper = new PEFileDumper(peData, targetDir);
targetDir.mkdirs();
doAndCheckForKnownIssues(() -> peFileDumper.dumpSections(), archiveFile);
doAndCheckForKnownIssues(() -> peFileDumper.dumpIcons(), archiveFile);
doAndCheckForKnownIssues(() -> peFileDumper.dumpOverlay(), archiveFile);
doAndCheckForKnownIssues(() -> peFileDumper.dumpResources(), archiveFile);
return true;
}
} catch (Exception ex) {
throw new IllegalStateException("EM2: " + ex.getMessage(), ex);
}
return org.metaeffekt.core.util.ArchiveUtils.unpackIfPossible(archiveFile, targetDir, issues);
}
private static void doAndCheckForKnownIssues(Runnable runnable, File archiveFile) {
try {
runnable.run();
} catch (Exception e) {
if (e.getClass().isAssignableFrom(FileNotFoundException.class)) {
// NOTE: this is a known issue in the PEDumper; we need to accept this as not severe issue;
LOG.warn("Cannot fully unwrap [{}]. Issue acceptable: {}", archiveFile.getPath(), e.getMessage());
return;
} else if (e.getClass().isAssignableFrom(FileFormatException.class)) {
LOG.warn("Cannot fully unwrap [{}]. Issue acceptable: {}", archiveFile.getPath(), e.getMessage());
return;
}
throw e;
}
}
public static void unpackArchiveOrCopyFile(File archiveFile, File unpackTargetDir) {
if (!archiveFile.exists()) return;
if (unpackTargetDir == null) return;
final File marker = new File(unpackTargetDir.getParentFile(), unpackTargetDir.getName() + ".lock");
if (marker.exists()) {
LOG.warn("Detected .lock file. Previous unpack process incomplete or not successful for [{}].", unpackTargetDir.getPath());
// when the marker exists, the previous unpack attempt was not successful and may have produced incomplete
// folder content. In this case, we delete the complete folder to reattempt unpacking
FileUtils.deleteDir(unpackTargetDir);
}
// if the folder exists and showed no extraction issue we may want to skip all further steps
if (unpackTargetDir.exists()) {
// check also content; an empty folder may indicate a previous issue (e.g. no space issue)
final File[] array = unpackTargetDir.listFiles();
if (array != null && array.length > 0) {
// skip all if the folder looks healthy unpacked
return;
}
}
// generate temporary marker file
try {
FileUtils.touch(marker);
} catch (IOException e) {
LOG.debug("Cannot touch/create marker file: " + marker);
}
// create directory structure
if (!unpackTargetDir.exists()) {
unpackTargetDir.mkdirs();
}
boolean failure = false;
LOG.debug(String.format("Unpacking artifact %s.", archiveFile.getName()));
boolean unpacked = ArchiveUtils.unpackIfPossible(archiveFile, unpackTargetDir, new ArrayList<>());
if (!unpacked) {
final File destFile = new File(unpackTargetDir, archiveFile.getName());
try {
FileUtils.copyFile(archiveFile, destFile);
} catch (IOException e) {
LOG.warn(String.format("Cannot copy [%s]. %s", archiveFile, e.getMessage()), e);
failure = true;
}
}
try {
ArchiveUtils.recursiveUnpack(unpackTargetDir, new ArrayList<>());
} catch (Exception e) {
LOG.warn("Exception during recursive unpack: {}", e.getMessage(), e);
failure = true;
}
// in case a failure was detected, we DO NOT delete the marker file; the next run will hopefully correct the
// situation; the folder structure is however preserved to support the analysis.
if (!failure) {
FileUtils.deleteQuietly(marker);
}
}
public static void tarDirectory(final File directory, final File targetTar) throws IOException {
if (targetTar.exists()) {
FileUtils.forceDelete(targetTar);
}
if (!targetTar.getParentFile().exists()) {
targetTar.getParentFile().mkdirs();
}
try (final TarOutputStream outputStream = new TarOutputStream(new BufferedOutputStream(Files.newOutputStream(targetTar.toPath())))) {
tarDirectory(directory, outputStream, "");
}
}
private static void tarDirectory(final File directory, final TarOutputStream outputStream, final String parent) throws IOException {
final File[] files = directory.listFiles();
if (files != null) {
for (final File file : files) {
final String entryName = parent + file.getName();
if (file.isDirectory()) {
tarDirectory(file, outputStream, entryName + "/");
} else {
addFileToTar(outputStream, file, entryName);
}
}
}
}
private static void addFileToTar(final TarOutputStream outputStream, final File file, final String entryName) throws IOException {
outputStream.putNextEntry(new TarEntry(file, entryName));
try (FileInputStream inputStream = new FileInputStream(file)) {
IOUtils.copy(inputStream, outputStream);
}
outputStream.closeEntry();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy