Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.whitesource.utils.files.FilesScanner Maven / Gradle / Ivy
/**
* The project is licensed under the WhiteSource Software End User License Agreement .
* Copyright (C) 2015 WhiteSource Ltd.
* Licensed under the WhiteSource Software End User License Agreement;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://s3.amazonaws.com/unified-agent/LICENSE.txt
*/
package org.whitesource.utils.files;
import org.apache.tools.ant.DirectoryScanner;
import org.slf4j.Logger;
import org.whitesource.utils.Constants;
import org.whitesource.utils.logger.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author eugen.horovitz
*/
public class FilesScanner {
private FileSystem FS = FileSystems.getDefault();
/* --- Static members --- */
private Logger logger = LoggerFactory.getLogger(FilesScanner.class);
/* --- Public methods --- */
public Map> findTopFoldersFileWalker(Collection pathsToScan, Map> dependencyExcludesMap, Map dependencyIncludesMap) {
Map> dependencyNamePathIncludedFilesMap = new HashMap<>();
pathsToScan.stream().forEach(scanFolder -> {
Map> includedFiles = getDirectoryBomFiles(new File(scanFolder).getPath(), dependencyExcludesMap, dependencyIncludesMap);
includedFiles.forEach((dependencyName, bomFile) -> {
// get top folders with boms (the parent of each project)
Collection resolvedFolders = new ArrayList<>();
Map> topFolders = getTopFoldersWithIncludedFiles(Constants.EMPTY_STRING, bomFile.toArray(new String[0]), true);
resolvedFolders.add(new ResolvedFolder(new File(scanFolder).getAbsolutePath(), topFolders));
dependencyNamePathIncludedFilesMap.putIfAbsent(dependencyName, new ArrayList<>());
dependencyNamePathIncludedFilesMap.get(dependencyName).addAll(resolvedFolders);
});
});
return dependencyNamePathIncludedFilesMap;
}
public void createIncludesExcludesMaps(Map> dependencyIncludePatterns, Map> dependencyExcludePatterns, Map dependencyIncludesMap, Map> dependencyExcludeMap) {
dependencyIncludesMap.forEach((dependencyName, bomPatterns) -> {
Set listOfIncludePatterns = new HashSet<>();
for (String include : bomPatterns) {
Glob glob = new AntGlob(include);
PathMatcher matcher = null;
try {
// logger.debug("include ant pattern is : " + include + ", regex pattern is: " + glob.pattern());
matcher = FS.getPathMatcher(Constants.REGEX + glob.pattern());
} catch (Exception e) {
logger.debug("include: " + include + "failed to be converted to regex, pattern will not be searched");
}
if (matcher != null) {
listOfIncludePatterns.add(matcher);
}
}
dependencyIncludePatterns.put(dependencyName, listOfIncludePatterns);
//handle of exludes map
Set listOfExcludePatterns = new HashSet<>();
for (String exclude : dependencyExcludeMap.get(dependencyName)) {
Glob glob = new AntGlob(exclude);
PathMatcher matcher = null;
try {
matcher = FS.getPathMatcher(Constants.REGEX + glob.pattern());
} catch (Exception e) {
logger.debug("exclude: " + exclude + "failed to be converted to regex, pattern will not be searched");
}
if (matcher != null) {
listOfExcludePatterns.add(matcher);
}
}
dependencyExcludePatterns.put(dependencyName, listOfExcludePatterns);
});
}
public void createIncludeExcludePatterns(Set includePatterns, Set excludePatterns, String[] includes, String[] excludes, String patternType, boolean globCaseSensitive) {
for (String include : includes) {
PathMatcher matcher = null;
if (patternType.equals(Constants.ANT)) {
Glob glob = globCaseSensitive ? new AntGlob(include) : new AntGlob(include.toLowerCase());
try {
matcher = FS.getPathMatcher(Constants.REGEX + glob.pattern());
} catch (Exception e) {
logger.debug("include: " + include + "failed to be converted to regex, pattern failed to be parsed and will not be searched");
}
}
if (matcher != null) {
includePatterns.add(matcher);
}
}
//handle of exludes map
for (String exclude : excludes) {
PathMatcher matcher = null;
if (patternType.equals(Constants.ANT)) {
Glob glob = globCaseSensitive ? new AntGlob(exclude) : new AntGlob(exclude.toLowerCase());
try {
matcher = FS.getPathMatcher(Constants.REGEX + glob.pattern());
} catch (Exception e) {
logger.debug("exclude: " + exclude + "failed to be converted to regex, pattern failed to be parsed and will not be searched");
}
}
if (matcher != null) {
excludePatterns.add(matcher);
}
}
}
public Map> getDirectoryBomFiles(String path, Map> dependencyExcludeMap, Map dependencyIncludesMap) {
Path mainPath = Paths.get(path);
Map> dependencyIncludePatterns = new HashMap<>();
Map> dependencyExcludePatterns = new HashMap<>();
Map> dependencyNamePerfolderMap = new HashMap<>();
//handle includes and exludes maps
createIncludesExcludesMaps(dependencyIncludePatterns, dependencyExcludePatterns, dependencyIncludesMap, dependencyExcludeMap);
FileVisitor visitor = new SimpleFileVisitor() {
@Override
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
if (filePath.getFileName() == null) {
return FileVisitResult.CONTINUE;
}
Path compareFilePath = mainPath.relativize(filePath);
dependencyIncludePatterns.forEach((dependencyName, listOfBomPatterns) -> {
for (PathMatcher matcher : listOfBomPatterns) {
if (matcher.matches(compareFilePath)) {
if (checkIfExcluded(dependencyName, compareFilePath)) {
return;
}
String absolutePath = filePath.toFile().getAbsolutePath();
logger.debug("Dependency file found location: {}", absolutePath);
dependencyNamePerfolderMap.putIfAbsent(dependencyName, new HashSet<>());
dependencyNamePerfolderMap.get(dependencyName).add(absolutePath);
}
}
});
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
logger.debug("file could not be visited {}, reason: {}", file.toFile().getAbsolutePath(), exc.getMessage());
return FileVisitResult.CONTINUE;
}
private boolean checkIfExcluded(String dependencyName, Path compareFilePath) {
for (PathMatcher excludesMatcher : dependencyExcludePatterns.get(dependencyName)) {
if (excludesMatcher.matches(compareFilePath)) {
return true;
}
}
return false;
}
};
try {
Files.walkFileTree(Paths.get(path), visitor);
} catch (IOException e) {
// we already ignore errors in the visitor
}
return dependencyNamePerfolderMap;
}
public String[] getDirectoryContentWithAbsolutePath(String scannerBaseDir, String[] includes, String[] excludes, boolean followSymlinks, boolean globCaseSensitive) {
String[] directoryContent = getDirectoryContent(scannerBaseDir, includes, excludes, followSymlinks, globCaseSensitive, false);
for (int i = 0; i < directoryContent.length; i++) {
directoryContent[i] = scannerBaseDir + Constants.SYS_FILE_SEPARATOR + directoryContent[i];
}
return directoryContent;
}
public String[] getDirectoryContent(String scannerBaseDir, String[] includes, String[] excludes, boolean followSymlinks, boolean globCaseSensitive) {
return getDirectoryContent(scannerBaseDir, includes, excludes, followSymlinks, globCaseSensitive, false);
}
public String[] getDirectoryContentByFileWalker(String scannerBaseDir, String[] includes, String[] excludes, boolean followSymlinks, boolean scanDirectories, boolean globCaseSensitive) {
Path mainPath = Paths.get(scannerBaseDir);
Set includePatterns = new HashSet<>();
Set excludePatterns = new HashSet<>();
Set fileNamesList = new HashSet<>();
//NOTE this is an additional constant just for future use, maybe we will support other patterns later on with this flag.
String patternType = Constants.ANT;
createIncludeExcludePatterns(includePatterns, excludePatterns, includes, excludes, patternType.toUpperCase(), globCaseSensitive);
FileVisitor visitor = new SimpleFileVisitor() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
if (scanDirectories) {
fileNamesList.add(mainPath.relativize(dir).toString());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
if (!scanDirectories) {
if (filePath.getFileName() == null) {
return FileVisitResult.CONTINUE;
}
String pathToAdd = mainPath.relativize(filePath).toFile().getPath();
Path compareFilePath = globCaseSensitive ? Paths.get(pathToAdd) : Paths.get(pathToAdd.toLowerCase());
for (PathMatcher matcher : includePatterns) {
if (matcher.matches(compareFilePath)) {
if (checkIfExcluded(compareFilePath)) {
continue;
}
logger.debug("Dependency file found location: {}", mainPath.relativize(filePath).toFile().getAbsolutePath());
fileNamesList.add(pathToAdd);
}
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
logger.debug("file could not be visited {}, reason: {}", file.toFile().getAbsolutePath(), exc.getMessage());
return FileVisitResult.CONTINUE;
}
private boolean checkIfExcluded(Path compareFilePath) {
for (PathMatcher excludesMatcher : excludePatterns) {
if (excludesMatcher.matches(compareFilePath)) {
return true;
}
}
return false;
}
};
try {
if (scanDirectories) {
// take only max depth of 2 which means only subdirectories at high level.
Files.walkFileTree(Paths.get(scannerBaseDir), EnumSet.noneOf(FileVisitOption.class), 2, visitor);
} else {
if (followSymlinks) {
Files.walkFileTree(Paths.get(scannerBaseDir), EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor);
} else {
Files.walkFileTree(Paths.get(scannerBaseDir), visitor);
}
}
} catch (IOException e) {
// we already ignore errors in the visitor
}
if(scanDirectories) {
fileNamesList.remove(Constants.EMPTY_STRING);
}
return fileNamesList.toArray(new String[0]);
}
public static boolean isWindows() {
return System.getProperty(Constants.OS_NAME).toLowerCase().contains(Constants.WIN);
}
// get the content of directory by includes, excludes, followSymlinks and globCaseSensitive, the scanDirectories property define if the scanner will scan to find directories
public String[] getDirectoryContent(String scannerBaseDir, String[] includes, String[] excludes, boolean followSymlinks, boolean globCaseSensitive, boolean scanSubDirectories) {
//FileVisitor is way faster than DirectoryScanner, but it doesn't support glob.case.sensitive flag in windows, so we are keeping the directory scanner functionality in case flag is true and its windows.
if (isWindows() && globCaseSensitive) {
File file = new File(scannerBaseDir);
String[] fileNames;
if (file.exists() && file.isDirectory()) {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(scannerBaseDir);
scanner.setIncludes(includes);
scanner.setExcludes(excludes);
scanner.setFollowSymlinks(followSymlinks);
scanner.setCaseSensitive(globCaseSensitive);
scanner.scan();
if (!scanSubDirectories) {
fileNames = scanner.getIncludedFiles();
} else {
fileNames = scanner.getIncludedDirectories();
}
return fileNames;
} else {
logger.debug("{} is not a folder", scannerBaseDir);
return new String[0];
}
} else {
return getDirectoryContentByFileWalker(scannerBaseDir, includes, excludes, followSymlinks, scanSubDirectories, globCaseSensitive);
}
}
public Collection findTopFolders(Collection pathsToScan, String[] includesPattern, Collection excludes) {
Collection resolvedFolders = new ArrayList<>();
// get folders containing bom files
Map pathToBomFilesMap = findAllFiles(pathsToScan, includesPattern, excludes);
// resolve dependencies
pathToBomFilesMap.forEach((folder, bomFile) -> {
// get top folders with boms (the parent of each project)
Map> topFolders = getTopFoldersWithIncludedFiles(folder, bomFile, false);
resolvedFolders.add(new ResolvedFolder(folder, topFolders));
});
return resolvedFolders;
}
/* --- Private methods --- */
private Map findAllFiles(Collection pathsToScan, String[] includesPattern, Collection excludes) {
Map pathToIncludedFilesMap = new HashMap<>();
pathsToScan.forEach(scanFolder -> {
String[] includedFiles = getDirectoryContent(new File(scanFolder).getPath(), includesPattern,
excludes.toArray(new String[excludes.size()]), false, false);
pathToIncludedFilesMap.put(new File(scanFolder).getAbsolutePath(), includedFiles);
});
return pathToIncludedFilesMap;
}
private Map> getTopFoldersWithIncludedFiles(String rootFolder, String[] includedFiles, boolean isFileWalker) {
// collect all full paths
List fullPaths;
if (isFileWalker) {
fullPaths = Arrays.asList(includedFiles);
} else {
// else it's DirectoryScanner
fullPaths = Arrays.stream(includedFiles)
.map(file -> Paths.get(new File(rootFolder).getAbsolutePath(), file).toString())
.collect(Collectors.toList());
}
// get top folders
Map> foldersGroupedByLengthMap = fullPaths.stream()
.collect(Collectors.groupingBy(filename -> new File(filename).getParentFile().getParent()));
// create result map with only the top folder and the corresponding bom files
Map> resultMap = new HashMap<>();
logger.debug("found folders:" + System.lineSeparator());
foldersGroupedByLengthMap.keySet().forEach(folder -> logger.debug(folder));
logger.debug(System.lineSeparator());
while (foldersGroupedByLengthMap.entrySet().size() > 0) {
String shortestFolder = foldersGroupedByLengthMap.keySet().stream().min(Comparator.comparingInt(String::length)).get();
List foundShortestFolder = foldersGroupedByLengthMap.get(shortestFolder);
foldersGroupedByLengthMap.remove(shortestFolder);
List topFolders = foundShortestFolder.stream()
.map(file -> new File(file).getParent()).collect(Collectors.toList());
topFolders.forEach(folder -> {
resultMap.put(folder, fullPaths.stream().filter(fileName -> fileName.contains(folder)).collect(Collectors.toSet()));
// remove from list folders that are children of the one found so they will not be calculated twice
foldersGroupedByLengthMap.entrySet().removeIf(otherFolder -> {
Path otherFolderPath = Paths.get(otherFolder.getKey());
Path folderPath = Paths.get(folder);
boolean shouldRemove = false;
try {
shouldRemove = otherFolderPath.toFile().getCanonicalPath().startsWith(folderPath.toFile().getCanonicalPath());
} catch (Exception e) {
logger.debug("could not get file path " + otherFolderPath + folderPath, e.getStackTrace());
logger.warn("could not get file path " + otherFolderPath + folderPath, e.getMessage());
}
logger.debug(String.join(";", otherFolder.getKey(), folder, Boolean.toString(shouldRemove)));
if (shouldRemove) {
logger.debug("---> removed: " + otherFolder.getKey());
return true;
}
return false;
});
});
}
logger.debug(System.lineSeparator());
return resultMap;
}
public boolean isIncluded(File file, String[] includes, String[] excludes, boolean followSymlinks, boolean globCaseSensitive) {
SingleFileScanner scanner = new SingleFileScanner();
scanner.setIncludes(includes);
scanner.setExcludes(excludes);
scanner.setFollowSymlinks(followSymlinks);
scanner.setCaseSensitive(globCaseSensitive);
return scanner.isIncluded(file);
}
}
