org.owasp.dependencycheck.Engine Maven / Gradle / Ivy
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.jcs3.JCS;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.owasp.dependencycheck.analyzer.AnalysisPhase;
import org.owasp.dependencycheck.analyzer.Analyzer;
import org.owasp.dependencycheck.analyzer.AnalyzerService;
import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer;
import org.owasp.dependencycheck.data.nvdcve.DatabaseManager;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.CachedWebDataSource;
import org.owasp.dependencycheck.data.update.UpdateService;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.exception.ExceptionCollection;
import org.owasp.dependencycheck.exception.InitializationException;
import org.owasp.dependencycheck.exception.NoDataException;
import org.owasp.dependencycheck.exception.ReportException;
import org.owasp.dependencycheck.exception.WriteLockException;
import org.owasp.dependencycheck.reporting.ReportGenerator;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.WriteLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.NotThreadSafe;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.FINAL;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.FINDING_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.FINDING_ANALYSIS_PHASE2;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.IDENTIFIER_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.INFORMATION_COLLECTION;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.INFORMATION_COLLECTION2;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.INITIAL;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.POST_FINDING_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.POST_IDENTIFIER_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.POST_INFORMATION_COLLECTION1;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.POST_INFORMATION_COLLECTION2;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.POST_INFORMATION_COLLECTION3;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.PRE_FINDING_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.PRE_IDENTIFIER_ANALYSIS;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.PRE_INFORMATION_COLLECTION;
import org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer;
import org.owasp.dependencycheck.dependency.naming.Identifier;
import org.owasp.dependencycheck.utils.Utils;
/**
* Scans files, directories, etc. for Dependencies. Analyzers are loaded and
* used to process the files found by the scan, if a file is encountered and an
* Analyzer is associated with the file type then the file is turned into a
* dependency.
*
* @author Jeremy Long
*/
@NotThreadSafe
public class Engine implements FileFilter, AutoCloseable {
/**
* The Logger for use throughout the class.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class);
/**
* The list of dependencies.
*/
private final List dependencies = Collections.synchronizedList(new ArrayList<>());
/**
* A Map of analyzers grouped by Analysis phase.
*/
private final Map> analyzers = new EnumMap<>(AnalysisPhase.class);
/**
* A Map of analyzers grouped by Analysis phase.
*/
private final Set fileTypeAnalyzers = new HashSet<>();
/**
* The engine execution mode indicating it will either collect evidence or
* process evidence or both.
*/
private final Mode mode;
/**
* The ClassLoader to use when dynamically loading Analyzer and Update
* services.
*/
private final ClassLoader serviceClassLoader;
/**
* The configured settings.
*/
private final Settings settings;
/**
* A storage location to persist objects throughout the execution of ODC.
*/
private final Map objects = new HashMap<>();
/**
* The external view of the dependency list.
*/
private Dependency[] dependenciesExternalView = null;
/**
* A reference to the database.
*/
private CveDB database = null;
/**
* Used to store the value of
* System.getProperty("javax.xml.accessExternalSchema") - ODC may change the
* value of this system property at runtime. We store the value to reset the
* property to its original value.
*/
private final String accessExternalSchema;
/**
* Creates a new {@link Mode#STANDALONE} Engine.
*
* @param settings reference to the configured settings
*/
public Engine(@NotNull final Settings settings) {
this(Mode.STANDALONE, settings);
}
/**
* Creates a new Engine.
*
* @param mode the mode of operation
* @param settings reference to the configured settings
*/
public Engine(@NotNull final Mode mode, @NotNull final Settings settings) {
this(Thread.currentThread().getContextClassLoader(), mode, settings);
}
/**
* Creates a new {@link Mode#STANDALONE} Engine.
*
* @param serviceClassLoader a reference the class loader being used
* @param settings reference to the configured settings
*/
public Engine(@NotNull final ClassLoader serviceClassLoader, @NotNull final Settings settings) {
this(serviceClassLoader, Mode.STANDALONE, settings);
}
/**
* Creates a new Engine.
*
* @param serviceClassLoader a reference the class loader being used
* @param mode the mode of the engine
* @param settings reference to the configured settings
*/
public Engine(@NotNull final ClassLoader serviceClassLoader, @NotNull final Mode mode, @NotNull final Settings settings) {
this.settings = settings;
this.serviceClassLoader = serviceClassLoader;
this.mode = mode;
this.accessExternalSchema = System.getProperty("javax.xml.accessExternalSchema");
checkRuntimeVersion();
initializeEngine();
}
/**
* Creates a new Engine using the specified classloader to dynamically load
* Analyzer and Update services.
*
* @throws DatabaseException thrown if there is an error connecting to the
* database
*/
protected final void initializeEngine() {
loadAnalyzers();
}
/**
* Properly cleans up resources allocated during analysis.
*/
@Override
public void close() {
if (mode.isDatabaseRequired()) {
if (database != null) {
database.close();
database = null;
}
}
if (accessExternalSchema != null) {
System.setProperty("javax.xml.accessExternalSchema", accessExternalSchema);
} else {
System.clearProperty("javax.xml.accessExternalSchema");
}
JCS.shutdown();
}
/**
* Loads the analyzers specified in the configuration file (or system
* properties).
*/
private void loadAnalyzers() {
if (!analyzers.isEmpty()) {
return;
}
mode.getPhases().forEach((phase) -> analyzers.put(phase, new ArrayList<>()));
final AnalyzerService service = new AnalyzerService(serviceClassLoader, settings);
final List iterator = service.getAnalyzers(mode.getPhases());
iterator.forEach((a) -> {
a.initialize(this.settings);
analyzers.get(a.getAnalysisPhase()).add(a);
if (a instanceof FileTypeAnalyzer) {
this.fileTypeAnalyzers.add((FileTypeAnalyzer) a);
}
});
}
/**
* Get the List of the analyzers for a specific phase of analysis.
*
* @param phase the phase to get the configured analyzers.
* @return the analyzers loaded
*/
public List getAnalyzers(AnalysisPhase phase) {
return analyzers.get(phase);
}
/**
* Adds a dependency. In some cases, when adding a virtual dependency, the
* method will identify if the virtual dependency was previously added and
* update the existing dependency rather then adding a duplicate.
*
* @param dependency the dependency to add
*/
public synchronized void addDependency(Dependency dependency) {
if (dependency.isVirtual()) {
for (Dependency existing : dependencies) {
if (existing.isVirtual()
&& existing.getSha256sum() != null
&& existing.getSha256sum().equals(dependency.getSha256sum())
&& existing.getDisplayFileName() != null
&& existing.getDisplayFileName().equals(dependency.getDisplayFileName())
&& identifiersMatch(existing.getSoftwareIdentifiers(), dependency.getSoftwareIdentifiers())) {
DependencyBundlingAnalyzer.mergeDependencies(existing, dependency, null);
return;
}
}
}
dependencies.add(dependency);
dependenciesExternalView = null;
}
/**
* Sorts the dependency list.
*/
public synchronized void sortDependencies() {
//TODO - is this actually necassary????
// Collections.sort(dependencies);
// dependenciesExternalView = null;
}
/**
* Removes the dependency.
*
* @param dependency the dependency to remove.
*/
public synchronized void removeDependency(@NotNull final Dependency dependency) {
dependencies.remove(dependency);
dependenciesExternalView = null;
}
/**
* Returns a copy of the dependencies as an array.
*
* @return the dependencies identified
*/
@SuppressFBWarnings(justification = "This is the intended external view of the dependencies", value = {"EI_EXPOSE_REP"})
public synchronized Dependency[] getDependencies() {
if (dependenciesExternalView == null) {
dependenciesExternalView = dependencies.toArray(new Dependency[0]);
}
return dependenciesExternalView;
}
/**
* Sets the dependencies.
*
* @param dependencies the dependencies
*/
public synchronized void setDependencies(@NotNull final List dependencies) {
this.dependencies.clear();
this.dependencies.addAll(dependencies);
dependenciesExternalView = null;
}
/**
* Scans an array of files or directories. If a directory is specified, it
* will be scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param paths an array of paths to files or directories to be analyzed
* @return the list of dependencies scanned
* @since v0.3.2.5
*/
public List scan(@NotNull final String[] paths) {
return scan(paths, null);
}
/**
* Scans an array of files or directories. If a directory is specified, it
* will be scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param paths an array of paths to files or directories to be analyzed
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of dependencies scanned
* @since v1.4.4
*/
public List scan(@NotNull final String[] paths, @Nullable final String projectReference) {
final List deps = new ArrayList<>();
for (String path : paths) {
final List d = scan(path, projectReference);
if (d != null) {
deps.addAll(d);
}
}
return deps;
}
/**
* Scans a given file or directory. If a directory is specified, it will be
* scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param path the path to a file or directory to be analyzed
* @return the list of dependencies scanned
*/
public List scan(@NotNull final String path) {
return scan(path, null);
}
/**
* Scans a given file or directory. If a directory is specified, it will be
* scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param path the path to a file or directory to be analyzed
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of dependencies scanned
* @since v1.4.4
*/
public List scan(@NotNull final String path, String projectReference) {
final File file = new File(path);
return scan(file, projectReference);
}
/**
* Scans an array of files or directories. If a directory is specified, it
* will be scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param files an array of paths to files or directories to be analyzed.
* @return the list of dependencies
* @since v0.3.2.5
*/
public List scan(File[] files) {
return scan(files, null);
}
/**
* Scans an array of files or directories. If a directory is specified, it
* will be scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param files an array of paths to files or directories to be analyzed.
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of dependencies
* @since v1.4.4
*/
public List scan(File[] files, String projectReference) {
final List deps = new ArrayList<>();
for (File file : files) {
final List d = scan(file, projectReference);
if (d != null) {
deps.addAll(d);
}
}
return deps;
}
/**
* Scans a collection of files or directories. If a directory is specified,
* it will be scanned recursively. Any dependencies identified are added to
* the dependency collection.
*
* @param files a set of paths to files or directories to be analyzed
* @return the list of dependencies scanned
* @since v0.3.2.5
*/
public List scan(Collection files) {
return scan(files, null);
}
/**
* Scans a collection of files or directories. If a directory is specified,
* it will be scanned recursively. Any dependencies identified are added to
* the dependency collection.
*
* @param files a set of paths to files or directories to be analyzed
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of dependencies scanned
* @since v1.4.4
*/
public List scan(Collection files, String projectReference) {
final List deps = new ArrayList<>();
files.stream().map((file) -> scan(file, projectReference))
.filter(Objects::nonNull)
.forEach(deps::addAll);
return deps;
}
/**
* Scans a given file or directory. If a directory is specified, it will be
* scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param file the path to a file or directory to be analyzed
* @return the list of dependencies scanned
* @since v0.3.2.4
*/
public List scan(File file) {
return scan(file, null);
}
/**
* Scans a given file or directory. If a directory is specified, it will be
* scanned recursively. Any dependencies identified are added to the
* dependency collection.
*
* @param file the path to a file or directory to be analyzed
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of dependencies scanned
* @since v1.4.4
*/
@Nullable
public List scan(@NotNull final File file, String projectReference) {
if (file.exists()) {
if (file.isDirectory()) {
return scanDirectory(file, projectReference);
} else {
final Dependency d = scanFile(file, projectReference);
if (d != null) {
final List deps = new ArrayList<>();
deps.add(d);
return deps;
}
}
}
return null;
}
/**
* Recursively scans files and directories. Any dependencies identified are
* added to the dependency collection.
*
* @param dir the directory to scan
* @return the list of Dependency objects scanned
*/
protected List scanDirectory(File dir) {
return scanDirectory(dir, null);
}
/**
* Recursively scans files and directories. Any dependencies identified are
* added to the dependency collection.
*
* @param dir the directory to scan
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the list of Dependency objects scanned
* @since v1.4.4
*/
protected List scanDirectory(@NotNull final File dir, @Nullable final String projectReference) {
final File[] files = dir.listFiles();
final List deps = new ArrayList<>();
if (files != null) {
for (File f : files) {
if (f.isDirectory()) {
final List d = scanDirectory(f, projectReference);
if (d != null) {
deps.addAll(d);
}
} else {
final Dependency d = scanFile(f, projectReference);
if (d != null) {
deps.add(d);
}
}
}
}
return deps;
}
/**
* Scans a specified file. If a dependency is identified it is added to the
* dependency collection.
*
* @param file The file to scan
* @return the scanned dependency
*/
protected Dependency scanFile(@NotNull final File file) {
return scanFile(file, null);
}
//CSOFF: NestedIfDepth
/**
* Scans a specified file. If a dependency is identified it is added to the
* dependency collection.
*
* @param file The file to scan
* @param projectReference the name of the project or scope in which the
* dependency was identified
* @return the scanned dependency
* @since v1.4.4
*/
protected synchronized Dependency scanFile(@NotNull final File file, @Nullable final String projectReference) {
Dependency dependency = null;
if (file.isFile()) {
if (accept(file)) {
dependency = new Dependency(file);
if (projectReference != null) {
dependency.addProjectReference(projectReference);
}
final String sha1 = dependency.getSha1sum();
boolean found = false;
if (sha1 != null) {
for (Dependency existing : dependencies) {
if (sha1.equals(existing.getSha1sum())) {
if (existing.getDisplayFileName().contains(": ")
|| dependency.getDisplayFileName().contains(": ")
|| dependency.getActualFilePath().contains("dctemp")) {
continue;
}
found = true;
if (projectReference != null) {
existing.addProjectReference(projectReference);
}
if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
&& !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
if (DependencyBundlingAnalyzer.firstPathIsShortest(existing.getFilePath(), dependency.getFilePath())) {
DependencyBundlingAnalyzer.mergeDependencies(existing, dependency, null);
//return null;
return existing;
} else {
//Merging dependency<-existing could be complicated. Instead analyze them seperately
//and possibly merge them at the end.
found = false;
}
} else { //somehow we scanned the same file twice?
//return null;
return existing;
}
break;
}
}
}
if (!found) {
dependencies.add(dependency);
dependenciesExternalView = null;
}
}
} else {
LOGGER.debug("Path passed to scanFile(File) is not a file that can be scanned by dependency-check: {}. Skipping the file.", file);
}
return dependency;
}
//CSON: NestedIfDepth
/**
* Runs the analyzers against all of the dependencies. Since the mutable
* dependencies list is exposed via {@link #getDependencies()}, this method
* iterates over a copy of the dependencies list. Thus, the potential for
* {@link java.util.ConcurrentModificationException}s is avoided, and
* analyzers may safely add or remove entries from the dependencies list.
*
* Every effort is made to complete analysis on the dependencies. In some
* cases an exception will occur with part of the analysis being performed
* which may not affect the entire analysis. If an exception occurs it will
* be included in the thrown exception collection.
*
* @throws ExceptionCollection a collections of any exceptions that occurred
* during analysis
*/
public void analyzeDependencies() throws ExceptionCollection {
final List exceptions = Collections.synchronizedList(new ArrayList<>());
initializeAndUpdateDatabase(exceptions);
//need to ensure that data exists
try {
ensureDataExists();
} catch (NoDataException ex) {
throwFatalExceptionCollection("Unable to continue dependency-check analysis.", ex, exceptions);
}
LOGGER.info("\n\nDependency-Check is an open source tool performing a best effort analysis of 3rd party dependencies; false positives and "
+ "false negatives may exist in the analysis performed by the tool. Use of the tool and the reporting provided constitutes "
+ "acceptance for use in an AS IS condition, and there are NO warranties, implied or otherwise, with regard to the analysis "
+ "or its use. Any use of the tool and the reporting provided is at the user's risk. In no event shall the copyright holder "
+ "or OWASP be held liable for any damages whatsoever arising out of or in connection with the use of this tool, the analysis "
+ "performed, or the resulting report.\n\n\n"
+ " About ODC: https://jeremylong.github.io/DependencyCheck/general/internals.html\n"
+ " False Positives: https://jeremylong.github.io/DependencyCheck/general/suppression.html\n"
+ "\n"
+ "💖 Sponsor: https://github.com/sponsors/jeremylong\n\n");
LOGGER.debug("\n----------------------------------------------------\nBEGIN ANALYSIS\n----------------------------------------------------");
LOGGER.info("Analysis Started");
final long analysisStart = System.currentTimeMillis();
// analysis phases
for (AnalysisPhase phase : mode.getPhases()) {
final List analyzerList = analyzers.get(phase);
for (final Analyzer analyzer : analyzerList) {
final long analyzerStart = System.currentTimeMillis();
try {
initializeAnalyzer(analyzer);
} catch (InitializationException ex) {
exceptions.add(ex);
if (ex.isFatal()) {
continue;
}
}
if (analyzer.isEnabled()) {
executeAnalysisTasks(analyzer, exceptions);
final long analyzerDurationMillis = System.currentTimeMillis() - analyzerStart;
final long analyzerDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(analyzerDurationMillis);
LOGGER.info("Finished {} ({} seconds)", analyzer.getName(), analyzerDurationSeconds);
} else {
LOGGER.debug("Skipping {} (not enabled)", analyzer.getName());
}
}
}
mode.getPhases().stream()
.map(analyzers::get)
.forEach((analyzerList) -> analyzerList.forEach(this::closeAnalyzer));
LOGGER.debug("\n----------------------------------------------------\nEND ANALYSIS\n----------------------------------------------------");
final long analysisDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - analysisStart);
LOGGER.info("Analysis Complete ({} seconds)", analysisDurationSeconds);
if (exceptions.size() > 0) {
throw new ExceptionCollection(exceptions);
}
}
/**
* Performs any necessary updates and initializes the database.
*
* @param exceptions a collection to store non-fatal exceptions
* @throws ExceptionCollection thrown if fatal exceptions occur
*/
private void initializeAndUpdateDatabase(@NotNull final List exceptions) throws ExceptionCollection {
if (!mode.isDatabaseRequired()) {
return;
}
final boolean autoUpdate;
autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true);
if (autoUpdate) {
try {
doUpdates(true);
} catch (UpdateException ex) {
exceptions.add(ex);
LOGGER.warn("Unable to update 1 or more Cached Web DataSource, using local "
+ "data instead. Results may not include recent vulnerabilities.");
LOGGER.debug("Update Error", ex);
} catch (DatabaseException ex) {
throwFatalDatabaseException(ex, exceptions);
}
} else {
try {
if (DatabaseManager.isH2Connection(settings) && !DatabaseManager.h2DataFileExists(settings)) {
throw new ExceptionCollection(new NoDataException("Autoupdate is disabled and the database does not exist"), true);
} else {
openDatabase(true, true);
}
} catch (IOException ex) {
throw new ExceptionCollection(new DatabaseException("Autoupdate is disabled and unable to connect to the database"), true);
} catch (DatabaseException ex) {
throwFatalDatabaseException(ex, exceptions);
}
}
}
/**
* Utility method to throw a fatal database exception.
*
* @param ex the exception that was caught
* @param exceptions the exception collection
* @throws ExceptionCollection the collection of exceptions is always thrown
* as a fatal exception
*/
private void throwFatalDatabaseException(DatabaseException ex, final List exceptions) throws ExceptionCollection {
final String msg;
if (ex.getMessage().contains("Unable to connect") && DatabaseManager.isH2Connection(settings)) {
msg = "Unable to connect to the database - if this error persists it may be "
+ "due to a corrupt database. Consider running `purge` to delete the existing database";
} else {
msg = "Unable to connect to the dependency-check database";
}
exceptions.add(new DatabaseException(msg, ex));
throw new ExceptionCollection(exceptions, true);
}
/**
* Executes executes the analyzer using multiple threads.
*
* @param exceptions a collection of exceptions that occurred during
* analysis
* @param analyzer the analyzer to execute
* @throws ExceptionCollection thrown if exceptions occurred during analysis
*/
protected void executeAnalysisTasks(@NotNull final Analyzer analyzer, List exceptions) throws ExceptionCollection {
LOGGER.debug("Starting {}", analyzer.getName());
final List analysisTasks = getAnalysisTasks(analyzer, exceptions);
final ExecutorService executorService = getExecutorService(analyzer);
try {
final int timeout = settings.getInt(Settings.KEYS.ANALYSIS_TIMEOUT, 180);
final List> results = executorService.invokeAll(analysisTasks, timeout, TimeUnit.MINUTES);
// ensure there was no exception during execution
for (Future result : results) {
try {
result.get();
} catch (ExecutionException e) {
throwFatalExceptionCollection("Analysis task failed with a fatal exception.", e, exceptions);
} catch (CancellationException e) {
throwFatalExceptionCollection("Analysis task was cancelled.", e, exceptions);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throwFatalExceptionCollection("Analysis has been interrupted.", e, exceptions);
} finally {
executorService.shutdown();
}
}
/**
* Returns the analysis tasks for the dependencies.
*
* @param analyzer the analyzer to create tasks for
* @param exceptions the collection of exceptions to collect
* @return a collection of analysis tasks
*/
protected synchronized List getAnalysisTasks(Analyzer analyzer, List exceptions) {
final List result = new ArrayList<>();
dependencies.stream().map((dependency) -> new AnalysisTask(analyzer, dependency, this, exceptions)).forEach(result::add);
return result;
}
/**
* Returns the executor service for a given analyzer.
*
* @param analyzer the analyzer to obtain an executor
* @return the executor service
*/
protected ExecutorService getExecutorService(Analyzer analyzer) {
if (analyzer.supportsParallelProcessing()) {
final int maximumNumberOfThreads = Runtime.getRuntime().availableProcessors();
LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
return Executors.newFixedThreadPool(maximumNumberOfThreads);
} else {
LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
return Executors.newSingleThreadExecutor();
}
}
/**
* Initializes the given analyzer.
*
* @param analyzer the analyzer to prepare
* @throws InitializationException thrown when there is a problem
* initializing the analyzer
*/
protected void initializeAnalyzer(@NotNull final Analyzer analyzer) throws InitializationException {
try {
LOGGER.debug("Initializing {}", analyzer.getName());
analyzer.prepare(this);
} catch (InitializationException ex) {
LOGGER.error("Exception occurred initializing {}.", analyzer.getName());
LOGGER.debug("", ex);
if (ex.isFatal()) {
try {
analyzer.close();
} catch (Throwable ex1) {
LOGGER.trace("", ex1);
}
}
throw ex;
} catch (Throwable ex) {
LOGGER.error("Unexpected exception occurred initializing {}.", analyzer.getName());
LOGGER.debug("", ex);
try {
analyzer.close();
} catch (Throwable ex1) {
LOGGER.trace("", ex1);
}
throw new InitializationException("Unexpected Exception", ex);
}
}
/**
* Closes the given analyzer.
*
* @param analyzer the analyzer to close
*/
protected void closeAnalyzer(@NotNull final Analyzer analyzer) {
LOGGER.debug("Closing Analyzer '{}'", analyzer.getName());
try {
analyzer.close();
} catch (Throwable ex) {
LOGGER.trace("", ex);
}
}
/**
* Cycles through the cached web data sources and calls update on all of
* them.
*
* @throws UpdateException thrown if the operation fails
* @throws DatabaseException if the operation fails due to a local database
* failure
* @return Whether any updates actually happened
*/
public boolean doUpdates() throws UpdateException, DatabaseException {
return doUpdates(false);
}
/**
* Cycles through the cached web data sources and calls update on all of
* them.
*
* @param remainOpen whether or not the database connection should remain
* open
* @throws UpdateException thrown if the operation fails
* @throws DatabaseException if the operation fails due to a local database
* failure
* @return Whether any updates actually happened
*/
public boolean doUpdates(boolean remainOpen) throws UpdateException, DatabaseException {
if (mode.isDatabaseRequired()) {
try (WriteLock dblock = new WriteLock(getSettings(), DatabaseManager.isH2Connection(getSettings()))) {
//lock is not needed as we already have the lock held
openDatabase(false, false);
LOGGER.info("Checking for updates");
final long updateStart = System.currentTimeMillis();
final UpdateService service = new UpdateService(serviceClassLoader);
final Iterator iterator = service.getDataSources();
boolean dbUpdatesMade = false;
UpdateException updateException = null;
while (iterator.hasNext()) {
try {
final CachedWebDataSource source = iterator.next();
dbUpdatesMade |= source.update(this);
} catch (UpdateException ex) {
updateException = ex;
LOGGER.error(ex.getMessage(), ex);
}
}
if (dbUpdatesMade) {
database.defrag();
}
database.close();
database = null;
if (updateException != null) {
throw updateException;
}
LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
if (remainOpen) {
//lock is not needed as we already have the lock held
openDatabase(true, false);
}
return dbUpdatesMade;
} catch (WriteLockException ex) {
throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex);
}
} else {
LOGGER.info("Skipping update check in evidence collection mode.");
return false;
}
}
/**
* Purges the cached web data sources.
*
* @return true if the purge was successful; otherwise
* false
*/
public boolean purge() {
boolean result = true;
final UpdateService service = new UpdateService(serviceClassLoader);
final Iterator iterator = service.getDataSources();
while (iterator.hasNext()) {
result &= iterator.next().purge(this);
}
try {
final File cache = new File(settings.getDataDirectory(), "cache");
if (cache.exists()) {
if (FileUtils.delete(cache)) {
LOGGER.info("Cache directory purged");
}
}
} catch (IOException ex) {
throw new RuntimeException(ex);
}
try {
final File cache = new File(settings.getDataDirectory(), "oss_cache");
if (cache.exists()) {
if (FileUtils.delete(cache)) {
LOGGER.info("OSS Cache directory purged");
}
}
} catch (IOException ex) {
throw new RuntimeException(ex);
}
return result;
}
/**
*
* This method is only public for unit/integration testing. This method
* should not be called by any integration that uses
* dependency-check-core.
*
* Opens the database connection.
*
* @throws DatabaseException if the database connection could not be created
*/
public void openDatabase() throws DatabaseException {
openDatabase(false, true);
}
/**
*
* This method is only public for unit/integration testing. This method
* should not be called by any integration that uses
* dependency-check-core.
*
* Opens the database connection; if readOnly is true a copy of the database
* will be made.
*
* @param readOnly whether or not the database connection should be readonly
* @param lockRequired whether or not a lock needs to be acquired when
* opening the database
* @throws DatabaseException if the database connection could not be created
*/
@SuppressWarnings("try")
public void openDatabase(boolean readOnly, boolean lockRequired) throws DatabaseException {
if (mode.isDatabaseRequired() && database == null) {
try (WriteLock dblock = new WriteLock(getSettings(), lockRequired && DatabaseManager.isH2Connection(settings))) {
if (readOnly
&& DatabaseManager.isH2Connection(settings)
&& settings.getString(Settings.KEYS.DB_CONNECTION_STRING).contains("file:%s")) {
final File db = DatabaseManager.getH2DataFile(settings);
if (db.isFile()) {
final File temp = settings.getTempDirectory();
final File tempDB = new File(temp, db.getName());
LOGGER.debug("copying database {} to {}", db.toPath(), temp.toPath());
Files.copy(db.toPath(), tempDB.toPath());
settings.setString(Settings.KEYS.H2_DATA_DIRECTORY, temp.getPath());
final String connStr = settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
if (!connStr.contains("ACCESS_MODE_DATA")) {
settings.setString(Settings.KEYS.DB_CONNECTION_STRING, connStr + "ACCESS_MODE_DATA=r");
}
settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
database = new CveDB(settings);
} else {
throw new DatabaseException("Unable to open database - configured database file does not exist: " + db);
}
} else {
database = new CveDB(settings);
}
} catch (IOException ex) {
throw new DatabaseException("Unable to open database in read only mode", ex);
} catch (WriteLockException ex) {
throw new DatabaseException("Failed to obtain lock - unable to open database", ex);
}
database.open();
}
}
/**
* Returns a reference to the database.
*
* @return a reference to the database
*/
public CveDB getDatabase() {
return this.database;
}
/**
* Returns a full list of all of the analyzers. This is useful for reporting
* which analyzers where used.
*
* @return a list of Analyzers
*/
@NotNull
public List getAnalyzers() {
final List analyzerList = new ArrayList<>();
//insteae of forEach - we can just do a collect
mode.getPhases().stream()
.map(analyzers::get)
.forEachOrdered(analyzerList::addAll);
return analyzerList;
}
/**
* Checks all analyzers to see if an extension is supported.
*
* @param file a file extension
* @return true or false depending on whether or not the file extension is
* supported
*/
@Override
public boolean accept(@Nullable final File file) {
if (file == null) {
return false;
}
/* note, we can't break early on this loop as the analyzers need to know if
they have files to work on prior to initialization */
return this.fileTypeAnalyzers.stream().map((a) -> a.accept(file)).reduce(false, (accumulator, result) -> accumulator || result);
}
/**
* Returns the set of file type analyzers.
*
* @return the set of file type analyzers
*/
public Set getFileTypeAnalyzers() {
return this.fileTypeAnalyzers;
}
/**
* Returns the configured settings.
*
* @return the configured settings
*/
public Settings getSettings() {
return settings;
}
/**
* Retrieve an object from the objects collection.
*
* @param key the key to retrieve the object
* @return the object
*/
public Object getObject(String key) {
return objects.get(key);
}
/**
* Put an object in the object collection.
*
* @param key the key to store the object
* @param object the object to store
*/
public void putObject(String key, Object object) {
objects.put(key, object);
}
/**
* Verifies if the object exists in the object store.
*
* @param key the key to retrieve the object
* @return true if the object exists; otherwise
* false
*/
public boolean hasObject(String key) {
return objects.containsKey(key);
}
/**
* Removes an object from the object store.
*
* @param key the key to the object
*/
public void removeObject(String key) {
objects.remove(key);
}
/**
* Returns the mode of the engine.
*
* @return the mode of the engine
*/
public Mode getMode() {
return mode;
}
/**
* Adds a file type analyzer. This has been added solely to assist in unit
* testing the Engine.
*
* @param fta the file type analyzer to add
*/
protected void addFileTypeAnalyzer(@NotNull final FileTypeAnalyzer fta) {
this.fileTypeAnalyzers.add(fta);
}
/**
* Checks the CPE Index to ensure documents exists. If none exist a
* NoDataException is thrown.
*
* @throws NoDataException thrown if no data exists in the CPE Index
*/
private void ensureDataExists() throws NoDataException {
if (mode.isDatabaseRequired() && (database == null || !database.dataExists())) {
throw new NoDataException("No documents exist");
}
}
/**
* Constructs and throws a fatal exception collection.
*
* @param message the exception message
* @param throwable the cause
* @param exceptions a collection of exception to include
* @throws ExceptionCollection a collection of exceptions that occurred
* during analysis
*/
private void throwFatalExceptionCollection(String message, @NotNull final Throwable throwable,
@NotNull final List exceptions) throws ExceptionCollection {
LOGGER.error(message);
LOGGER.debug("", throwable);
exceptions.add(throwable);
throw new ExceptionCollection(exceptions, true);
}
/**
* Writes the report to the given output directory.
*
* @param applicationName the name of the application/project
* @param outputDir the path to the output directory (can include the full
* file name if the format is not ALL)
* @param format the report format (see {@link ReportGenerator.Format})
* @throws ReportException thrown if there is an error generating the report
* @deprecated use
* {@link #writeReports(java.lang.String, java.io.File, java.lang.String, org.owasp.dependencycheck.exception.ExceptionCollection)}
*/
@Deprecated
public void writeReports(String applicationName, File outputDir, String format) throws ReportException {
writeReports(applicationName, null, null, null, outputDir, format, null);
}
//CSOFF: LineLength
/**
* Writes the report to the given output directory.
*
* @param applicationName the name of the application/project
* @param outputDir the path to the output directory (can include the full
* file name if the format is not ALL)
* @param format the report format (see {@link ReportGenerator.Format})
* @param exceptions a collection of exceptions that may have occurred
* during the analysis
* @throws ReportException thrown if there is an error generating the report
*/
public void writeReports(String applicationName, File outputDir, String format, ExceptionCollection exceptions) throws ReportException {
writeReports(applicationName, null, null, null, outputDir, format, exceptions);
}
//CSON: LineLength
/**
* Writes the report to the given output directory.
*
* @param applicationName the name of the application/project
* @param groupId the Maven groupId
* @param artifactId the Maven artifactId
* @param version the Maven version
* @param outputDir the path to the output directory (can include the full
* file name if the format is not ALL)
* @param format the report format (see {@link ReportGenerator.Format})
* @throws ReportException thrown if there is an error generating the report
* @deprecated use
* {@link #writeReports(String, String, String, String, File, String, ExceptionCollection)}
*/
@Deprecated
public synchronized void writeReports(String applicationName, @Nullable final String groupId,
@Nullable final String artifactId, @Nullable final String version,
@NotNull final File outputDir, String format) throws ReportException {
writeReports(applicationName, groupId, artifactId, version, outputDir, format, null);
}
//CSOFF: LineLength
/**
* Writes the report to the given output directory.
*
* @param applicationName the name of the application/project
* @param groupId the Maven groupId
* @param artifactId the Maven artifactId
* @param version the Maven version
* @param outputDir the path to the output directory (can include the full
* file name if the format is not ALL)
* @param format the report format (see {@link ReportGenerator.Format})
* @param exceptions a collection of exceptions that may have occurred
* during the analysis
* @throws ReportException thrown if there is an error generating the report
*/
public synchronized void writeReports(String applicationName, @Nullable final String groupId,
@Nullable final String artifactId, @Nullable final String version,
@NotNull final File outputDir, String format, ExceptionCollection exceptions) throws ReportException {
if (mode == Mode.EVIDENCE_COLLECTION) {
throw new UnsupportedOperationException("Cannot generate report in evidence collection mode.");
}
final DatabaseProperties prop = database.getDatabaseProperties();
final ReportGenerator r = new ReportGenerator(applicationName, groupId, artifactId, version,
dependencies, getAnalyzers(), prop, settings, exceptions);
try {
r.write(outputDir.getAbsolutePath(), format);
} catch (ReportException ex) {
final String msg = String.format("Error generating the report for %s", applicationName);
LOGGER.debug(msg, ex);
throw new ReportException(msg, ex);
}
}
//CSON: LineLength
private boolean identifiersMatch(Set left, Set right) {
if (left != null && right != null && left.size() > 0 && left.size() == right.size()) {
int count = 0;
for (Identifier l : left) {
for (Identifier r : right) {
if (l.getValue().equals(r.getValue())) {
count += 1;
break;
}
}
}
return count == left.size();
}
return false;
}
/**
* Checks that if Java 8 is being used, it is at least update 251. This is
* required as a new method was introduced that is used by Apache HTTP
* Client. See
* https://stackoverflow.com/questions/76226322/exception-in-thread-httpclient-dispatch-1-java-lang-nosuchmethoderror-javax-n#comment134427003_76226322
*/
private void checkRuntimeVersion() {
if (Utils.getJavaVersion() == 8 && Utils.getJavaUpdateVersion() < 251) {
LOGGER.error("Non-supported Java Runtime: dependency-check requires at least Java 8 update 251 or higher.");
throw new RuntimeException("dependency-check requires Java 8 update 251 or higher");
}
}
/**
* {@link Engine} execution modes.
*/
public enum Mode {
/**
* In evidence collection mode the {@link Engine} only collects evidence
* from the scan targets, and doesn't require a database.
*/
EVIDENCE_COLLECTION(
false,
INITIAL,
PRE_INFORMATION_COLLECTION,
INFORMATION_COLLECTION,
INFORMATION_COLLECTION2,
POST_INFORMATION_COLLECTION1,
POST_INFORMATION_COLLECTION2,
POST_INFORMATION_COLLECTION3
),
/**
* In evidence processing mode the {@link Engine} processes the evidence
* collected using the {@link #EVIDENCE_COLLECTION} mode. Dependencies
* should be injected into the {@link Engine} using
* {@link Engine#setDependencies(List)}.
*/
EVIDENCE_PROCESSING(
true,
PRE_IDENTIFIER_ANALYSIS,
IDENTIFIER_ANALYSIS,
POST_IDENTIFIER_ANALYSIS,
PRE_FINDING_ANALYSIS,
FINDING_ANALYSIS,
POST_FINDING_ANALYSIS,
FINDING_ANALYSIS_PHASE2,
FINAL
),
/**
* In standalone mode the {@link Engine} will collect and process
* evidence in a single execution.
*/
STANDALONE(true, AnalysisPhase.values());
/**
* Whether the database is required in this mode.
*/
private final boolean databaseRequired;
/**
* The analysis phases included in the mode.
*/
private final List phases;
/**
* Constructs a new mode.
*
* @param databaseRequired if the database is required for the mode
* @param phases the analysis phases to include in the mode
*/
Mode(boolean databaseRequired, AnalysisPhase... phases) {
this.databaseRequired = databaseRequired;
this.phases = Collections.unmodifiableList(Arrays.asList(phases));
}
/**
* Returns true if the database is required; otherwise false.
*
* @return whether or not the database is required
*/
private boolean isDatabaseRequired() {
return databaseRequired;
}
/**
* Returns the phases for this mode.
*
* @return the phases for this mode
*/
public List getPhases() {
return phases;
}
}
}