org.whitesource.config.FSAConfigurationManager Maven / Gradle / Ivy
The newest version!
package org.whitesource.config;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.whitesource.agent.client.ClientConstants;
import org.whitesource.config.FSAConfiguration.ProxyAuthenticator;
import org.whitesource.config.enums.EuaOfflineMode;
import org.whitesource.config.interfaces.FSAConfigProperty;
import org.whitesource.config.scan.config.*;
import org.whitesource.config.utils.*;
import org.whitesource.statistics.Statistics;
import org.whitesource.statistics.StatisticsTypes.FetchConfigurationStatistics;
import org.whitesource.utils.Constants;
import org.whitesource.utils.OsUtils;
import org.whitesource.utils.Pair;
import org.whitesource.utils.cli.Cli;
import org.whitesource.utils.cli.CommandLineProcess;
import org.whitesource.utils.logger.LoggerFactory;
import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import static org.whitesource.config.utils.ConfigPropertyKeys.SERVERLESS_SCAN_FUNCTIONS;
public class FSAConfigurationManager {
private static final Logger logger = LoggerFactory.getLogger(FSAConfiguration.class);
private final String EUA_ANALYZE_FRAMEWORKS = "analyze frameworks";
private String[] args;
private CommandLineArgs cmdArgs = null;
private Map cmdPropsMap = null;
private Map filePropsMap = null;
private Map propsMap = null;
private List errors;
private Statistics statistics;
private boolean useCommandLineRequestFiles = false;
private boolean setUpMuiltiModuleFile = false;
@FSAConfigProperty
private AgentConfiguration agent;
private EndPointConfiguration endpoint;
@FSAConfigProperty
private OfflineConfiguration offline;
@FSAConfigProperty
private RemoteDockerConfiguration remoteDockerConfiguration;
@FSAConfigProperty
private RequestConfiguration request;
private RequestConfiguration offlineRequestFilesRequest;
@FSAConfigProperty
private ResolverConfiguration resolver;
private ScmConfiguration scm;
@FSAConfigProperty
private SenderConfiguration sender;
@FSAConfigProperty
private ServerlessConfiguration serverlessConfiguration;
private ConfigPropertyDefinitions definitions;
public static String failErrorLevel; // WorkAround
// *****************************************************************
// temporary for logs printing
@FSAConfigProperty
private static final long DEFAULT_TIMEOUT_PROCESS_MINUTES = 15;
@FSAConfigProperty
private boolean projectPerFolder;
@FSAConfigProperty
private int connectionTimeOut;
@FSAConfigProperty
private String fileListPath;
@FSAConfigProperty
private List dependencyDirs;
@FSAConfigProperty
private String configFilePath;
@FSAConfigProperty
private boolean scanPackageManager;
@FSAConfigProperty
private boolean scanDockerImages;
@FSAConfigProperty
private boolean scanDockerContainers;
@FSAConfigProperty
private boolean scanServerlessFunctions;
@FSAConfigProperty
private String logLevel;
// *****************************************************************
/* *** Constructors - START *** */
public FSAConfigurationManager() {
init(null, null);
}
public FSAConfigurationManager(String[] args) {
init(args, null);
}
public FSAConfigurationManager(Map properties) {
init(null, properties);
}
public FSAConfigurationManager(FSAConfigProperties config) {
Map convertedMap = new HashMap<>();
config.forEach((k, v) -> convertedMap.put((String) k, v));
init(null, convertedMap);
}
public FSAConfigurationManager(FSAConfigProperties config, String[] args) {
Map convertedMap = new HashMap<>();
config.forEach((k, v) -> convertedMap.put((String) k, v));
init(args, convertedMap);
}
public FSAConfigurationManager(String[] args, Map properties) {
init(args, properties);
}
/* *** Constructors - END *** */
private void init(String[] args, Map properties){
this.statistics = new FetchConfigurationStatistics();
this.statistics.getStartStatisticLog();
this.errors = new ArrayList<>();
this.propsMap = new HashMap<>();
this.args = args;
if (properties == null) {
// populates this.cmdPropsMap and this.filePropsMap
loadAndParseBaseConfigurations(args);
} else {
// init maps using values from properties
this.cmdPropsMap = new HashMap<>();
this.filePropsMap = properties;
}
margeCmdAndFileProperties();
loadAdditionalConfigurations(); // currently empty method
definitions = new ConfigPropertyDefinitions();
convertPropertiesValuesType();
populateValues();
populateDefaultValues();
populateStaticValues();
runPropertiesValidations();
initConfigWrappers();
}
/**
* @param args the command line arguments
*/
private void loadAndParseBaseConfigurations(String[] args) {
this.cmdArgs = new CommandLineArgs();
if (args != null) {
// parse command line arguments
this.cmdArgs.parseCommandLine(args);
this.cmdPropsMap = cmdArgs.getValuesAsMap();
}
// check if allowed to parse config file
if (!Boolean.valueOf(cmdArgs.noConfig)) {
Pair> configFileStream = getConfigFileDataStream();
List fileReadErrors = configFileStream.getValue();
if (fileReadErrors != null && fileReadErrors.size() > 0) {
fileReadErrors.forEach(s -> logger.warn("loadAndParseBaseConfigurations - error: {}", s));
// TODO handle errors ??
errors.addAll(fileReadErrors);
} else {
this.filePropsMap = loadAndParseConfFile(configFileStream.getKey());
}
}
if (this.cmdPropsMap == null) {
this.cmdPropsMap = new HashMap<>();
}
if (this.filePropsMap == null) {
this.filePropsMap = new HashMap<>();
}
}
/**
* 1. if configuration file flag is set, then try to read the file (local or remote file)
* 2. else try to open:
* 2.1. CommandLineArgs.CONFIG_FILE
* 2.1. CommandLineArgs.OLD_CONFIG_FILE
*
* @return
*/
private Pair> getConfigFileDataStream() {
InputStream configFileStream = null;
List readErrors = new ArrayList<>();
// not default
if (!Constants.DEFAULT.equals(cmdArgs.configFilePath)) {
/*
* since we don't know if it's a local path or remote URL
* we first try to read from local file, then in case of failure we try to resolve a URL
*/
configFileStream = readLocalFile(cmdArgs.configFilePath);
if (configFileStream == null) {
configFileStream = readRemoteFile(cmdArgs.configFilePath, readErrors);
}
if (configFileStream == null) {
readErrors.add("Failed to find file " + cmdArgs.configFilePath);
}
} else { // check if a default option exists
// try option 1
configFileStream = readLocalFile(CommandLineArgs.CONFIG_FILE);
if (configFileStream == null) {
// try option 2
configFileStream = readLocalFile(CommandLineArgs.OLD_CONFIG_FILE);
}
if (configFileStream == null) {
readErrors.add("Failed to find default files " + CommandLineArgs.CONFIG_FILE + " or " + CommandLineArgs.OLD_CONFIG_FILE);
}
}
return new Pair>(configFileStream, readErrors);
}
/**
* @param filePath
* @return
*/
private InputStream readLocalFile(String filePath) {
InputStream stream = null;
try {
stream = new FileInputStream(filePath);
} catch (FileNotFoundException e) {
logger.debug("Failed to find file: {}, error message:", filePath, e.getMessage());
}
return stream;
}
/**
* @param fileUrl
* @param errorsList
* @return
*/
private InputStream readRemoteFile(String fileUrl, List errorsList) {
List proxyErrors = new ArrayList<>();
BufferedReader readFileFromUrl = null;
StringBuffer writeUrlFileContent = null;
Proxy proxy = null;
String inputSteamLine = null;
InputStream inputStream = null;
ProxyAuthenticator proxyAuthenticator = null;
String[] parsedProxy = null;
parsedProxy = getProxyProperties();
try {
// assign the url to point to config path url
URL url = new URL(fileUrl);
URLConnection urlConnection;
if (parsedProxy != null) {
if (parsedProxy[1] != null && Integer.valueOf(parsedProxy[1]) > 0) {
proxyAuthenticator = new ProxyAuthenticator(parsedProxy[2], parsedProxy[3]);
proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(parsedProxy[0], Integer.valueOf(parsedProxy[1])));
} else {
proxyErrors.add("Port must be set or greater than 0");
}
}
// if proxy is set, open the connection with proxy
if (proxy != null) {
Authenticator.setDefault(proxyAuthenticator);
/*
* The 'jdk.http.auth.tunneling.disabledSchemes' property lists the authentication
* schemes that will be disabled when tunneling HTTPS over a proxy, HTTP CONNECT.
* so setting it to empty for this run only
*/
System.setProperty("jdk.http.auth.tunneling.disabledSchemes", Constants.EMPTY_STRING);
urlConnection = url.openConnection(proxy);
} else {
urlConnection = url.openConnection();
}
urlConnection.connect();
readFileFromUrl = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));
writeUrlFileContent = new StringBuffer();
// write data of the file to string buffer
while ((inputSteamLine = readFileFromUrl.readLine()) != null) {
writeUrlFileContent.append(inputSteamLine + Constants.DOWN_LINE);
}
} catch (MalformedURLException ex) {
// this is last option of reading configuration file
// TODO handle error
logger.debug("FSAConfigurationRework - readRemoteFile - ERROR", ex);
} catch (IOException ex) {
// if error occurred that means it is not a url that we can resolve
// this is last option of reading configuration file
// TODO handle error
logger.debug("FSAConfigurationRework - readRemoteFile - ERROR", ex);
}
// if there is any data written to the buffer, convert it to input stream
if (writeUrlFileContent != null) {
inputStream = IOUtils.toInputStream(writeUrlFileContent, StandardCharsets.UTF_8);
}
if (inputStream == null) {
errorsList.addAll(proxyErrors);
}
return inputStream;
}
/**
* @return proxy details in a String[] in the order: { Host, Port, User, Password }
*/
private String[] getProxyProperties() {
String[] proxy = null;
Map mapToUse = null;
if (this.propsMap != null) {
if (propsMap.get(ConfigPropertyKeys.PROXY_HOST_PROPERTY_KEY) != null) {
mapToUse = propsMap;
}
}
if (mapToUse == null && this.cmdPropsMap != null) {
if (cmdPropsMap.get(ConfigPropertyKeys.PROXY_HOST_PROPERTY_KEY) != null) {
mapToUse = cmdPropsMap;
}
}
if (mapToUse == null && this.filePropsMap != null) {
if (filePropsMap.get(ConfigPropertyKeys.PROXY_HOST_PROPERTY_KEY) != null) {
mapToUse = filePropsMap;
}
}
if (mapToUse != null) {
proxy = new String[4];
proxy[0] = (String) mapToUse.get(ConfigPropertyKeys.PROXY_HOST_PROPERTY_KEY);
proxy[1] = (String) mapToUse.get(ConfigPropertyKeys.PROXY_PORT_PROPERTY_KEY);
proxy[2] = (String) mapToUse.get(ConfigPropertyKeys.PROXY_USER_PROPERTY_KEY);
proxy[3] = (String) mapToUse.get(ConfigPropertyKeys.PROXY_PASS_PROPERTY_KEY);
}
return proxy;
}
/**
* @param configFileStream
* @return
*/
private Map loadAndParseConfFile(InputStream configFileStream) {
if (configFileStream == null) {
return null;
}
Map result = new HashMap<>();
try {
Properties props = new Properties();
props.load(new InputStreamReader(configFileStream, StandardCharsets.UTF_8));
// Remove extra spaces from the values
Set keys = props.stringPropertyNames();
for (String key : keys) {
String value = props.getProperty(key);
if (value != null) {
result.put(key, value.trim());
}
}
} catch (Exception e) {
logger.info("Error occurred when reading from configuration file stream", e);
errors.add("Error occurred when reading from configuration file stream, error: " + e.getMessage());
}
return result;
}
/**
*
*/
private void margeCmdAndFileProperties() {
this.propsMap = new HashMap<>();
if (this.cmdPropsMap != null) {
cmdPropsMap.forEach((k, v) -> propsMap.putIfAbsent(k, v));
}
if (this.filePropsMap != null) {
filePropsMap.forEach((k, v) -> propsMap.putIfAbsent(k, v));
}
}
/**
*
*/
private void convertPropertiesValuesType() {
for (PropertyDefinition pd : definitions.getIntegerTypesList()) {
Object value = propsMap.get(pd.getName());
if (value != null) {
if (value instanceof String && StringUtils.isNotBlank((String) value)) {
propsMap.put(pd.getName(), Integer.valueOf((String) value));
} else if (!(value instanceof Integer)) {
errors.add("Property " + pd.getName() + " type error! expected: Integer found:" + value.getClass().getName());
}
}
}
for (PropertyDefinition pd : definitions.getBooleanTypesList()) {
String value = (String) propsMap.get(pd.getName());
if (StringUtils.isNotBlank(value)) {
propsMap.put(pd.getName(), Boolean.valueOf(value));
}
}
for (PropertyDefinition pd : definitions.getArrayTypesList()) {
Object value = propsMap.get(pd.getName());
if (value!=null) {
if (value instanceof String) {
if (StringUtils.isNotBlank((String) value)) {
propsMap.put(pd.getName(), ((String) value).split(pd.getSplitDelimiter()));
}
} else if (!(value instanceof String[])) {
errors.add("Property " + pd.getName() + " type error! expected: String[] found:" + value.getClass().getName());
}
}
}
for (PropertyDefinition pd : definitions.getListTypesList()) {
Object value = propsMap.get(pd.getName());
if (value!=null) {
if (value instanceof String ) {
if (StringUtils.isNotBlank((String) value)) {
propsMap.put(pd.getName(), Arrays.asList(((String) value).split(pd.getSplitDelimiter())));
}
} else if (!(value instanceof List)) {
errors.add("Property " + pd.getName() + " type error! expected: List found:" + value.getClass().getName());
}
}
}
}
/**
*
*/
private void populateValues() {
// place holders from previous implementation
List offlineRequestFiles = (List) propsMap.get(ConfigPropertyKeys.CMD_OFFLINE_REQUEST_FILES);
String fileListPath = (String) propsMap.get(ConfigPropertyKeys.CMD_FILE_LIST_PATH);
String scmUrl = (String) propsMap.get(ConfigPropertyKeys.SCM_URL_PROPERTY_KEY);
List dependencyDirs = new ArrayList<>();
// case 1
String scannedFolders = (String) propsMap.get(ConfigPropertyKeys.SCANNED_FOLDERS);
if (StringUtils.isNotBlank(scannedFolders)) {
String[] libsList = scannedFolders.split(Constants.COMMA);
// Trim all elements in libsList
String[] libsListTrimmed = Arrays.stream(libsList).map(String::trim).toArray(unused -> libsList);
dependencyDirs = Arrays.asList(libsListTrimmed);
}
// case 2
List scanDirs = (List) propsMap.get(ConfigPropertyKeys.CMD_D_SCAN_DIRS);
if (scanDirs == null) {
scanDirs = new LinkedList<>();
}
if (!scanDirs.isEmpty()) {
dependencyDirs = scanDirs;
}
// case 3
Object serverlessScanObj = propsMap.get(SERVERLESS_SCAN_FUNCTIONS);
boolean serverlessScan = serverlessScanObj != null && (boolean) serverlessScanObj;
if (!serverlessScan&& dependencyDirs.isEmpty() && StringUtils.isBlank(fileListPath) &&
(offlineRequestFiles == null || offlineRequestFiles.isEmpty()) && StringUtils.isBlank(scmUrl)) {
dependencyDirs.add(Constants.DOT);
}
propsMap.put(ConfigPropertyKeys.CMD_D_SCAN_DIRS, dependencyDirs);
// cloned value
// TODO: remove later (after config refactor)
propsMap.put(ConfigPropertyKeys.WHITESOURCE_CONFIGURATION, propsMap.get(ConfigPropertyKeys.PROJECT_CONFIGURATION_PATH));
/* reading order:
* 1. api-Key-File
* 2. command line
* 3. config file
* 4. environment variable
*/
String apiKeyFile = (String) filePropsMap.get(ConfigPropertyKeys.ORG_TOKEN_FILE);
if (StringUtils.isNotBlank(apiKeyFile)) {
String keyFromFile = readSecurityKeyFromFile(apiKeyFile);
if (StringUtils.isBlank(keyFromFile)) {
keyFromFile = (String) propsMap.get(ConfigPropertyKeys.ORG_TOKEN_PROPERTY_KEY);
}
if (StringUtils.isBlank(keyFromFile)) {
// apiToken was not found in the command line or in the config file - so we try to read it from env
keyFromFile = System.getenv(ConfigPropertyKeys.ORG_TOKEN_PROPERTY_KEY);
}
if (StringUtils.isNotBlank(keyFromFile)) {
propsMap.put(ConfigPropertyKeys.ORG_TOKEN_PROPERTY_KEY, keyFromFile);
}
}
/* reading order:
* 1. use-Key-File
* 2. command line
* 3. config file
* 4. environment variable
*/
String userKeyFile = (String) filePropsMap.get(ConfigPropertyKeys.USER_KEY_FILE);
if (StringUtils.isNotBlank(userKeyFile)) {
String keyFromFile = readSecurityKeyFromFile(userKeyFile);
if (StringUtils.isBlank(keyFromFile)) {
keyFromFile = (String) propsMap.get(ConfigPropertyKeys.USER_KEY_PROPERTY_KEY);
}
if (StringUtils.isBlank(keyFromFile)) {
// userKey was not found in the command line or in the config file - so we try to read it from env
keyFromFile = System.getenv(ConfigPropertyKeys.USER_KEY_PROPERTY_KEY);
}
if (StringUtils.isNotBlank(keyFromFile)) {
propsMap.put(ConfigPropertyKeys.USER_KEY_PROPERTY_KEY, keyFromFile);
}
}
List reqsFileIncludes = (List) propsMap.get(ConfigPropertyKeys.CMD_REQUIREMENTS_FILE_INCLUDES);
if (reqsFileIncludes!= null && !reqsFileIncludes.isEmpty()) {
StringBuilder requirements = new StringBuilder();
for (String requirementFileIncludes : reqsFileIncludes) {
requirements.append(requirementFileIncludes);
requirements.append(Constants.WHITESPACE);
}
propsMap.put(ConfigPropertyKeys.PYTHON_REQUIREMENTS_FILE_INCLUDES, requirements.toString().trim());
}
List files = (List) cmdPropsMap.get(ConfigPropertyKeys.CMD_OFFLINE_REQUEST_FILES);
boolean useEUA = false;
if (this.cmdArgs != null) {
useEUA = (this.cmdArgs.euaOffline != null && (this.cmdArgs.euaOffline.equals(EuaOfflineMode.VUL.toString()) ||
this.cmdArgs.euaOffline.equals(EuaOfflineMode.VUL.toString()) ||
this.cmdArgs.euaOffline.equals(EuaOfflineMode.VUL.toString())));
}
useCommandLineRequestFiles = (files != null && !files.isEmpty()) || useEUA;
// handle RESOLVE_ALL_DEPENDENCIES=false (since by default it's true)
Object resolveAll = propsMap.get(ConfigPropertyKeys.RESOLVE_ALL_DEPENDENCIES);
if (resolveAll != null && !((boolean)resolveAll)) {
for (String flagKey : this.definitions.getEnableResolutionFlags()) {
propsMap.putIfAbsent(flagKey, false);
}
}
// handle IGNORE_SOURCE_FILES=true (by default it's false)
Object ignoreSourceFile = propsMap.get(ConfigPropertyKeys.IGNORE_SOURCE_FILES);
if (ignoreSourceFile != null && ((boolean)ignoreSourceFile)) {
for (String flagKey : this.definitions.getIgnoreSourceFilesFlags()) {
propsMap.putIfAbsent(flagKey, true);
}
}
initializeDependencyDirsForEUA();
}
private void populateDefaultValues() {
for (PropertyDefinition pd : definitions.getStringTypesList()) {
Object o = propsMap.get(pd.getName());
if (o == null || o instanceof String && StringUtils.isBlank((String) o)) {
propsMap.put(pd.getName(), pd.getDefaultValue());
}
}
for (PropertyDefinition pd : definitions.getIntegerTypesList()) {
Object o = propsMap.get(pd.getName());
if (o == null || o instanceof String && StringUtils.isBlank((String) o)) {
propsMap.put(pd.getName(), pd.getDefaultValue());
}
}
for (PropertyDefinition pd : definitions.getBooleanTypesList()) {
Object o = propsMap.get(pd.getName());
if (o == null || o instanceof String && StringUtils.isBlank((String) o)) {
propsMap.put(pd.getName(), pd.getDefaultValue());
}
}
for (PropertyDefinition pd : definitions.getArrayTypesList()) {
Object o = propsMap.get(pd.getName());
if (o == null || o instanceof String && StringUtils.isBlank((String) o)) {
propsMap.put(pd.getName(), pd.getDefaultValue());
}
}
for (PropertyDefinition pd : definitions.getListTypesList()) {
Object o = propsMap.get(pd.getName());
if (o == null || o instanceof String && StringUtils.isBlank((String) o)) {
propsMap.put(pd.getName(), pd.getDefaultValue());
}
}
}
private void populateStaticValues() {
boolean printCliErrors = (boolean) propsMap.get(ConfigPropertyKeys.PRINT_COMMAND_LINE_WARNINGS);
String logContext = (String) propsMap.get(ConfigPropertyKeys.LOG_CONTEXT);
CommandLineProcess.setIncludeErrorLines(printCliErrors);
if (StringUtils.isNotBlank(logContext)) {
LoggerFactory.contextId = logContext;
}
failErrorLevel = (String) propsMap.get(ConfigPropertyKeys.FAIL_ERROR_LEVEL);
}
/**
* @param keyFile
* @return
*/
private String readSecurityKeyFromFile(String keyFile) {
String key = null;
InputStream fileStream = readLocalFile(keyFile);
if (fileStream == null) {
fileStream = readRemoteFile(keyFile, errors);
}
if (fileStream == null) {
errors.add("Failed to find file " + keyFile);
return null;
}
InputStreamReader inputStreamReader = new InputStreamReader(fileStream);
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
try {
key = bufferedReader.readLine();
} catch (IOException e1) {
errors.add("Error occurred when reading from file: " + keyFile + " " + e1.getMessage());
} finally {
try {
bufferedReader.close();
} catch (IOException e2) {
logger.debug("error closing bufferedReader at readSecurityKeyFromFile", e2);
}
try {
inputStreamReader.close();
} catch (IOException e3) {
logger.debug("error closing inputStreamReader at readSecurityKeyFromFile", e3);
}
try {
fileStream.close();
} catch (IOException e4) {
logger.debug("error closing fileStream at readSecurityKeyFromFile", e4);
}
}
return key;
}
/**
*
*/
private void loadAdditionalConfigurations() {
// TODO Auto-generated method stub
}
/**
*
*/
private void runPropertiesValidations() {
String configPath = (String) propsMap.get(ConfigPropertyKeys.PROJECT_CONFIGURATION_PATH);
String productToken = (String) propsMap.get(ConfigPropertyKeys.PRODUCT_TOKEN_PROPERTY_KEY);
String productName = (String) propsMap.get(ConfigPropertyKeys.PRODUCT_NAME_PROPERTY_KEY);
String projectToken = (String) propsMap.get(ConfigPropertyKeys.PROJECT_TOKEN_PROPERTY_KEY);
String failErrorLevel = (String) propsMap.get(ConfigPropertyKeys.FAIL_ERROR_LEVEL);
String apiKey = (String) propsMap.get(ConfigPropertyKeys.ORG_TOKEN_PROPERTY_KEY);
String scanComment = (String) propsMap.get(ConfigPropertyKeys.SCAN_COMMENT);
boolean noConfigFlag = (boolean) propsMap.get(ConfigPropertyKeys.CMD_NO_CONFIG);
boolean scanDockerImages = (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_IMAGES);
boolean scanDockerContainers = (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_CONTAINERS);
boolean projectPerFolder = (boolean) propsMap.get(ConfigPropertyKeys.PROJECT_PER_SUBFOLDER);
boolean resolveAllDeps = (boolean) propsMap.get(ConfigPropertyKeys.RESOLVE_ALL_DEPENDENCIES);
boolean mavenResolveDeps = (boolean) propsMap.get(ConfigPropertyKeys.MAVEN_RESOLVE_DEPENDENCIES);
boolean mavenAggregateModules = (boolean) propsMap.get(ConfigPropertyKeys.MAVEN_AGGREGATE_MODULES);
boolean gradleResolveDeps = (boolean) propsMap.get(ConfigPropertyKeys.GRADLE_RESOLVE_DEPENDENCIES);
boolean gradleAggregateModules = (boolean) propsMap.get(ConfigPropertyKeys.GRADLE_AGGREGATE_MODULES);
boolean npmIdentifyByName = (boolean) propsMap.get(ConfigPropertyKeys.NPM_IDENTIFY_BY_NAME_AND_VERSION);
int archiveDepth = (int) propsMap.get(ConfigPropertyKeys.ARCHIVE_EXTRACTION_DEPTH_KEY);
String[] includes = (String[]) propsMap.get(ConfigPropertyKeys.INCLUDES_PATTERN_PROPERTY_KEY);
String[] preProjectIncludes = (String[]) propsMap.get(ConfigPropertyKeys.PROJECT_PER_FOLDER_INCLUDES);
String[] pythonIncludes = (String[]) propsMap.get(ConfigPropertyKeys.PYTHON_REQUIREMENTS_FILE_INCLUDES);
List scanDirs = (List) propsMap.get(ConfigPropertyKeys.CMD_D_SCAN_DIRS);
List offlineRequestFiles = (List) propsMap.get(ConfigPropertyKeys.CMD_OFFLINE_REQUEST_FILES);
ConfigurationValidation configValidation = new ConfigurationValidation();
if (noConfigFlag) {
// TODO: confirm this validation
/* old checkCmdArgsWithoutConfig()
* check if the minimum required settings for running without config file exist
* apiKey & projectName/projectToken & productName/productToken & scannedDirectory
*/
if ((StringUtils.isBlank(apiKey) || scanDirs == null || scanDirs.isEmpty() ||
(StringUtils.isBlank(projectToken) && StringUtils.isBlank(productName))) &&
EuaOfflineMode.VUL.toString().equals(propsMap.get(ConfigPropertyKeys.EUA_OFFLINE))) {
errors.add("The apiKey and project/projectToken parameters are required to perform a scan without a configuration file");
}
} else {
boolean useCommandLineRequestFiles = offlineRequestFiles != null && !offlineRequestFiles.isEmpty();
boolean isMavenMultiModule = !resolveAllDeps && mavenResolveDeps && !mavenAggregateModules;
boolean isGradleMultiModule = !resolveAllDeps && gradleResolveDeps && !gradleAggregateModules;
List configurationErrors = configValidation.getConfigurationErrors(projectPerFolder, projectToken, productName, apiKey, configPath, archiveDepth,
includes, preProjectIncludes, pythonIncludes, scanComment, useCommandLineRequestFiles,
npmIdentifyByName, isMavenMultiModule, isGradleMultiModule);
errors.addAll(configurationErrors);
}
if (scanDockerImages && scanDockerContainers) {
errors.add("Please use either 'docker.scanImages' or 'docker.scanContainers' (not both)");
}
// FAIL_ERROR_LEVEL
if (Constants.ALL.equals(failErrorLevel)) {
if (StringUtils.isBlank(productToken) && StringUtils.isBlank(productName) && StringUtils.isBlank(projectToken)) {
errors.add("failErrorLevel=ALL, Missing Product Identification (productName, productToken or projectToken)");
}
}
/* *********************************************** */
/* **** EUA (Via) properties validations start *** */
/* *********************************************** */
String iaLanguage = (String) propsMap.get(ConfigPropertyKeys.IA_LANGUAGE);
boolean enableIA = (boolean) propsMap.get(ConfigPropertyKeys.ENABLE_IMPACT_ANALYSIS);
errors.addAll(configValidation.validateIaLanguage(iaLanguage, enableIA));
String analyzeMultiModule = (String) propsMap.get(ConfigPropertyKeys.ANALYZE_MULTI_MODULE);
// The config file is not necessary if there is the analyzeMultiModule parameter
if (StringUtils.isBlank(analyzeMultiModule)) {
// if the errors array already contains error message 'Error: the '-appPath' parameter cannot follow the parameter '-d'
// or the error raised if analyzeFrameworksReference is not valid -
// no point in checking for other errors
if (errors.isEmpty() || errors.stream().noneMatch(s -> s.contains(Constants.APP_PATH) || s.contains(Constants.DASH_D) || s.contains(EUA_ANALYZE_FRAMEWORKS))) {
// check properties to ensure via is ready to run
boolean isEnableImpactAnalysis = (boolean) propsMap.get(ConfigPropertyKeys.ENABLE_IMPACT_ANALYSIS);
Map> appPathsToDependencyDirs = (Map>) propsMap.get(ConfigPropertyKeys.APP_PATH_TO_DEPENDENCIES_DIRS);
Set viaAppPaths = appPathsToDependencyDirs.keySet();
if (isEnableImpactAnalysis || EuaOfflineMode.RES.toString().equals(propsMap.get(ConfigPropertyKeys.EUA_OFFLINE))) {
// WSE-1718 - making sure 'jdeps' is installed (instead of checking if the PATH contains jdk/bin)
if (isJdepsFound()) {
if (viaAppPaths.size() <= 1) {// the default appPath size is one (defaultKey = -d property), this one check if the user set more then one appPath
errors.add(Constants.EUA_ERROR + " the command line parameters -appPath and -d are not specified");
} else {
// at this point, checking only the appPath parameters;
// all the other parameters (maven.ignoredScopes, maven.aggregateModules, gradle.aggregateModules OR npm.includeDevDependencies)
// are checked later, only if valid maven or NPM code is found and about to be scanned
checkAppPathsForVia(viaAppPaths, errors);
}
} else {
errors.add(Constants.EUA_ERROR + " Jdeps is not installed. Verify that it is installed");
}
} else if (viaAppPaths.size() > 1) {
errors.add(Constants.EUA_ERROR + " the configuration file parameter enableImpactAnalysis is not set to 'true'");
}
}
} else {
errors.clear();
if ((args.length == 4 || args.length == 6 || args.length == 8) && scanDirs.size() == 1) {
Path path = Paths.get(analyzeMultiModule);
try {
boolean fileExists = path.toFile().exists();
boolean overrideExistingSetup = (boolean) propsMap.get(ConfigPropertyKeys.OVERRIDE_EXISTING_SETUP);
if (!fileExists || overrideExistingSetup) {
File setUpFile = new File(analyzeMultiModule);
boolean fileCreated = true;
if (!fileExists) {
fileCreated = setUpFile.createNewFile();
}
if (fileCreated) {
// Updating property value during validation
setUpMuiltiModuleFile = true;
} else {
errors.add("The system could not create the multi-project setup file. Please contact support.");
}
} else {
errors.add("The file specified for storing multi-module analysis results already exists. Please specify a new file name.");
}
} catch (IOException e) {
errors.add("The system could not create the multi-project setup file : " + path + " " + "Please contact support.");
}
} else {
errors.add("Multi-module analysis could not run due to specified invalid parameters.");
}
}
}
/**
*
*/
private void initConfigWrappers() {
agent = new AgentConfiguration(this.propsMap);
endpoint = new EndPointConfiguration(this.propsMap);
offline = new OfflineConfiguration(this.propsMap);
resolver = new ResolverConfiguration(this.propsMap);
scm = new ScmConfiguration(this.propsMap);
sender = new SenderConfiguration(this.propsMap);
request = new RequestConfiguration(this.propsMap);
remoteDockerConfiguration = new RemoteDockerConfiguration(this.propsMap);
if ((boolean) propsMap.get(SERVERLESS_SCAN_FUNCTIONS)) {
serverlessConfiguration = new ServerlessConfiguration(this.propsMap);
}
}
/**
*
*/
private boolean getBooleanValue(Object obj) {
return Boolean.valueOf(String.valueOf(obj));
}
/**
*
*/
private boolean isJdepsFound() {
Cli cli = new Cli();
boolean installed = cli.runCmdWithErrorOutput(Constants.DOT, cli.getCommandParams("jdeps", Constants.DASH + Constants.VERSION));
return installed;
}
/**
* check validation for appPath property, first check if the path is exist and then if this path is not a directory
*/
private boolean checkAppPathsForVia(Set keySet, List errors) {
for (String key : keySet) {
if (!key.equals(Constants.DEFAULT_KEY)) {
File file = new File(key);
if (!file.exists() || !file.isFile()) {
errors.add(Constants.EUA_ERROR + " the -appPath parameter references an invalid file path. Check that the -appPath parameter specifies a valid path");
return false;
} else if (!file.getName().endsWith(Constants.DOT + Constants.JAR) && !file.getName().endsWith(Constants.DOT + Constants.WAR)
&& !file.getName().endsWith(Constants.DOT + Constants.EAR) && !file.getName().equals(Constants.PACKAGE_JSON)
&& !file.getName().equalsIgnoreCase(Constants.PIPFILE) && !file.getName().equalsIgnoreCase(Constants.PYTHON_REQUIREMENTS)) {
errors.add(Constants.EUA_ERROR + " the system cannot locate a valid analysis target. Check that the -appPath parameter specifies a path to a valid file");
return false;
} else {
return true;
}
}
}
return false;
}
/**
* Effective Usage Analysis parameters -xPath and -appPath
*/
@SuppressWarnings("unchecked")
private void initializeDependencyDirsForEUA() {
Map> appPaths = new HashMap<>();
String xPaths = (String) propsMap.get(ConfigPropertyKeys.X_PATHS);
List dependencyDirs = (List) propsMap.get(ConfigPropertyKeys.CMD_D_SCAN_DIRS);
if (StringUtils.isNotBlank(xPaths)) {
try {
String textFromFile = new String(Files.readAllBytes(Paths.get(xPaths)), StandardCharsets.UTF_8);
textFromFile = textFromFile.replaceAll(Constants.COMMA + Constants.WHITESPACE, Constants.COMMA);
textFromFile = textFromFile.replaceAll(System.lineSeparator(), Constants.WHITESPACE);
String[] extractedArgs = textFromFile.split(Constants.WHITESPACE);
if (extractedArgs != null && extractedArgs.length > 0) {
initializeDependencyDirsToAppPathForEUA(extractedArgs, appPaths, dependencyDirs);
}
for (Map.Entry> entry : appPaths.entrySet()) {
dependencyDirs.addAll(entry.getValue());
}
} catch (IOException e) {
errors.add("Error: Could not read the xPaths file: " + xPaths);
}
} else if (this.args != null && this.args.length > 0) {
initializeDependencyDirsToAppPathForEUA(this.args, appPaths, dependencyDirs);
} else { // WSE-1386 - if this there's no param in the cli - this collection must not be empty
appPaths.put(Constants.DEFAULT_KEY, new HashSet<>(dependencyDirs));
}
propsMap.put(ConfigPropertyKeys.APP_PATH_TO_DEPENDENCIES_DIRS, appPaths);
}
/**
* Effective Usage Analysis related
*/
private void initializeDependencyDirsToAppPathForEUA(String[] args, Map> appPathsToDependencyDirs, List dependencyDirs) {
if (Arrays.asList(args).stream().filter(s -> s.equals(Constants.APP_PATH)).collect(Collectors.toList()).size() > 1) {
errors.add(Constants.EUA_ERROR + " the command line parameter -appPath is specified more than once");
return;
}
if (Arrays.asList(args).stream().filter(s -> s.equals(Constants.DASH_D)).collect(Collectors.toList()).size() > 1) {
errors.add(Constants.EUA_ERROR + " the command line parameter -d is specified more than once");
return;
}
boolean wasDir = false;
for (int i = 0; i < args.length; i++) {
if (!wasDir && args[i].equals(Constants.APP_PATH)) {
if (i + 3 < args.length && args[i + 2].equals(Constants.DASH_D)) {
List paths = Arrays.asList(args[i + 3].split(Constants.COMMA));
Set value = new HashSet<>();
value.addAll(paths);
appPathsToDependencyDirs.put(args[i + 1], value);
i = i + 3;
} else {
// WSE-2036
errors.add("Error: the '-appPath' parameter must have a '-d' parameter next to it.");
return;
}
} else if (wasDir && args[i].equals(Constants.APP_PATH)) {
errors.add("Error: the '-appPath' parameter cannot follow the parameter '-d'.");
break;
} else if (args[i].equals(Constants.DASH + Constants.DIRECTORY)) {
if (i + 1 < args.length) {
if (appPathsToDependencyDirs.containsKey(Constants.DEFAULT_KEY)) {
appPathsToDependencyDirs.get(Constants.DEFAULT_KEY).addAll(Arrays.asList(args[i + 1].split(Constants.COMMA)));
} else {
List paths = Arrays.asList(args[i + 1].split(Constants.COMMA));
Set value = new HashSet<>();
value.addAll(paths);
appPathsToDependencyDirs.put(Constants.DEFAULT_KEY, value);
}
i++;
} else {
errors.add("Error: there is not path after the '-d' parameter.");
return;
}
wasDir = true;
}
}
if (!wasDir) {
appPathsToDependencyDirs.put(Constants.DEFAULT_KEY, new HashSet<>(dependencyDirs));
}
}
/* ******************************************************************************************** */
/* ************************************** Public Getters ************************************** */
/* ******************************************************************************************** */
public Map getPropertiesMap() {
return this.propsMap;
}
public String getPropertyAsString(String propertyName) {
String value = (String) this.propsMap.get(propertyName);
if (StringUtils.isNotBlank(value)) {
return value;
}
return Constants.EMPTY_STRING;
}
public boolean getPropertyAsBoolean(String propertyName) {
return (boolean) this.propsMap.get(propertyName);
}
public int getPropertyAsInteger(String propertyName) {
return (int) this.propsMap.get(propertyName);
}
public AgentConfiguration getAgent() {
return agent;
}
public EndPointConfiguration getEndpoint() {
return endpoint;
}
public OfflineConfiguration getOffline() {
return offline;
}
public RemoteDockerConfiguration getRemoteDocker() {
return remoteDockerConfiguration;
}
public RequestConfiguration getRequest() {
return request;
}
public ResolverConfiguration getResolver() {
return resolver;
}
public ScmConfiguration getScm() {
return scm;
}
public SenderConfiguration getSender() {
return sender;
}
public ServerlessConfiguration getServerlessConfiguration() {
return serverlessConfiguration;
}
public RequestConfiguration getOfflineRequestsFilesRequest() {
return this.offlineRequestFilesRequest;
}
public void setOfflineRequestsFilesRequest(RequestConfiguration offlineRequestFilesRequest) {
this.offlineRequestFilesRequest = offlineRequestFilesRequest;
}
public boolean getUseCommandLineRequestFiles() {
return useCommandLineRequestFiles;
}
public List getErrors() {
return errors;
}
public Statistics getStatistics() {
return statistics;
}
/* ********** backward compatibility ********** */
@Override
public String toString() {
try {
projectPerFolder = (boolean) propsMap.get(ConfigPropertyKeys.PROJECT_PER_SUBFOLDER);
connectionTimeOut = (int) propsMap.get(ClientConstants.CONNECTION_TIMEOUT_KEYWORD);
fileListPath = (String) propsMap.get(ConfigPropertyKeys.CMD_FILE_LIST_PATH);
dependencyDirs = (List) propsMap.get(ConfigPropertyKeys.CMD_D_SCAN_DIRS);
configFilePath = (String) propsMap.get(ConfigPropertyKeys.PROJECT_CONFIGURATION_PATH);
scanPackageManager = (boolean) propsMap.get(ConfigPropertyKeys.SCAN_PACKAGE_MANAGER);
scanDockerImages = (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_IMAGES);
scanDockerContainers = (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_CONTAINERS);
scanServerlessFunctions = (boolean) propsMap.get(SERVERLESS_SCAN_FUNCTIONS);
logLevel = (String) propsMap.get(ConfigPropertyKeys.LOG_LEVEL_KEY);
return "UA Configuration {" + OsUtils.NEW_LINE + ConfigurationsStringUtils.toString(this) + "}";
}
catch (NullPointerException e){
return "UA Configuration { toString error:{+ " + e.getMessage() + "}";
}
}
@SuppressWarnings("unchecked")
public void setOfflineRequestFiles(List offlineRequestFiles) {
List files = (List) propsMap.get(ConfigPropertyKeys.CMD_OFFLINE_REQUEST_FILES);
files.addAll(offlineRequestFiles);
}
@SuppressWarnings("unchecked")
public List getOfflineRequestFiles() {
return (List) propsMap.get(ConfigPropertyKeys.CMD_OFFLINE_REQUEST_FILES);
}
public boolean isScanProjectManager() {
return (boolean) propsMap.get(ConfigPropertyKeys.SCAN_PACKAGE_MANAGER);
}
public boolean isScanDockerImages() {
return (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_IMAGES);
}
public boolean isScanDockerContainers() {
return (boolean) propsMap.get(ConfigPropertyKeys.SCAN_DOCKER_CONTAINERS);
}
public boolean isScanServerlessFunctions() {
return (boolean) propsMap.get(ConfigPropertyKeys.SERVERLESS_SCAN_FUNCTIONS);
}
public boolean isSetUpMuiltiModuleFile() {
return this.setUpMuiltiModuleFile;
}
public boolean isScanImagesTar() {
return (boolean) propsMap.get(ConfigPropertyKeys.SCAN_TAR_IMAGES);
}
public boolean deleteTarImages() {
return (boolean) propsMap.get(ConfigPropertyKeys.DELETE_TAR_FILES);
}
@SuppressWarnings("unchecked")
public List getDependencyDirs() {
return (List) propsMap.get(ConfigPropertyKeys.CMD_D_SCAN_DIRS);
}
@SuppressWarnings("unchecked")
public Map> getAppPathsToDependencyDirs() {
return (Map>) propsMap.get(ConfigPropertyKeys.APP_PATH_TO_DEPENDENCIES_DIRS);
}
public String[] getAnalyzeMultiModulePackageManagers() {
return (String[]) propsMap.get(ConfigPropertyKeys.ANALYZE_MULTI_MODULE_PACKAGE_MANAGERS);
}
public String[] getAnalyzeMultiModuleExclusions() {
String val = (String) propsMap.get(ConfigPropertyKeys.CMD_ANALYZE_MULTI_MODULE_EXCLUSIONS);
return val.split(ConfigPropertyDefinitions.INCLUDES_EXCLUDES_SEPARATOR_REGEX);
}
public String getAnalyzeMultiModule() {
return (String) propsMap.get(ConfigPropertyKeys.ANALYZE_MULTI_MODULE);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy