com.metaeffekt.artifact.enrichment.matching.VulnerabilitiesFromCpeEnrichment Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.enrichment.matching;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.version.AllCategorizedPartsVersionImpl;
import com.metaeffekt.artifact.analysis.version.Version;
import com.metaeffekt.artifact.analysis.version.curation.VersionContext;
import com.metaeffekt.artifact.analysis.vulnerability.CommonEnumerationUtil;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.InventoryAttribute;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.warnings.InventoryWarningEntry;
import com.metaeffekt.artifact.enrichment.InventoryEnricher;
import com.metaeffekt.artifact.enrichment.configurations.VulnerabilitiesFromCpeEnrichmentConfiguration;
import com.metaeffekt.mirror.contents.base.AmbDataClass;
import com.metaeffekt.mirror.contents.base.DataSourceIndicator;
import com.metaeffekt.mirror.contents.base.VulnerabilityContextInventory;
import com.metaeffekt.mirror.contents.store.ContentIdentifierStore;
import com.metaeffekt.mirror.contents.store.VulnerabilityTypeIdentifier;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import com.metaeffekt.mirror.contents.vulnerability.VulnerableSoftwareVersionRangeCpe;
import com.metaeffekt.mirror.query.VulnerabilityIndexQuery;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.springett.parsers.cpe.Cpe;
import us.springett.parsers.cpe.exceptions.CpeValidationException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.metaeffekt.core.inventory.processor.model.Constants.ASTERISK;
public abstract class VulnerabilitiesFromCpeEnrichment extends InventoryEnricher {
private final static Logger LOG = LoggerFactory.getLogger(VulnerabilitiesFromCpeEnrichment.class);
protected VulnerabilitiesFromCpeEnrichmentConfiguration configuration;
public VulnerabilitiesFromCpeEnrichment(VulnerabilitiesFromCpeEnrichmentConfiguration configuration) {
this.configuration = configuration;
}
public void setConfiguration(VulnerabilitiesFromCpeEnrichmentConfiguration configuration) {
this.configuration = configuration;
}
protected abstract VulnerabilityIndexQuery getVulnerabilityQuery();
protected abstract ContentIdentifierStore.ContentIdentifier getVulnerabilitySource();
protected void enrichVulnerabilitiesForCpe(VulnerabilityContextInventory vInventory, Artifact artifact) {
final Map> vulnerabilitiesForCpes = this.queryVulnerabilitiesForArtifact(vInventory, artifact);
// collect matched cpes
final Set matchedCpes = new HashSet<>();
vulnerabilitiesForCpes.keySet().forEach(cpe -> matchedCpes.add(CommonEnumerationUtil.toCpe22UriOrFallbackToCpe23FS(cpe)));
// add matched cpes as field in the artifact
if (!matchedCpes.isEmpty()) {
artifact.set(InventoryAttribute.MATCHED_CPES,
matchedCpes.stream().filter(cpe -> cpe != null && !cpe.equals("null")).collect(Collectors.joining(", ")));
} else {
artifact.set(InventoryAttribute.MATCHED_CPES, null);
}
}
public Map> queryVulnerabilitiesForArtifact(VulnerabilityContextInventory vInventory, Artifact artifact) {
// represents return value: collects vulnerabilities for artifact
final Map> aggregatedVulnerabilitiesForCpes = new LinkedHashMap<>();
// only used for tracking the amount of vulnerabilities for the config parameter
final Set aggregatedVulnerabilities = new HashSet<>();
// The aggregation of CPEs is often version agnostic. This is why a specification on the artifact version is
// required below.
final List artifactCpes = CommonEnumerationUtil.parseEffectiveCpe(artifact);
// query CVEs for CPE with matching version
for (Cpe cpe : artifactCpes) {
try {
final Optional optionalQueryCpe = deriveQueryCpe(vInventory, this.getEnrichmentName(), artifact, cpe);
if (!optionalQueryCpe.isPresent()) {
continue;
}
final Cpe queryCpe = optionalQueryCpe.get();
final Map vulnerabilitiesWithSources = getVulnerabilityQuery().findVulnerabilitiesByFlatAffectedConfigurationRetainSource(queryCpe);
final List vulnerabilitiesForCpes = vulnerabilitiesWithSources.keySet().stream()
.map(AmbDataClass::getId)
.map(vInventory::findOrCreateVulnerabilityByName)
.collect(Collectors.toList());
if (!vulnerabilitiesForCpes.isEmpty()) {
for (Vulnerability vulnerability : vulnerabilitiesForCpes) {
final VulnerableSoftwareVersionRangeCpe affectedConfiguration = vulnerabilitiesWithSources.get(vulnerability);
final DataSourceIndicator matchingSource = DataSourceIndicator.cpe(artifact, this.getVulnerabilitySource(), queryCpe, affectedConfiguration != null ? affectedConfiguration.toString() : null);
vulnerability.addMatchingSource(matchingSource);
}
}
aggregatedVulnerabilities.addAll(vulnerabilitiesForCpes.stream().map(Vulnerability::getId).collect(Collectors.toSet()));
aggregatedVulnerabilitiesForCpes.put(cpe, vulnerabilitiesForCpes);
if (aggregatedVulnerabilities.size() > configuration.getMaxCorrelatedVulnerabilitiesPerArtifact()) {
break;
}
} catch (Exception e) {
throw new RuntimeException("Failed to query vulnerabilities for artifact [" + artifact.getId() + "] on CPE [" + CommonEnumerationUtil.toCpe22UriOrFallbackToCpe23FS(cpe) + "]: " + e.getMessage(), e);
}
}
if (aggregatedVulnerabilities.size() > configuration.getMaxCorrelatedVulnerabilitiesPerArtifact()) {
LOG.warn("Found [{}] vulnerabilities for artifact [{}] but only the first [{}] will be considered.",
aggregatedVulnerabilities.size(), artifact.getId(), configuration.getMaxCorrelatedVulnerabilitiesPerArtifact());
vInventory.getInventoryWarnings().addArtifactWarning(new InventoryWarningEntry<>(artifact,
"Found " + aggregatedVulnerabilities.size() + " vulnerabilities but only the first "
+ configuration.getMaxCorrelatedVulnerabilitiesPerArtifact() + " will be considered.",
this.getEnrichmentName()
));
int count = 0;
boolean clearAll = false;
for (Map.Entry> entry : aggregatedVulnerabilitiesForCpes.entrySet()) {
final List vulnerabilities = entry.getValue();
count += vulnerabilities.size();
if (count > configuration.getMaxCorrelatedVulnerabilitiesPerArtifact()) {
if (clearAll) {
vulnerabilities.clear();
} else {
final int removeCount = count - configuration.getMaxCorrelatedVulnerabilitiesPerArtifact();
vulnerabilities.subList(vulnerabilities.size() - removeCount, vulnerabilities.size()).clear();
clearAll = true;
}
}
}
}
final VulnerabilityTypeIdentifier vulnerabilityIdentifier = this.getVulnerabilityQuery().getVulnerabilityType();
for (List vulnerabilities : aggregatedVulnerabilitiesForCpes.values()) {
for (Vulnerability vulnerability : vulnerabilities) {
vulnerability.setSourceIdentifier(vulnerabilityIdentifier);
}
}
return aggregatedVulnerabilitiesForCpes;
}
/**
* A CPE that can be used to query an index for vulnerabilities. The version and update are derived from the
* artifact and the cpe in combination.
* This differs from a normal {@link Version#of(String, String)} in that it replaces certain characters in the
* version and update before parsing it.
*
* @param vInventory may be null. If set, warnings will be added to the inventory.
* @param enrichmentName The name of the enrichment.
* @param artifact The artifact to use the version from to generate the query version.
* @param cpe The cpe to use the version and update from to generate the query version.
* @return The query cpe.
*/
public static Optional deriveQueryCpe(VulnerabilityContextInventory vInventory, String enrichmentName, Artifact artifact, Cpe cpe) {
final Version queryVersion = deriveQueryVersion(artifact, cpe);
try {
return Optional.of(CommonEnumerationUtil.builder()
.from(cpe)
.version(replaceIfNotNull(queryVersion.getVersion(), " ", "_"))
.update(replaceIfNotNull(queryVersion.getUpdate(), " ", "_"))
.build());
} catch (CpeValidationException e) {
LOG.warn("Failed to build CPE for querying vulnerability data: [{}] [{}]: {}", cpe, queryVersion, e.getMessage());
if (vInventory != null) {
vInventory.getInventoryWarnings().addArtifactWarning(new InventoryWarningEntry<>(artifact,
"Failed to build CPE for querying vulnerability data: " + cpe + " " + queryVersion + ": " + e.getMessage(),
enrichmentName != null ? enrichmentName : VulnerabilitiesFromCpeEnrichment.class.getName()
));
}
return Optional.empty();
}
}
private static String replaceIfNotNull(String str, String value, String replacement) {
if (str != null) {
return str.replace(value, replacement);
} else {
return null;
}
}
/**
* Derive the query version. The artifact version and the cpeVersion are combined.
*
* @param artifact The artifact to get the version from.
* @param cpe The cpe to get the version from.
* @return The derived version.
*/
public static Version deriveQueryVersion(Artifact artifact, Cpe cpe) {
final Version artifactVersion = deriveArtifactVersion(artifact);
final Version cpeVersion = Version.of(cpe.getVersion(), cpe.getUpdate(), VersionContext.fromCpe(cpe));
final Version modulateVersion = modulateVersions(artifactVersion, cpeVersion);
if (!artifactVersion.toString().equals(modulateVersion.toString())) {
LOG.info("Derived query version for artifact [{}] and CPE [{}]: [{}] + [{}] = [{}]", artifact.getId(), CommonEnumerationUtil.toCpe22UriOrFallbackToCpe23FS(cpe), artifactVersion, cpeVersion, modulateVersion);
}
return modulateVersion;
}
public static Version deriveArtifactVersion(Artifact artifact) {
String version = extractCpeVersion(artifact);
String update = null;
final Pattern compile = Pattern.compile("^.*p[0-9]+$");
final Matcher matcher = compile.matcher(version);
if (matcher.matches()) {
int pIndex = version.lastIndexOf("p");
update = version.substring(pIndex);
version = version.substring(0, pIndex);
}
return Version.of(version, update, VersionContext.fromArtifact(artifact));
}
private static Version modulateVersions(Version artifactVersion, Version cpeQueryVersion) {
if (artifactVersion instanceof AllCategorizedPartsVersionImpl && (cpeQueryVersion == null || cpeQueryVersion instanceof AllCategorizedPartsVersionImpl)) {
final AllCategorizedPartsVersionImpl artifactVersionImpl = (AllCategorizedPartsVersionImpl) artifactVersion;
final AllCategorizedPartsVersionImpl cpeQueryVersionImpl = (AllCategorizedPartsVersionImpl) cpeQueryVersion;
final String artifactVersionPart = artifactVersionImpl.toStringPreModifierPart();
final String artifactUpdatePart = artifactVersionImpl.toStringModifierPart();
// check whether query was specific
if (cpeQueryVersionImpl == null) {
return Version.of(artifactVersionPart, artifactUpdatePart);
} else {
final String cpeQueryVersionPart = cpeQueryVersionImpl.toStringPreModifierPart();
final String cpeQueryUpdatePart = cpeQueryVersionImpl.toStringModifierPart();
final String effectiveVersion;
final String effectiveUpdate;
if (!ASTERISK.equals(cpeQueryVersion.getVersion()) && StringUtils.hasText(cpeQueryVersion.getVersion()) && StringUtils.hasText(cpeQueryVersionPart)) {
effectiveVersion = cpeQueryVersionPart;
} else {
effectiveVersion = artifactVersionPart;
}
if (!ASTERISK.equals(cpeQueryVersionImpl.getUpdate()) && StringUtils.hasText(cpeQueryVersion.getUpdate()) && StringUtils.hasText(cpeQueryUpdatePart)) {
effectiveUpdate = cpeQueryUpdatePart;
} else {
effectiveUpdate = artifactUpdatePart;
}
if (StringUtils.hasText(effectiveVersion) && StringUtils.hasText(effectiveUpdate)) {
return Version.of(effectiveVersion, effectiveUpdate);
} else if (StringUtils.hasText(effectiveVersion)) {
return Version.of(effectiveVersion, artifactVersion.getUpdate());
} else {
return Version.of(artifactVersion.getVersion(), artifactVersion.getUpdate());
}
}
} else {
// old implementation
LOG.warn("Using old implementation for version modulation. Please update to the new implementation [{}] [{}]", artifactVersion, cpeQueryVersion);
String version = artifactVersion.getVersion();
String update = artifactVersion.getUpdate();
// check whether query was specific
if (cpeQueryVersion == null) {
return artifactVersion;
} else if (!ASTERISK.equals(cpeQueryVersion.getVersion())) { // also checks for null
version = cpeQueryVersion.getVersion();
if (!ASTERISK.equals(cpeQueryVersion.getUpdate())) {
update = cpeQueryVersion.getUpdate();
}
}
return Version.of(version, update);
}
}
public static String extractCpeVersion(Artifact artifact) {
final String strippedVersion = StringUtils.isEmpty(artifact.getVersion()) ? ASTERISK : artifact.getVersion().trim();
if (strippedVersion.equalsIgnoreCase("unspecific") || strippedVersion.equalsIgnoreCase("undefined")) {
return ASTERISK;
}
if (strippedVersion.contains(":")) {
return strippedVersion.substring(strippedVersion.indexOf(":") + 1);
}
if (strippedVersion.contains("+")) {
return strippedVersion.substring(0, strippedVersion.indexOf("+"));
}
if (strippedVersion.contains("~")) {
return strippedVersion.substring(0, strippedVersion.indexOf("~"));
}
if (strippedVersion.contains("-")) {
return strippedVersion.substring(0, strippedVersion.indexOf("-"));
}
if (strippedVersion.contains(".RELEASE")) {
return strippedVersion.substring(0, strippedVersion.indexOf(".RELEASE"));
}
if (strippedVersion.contains(".FINAL")) {
return strippedVersion.substring(0, strippedVersion.indexOf(".FINAL"));
}
if (strippedVersion.contains(".Release")) {
return strippedVersion.substring(0, strippedVersion.indexOf(".Release"));
}
if (strippedVersion.contains(".Final")) {
return strippedVersion.substring(0, strippedVersion.indexOf(".Final"));
}
return strippedVersion;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy