com.metaeffekt.artifact.enrichment.other.timeline.VulnerabilityTimeline Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.enrichment.other.timeline;
import com.metaeffekt.artifact.analysis.utils.CountdownTimer;
import com.metaeffekt.artifact.analysis.utils.LruLinkedHashMap;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.version.Version;
import com.metaeffekt.artifact.analysis.version.curation.VersionContext;
import com.metaeffekt.artifact.analysis.vulnerability.CommonEnumerationUtil;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.VersionComparator;
import com.metaeffekt.artifact.enrichment.matching.VulnerabilitiesFromCpeEnrichment;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import lombok.Getter;
import org.apache.commons.lang3.ObjectUtils;
import org.json.JSONObject;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.metaeffekt.core.security.cvss.CvssSeverityRanges;
import org.metaeffekt.core.security.cvss.v2.Cvss2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.springett.parsers.cpe.Cpe;
import us.springett.parsers.cpe.exceptions.CpeValidationException;
import us.springett.parsers.cpe.values.Part;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Used for the vulnerability assessment dashboard.
* Generates a timeline of cpes to a given vendor product pair which can be used to help decide which version of the
* artifact should be used in the future.
*/
public class VulnerabilityTimeline {
private final static Logger LOG = LoggerFactory.getLogger(VulnerabilityTimeline.class);
private final static Cvss2 DUMMY_CVSS = new Cvss2();
/**
* This regular expression pattern is used to match version numbers in a string.
* The pattern supports version numbers with any number of segments, and each segment can be separated by either a period (".") or a colon (":").
* The pattern matches version numbers that consist of alphanumeric characters, separated by either a period (".") or a colon (":").
* Each segment is captured in a group.
*/
private final static Pattern VERSION_PATTERN = Pattern.compile("([0-9a-zA-Z]+)(?:[.:]+)?([0-9a-zA-Z]+)?(?:[.:]+)?((?:(?:[0-9a-zA-Z]+)?(?:[.:]+)?)+)");
private final String vendor;
private final String product;
private final VulnerabilityTimelineGenerator configuration;
private final List versions;
private final List relevantTimelineVulnerabilities;
public VulnerabilityTimeline(String vendor, String product, VulnerabilityTimelineGenerator configuration) {
this.vendor = vendor;
this.product = product;
this.configuration = configuration;
this.versions = this.generateTimelineVersions(vendor, product);
this.relevantTimelineVulnerabilities = this.findVulnerabilitiesForAnyVersion(vendor, product).stream()
.map(Vulnerability::getId)
.filter(v -> configuration.getRelevantVulnerabilities().contains(v))
.collect(Collectors.toList());
}
public String getVendor() {
return vendor;
}
public String getProduct() {
return product;
}
private List generateTimelineVersions(String vendor, String product) {
final List versions = new ArrayList<>();
final CountdownTimer perTimelineTimer = new CountdownTimer(configuration.getVadConfiguration().getMaximumTimeSpentPerTimeline() * 1000L);
perTimelineTimer.start();
configuration.getTimelineGenerationTime().start();
try {
final List unsortedCpeSet = configuration.getCpeDictionaryQuery().findCpeByVendorProduct(vendor, product);
if (unsortedCpeSet == null || unsortedCpeSet.isEmpty()) {
LOG.error("NVD mirror did not contain any CPE for vendor/product [{} {}]", vendor, product);
return versions;
}
LOG.info("Generating vulnerability timeline for [{} {}] with [~{}] versions", vendor, product, unsortedCpeSet.size());
final List cpeList = unsortedCpeSet.stream()
.sorted((o1, o2) -> VersionComparator.INSTANCE.compare(o1.getVersion(), o2.getVersion()))
.distinct()
.collect(Collectors.toList());
int totalVulnerabilities = 0;
final long startTime = System.currentTimeMillis();
long lastLogTime = 0;
// iterate the list backwards to make sure the vulnerabilityTimelineVersionLimit does not cut off the latter versions
for (int i = cpeList.size() - 1; i >= 0; i--) {
if (configuration.getVadConfiguration().getMaximumVersionsPerTimeline() != -1 && versions.size() > configuration.getVadConfiguration().getMaximumVersionsPerTimeline()) {
LOG.info("Configured limit of [{}] versions has been reached for vulnerability timeline [{} {}], skipping [{}/{}]",
configuration.getVadConfiguration().getMaximumVersionsPerTimeline(), vendor, product, i, cpeList.size());
break;
}
if (i % 100 == 0) {
final long passedTime = System.currentTimeMillis() - startTime;
if (passedTime - lastLogTime > 1000 * 10) {
lastLogTime = passedTime;
final int progressI = cpeList.size() - i;
final long estimatedTime = (passedTime / progressI) * (cpeList.size() - progressI);
LOG.info("Generating vulnerability timeline for [{} {}] with [{}] versions, currently at [{}/{}] ({}s passed, ~{}s remaining)",
vendor, product, cpeList.size(), progressI, cpeList.size(), passedTime / 1000, estimatedTime / 1000);
if (configuration.getTimelineGenerationTime().isEndReached()) {
LOG.info("Timeline generation time limit of [{}]s has been reached, skipping [{}/{}]",
configuration.getVadConfiguration().getMaximumTimeSpentOnTimelines(), i, cpeList.size());
break;
} else if (perTimelineTimer.isEndReached()) {
LOG.info("Timeline generation time limit of [{}]s has been reached for timeline [{} {}], skipping [{}/{}]",
configuration.getVadConfiguration().getMaximumTimeSpentPerTimeline(), vendor, product, i, cpeList.size());
break;
}
}
}
final Cpe cpe = cpeList.get(i);
final TimelineVersion version = generateTimelineVersion(null, cpe);
if (version == null) continue;
totalVulnerabilities += version.getVulnerabilityCount();
versions.add(version);
if (totalVulnerabilities > configuration.getVadConfiguration().getMaximumVulnerabilitiesPerTimeline()) {
LOG.info("Configured limit of [{}] vulnerabilities has been reached for vulnerability timeline [{} {}], skipping [{}/{}]",
configuration.getVadConfiguration().getMaximumVulnerabilitiesPerTimeline(), vendor, product, i, cpeList.size());
break;
}
}
// since the user might specify a max timeline size, the overhanging vulnerabilities have to be cropped from
// the right, which is why we have to go through in the reverse order when generating the timeline above.
// this is why the versions have to be reversed afterwards:
Collections.reverse(versions);
LOG.info("Generated vulnerability timeline for [{} {}] with [{}] versions and [{}] vulnerabilities",
vendor, product, versions.size(), totalVulnerabilities);
} catch (Exception e) {
LOG.error("Failed to generate vulnerability timeline for [{} {}]", vendor, product, e);
} finally {
configuration.getTimelineGenerationTime().stop();
perTimelineTimer.stop();
}
return versions;
}
private List findVulnerabilitiesForAnyVersion(String vendor, String product) {
final Cpe checkCpe;
try {
checkCpe = CommonEnumerationUtil.builder()
.part(Part.ANY)
.vendor(vendor)
.product(product)
.version("*")
.build();
} catch (CpeValidationException e) {
LOG.warn("Failed to generate CPE for vendor/product [{} {}]", vendor, product, e);
return new ArrayList<>();
}
return configuration.getVulnerabilityQuery().findVulnerabilitiesByFlatAffectedConfiguration(checkCpe);
}
/**
* Manages to get a hit rate of 10-50%, depending on the specific inventory and how repetitive it's contents are.
*/
private final static LruLinkedHashMap timelineVersionCache = new LruLinkedHashMap<>(1000);
private TimelineVersion generateTimelineVersion(Artifact artifact, Cpe cpe) {
if ((artifact == null || StringUtils.isEmpty(artifact.getVersion()) || artifact.getVersion().equals("*") || artifact.getVersion().equals("-"))
&& (cpe.getVersion().equals("*") || cpe.getVersion().equals("-"))) {
return null;
}
final Cpe queryCpe;
final TimelineVersion version;
if (artifact != null) {
queryCpe = VulnerabilitiesFromCpeEnrichment.deriveQueryCpe(null, null, artifact, cpe).orElse(cpe);
version = new TimelineVersion(Version.of(artifact.getVersion(), VersionContext.fromArtifact(artifact)));
} else {
queryCpe = cpe;
version = new TimelineVersion(Version.of(queryCpe.getVersion(), queryCpe.getUpdate(), VersionContext.fromCpe(queryCpe)));
}
final String cacheKey = CommonEnumerationUtil.toCpe22UriOrFallbackToCpe23FS(queryCpe);
if (timelineVersionCache.containsKey(cacheKey)) {
return timelineVersionCache.get(cacheKey);
}
final List vulnerabilities = configuration.getVulnerabilityQuery().findVulnerabilitiesByFlatAffectedConfiguration(queryCpe);
for (Vulnerability vulnerability : vulnerabilities) {
vulnerability.selectEffectiveCvssVectors(configuration.getCentralSecurityPolicyConfiguration());
final CvssSeverityRanges.SeverityRange severity = configuration.getCentralSecurityPolicyConfiguration().getCvssSeverityRanges().getRange(
ObjectUtils.firstNonNull(
vulnerability.getCvssSelectionResult().getSelectedInitialCvss(),
new Cvss2()
).getBakedScores().getNormalizedBaseScore()
);
if (configuration.getRelevantVulnerabilities().contains(vulnerability.getId())) {
version.putSeverityWithVulnerabilityIdentifier(severity, vulnerability.getId());
} else {
version.putSeverity(severity);
}
}
timelineVersionCache.put(cacheKey, version);
return version;
}
public boolean containsVulnerability(String vulnerabilityId) {
return relevantTimelineVulnerabilities.contains(vulnerabilityId);
}
public boolean containsVulnerability(Vulnerability vulnerability) {
return relevantTimelineVulnerabilities.contains(vulnerability.getId());
}
public List getVersions() {
return versions;
}
@Override
public String toString() {
return new JSONObject()
.put("vendor", vendor)
.put("product", product)
.put("versions", versions.size())
.toString();
}
public List generateCustomVersionsFromArtifacts(Collection artifacts) {
final Set artifactCpes = new HashSet<>();
final Map distinctArtifactVersions = new LinkedHashMap<>();
for (Artifact artifact : artifacts) {
try {
artifactCpes.add(
CommonEnumerationUtil.builder()
.part(Part.APPLICATION)
.vendor(vendor)
.product(product)
// .version(artifact.getVersion())
.build()
);
} catch (CpeValidationException e) {
LOG.warn("Cannot create CPE from artifact [{}] for vulnerability timeline [{} {}]: {}",
artifact.getId(), vendor, product, e.getMessage());
}
if (StringUtils.hasText(artifact.getVersion()) && !distinctArtifactVersions.containsKey(artifact.getVersion())) {
distinctArtifactVersions.put(artifact.getVersion(), artifact);
}
}
final List versions = new ArrayList<>();
for (Cpe artifactCpe : artifactCpes) {
for (Artifact artifact : distinctArtifactVersions.values()) {
final TimelineVersion version = generateTimelineVersion(artifact, artifactCpe);
if (version == null) continue;
versions.add(version);
}
}
return versions;
}
public List getOfficialAndArtifactMergedVersions(Collection additionalTimelineVersions) {
final List mergedVersions = new ArrayList<>(this.versions);
// only add version if not already present
for (TimelineVersion addVersion : additionalTimelineVersions) {
if (mergedVersions.stream().noneMatch(v -> v.getVersion().matchesVersionOf(addVersion.getVersion()))) {
mergedVersions.add(addVersion);
}
}
mergedVersions.sort(((o1, o2) -> VersionComparator.INSTANCE.compare(o1.getVersionUpdate(), o2.getVersionUpdate())));
return mergedVersions;
}
/**
* Not all Cpe versions are relevant. Only show the ones that:
*
* - Have a version that is contained in the artifacts set
* - Is the latest Cpe version
* - The next Cpe version has a different major version
* - The next Cpe version has a different amount of vulnerabilities
*
*
* @param artifacts The artifacts to check for.
* @param additionalTimelineVersions The Cpe versions that are already present in the timeline.
* @return A list with only the relevant Cpe versions.
*/
public List getReducedOfficialAndArtifactMergedCpeVersions(Set artifacts, List additionalTimelineVersions) {
final List versions = getOfficialAndArtifactMergedVersions(additionalTimelineVersions);
// if there are only few versions anyway, return the whole list
if (versions.size() <= 10)
return versions;
// build a set of Cpes that are required no matter what:
// have a version that is contained in the artifacts set OR is the latest Cpe version
final Set requiredCpeVersions = new HashSet<>(createVersionsForArtifactVersion(artifacts, additionalTimelineVersions));
requiredCpeVersions.add(versions.get(versions.size() - 1));
requiredCpeVersions.addAll(additionalTimelineVersions);
final List reducedCpeVersions = new ArrayList<>();
String majorVersion = null;
int amountVulnerabilities = -1;
// iterate over all versions and decide whether to include the version or not
for (int i = 0; i < versions.size(); i++) {
if (requiredCpeVersions.contains(versions.get(i))) {
reducedCpeVersions.add(versions.get(i));
continue;
}
final Matcher versionMatcher = VERSION_PATTERN.matcher(versions.get(i).getVersionUpdate());
if (versionMatcher.find()) {
if (majorVersion == null) {
reducedCpeVersions.add(versions.get(i));
} else if (!majorVersion.equals(versionMatcher.group(1)) || amountVulnerabilities != versions.get(i).getVulnerabilityCount()) {
reducedCpeVersions.add(versions.get(i - 1));
}
majorVersion = versionMatcher.group(1);
amountVulnerabilities = versions.get(i).getVulnerabilityCount();
} else {
reducedCpeVersions.add(versions.get(i));
majorVersion = versions.get(i).getVersion().getVersion();
}
}
// if there are too few remaining Cpe, return all Cpe if there are less or equals 20
if (reducedCpeVersions.size() <= 10 && versions.size() <= 20)
return versions;
// remove doubles
removeDoubles(reducedCpeVersions);
// remove versions that suddenly drop to a low amount of vulnerabilities and go back up to the previous amount
for (int i = 2; i < reducedCpeVersions.size() - 2; i++) {
if (requiredCpeVersions.contains(reducedCpeVersions.get(i))) {
continue;
}
int currentVulnerabilityCount = reducedCpeVersions.get(i).getVulnerabilityCount();
if (currentVulnerabilityCount < 4) {
double averageBefore = 0;
int count = 0;
for (int j = Math.max(0, i - 2); j < i; j++) {
averageBefore += reducedCpeVersions.get(j).getVulnerabilityCount();
count++;
}
averageBefore = averageBefore / Math.max(1.0, count);
double averageAfter = 0;
count = 0;
for (int j = i + 1; j < Math.min(reducedCpeVersions.size() - 1, i + 3); j++) {
averageAfter += reducedCpeVersions.get(j).getVulnerabilityCount();
count++;
}
averageAfter = averageAfter / Math.max(1.0, count);
boolean isPatchVersion = StringUtils.hasText(reducedCpeVersions.get(i).getVersion().getUpdate());
double threshold = Math.max(5, ((averageBefore + averageAfter) / 2) * 0.2);
int distanceToBefore = (int) Math.abs(currentVulnerabilityCount - averageBefore);
int distanceToAfter = (int) Math.abs(currentVulnerabilityCount - averageAfter);
boolean outsideThresholdBefore = distanceToBefore > threshold;
boolean outsideThresholdAfter = distanceToAfter > threshold;
// require both thresholds not to be met or only one of them if the version is a patch version
if ((outsideThresholdBefore && outsideThresholdAfter) || (isPatchVersion && (outsideThresholdBefore || outsideThresholdAfter))) {
reducedCpeVersions.remove(i);
i--;
}
}
}
return reducedCpeVersions;
}
/**
* @param artifacts The artifacts to check for.
* @param artifactCpeVersions The Cpe versions that are already present in the timeline.
* @return A list of all Versions that contain at least one artifact's version.
*/
public List createVersionsForArtifactVersion(Collection artifacts, Collection artifactCpeVersions) {
final List artifactVersions = artifacts.stream()
.map(artifact -> Version.of(artifact.getVersion(), VersionContext.fromArtifact(artifact)))
.collect(Collectors.toList());
final List mergedVersions = getOfficialAndArtifactMergedVersions(artifactCpeVersions);
final List filteredVersions = new ArrayList<>();
for (TimelineVersion cpe : mergedVersions) {
for (Version artifactVersion : artifactVersions) {
if (cpe.getVersion().matchesVersionOf(artifactVersion)) {
filteredVersions.add(cpe);
break;
}
}
}
return filteredVersions;
}
/**
* Removes Cpes that appear multiple times in the given List.
*
* @param versions The List to remove doubles from.
*/
private void removeDoubles(List versions) {
for (int i = versions.size() - 1; i >= 0; i--) {
for (int j = versions.size() - 1; j >= 0; j--) {
if (i == j) continue;
if (i >= versions.size()) break;
if (j >= versions.size()) continue;
if (versions.get(i).getVersionUpdate().equals(versions.get(j).getVersionUpdate())) {
if (versions.get(i).getVulnerabilityCount() >= versions.get(j).getVulnerabilityCount())
versions.remove(j);
else versions.remove(i);
}
}
}
}
@Getter
public static class TimelineVersion {
private final Version version;
private final Map vulnerabilitiesPerSeverity = new HashMap<>();
private final List vulnerabilityIdentifiers = new ArrayList<>();
private TimelineVersion(Version version) {
this.version = version;
}
public int getCountForSeverity(CvssSeverityRanges.SeverityRange severity) {
return vulnerabilitiesPerSeverity.getOrDefault(severity, 0);
}
public boolean containsVulnerability(String vulnerabilityId) {
return vulnerabilityIdentifiers.contains(vulnerabilityId);
}
public boolean containsVulnerability(Vulnerability vulnerability) {
return vulnerabilityIdentifiers.contains(vulnerability.getId());
}
public int getVulnerabilityCount() {
int count = 0;
for (Integer value : vulnerabilitiesPerSeverity.values()) {
count += value;
}
return count;
}
public void putSeverityWithVulnerabilityIdentifier(CvssSeverityRanges.SeverityRange severity, String vulnerabilityId) {
putSeverity(severity);
vulnerabilityIdentifiers.add(vulnerabilityId);
}
public void putSeverity(CvssSeverityRanges.SeverityRange severity) {
if (vulnerabilitiesPerSeverity.containsKey(severity)) {
vulnerabilitiesPerSeverity.put(severity, vulnerabilitiesPerSeverity.get(severity) + 1);
} else {
vulnerabilitiesPerSeverity.put(severity, 1);
}
}
public String getVersionUpdate() {
return version.getVersion() + (shouldUpdatePartBeIncludedInLabel(version.getUpdate()) ? ":" + version.getUpdate() : "");
}
private boolean shouldUpdatePartBeIncludedInLabel(String updatePart) {
return StringUtils.hasText(updatePart) && !"*".equals(updatePart) && !"-".equals(updatePart);
}
@Override
public String toString() {
return "[" + getVersionUpdate() + ": " + vulnerabilitiesPerSeverity + "]";
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy