com.metaeffekt.artifact.analysis.cert.PeriodicDataSourcesOperations Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.cert;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.vulnerabilitystatus.VulnerabilityStatusReviewedEntry;
import com.metaeffekt.mirror.contents.advisory.AdvisoryEntry;
import com.metaeffekt.mirror.contents.base.VulnerabilityContextInventory;
import com.metaeffekt.mirror.contents.store.*;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import com.metaeffekt.mirror.query.AdvisorIndexQuery;
import com.metaeffekt.mirror.query.VulnerabilityIndexQuery;
import lombok.Data;
import org.metaeffekt.core.inventory.processor.model.AdvisoryMetaData;
import org.metaeffekt.core.inventory.processor.report.configuration.CentralSecurityPolicyConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.metaeffekt.core.inventory.processor.model.AdvisoryMetaData.Attribute.REVIEW_STATUS;
import static org.metaeffekt.core.inventory.processor.model.AdvisoryMetaData.STATUS_VALUE_UNAFFECTED;
public class PeriodicDataSourcesOperations {
private static final Logger LOG = LoggerFactory.getLogger(PeriodicDataSourcesOperations.class);
private final List> advisorIndexQueries = new ArrayList<>();
private final List vulnerabilityIndexQueries = new ArrayList<>();
/**
* Adds an advisor index query to the list of queries.
* If an advisor index query of the same class already exists, it will be replaced.
*
* @param advisorIndexQuery The advisor index query to add.
*/
public void addAdvisorIndexQuery(AdvisorIndexQuery advisorIndexQuery) {
this.advisorIndexQueries.removeIf(a -> a.getClass().equals(advisorIndexQuery.getClass()));
this.advisorIndexQueries.add(advisorIndexQuery);
}
public void addAdvisorIndexQueriesForContentIdentifiers(File baseMirrorDirectory, Collection> advisoryProviders) {
advisoryProviders = this.computeEffectiveAdvisoryIdentifiers(advisoryProviders);
for (AdvisoryTypeIdentifier advisoryProvider : advisoryProviders) {
final Function> factory = advisoryProvider.getAdvisorIndexQueryFactory();
if (factory == null) {
LOG.warn("No advisor index query factory found for [{}] during registration, skipping", advisoryProvider.toExtendedString());
continue;
}
final AdvisorIndexQuery query = factory.apply(baseMirrorDirectory);
this.addAdvisorIndexQuery(query);
}
}
private AdvisorIndexQuery getAdvisorIndexQuery(AdvisoryTypeIdentifier identifier) {
if (identifier == null) {
return null;
}
for (AdvisorIndexQuery advisorIndexQuery : this.advisorIndexQueries) {
if (advisorIndexQuery.getClass().equals(identifier.getAdvisorIndexQueryClass())) {
return advisorIndexQuery;
}
}
return null;
}
public void addVulnerabilityIndexQuery(VulnerabilityIndexQuery vulnerabilityIndexQuery) {
this.vulnerabilityIndexQueries.removeIf(v -> v.getClass().equals(vulnerabilityIndexQuery.getClass()));
this.vulnerabilityIndexQueries.add(vulnerabilityIndexQuery);
}
public void addVulnerabilityIndexQueriesForContentIdentifiers(File baseMirrorDirectory, Collection> vulnerabilityProviders) {
vulnerabilityProviders = this.computeEffectiveVulnerabilityIdentifiers(vulnerabilityProviders);
for (VulnerabilityTypeIdentifier vulnerabilityProvider : vulnerabilityProviders) {
final Function factory = vulnerabilityProvider.getVulnerabilityIndexQueryFactory();
if (factory == null) {
LOG.warn("No vulnerability index query factory found for [{}] during registration, skipping", vulnerabilityProvider.toExtendedString());
continue;
}
final VulnerabilityIndexQuery query = factory.apply(baseMirrorDirectory);
this.addVulnerabilityIndexQuery(query);
}
}
private VulnerabilityIndexQuery getVulnerabilityIndexQuery(VulnerabilityTypeIdentifier identifier) {
if (identifier == null) {
return null;
}
for (VulnerabilityIndexQuery vulnerabilityIndexQuery : this.vulnerabilityIndexQueries) {
if (vulnerabilityIndexQuery.getClass().equals(identifier.getVulnerabilityIndexQueryClass())) {
return vulnerabilityIndexQuery;
}
}
return null;
}
// START: SECURITY ADVISORIES
/**
* Finds all security advisories that have been created or updated since the lastModified.
*
* @param queryPeriod The timestamp in milliseconds to start looking up security advisories from.
* @param advisoryProviders The security advisory providers to include.
* @param includeAdvisoryTypes If set to a non-empty string, the advisory types not included in the list will be
* excluded from the results.
* @return A list of security advisories that have been created or updated since the lastModified.
*/
public List findSecurityAdvisoriesUpdatedOrModifiedSince(QueryTimePeriod queryPeriod, List> advisoryProviders, List includeAdvisoryTypes) {
advisoryProviders = this.computeEffectiveAdvisoryIdentifiers(advisoryProviders);
final List foundSecurityAdvisories = new ArrayList<>();
for (AdvisoryTypeIdentifier provider : advisoryProviders) {
LOG.info("Querying the advisory index for [types={}] [provider={}] [queryPeriod={}]", includeAdvisoryTypes, provider, queryPeriod);
final AdvisorIndexQuery index = getAdvisorIndexQuery(provider);
if (index == null) {
LOG.warn("No advisory index query has been provided for [{}], skipping", provider.toExtendedString());
continue;
}
final List advisoriesForProvider = findAdvisorsForProvider(index, queryPeriod, includeAdvisoryTypes);
foundSecurityAdvisories.addAll(advisoriesForProvider);
}
LOG.info("Found a total of [{}] advisories", foundSecurityAdvisories.size());
return foundSecurityAdvisories;
}
private List findAdvisorsForProvider(AdvisorIndexQuery index, QueryTimePeriod queryPeriod, List includeAdvisoryTypes) {
final List advisors = getAdvisoryMetaDataLastModifiedSince(index, queryPeriod, includeAdvisoryTypes);
if (advisors.isEmpty()) {
LOG.info("No security advisories found for [{}] in the given time range", index.getClass().getSimpleName());
} else {
LOG.info("Found [{}] security advisories for [{}]", advisors.size(), index.getClass().getSimpleName());
}
return advisors;
}
public List getAdvisoryMetaDataLastModifiedSince(AdvisorIndexQuery query, QueryTimePeriod queryPeriod, List includeAdvisoryTypes) {
final List entries = query.findCreatedOrUpdatedInRange(queryPeriod.getStart(), queryPeriod.getEnd());
return entries.stream()
.sorted(AdvisoryEntry.UPDATE_CREATE_TIME_COMPARATOR)
.filter(e -> checkForAdvisoryTypeInclude(e, includeAdvisoryTypes))
.collect(Collectors.toList());
}
private boolean checkForAdvisoryTypeInclude(AdvisoryEntry advisory, List includeAdvisoryTypes) {
if (includeAdvisoryTypes == null || includeAdvisoryTypes.isEmpty() || CentralSecurityPolicyConfiguration.containsAny(includeAdvisoryTypes)) {
return true;
}
final String type = advisory.getType();
for (String a : includeAdvisoryTypes) {
if (Objects.equals(type, a)) {
return true;
}
}
return false;
}
public void removeSecurityAdvisoryIfStatus(VulnerabilityContextInventory vInventory, String status) {
for (AdvisoryEntry securityAdvisory : vInventory.getShallowCopySecurityAdvisories()) {
if (status.equals(securityAdvisory.getAdditionalAttribute(REVIEW_STATUS))) {
vInventory.remove(securityAdvisory);
}
}
}
public void filterVulnerabilitiesByAdvisoryFilter(VulnerabilityContextInventory vInventory, List> advisoryProviders) {
if (advisoryProviders != null && !advisoryProviders.isEmpty()) {
final Set vulnerabilities = vInventory.getShallowCopyVulnerabilities();
final int sizeBefore = vulnerabilities.size();
for (Vulnerability vulnerability : vulnerabilities) {
boolean containsAllowedAdvisoryProvider = false;
for (AdvisoryEntry advisory : vulnerability.getSecurityAdvisories()) {
if (advisoryProviders.contains(advisory.getSourceIdentifier())) {
containsAllowedAdvisoryProvider = true;
break;
} else if (advisory.getDataSources().stream().anyMatch(advisoryProviders::contains)) {
containsAllowedAdvisoryProvider = true;
break;
}
}
if (!containsAllowedAdvisoryProvider) {
vInventory.remove(vulnerability);
}
}
int diff = sizeBefore - vulnerabilities.size();
if (diff > 0) {
LOG.info("Removed [{}] vulnerabilities that do not contain advisories from {}", diff, advisoryProviders);
}
}
}
// END: SECURITY ADVISORIES
// START: VULNERABILITIES
/**
* Finds all vulnerabilities that have been created or updated since the lastModified.
*
* @param lastModified The timestamp in milliseconds to start looking up vulnerabilities from.
* @param vulnerabilityProviders The vulnerabilities providers to include.
* @return A list of vulnerabilities that have been created or updated since the lastModified.
*/
public List findVulnerabilitiesUpdatedOrModifiedSince(long lastModified, List> vulnerabilityProviders) {
vulnerabilityProviders = this.computeEffectiveVulnerabilityIdentifiers(vulnerabilityProviders);
final List foundVulnerabilities = new ArrayList<>();
for (VulnerabilityTypeIdentifier provider : vulnerabilityProviders) {
LOG.info("Querying the advisory index for [provider={}] [changed-since={}]", provider, lastModified);
final VulnerabilityIndexQuery index = getVulnerabilityIndexQuery(provider);
if (index == null) {
LOG.warn("No vulnerability index query has been provided for [{}], skipping", provider.toExtendedString());
continue;
}
final List vulnerabilitiesForProvider = findVulnerabilitiesForProvider(index, lastModified);
foundVulnerabilities.addAll(vulnerabilitiesForProvider);
}
LOG.info("Found a total of [{}] advisories", foundVulnerabilities.size());
return foundVulnerabilities;
}
private List findVulnerabilitiesForProvider(VulnerabilityIndexQuery index, long lastModified) {
final List vulnerabilities = getVulnerabilityLastModifiedSince(index, lastModified);
if (vulnerabilities.isEmpty()) {
LOG.info("No vulnerabilities found for [{}] in the given time range", index.getClass().getSimpleName());
} else {
LOG.info("Found [{}] vulnerabilities for [{}]", vulnerabilities.size(), index.getClass().getSimpleName());
}
return vulnerabilities;
}
public List getVulnerabilityLastModifiedSince(VulnerabilityIndexQuery query, long lastModified) {
final List entries = query.findCreatedOrUpdatedInRange(lastModified, System.currentTimeMillis() + 1000L * 60 * 60 * 24);
return entries.stream()
.sorted(Vulnerability.UPDATE_CREATE_TIME_COMPARATOR)
.collect(Collectors.toList());
}
// END: VULNERABILITIES
private List> computeEffectiveAdvisoryIdentifiers(Collection> sourceIdentifiers) {
if (sourceIdentifiers == null) {
return new ArrayList<>();
}
if (sourceIdentifiers.contains(AdvisoryTypeStore.ANY_ADVISORY_FILTER_WILDCARD)) {
final List> identifiers = new ArrayList<>(AdvisoryTypeStore.get().values());
identifiers.remove(AdvisoryTypeStore.ANY_ADVISORY_FILTER_WILDCARD);
return identifiers;
}
return new ArrayList<>(sourceIdentifiers);
}
private List> computeEffectiveVulnerabilityIdentifiers(Collection> sourceIdentifiers) {
if (sourceIdentifiers == null) {
return new ArrayList<>();
}
if (sourceIdentifiers.contains(VulnerabilityTypeStore.ANY_VULNERABILITY_FILTER_WILDCARD)) {
final List> identifiers = new ArrayList<>(VulnerabilityTypeStore.get().values());
identifiers.remove(VulnerabilityTypeStore.ANY_VULNERABILITY_FILTER_WILDCARD);
return identifiers;
}
return new ArrayList<>(sourceIdentifiers);
}
/**
* State transition diagram (online viewer, remove \ from the code block below):
*
* \@startuml
* [*] --> UNAFFECTED
* state UNAFFECTED
* state NEW
* state REVIEWED
* state IN_REVIEW
* UNAFFECTED --> NEW : Not in reviewed list,\nbut present in inventory
* UNAFFECTED --> REVIEWED : In reviewed list
* NEW --> REVIEWED : In reviewed list
* REVIEWED --> IN_REVIEW : Not in reviewed list,\nbut present in inventory
* \@enduml
*
*
* @param checkSecurityAdvisories The security advisories to traverse the state transition diagram and set the
* status accordingly.
* @param vReferenceInventories The reference inventories to use for the traversal.
*
* @throws RuntimeException If something goes wrong.
*/
public void setSecurityAdvisoryReviewedStatusFromReferenceInventories(
Collection checkSecurityAdvisories,
Collection vReferenceInventories
) throws RuntimeException {
// mark all entries as unaffected initially to be overwritten later
checkSecurityAdvisories.forEach(checkSecurityAdvisory -> checkSecurityAdvisory.setAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_UNAFFECTED));
if (vReferenceInventories == null || vReferenceInventories.isEmpty()) {
LOG.info("No reference inventories provided, skipping reference inventory processing");
return;
}
LOG.info("Using [{}] reference inventories to calculate the review status", vReferenceInventories.size());
for (VulnerabilityContextInventory vReferenceInventory : vReferenceInventories) {
LOG.info("Processing [{}] vulnerabilities from reference inventory", vReferenceInventory.getVulnerabilities().size());
for (Vulnerability referenceVulnerability : vReferenceInventory.getVulnerabilities()) {
final List allAdvisories = new ArrayList<>(referenceVulnerability.getSecurityAdvisories());
final List reviewedAdvisories = referenceVulnerability.getOrCreateNewVulnerabilityStatus().getReviewedAdvisories().stream()
.map(VulnerabilityStatusReviewedEntry::getId)
.map(id -> {
final Optional foundAdvisoryEntry = vReferenceInventory.findAdvisoryEntryByName(id);
if (foundAdvisoryEntry.isPresent()) {
return foundAdvisoryEntry.get();
} else {
final Optional>> source = AdvisoryTypeStore.get().fromEntryIdentifier(id);
if (!source.isPresent()) {
throw new RuntimeException("Parsing reviewed advisory entry identifier [" + id + "] from assessment. Check your assessment files for the 'reviewed' > 'id' attributes.");
}
return source.get().getIdentifier().getAdvisoryFactory().get().setId(id);
}
})
.collect(Collectors.toList());
allAdvisories.removeIf(reviewedAdvisories::contains);
if (allAdvisories.size() + reviewedAdvisories.size() == 0) {
// there are no advisories for this vulnerability
continue;
}
final Set allAdvisoryIds = allAdvisories.stream().map(AdvisoryEntry::getId).collect(Collectors.toSet());
final Set reviewedAdvisoryIds = reviewedAdvisories.stream().map(AdvisoryEntry::getId).collect(Collectors.toSet());
for (AdvisoryEntry checkSecurityAdvisory : checkSecurityAdvisories) {
if (checkSecurityAdvisory.getAdditionalAttribute(REVIEW_STATUS) == null) {
checkSecurityAdvisory.setAdditionalAttribute(REVIEW_STATUS, STATUS_VALUE_UNAFFECTED);
}
final String currentStatus = checkSecurityAdvisory.getAdditionalAttribute(REVIEW_STATUS);
if (reviewedAdvisoryIds.contains(checkSecurityAdvisory.getId())) {
switch (currentStatus) {
case STATUS_VALUE_UNAFFECTED:
checkSecurityAdvisory.setAdditionalAttribute(REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_REVIEWED);
break;
case AdvisoryMetaData.STATUS_VALUE_NEW:
checkSecurityAdvisory.setAdditionalAttribute(REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_IN_REVIEW);
break;
}
} else if (allAdvisoryIds.contains(checkSecurityAdvisory.getId())) {
switch (currentStatus) {
case STATUS_VALUE_UNAFFECTED:
checkSecurityAdvisory.setAdditionalAttribute(REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_NEW);
break;
case AdvisoryMetaData.STATUS_VALUE_REVIEWED:
checkSecurityAdvisory.setAdditionalAttribute(REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_IN_REVIEW);
break;
}
}
if (!currentStatus.equals(checkSecurityAdvisory.getAdditionalAttribute(REVIEW_STATUS))) {
LOG.info(" [{}] - [{}] status change [{} --> {}]",
referenceVulnerability.getId(),
checkSecurityAdvisory.getId(),
currentStatus,
checkSecurityAdvisory.getAdditionalAttribute(REVIEW_STATUS));
}
}
}
}
LOG.info("Done setting status via reference inventory files");
}
@Data
public static class QueryTimePeriod {
private final long start;
private final long end;
@Override
public String toString() {
final Date startDate = TimeUtils.tryParse(String.valueOf(start));
final Date endDate = TimeUtils.tryParse(String.valueOf(end));
final long now = System.currentTimeMillis();
return "[" + TimeUtils.formatNormalizedDate(startDate) +
" (" + TimeUtils.formatTimeUntilOrAgoDefault(start - now) + ") - " +
TimeUtils.formatNormalizedDate(endDate) +
" (" + TimeUtils.formatTimeUntilOrAgoDefault(end - now) + ")]";
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy