com.metaeffekt.artifact.enrichment.other.periodic.AdvisorPeriodicEnrichment Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.enrichment.other.periodic;
import com.metaeffekt.artifact.analysis.cert.PeriodicDataSourcesOperations;
import com.metaeffekt.artifact.analysis.diffmerge.InventoryMerger;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.artifact.enrichment.InventoryEnricher;
import com.metaeffekt.artifact.enrichment.configurations.AdvisorPeriodicEnrichmentConfiguration;
import com.metaeffekt.mirror.contents.advisory.AdvisoryEntry;
import com.metaeffekt.mirror.contents.base.VulnerabilityContextInventory;
import com.metaeffekt.mirror.contents.store.VulnerabilityTypeIdentifier;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import com.metaeffekt.mirror.download.documentation.EnricherMetadata;
import com.metaeffekt.mirror.download.documentation.InventoryEnrichmentPhase;
import lombok.Setter;
import org.metaeffekt.core.inventory.processor.model.AdvisoryMetaData;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import org.metaeffekt.core.inventory.processor.model.InventoryInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
/**
* See data flow diagram.
*
* Important Parameter combinations
*
* The parameters {@link AdvisorPeriodicEnrichmentConfiguration}#filterUnclassified and
* {@link AdvisorPeriodicEnrichmentConfiguration}#filterUnaffectedVulnerabilities must both be set to the
* correct value to correctly track vulnerabilities.
* If filterUnclassified is set to false and
* filterUnaffectedVulnerabilities is set to true, vulnerabilities we might want to remove from an inventory
* will still be tracked, as they might still be affected by existing advisories.
*
*/
@EnricherMetadata(
name = "Advisor periodic", phase = InventoryEnrichmentPhase.STANDALONE,
intermediateFileSuffix = "advisor-periodic", mavenPropertyName = "advisorPeriodicEnrichment"
)
public class AdvisorPeriodicEnrichment extends InventoryEnricher {
private static final Logger LOG = LoggerFactory.getLogger(AdvisorPeriodicEnrichment.class);
public final static String ADVISOR_PERIODIC_QUERY_KEY = "cert-periodic-query";
public final static String ADVISOR_PERIODIC_QUERY_RANGE_START_KEY = "Range Start";
public final static String ADVISOR_PERIODIC_QUERY_RANGE_END_KEY = "Range End";
@Setter
private AdvisorPeriodicEnrichmentConfiguration configuration = new AdvisorPeriodicEnrichmentConfiguration();
private final File baseMirrorDirectory;
public AdvisorPeriodicEnrichment(File baseMirrorDirectory) {
this.baseMirrorDirectory = baseMirrorDirectory;
}
@Override
public AdvisorPeriodicEnrichmentConfiguration getConfiguration() {
return configuration;
}
@Override
protected void performEnrichment(Inventory inventory) {
final Map parsedReferenceInventories = new LinkedHashMap<>();
// reference inventory must be first for merging later on
if (configuration.hasInitialInventoryContextName()) {
parsedReferenceInventories.put(configuration.getInitialInventoryContextName(), inventory);
}
parsedReferenceInventories.putAll(configuration.parseReferenceInventories());
if (parsedReferenceInventories.isEmpty()) {
LOG.warn("No reference inventories have been provided.");
} else if (parsedReferenceInventories.size() == 1) {
LOG.info("Using [{}] as the only reference inventory", parsedReferenceInventories.keySet().iterator().next());
} else {
LOG.info("Using [{}] as the reference inventories", parsedReferenceInventories.keySet());
}
final Map vParsedReferenceInventories =
VulnerabilityContextInventory.fromInventories(parsedReferenceInventories);
final Inventory outputInventory = new Inventory();
// transfer metadata from input inventory; please note that assets are not taken from the reference inventories
outputInventory.setAssetMetaData(inventory.getAssetMetaData());
final VulnerabilityContextInventory vOutputInventory =
VulnerabilityContextInventory.fromInventory(outputInventory);
final PeriodicDataSourcesOperations.QueryTimePeriod queryTimePeriod = configuration.getAdvisoryQueryPeriod();
LOG.info("Listing all advisories for changed entries of {} in period {}", configuration.getAdvisoryProviders(), queryTimePeriod);
final InventoryInfo info = outputInventory.findOrCreateInventoryInfo(ADVISOR_PERIODIC_QUERY_KEY);
info.set(ADVISOR_PERIODIC_QUERY_RANGE_START_KEY, new SimpleDateFormat("yyyy-MM-dd").format(queryTimePeriod.getStart()));
info.set(ADVISOR_PERIODIC_QUERY_RANGE_END_KEY, new SimpleDateFormat("yyyy-MM-dd").format(queryTimePeriod.getEnd()));
// set up the index for the advisory providers
final PeriodicDataSourcesOperations periodicDataSourcesOperations = new PeriodicDataSourcesOperations();
periodicDataSourcesOperations.addAdvisorIndexQueriesForContentIdentifiers(baseMirrorDirectory, configuration.getAdvisoryProviders());
// perform query
final List securityAdvisoriesUpdatedOrModifiedSince =
periodicDataSourcesOperations.findSecurityAdvisoriesUpdatedOrModifiedSince(
queryTimePeriod, configuration.getAdvisoryProviders(), configuration.getIncludeAdvisoryTypes());
// add all found advisories to output inventory
vOutputInventory.addAllAdvisories(securityAdvisoriesUpdatedOrModifiedSince);
// if there are reference inventories, mark the AdvisoryMetaData entries as [unaffected, new, in progress, reviewed] based on whether they are listed as reviewed or not.
periodicDataSourcesOperations.setSecurityAdvisoryReviewedStatusFromReferenceInventories(
vOutputInventory.getSecurityAdvisories(), vParsedReferenceInventories.values());
this.filterInventoryDataPreMerge(vOutputInventory, periodicDataSourcesOperations);
this.mergeReferenceInventoriesToSourceInventory(vOutputInventory, parsedReferenceInventories, outputInventory);
// mark all the advisories that have been added by the merger as 'unclassified'
for (AdvisoryEntry securityAdvisory : vOutputInventory.getShallowCopySecurityAdvisories()) {
if (securityAdvisory.getAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS) == null) {
securityAdvisory.setAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS, AdvisoryMetaData.STATUS_VALUE_UNCLASSIFIED);
}
}
this.filterInventoryDataPostMerge(vOutputInventory, periodicDataSourcesOperations);
vOutputInventory.writeBack(true);
this.logReviewStateCounts(vOutputInventory.getSecurityAdvisories());
// move all data from the output inventory to the original inventory so that the original inventory is enriched
super.moveInventoryData(outputInventory, inventory);
}
private void mergeReferenceInventoriesToSourceInventory(VulnerabilityContextInventory vOutputInventory,
Map parsedReferenceInventories, Inventory outputInventory) {
vOutputInventory.writeBack(true);
final int initialAdvisoryCount = vOutputInventory.getSecurityAdvisories().size();
// merge the reference inventories into the output inventory
if (!parsedReferenceInventories.isEmpty()) {
final InventoryMerger merger = new InventoryMerger(outputInventory);
for (Map.Entry reference : parsedReferenceInventories.entrySet()) {
merger.addReferenceInventory(reference.getValue(), reference.getKey());
}
merger.includeArtifacts();
merger.includeVulnerabilities();
merger.includeAdvisories();
// NOTE: assets from reference inventories are not considered
// merger.includeAssets();
}
vOutputInventory.parse();
final int finalAdvisoryCount = vOutputInventory.getSecurityAdvisories().size();
final int addedAdvisories = finalAdvisoryCount - initialAdvisoryCount;
LOG.info(" --> Added [{} - {} = {}] security advisories from [{}] reference inventories",
finalAdvisoryCount, initialAdvisoryCount, addedAdvisories, parsedReferenceInventories.size());
}
private void filterInventoryDataPreMerge(VulnerabilityContextInventory vOutputInventory, PeriodicDataSourcesOperations periodicDataSourcesOperations) {
// remove all security advisories that are marked as 'unaffected'
if (configuration.isFilterUnaffected()) {
this.filterSecurityAdvisoriesIfStatus(vOutputInventory, periodicDataSourcesOperations, AdvisoryMetaData.STATUS_VALUE_UNAFFECTED);
}
}
private void filterInventoryDataPostMerge(VulnerabilityContextInventory vOutputInventory, PeriodicDataSourcesOperations periodicDataSourcesOperations) {
final Set unfilteredVulnerabilities = vOutputInventory.getShallowCopyVulnerabilities();
final int unfilteredSecurityAdvisoriesCount = vOutputInventory.getSecurityAdvisories().size();
// remove all security advisories that are marked as 'unclassified'
if (configuration.isFilterUnclassified()) {
this.filterSecurityAdvisoriesIfStatus(vOutputInventory, periodicDataSourcesOperations, AdvisoryMetaData.STATUS_VALUE_UNCLASSIFIED);
}
// remove all vulnerabilities that do not contain at least one vulnerability with a specified security advisory provider
if (configuration.isFilterVulnerabilitiesWithoutSpecifiedAdvisory()) {
if (!configuration.getVulnerabilityAdvisoryFilter().isEmpty()) {
LOG.info("Removing all vulnerabilities that do not contain at least one vulnerability with a specified advisory provider of {}", configuration.getVulnerabilityAdvisoryFilter());
final int initialVulnerabilityCount = vOutputInventory.getVulnerabilities().size();
periodicDataSourcesOperations.filterVulnerabilitiesByAdvisoryFilter(
vOutputInventory,
configuration.getVulnerabilityAdvisoryFilter()
);
final int finalVulnerabilityCount = vOutputInventory.getVulnerabilities().size();
final int removedVulnerabilities = initialVulnerabilityCount - finalVulnerabilityCount;
LOG.info(" --> Removed [{} - {} = {}] vulnerabilities that do not contain at least one vulnerability with a specified advisory provider",
initialVulnerabilityCount, removedVulnerabilities, finalVulnerabilityCount);
} else {
LOG.warn("Filtering vulnerabilities without specified advisory is enabled, but no advisory providers are specified. No vulnerabilities will be removed.");
}
}
// remove all vulnerabilities if not affected by security advisors
if (configuration.isFilterUnaffectedVulnerabilities()) {
LOG.info("Removing all vulnerabilities that are not affected by any advisory");
final Set allAffectedVulnerabilities = new HashSet<>();
for (AdvisoryEntry securityAdvisory : vOutputInventory.getShallowCopySecurityAdvisories()) {
final Map, Set> referencedIds = securityAdvisory.getReferencedVulnerabilities();
for (Map.Entry, Set> referencedId : referencedIds.entrySet()) {
for (String referencedVulnerability : referencedId.getValue()) {
vOutputInventory.findVulnerabilityByName(referencedVulnerability).ifPresent(allAffectedVulnerabilities::add);
}
}
}
LOG.info(" --> Found [{}] vulnerabilities that are affected by at least one advisory", allAffectedVulnerabilities.size());
final int initialVulnerabilityCount = vOutputInventory.getVulnerabilities().size();
for (Vulnerability vulnerability : vOutputInventory.getShallowCopyVulnerabilities()) {
if (!allAffectedVulnerabilities.contains(vulnerability)) {
vOutputInventory.remove(vulnerability);
}
}
final int finalVulnerabilityCount = vOutputInventory.getVulnerabilities().size();
final int removedVulnerabilities = initialVulnerabilityCount - finalVulnerabilityCount;
LOG.info(" --> Removed [{} - {} = {}] vulnerabilities that are not affected by any advisory",
initialVulnerabilityCount, removedVulnerabilities, finalVulnerabilityCount);
}
// find what vulnerabilities must remain in the inventory no matter what and re-add them if they were to have been removed
final long forceVulnerabilitiesToBeIncludedChangedSinceTime = this.configuration.getIncludeVulnerabilitiesChangedSinceTimestamp();
final Date forceVulnerabilitiesToBeIncludedChangedSinceDate = new Date(forceVulnerabilitiesToBeIncludedChangedSinceTime);
if (forceVulnerabilitiesToBeIncludedChangedSinceTime != 0) {
int reAddedVulnerabilities = 0;
for (Vulnerability vulnerability : unfilteredVulnerabilities) {
if (vulnerability.isCreatedAfter(forceVulnerabilitiesToBeIncludedChangedSinceDate) || vulnerability.isUpdatedAfter(forceVulnerabilitiesToBeIncludedChangedSinceDate)) {
if (!vOutputInventory.contains(vulnerability)) {
vOutputInventory.add(vulnerability);
reAddedVulnerabilities++;
}
}
}
if (reAddedVulnerabilities > 0) {
LOG.info(" --> Re-added [{}] previously removed vulnerabilities that were created or updated after [{}]",
reAddedVulnerabilities, TimeUtils.formatNormalizedDate(forceVulnerabilitiesToBeIncludedChangedSinceDate));
} else {
LOG.info(" --> No vulnerabilities were re-added, as none were removed that were created or updated after [{}]",
TimeUtils.formatNormalizedDate(forceVulnerabilitiesToBeIncludedChangedSinceDate));
}
}
LOG.info("Counts after applying filtering: [vulnerabilities {} -> {}] [advisories {} -> {}]",
unfilteredVulnerabilities.size(), vOutputInventory.getVulnerabilities().size(),
unfilteredSecurityAdvisoriesCount, vOutputInventory.getSecurityAdvisories().size());
}
private void filterSecurityAdvisoriesIfStatus(VulnerabilityContextInventory vOutputInventory,
PeriodicDataSourcesOperations periodicDataSourcesOperations, String statusValueUnclassified) {
LOG.info("Removing all security advisories that are marked as [{}]", statusValueUnclassified);
final int initialAdvisoryCount = vOutputInventory.getSecurityAdvisories().size();
periodicDataSourcesOperations.removeSecurityAdvisoryIfStatus(vOutputInventory, statusValueUnclassified);
final int finalAdvisoryCount = vOutputInventory.getSecurityAdvisories().size();
final int removedAdvisories = initialAdvisoryCount - finalAdvisoryCount;
LOG.info(" --> Removed [{} - {} = {}] security advisories that are marked as [{}]",
initialAdvisoryCount, removedAdvisories, finalAdvisoryCount, statusValueUnclassified);
}
private void logReviewStateCounts(Collection securityAdvisories) {
final Map reviewStatusCounts = securityAdvisories.stream()
.collect(Collectors.groupingBy(
a -> a.getAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS) == null ?
"null" : a.getAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS),
Collectors.counting()
));
LOG.info("");
LOG.info("Inventory review status state summary:");
for (Map.Entry reviewStatusCount : reviewStatusCounts.entrySet()) {
LOG.info(" {}: {}", reviewStatusCount.getKey(), reviewStatusCount.getValue());
final StringJoiner matchingEntries = new StringJoiner(", ");
for (AdvisoryEntry securityAdvisory : securityAdvisories) {
final String value = securityAdvisory.getAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS) == null ?
"null" : securityAdvisory.getAdditionalAttribute(AdvisoryMetaData.Attribute.REVIEW_STATUS);
if (reviewStatusCount.getKey().equals(value)) {
matchingEntries.add(securityAdvisory.getId());
}
}
LOG.info(" {}", matchingEntries);
}
LOG.info("");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy