com.metaeffekt.artifact.enrichment.InventoryEnrichmentPipeline Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.enrichment;
import com.metaeffekt.artifact.analysis.utils.BuildProperties;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.artifact.enrichment.configurations.VulnerabilityAssessmentDashboardEnrichmentConfiguration;
import com.metaeffekt.artifact.enrichment.other.vad.VulnerabilityAssessmentDashboard;
import com.metaeffekt.mirror.contents.base.VulnerabilityContextInventory;
import com.metaeffekt.mirror.download.documentation.EnricherMetadata;
import com.metaeffekt.mirror.download.documentation.InventoryEnrichmentPhase;
import org.json.JSONArray;
import org.json.JSONObject;
import org.metaeffekt.core.inventory.processor.configuration.ProcessConfiguration;
import org.metaeffekt.core.inventory.processor.configuration.ProcessMisconfiguration;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import org.metaeffekt.core.inventory.processor.model.InventoryInfo;
import org.metaeffekt.core.inventory.processor.reader.InventoryReader;
import org.metaeffekt.core.inventory.processor.writer.InventoryWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@EnricherMetadata(
name = "Inventory Enrichment Pipeline", phase = InventoryEnrichmentPhase.STANDALONE,
intermediateFileSuffix = "result", mavenPropertyName = ""
)
public class InventoryEnrichmentPipeline extends InventoryEnricher {
private final static Logger LOG = LoggerFactory.getLogger(InventoryEnrichmentPipeline.class);
private final List enrichers = new ArrayList<>();
private final File baseMirrorDirectory;
private File inventorySourceFile;
private boolean writeIntermediateInventories = true;
private boolean storeIntermediateStepsInInventoryInfo = true;
private File intermediateInventoriesDirectory;
private File inventoryResultFile;
private Inventory inventory;
private InventoryEnricher resumeAtEnricher;
private List> progressListeners = new ArrayList<>();
public final static String INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT = "inventory-enrichment";
public final static String INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT_STEPS = "Steps";
public final static String INVENTORY_INFO_KEY_ARTIFACT_ANALYSIS_VERSION = "Artifact Analysis Version";
public final static String INVENTORY_INFO_KEY_VAD_VERSION = "VAD Version";
public InventoryEnrichmentPipeline(Inventory inventory, File baseMirrorDirectory) {
this.inventory = inventory;
this.baseMirrorDirectory = baseMirrorDirectory;
}
public InventoryEnrichmentPipeline(File inventorySourceFile, File baseMirrorDirectory) throws IOException {
this.inventory = new InventoryReader().readInventory(inventorySourceFile);
this.baseMirrorDirectory = baseMirrorDirectory;
setInventorySourceFile(inventorySourceFile);
}
public void setInventorySourceFile(File inventorySourceFile) {
this.inventorySourceFile = inventorySourceFile;
if (inventorySourceFile != null) {
intermediateInventoriesDirectory = new File(inventorySourceFile.getParentFile(), "intermediate-inventories");
}
}
public void setWriteIntermediateInventories(boolean writeIntermediateInventories) {
this.writeIntermediateInventories = writeIntermediateInventories;
}
public void setStoreIntermediateStepsInInventoryInfo(boolean storeIntermediateStepsInInventoryInfo) {
this.storeIntermediateStepsInInventoryInfo = storeIntermediateStepsInInventoryInfo;
}
public void setIntermediateInventoriesDirectory(File intermediateInventoriesDirectory) {
this.intermediateInventoriesDirectory = intermediateInventoriesDirectory;
}
public void setInventoryResultFile(File inventoryResultFile) {
this.inventoryResultFile = inventoryResultFile;
}
public void addEnrichment(InventoryEnricher enricher) {
if (enricher == this) {
throw new RuntimeException("Cannot add self as enrichment.");
}
enrichers.add(enricher);
}
public T addEnrichment(Class type) {
final InventoryEnricher enricher = constructEnricher(type);
enrichers.add(enricher);
return (T) enricher;
}
public void addProgressListener(BiConsumer progressListener) {
progressListeners.add(progressListener);
}
public void removeProgressListener(BiConsumer progressListener) {
progressListeners.remove(progressListener);
}
public void clearProgressListeners() {
progressListeners.clear();
}
/**
* Any changes made to the enrichment pipeline after this method is called will not be reflected in the inventory
* enrichment pipeline. This method is intended to be called after all enrichments have been added.
*
* @param id The ID of the enrichment pipeline.
*/
public void resumeAt(String id) {
if (id == null) {
throw new RuntimeException("Inventory Enricher ID must not be null");
}
if (inventorySourceFile == null) {
throw new RuntimeException("Inventory source file must be set to resume at enricher");
}
// this method will change the IDs of the enrichers to be unique, by appending a numerical suffix in case a duplicate exists
assignEffectiveIdsToEnrichers();
for (int i = 0; i < this.enrichers.size(); i++) {
final InventoryEnricher enricher = this.enrichers.get(i);
final InventoryEnricher previousEnricher = i > 0 ? this.enrichers.get(i - 1) : null;
if (previousEnricher != null && id.equals(enricher.getConfiguration().getId())) {
final File file = this.getInventoryFileForEnricher(previousEnricher);
if (!file.exists()) {
throw new RuntimeException("Cannot resume at enricher " + id + " as the inventory file " + file.getAbsolutePath() + " does not exist. Make sure that the process has run before with [writeIntermediateInventories = true] and [inventorySourceFile != null].");
}
try {
this.inventory = new InventoryReader().readInventory(file);
} catch (IOException e) {
throw new RuntimeException("Failed to read inventory from " + file, e);
}
resumeAtEnricher = enricher;
LOG.info("Resuming at enricher [{}], using inventory from previous step [{}] {}", resumeAtEnricher.getConfiguration().getId(), previousEnricher.getConfiguration().getId(), file.getAbsolutePath());
return;
}
}
throw new RuntimeException("Specified enrichment step ID [" + id + "] does not exist, pick one of:\n" + enrichers.stream().map(e -> e.getConfiguration().getId()).collect(Collectors.joining("\n")));
}
public void performEnrichmentIfActive() {
performEnrichmentIfActive(this.inventory);
}
@Override
public void performEnrichment(Inventory inventory) {
// this method will change the IDs of the enrichers to be unique, by appending a numerical suffix in case a duplicate exists
assignEffectiveIdsToEnrichers();
// copy over security policy configuration if not set on enricher
if (super.isSecurityPolicyConfigurationDefined()) {
for (InventoryEnricher enricher : enrichers) {
if (!enricher.isSecurityPolicyConfigurationDefined()) {
enricher.setSecurityPolicyConfiguration(super.getSecurityPolicyConfiguration());
}
}
}
LOG.info("Enrichment order:");
for (InventoryEnricher inventoryEnricher : enrichers) {
LOG.info(formatEnrichmentListEntry(inventoryEnricher, -1));
}
assertCorrectlyConfigured();
logCentralSecurityConfiguration();
RuntimeException enrichmentException = null;
InventoryEnricher failedEnricher = null;
// if resumeAtEnricher is null, do not skip any
boolean resumeAtEnricherFound = resumeAtEnricher == null;
final Map enrichmentDurations = new LinkedHashMap<>();
for (InventoryEnricher enricher : enrichers) {
if (!resumeAtEnricherFound) {
if (enricher == resumeAtEnricher) {
resumeAtEnricherFound = true;
} else {
continue;
}
}
if (storeIntermediateStepsInInventoryInfo) {
this.appendInventoryInfoStep(inventory, enricher);
}
final long startTime = TimeUtils.utcNow();
try {
enricher.performEnrichmentIfActive(inventory);
progressListeners.forEach(l -> l.accept(enricher, inventory));
if (writeIntermediateInventories && enricher.shouldWriteIntermediateInventory()) {
this.writeInventoryToFileAsEnricher(enricher);
}
} catch (Exception e) {
LOG.error("Failed to enrich inventory on step [{}], see stack trace below for more information.\n{}", enricher.getEnrichmentName(), e.getMessage());
try {
if (writeIntermediateInventories && enricher.shouldWriteIntermediateInventory()) {
this.writeInventoryToFileAsEnricher(enricher);
}
} catch (Exception inventoryWriteException) {
LOG.error("Failed to write intermediate inventory", e);
}
LOG.error(formatLogHeader("FAILED: " + enricher.getEnrichmentName()));
enrichmentException = new RuntimeException("Failed to enrich inventory on step " + enricher.getEnrichmentName() + "\n" + e.getMessage(), e);
failedEnricher = enricher;
break;
} finally {
enrichmentDurations.put(enricher, TimeUtils.utcNow() - startTime);
}
}
if (enrichmentException == null) {
LOG.info("All enrichment steps have been applied successfully:");
} else {
LOG.info("To resume from failed step, use id [{}]", failedEnricher.getConfiguration().getId());
}
for (InventoryEnricher enricher : enrichers) {
if (enricher == failedEnricher) {
LOG.info("Failed at:");
}
LOG.info(formatEnrichmentListEntry(enricher, enrichmentDurations.getOrDefault(enricher, 0L)));
}
LOG.info("");
if (enrichmentException != null) {
LOG.error("Failed to enrich inventory: {}", enrichmentException.getMessage(), enrichmentException);
throw enrichmentException;
}
{
final VulnerabilityContextInventory vInventory = VulnerabilityContextInventory.fromInventory(inventory);
vInventory.calculateEffectiveCvssVectorsForVulnerabilities(super.getSecurityPolicyConfiguration());
vInventory.writeBack();
vInventory.writeAdditionalInformationBack(super.getSecurityPolicyConfiguration());
}
if (writeIntermediateInventories) {
writeInventoryToFileAsEnricher(this);
}
if (inventoryResultFile != null) {
try {
if (!inventoryResultFile.getParentFile().exists()) {
inventoryResultFile.getParentFile().mkdirs();
}
new InventoryWriter().writeInventory(inventory, inventoryResultFile);
} catch (IOException e) {
throw new RuntimeException("Failed to write inventory to " + inventoryResultFile, e);
}
}
logInventoryResultStatistics(inventory);
}
private void logCentralSecurityConfiguration() {
LOG.info("");
LOG.info(formatLogHeader("Security Policy Configuration"));
super.getSecurityPolicyConfiguration().logConfiguration();
}
private void logInventoryResultStatistics(Inventory inventory) {
final int artifactCount = inventory.getArtifacts().size();
final int vulnerabilityCount = inventory.getVulnerabilityMetaData().size();
final int securityAdvisoriesCount = inventory.getAdvisoryMetaData().size();
final int licenseCount = inventory.getLicenseMetaData().size();
LOG.info("Pipeline result:");
if (artifactCount > 0) LOG.info(" Artifacts: {}", artifactCount);
if (vulnerabilityCount > 0) LOG.info(" Vulnerabilities: {}", vulnerabilityCount);
if (securityAdvisoriesCount > 0) LOG.info(" Security Advisories: {}", securityAdvisoriesCount);
if (licenseCount > 0) LOG.info(" Licenses: {}", licenseCount);
if (this.inventoryResultFile != null) {
LOG.info(" Result inventory: file://{}", this.inventoryResultFile.getAbsolutePath());
}
if (this.intermediateInventoriesDirectory != null) {
LOG.info(" Intermediate inventories: file://{}", this.intermediateInventoriesDirectory.getAbsolutePath());
}
// Vulnerability Assessment Dashboard file location
enrichers.stream()
.filter(e -> e instanceof VulnerabilityAssessmentDashboard)
.map(e -> (VulnerabilityAssessmentDashboard) e)
.findFirst()
.map(VulnerabilityAssessmentDashboard::getConfiguration)
.map(VulnerabilityAssessmentDashboardEnrichmentConfiguration::getOutputDashboardFile)
.ifPresent(file -> LOG.info(" Vulnerability Assessment Dashboard: file://{}", file.getAbsolutePath()));
}
private void assertCorrectlyConfigured() {
final Map> misconfigurations = new LinkedHashMap<>();
final Map> misconfigurationsWarnings = new LinkedHashMap<>();
for (InventoryEnricher enricher : enrichers) {
final List misconfiguration = enricher.collectMisconfigurations();
if (!misconfiguration.isEmpty()) {
if (enricher.getConfiguration().isActive()) {
misconfigurations.put(enricher, misconfiguration);
} else {
misconfigurationsWarnings.put(enricher, misconfiguration);
}
}
}
if (!misconfigurationsWarnings.isEmpty()) {
LOG.info("");
LOG.warn("Found misconfigurations in inactive enrichers:");
for (Map.Entry> entry : misconfigurationsWarnings.entrySet()) {
LOG.warn(" Enricher [{}] [{}]:", entry.getKey().getConfiguration().getId(), entry.getKey().getEnrichmentName());
for (ProcessMisconfiguration misconfiguration : entry.getValue()) {
LOG.warn(" - [{}] {}", misconfiguration.getField(), misconfiguration.getMessage());
}
}
}
if (!misconfigurations.isEmpty()) {
LOG.info("");
LOG.warn("Found misconfigurations in active enrichers:");
for (Map.Entry> entry : misconfigurations.entrySet()) {
LOG.warn(" Enricher [{}] [{}]:", entry.getKey().getConfiguration().getId(), entry.getKey().getEnrichmentName());
for (ProcessMisconfiguration misconfiguration : entry.getValue()) {
LOG.error(" - [{}] {}", misconfiguration.getField(), misconfiguration.getMessage());
}
}
throw new RuntimeException("Found misconfigurations in " + misconfigurations.size() + " enricher(s), see log for details");
}
}
private void appendInventoryInfoStep(Inventory inventory, InventoryEnricher enricher) {
final InventoryInfo info = inventory.findOrCreateInventoryInfo(INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT);
final JSONArray enrichmentInformation = findExistingOrCreateInventoryInfoStoredSteps(info);
final JSONObject step = new JSONObject()
.put("name", enricher.getEnrichmentName())
.put("id", enricher.getConfiguration().getId())
.put("active", enricher.getConfiguration().isActive())
.put("index", getEnricherIndex(enricher))
.put("configuration", enricher.getConfiguration().getProperties())
.put("time", TimeUtils.utcNow());
enrichmentInformation.put(step);
info.set(INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT_STEPS, enrichmentInformation.toString());
info.set(INVENTORY_INFO_KEY_ARTIFACT_ANALYSIS_VERSION, BuildProperties.getProjectVersion());
info.set(INVENTORY_INFO_KEY_VAD_VERSION, BuildProperties.getVulnerabilityAssessmentDashboardVersion());
}
private static JSONArray findExistingOrCreateInventoryInfoStoredSteps(InventoryInfo info) {
if (info.has(INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT_STEPS) && info.get(INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT_STEPS).startsWith("[")) {
return new JSONArray(info.get(INVENTORY_INFO_KEY_INVENTORY_ENRICHMENT_STEPS));
} else {
return new JSONArray();
}
}
private int getEnricherIndex(InventoryEnricher enricher) {
return IntStream.range(0, enrichers.size()).filter(i -> enrichers.get(i) == enricher).findFirst().orElse(-1);
}
private String formatEnrichmentListEntry(InventoryEnricher enricher, long duration) {
final StringBuilder sb = new StringBuilder();
final int digitCount = (int) Math.ceil(Math.log10(enrichers.size() + 1));
sb
.append(" ")
.append(String.format("%" + digitCount + "d", getEnricherIndex(enricher) + 1))
.append(". ");
sb.append(resumeAtEnricher != null && isBefore(resumeAtEnricher, enricher) ? "(skipped) " : "");
if (enricher.getConfiguration() == null) {
throw new RuntimeException("Missing configuration on " + enricher.getEnrichmentName());
}
sb.append(enricher.getConfiguration().isActive() ? "" : "(inactive) ");
sb.append(enricher.getEnrichmentName());
final String duplicateIndex = enricher.getConfiguration().getId().replaceAll(".+?(\\d+)$", "$1");
final boolean hasDuplicateIndex = StringUtils.hasText(duplicateIndex) && !duplicateIndex.equals(enricher.getConfiguration().getId());
if (hasDuplicateIndex) {
sb.append(" (").append(duplicateIndex).append(")");
} else {
final String initialId = enricher.getConfiguration().buildInitialId();
if (StringUtils.hasText(initialId) && !initialId.equals(enricher.getConfiguration().getId())) {
sb.append(" (").append(enricher.getConfiguration().getId()).append(")");
}
}
if (duration >= 0) {
if (sb.length() < 59 && sb.length() % 2 == 0) {
sb.append(" ");
}
while (sb.length() < 59) {
sb.append(" .");
}
sb.append(String.format(" [%9s]", TimeUtils.formatTimeDiff(duration)));
}
return sb.toString();
}
private boolean isBefore(InventoryEnricher pivot, InventoryEnricher checkBefore) {
return getEnricherIndex(pivot) > getEnricherIndex(checkBefore);
}
private void writeInventoryToFileAsEnricher(InventoryEnricher enricher) {
if (inventorySourceFile != null && intermediateInventoriesDirectory != null) {
final boolean isFinalStep = enricher instanceof InventoryEnrichmentPipeline;
final File destinationFile = this.getInventoryFileForEnricher(enricher);
if (!destinationFile.getParentFile().exists()) {
destinationFile.getParentFile().mkdirs();
}
try {
new InventoryWriter().writeInventory(inventory, destinationFile);
} catch (IOException e) {
throw new RuntimeException("Failed to write " + (isFinalStep ? "resulting" : "intermediate") + " inventory to file: " + destinationFile.getAbsolutePath(), e);
}
}
}
private File getInventoryFileForEnricher(InventoryEnricher enricher) {
return new File(this.intermediateInventoriesDirectory,
String.format("%s-%s-%s.xls",
inventorySourceFile.getName().replace(".xls", ""),
enricher.getInventoryFileNameSuffix(),
enricher.getConfiguration().getId()
));
}
private Map deriveEffectiveEnrichmentIds() {
final Map enrichmentIdDuplicateCount = new LinkedHashMap<>();
for (InventoryEnricher enricher : enrichers) {
final String id = enricher.getConfiguration().getId();
enrichmentIdDuplicateCount.compute(id, (k, v) -> v == null ? 1 : v + 1);
}
final Set duplicates = enrichmentIdDuplicateCount.entrySet().stream()
.filter(e -> e.getValue() > 1)
.map(Map.Entry::getKey)
.collect(Collectors.toSet());
final Map effectiveEnrichmentIds = new LinkedHashMap<>();
enrichmentIdDuplicateCount.clear();
for (InventoryEnricher enricher : enrichers) {
final String id = enricher.getConfiguration().getId();
final int occurrenceCount = enrichmentIdDuplicateCount.compute(id, (k, v) -> v == null ? 1 : v + 1);
final String effectiveId = occurrenceCount > 1 || duplicates.contains(id) ? id + "-" + occurrenceCount : id;
effectiveEnrichmentIds.put(effectiveId, enricher);
}
return effectiveEnrichmentIds;
}
private void assignEffectiveIdsToEnrichers() {
for (Map.Entry entry : deriveEffectiveEnrichmentIds().entrySet()) {
entry.getValue().getConfiguration().setId(entry.getKey());
}
}
@Override
public ProcessConfiguration getConfiguration() {
return new ProcessConfiguration() {
@Override
public LinkedHashMap getProperties() {
return new LinkedHashMap<>();
}
@Override
public void setProperties(LinkedHashMap properties) {
}
@Override
protected void collectMisconfigurations(List misconfigurations) {
}
};
}
private InventoryEnricher constructEnricher(Class clazz) {
return constructEnricher(clazz, baseMirrorDirectory);
}
public static InventoryEnricher constructEnricher(Class clazz, File baseMirrorDirectory) {
try {
return clazz.getConstructor(File.class).newInstance(baseMirrorDirectory);
} catch (InstantiationException | IllegalAccessException | NoSuchMethodException ignored) {
} catch (Exception e) {
throw new RuntimeException("Failed to instantiate enricher class due to constructor failing: " + clazz.getName(), e);
}
try {
return clazz.getConstructor().newInstance();
} catch (InstantiationException | IllegalAccessException | NoSuchMethodException e) {
throw new RuntimeException("Failed to instantiate enrichment class: " + clazz + ". (File baseMirrorDirectory) or () constructor must exist", e);
} catch (Exception e) {
throw new RuntimeException("Failed to instantiate enrichment class due to constructor failing: " + clazz, e);
}
}
public File getBaseMirrorDirectory() {
return baseMirrorDirectory;
}
public File getIntermediateInventoriesDirectory() {
return intermediateInventoriesDirectory;
}
public File getInventoryResultFile() {
return inventoryResultFile;
}
public File getInventorySourceFile() {
return inventorySourceFile;
}
public Inventory getInventory() {
return inventory;
}
public InventoryEnricher getResumeAtEnricher() {
return resumeAtEnricher;
}
public List getEnrichers() {
return enrichers;
}
public boolean isStoreIntermediateStepsInInventoryInfo() {
return storeIntermediateStepsInInventoryInfo;
}
public boolean isWriteIntermediateInventories() {
return writeIntermediateInventories;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy