com.metaeffekt.artifact.enrichment.configurations.AdvisorPeriodicEnrichmentConfiguration Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.artifact.enrichment.configurations;

import com.metaeffekt.artifact.analysis.cert.PeriodicDataSourcesOperations;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.mirror.contents.store.AdvisoryTypeIdentifier;
import com.metaeffekt.mirror.contents.store.AdvisoryTypeStore;
import lombok.Getter;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.json.JSONArray;
import org.json.JSONObject;
import org.metaeffekt.core.inventory.processor.configuration.ProcessMisconfiguration;
import org.metaeffekt.core.inventory.processor.model.AdvisoryMetaData;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import org.metaeffekt.core.inventory.processor.reader.InventoryReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

public class AdvisorPeriodicEnrichmentConfiguration extends VulnerabilitiesFromCpeEnrichmentConfiguration {

    private static final Logger LOG = LoggerFactory.getLogger(AdvisorPeriodicEnrichmentConfiguration.class);

    // limiting what advisories to consider

    /**
     * Represents a {@link List}<{@link Map}<{@link String}, {@link String}>>.

     * The key "name" is mandatory and can optionally be combined with an "implementation" value. If the implementation
     * is not specified, the name will be used as the implementation. Each list entry represents a single advisory type.
     * 
     * The {@link AdvisorPeriodicEnrichmentConfiguration#advisoryProviders} determines the data sources the advisories
     * are fetched from. If a specified advisory type does not provide a valid index to query advisories from, it will
     * be ignored.
     * 

     * Example:
     * 
     *     [{"name":"CERT_FR"},
     *      {"name":"CERT_SEI"},
     *      {"name":"RHSA","implementation":"CSAF"}]
     * 
     */
    private String advisoryProviders = new JSONArray()
            .put(new JSONObject().put("name", "all").put("implementation", "all")).toString();

    /**
     * A string that represents the start time for the period to query for advisories in.
     * This parameter determines how far back in time advisories should be considered.
     * The value can be a date in the format "yyyy-MM-dd" or a string like "3 months 5 hours".
     */
    private String changedSince = "3 months";

    /**
     * A string that represents the end time for the period to query for advisories in.
     * See {@link AdvisorPeriodicEnrichmentConfiguration#changedSince} for more information.
     * By default, this is set to "now", meaning the current time.
     */
    private String changedUntil = "now";

    // filtering the advisories

    /**
     * Is applied after all relevant advisories have been found for the selected time period.

     * If set to true, advisories with the {@link AdvisoryMetaData#STATUS_VALUE_UNAFFECTED} status will be filtered out
     * from the inventory.
     */
    @Getter
    private boolean filterUnaffected = false;

    /**
     * Is applied after all relevant advisories have been found for the selected time period.

     * If set to true, advisories with the {@link AdvisoryMetaData#STATUS_VALUE_UNCLASSIFIED} status will be filtered
     * out from the inventory.
     */
    @Getter
    private boolean filterUnclassified = false;

    /**
     * What TYPES of advisories to include. This is NOT the advisory provider or source.

     * Valid values include: notice, alert, news

     * To include all types, use all.
     */
    private List includeAdvisoryTypes = new ArrayList<>(Collections.singletonList("all"));

    // filtering the vulnerabilities

    /**
     * Is applied after all relevant advisories have been found for the selected time period and advisory filters have
     * been applied.

     * If set to true, vulnerabilities that are not affected by any of the found advisories will be filtered out from
     * the inventory.

     * Note: This is overwritten by the
     * {@link AdvisorPeriodicEnrichmentConfiguration#includeVulnerabilitiesChangedSince} parameter.
     */
    @Getter
    private boolean filterUnaffectedVulnerabilities = false;

    /**
     * Is applied after all relevant advisories have been found for the selected time period and advisory filters have
     * been applied.

     * If set to true, vulnerabilities that do not have an advisory from the sources listen in the
     * {@link AdvisorPeriodicEnrichmentConfiguration#vulnerabilityAdvisoryFilter} parameter will be filtered out from
     * the inventory.

     * Note: This is overwritten by the
     * {@link AdvisorPeriodicEnrichmentConfiguration#includeVulnerabilitiesChangedSince} parameter.
     */
    @Getter
    private boolean filterVulnerabilitiesWithoutSpecifiedAdvisory = false;

    /**
     * Represents a {@link List}<{@link Map}<{@link String}, {@link String}>>.

     * The key "name" is mandatory and can optionally be combined with an "implementation" value. If the implementation
     * is not specified, the name will be used as the implementation. Each list entry represents a single advisory type.
     * 
     * The {@link AdvisorPeriodicEnrichmentConfiguration#vulnerabilityAdvisoryFilter} is used to filter vulnerabilities
     * based on the advisory type. If the vulnerability does not have one of the specified advisory types, it will be
     * filtered out.
     * 

     * Example:
     * 
     *     [{"name":"CERT_FR"},
     *      {"name":"CERT_SEI"},
     *      {"name":"RHSA","implementation":"CSAF"}]
     * 
     */
    private String vulnerabilityAdvisoryFilter = new JSONArray().toString();

    /**
     * Can be set to a value similar to the {@link AdvisorPeriodicEnrichmentConfiguration#changedSince} value.
     * This parameter will force all vulnerabilities from the source (initial) or reference inventories to be included
     * that have been changed since the specified date, no matter if they are affected or would be filtered otherwise by
     * the other parameters.

     * Example: "2023-11-08" or "3 months"
     */
    private String includeVulnerabilitiesChangedSince = null;

    // reference inventories
    @Getter
    private String initialInventoryContextName = "initial";
    private List referenceInventories = new ArrayList<>();
    private final Map referenceInventoriesInventories = new HashMap<>();

    public AdvisorPeriodicEnrichmentConfiguration setAdvisoryProviders(Map advisoryProviders) {
        final JSONArray advisoryProvidersArray = new JSONArray();
        advisoryProviders.forEach((name, implementation) -> advisoryProvidersArray.put(new JSONObject().put("name", name).put("implementation", StringUtils.hasText(implementation) ? implementation : name)));
        this.advisoryProviders = advisoryProvidersArray.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setAdvisoryProviders(JSONArray advisoryProviders) {
        this.advisoryProviders = advisoryProviders.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setAdvisoryProviders(AdvisoryTypeIdentifier... advisoryProviders) {
        final JSONArray advisoryProvidersArray = new JSONArray();
        Arrays.stream(advisoryProviders).forEach(advisoryTypeIdentifier ->
            advisoryProvidersArray.put(new JSONObject()
                .put("name", advisoryTypeIdentifier.getName())
                .put("implementation", advisoryTypeIdentifier.getImplementation())));
        this.advisoryProviders = advisoryProvidersArray.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setChangedSince(String changedSince) {
        this.changedSince = changedSince.toLowerCase();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setChangedSince(long changedSince) {
        this.changedSince = String.valueOf(changedSince);
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setChangedUntil(String changedUntil) {
        this.changedUntil = changedUntil.toLowerCase();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setChangedUntil(long changedUntil) {
        this.changedUntil = String.valueOf(changedUntil);
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setIncludeVulnerabilitiesChangedSince(String includeVulnerabilitiesChangedSince) {
        this.includeVulnerabilitiesChangedSince = includeVulnerabilitiesChangedSince.toLowerCase();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setIncludeVulnerabilitiesChangedSince(long includeVulnerabilitiesChangedSince) {
        this.includeVulnerabilitiesChangedSince = String.valueOf(includeVulnerabilitiesChangedSince);
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setFilterUnaffected(boolean filterUnaffected) {
        this.filterUnaffected = filterUnaffected;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setFilterUnaffectedVulnerabilities(boolean filterUnaffectedVulnerabilities) {
        this.filterUnaffectedVulnerabilities = filterUnaffectedVulnerabilities;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setFilterUnclassified(boolean filterUnclassified) {
        this.filterUnclassified = filterUnclassified;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setFilterVulnerabilitiesWithoutSpecifiedAdvisory(boolean filterVulnerabilitiesWithoutSpecifiedAdvisory) {
        this.filterVulnerabilitiesWithoutSpecifiedAdvisory = filterVulnerabilitiesWithoutSpecifiedAdvisory;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setVulnerabilityAdvisoryFilter(Map vulnerabilityAdvisoryFilter) {
        final JSONArray vulnerabilityAdvisoryFilterArray = new JSONArray();
        vulnerabilityAdvisoryFilter.forEach((name, implementation) -> vulnerabilityAdvisoryFilterArray.put(new JSONObject().put("name", name).put("implementation", StringUtils.hasText(implementation) ? implementation : name)));
        this.vulnerabilityAdvisoryFilter = vulnerabilityAdvisoryFilterArray.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setVulnerabilityAdvisoryFilter(JSONArray vulnerabilityAdvisoryFilter) {
        this.vulnerabilityAdvisoryFilter = vulnerabilityAdvisoryFilter.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setVulnerabilityAdvisoryFilterCI(Collection> vulnerabilityAdvisoryFilter) {
        final JSONArray vulnerabilityAdvisoryFilterArray = new JSONArray();
        vulnerabilityAdvisoryFilter.forEach(advisoryTypeIdentifier -> vulnerabilityAdvisoryFilterArray.put(new JSONObject().put("name", advisoryTypeIdentifier.getName()).put("implementation", advisoryTypeIdentifier.getImplementation())));
        this.vulnerabilityAdvisoryFilter = vulnerabilityAdvisoryFilterArray.toString();
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setIncludeAdvisoryTypes(List includeAdvisoryTypes) {
        this.includeAdvisoryTypes = includeAdvisoryTypes;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setReferenceInventories(List referenceInventories) {
        this.referenceInventories = referenceInventories;
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration addReferenceInventory(File referenceInventory) {
        this.referenceInventories.add(referenceInventory);
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration addReferenceInventory(Inventory referenceInventory, String contextName) {
        this.referenceInventoriesInventories.put(contextName, referenceInventory);
        return this;
    }

    public AdvisorPeriodicEnrichmentConfiguration setInitialInventoryContextName(String initialInventoryContextName) {
        this.initialInventoryContextName = initialInventoryContextName;
        return this;
    }

    public List> getAdvisoryProviders() {
        return parseAdvisoryProviders(advisoryProviders);
    }

    public List getIncludeAdvisoryTypes() {
        return includeAdvisoryTypes;
    }

    public List> getVulnerabilityAdvisoryFilter() {
        return parseAdvisoryProviders(vulnerabilityAdvisoryFilter);
    }

    public static List> parseAdvisoryProviders(String jsonArrayProviders) {
        return parseAdvisoryProviders(new JSONArray(jsonArrayProviders));
    }

    public static List> parseAdvisoryProviders(JSONArray providers) {
        if (providers == null || providers.isEmpty()) {
            return AdvisoryTypeStore.get().values();
        } else {
            final List> parsed = AdvisoryTypeStore.get().fromJsonNamesAndImplementations(providers);
            if (parsed.contains(AdvisoryTypeStore.ANY_ADVISORY_FILTER_WILDCARD)) {
                return AdvisoryTypeStore.get().values();
            }
            return parsed;
        }
    }

    public long getChangedSinceTimestamp() {
        return TimeUtils.parseTimeFromInput(changedSince);
    }

    public long getChangedUntilTimestamp() {
        return TimeUtils.parseTimeFromInput(changedUntil);
    }

    public PeriodicDataSourcesOperations.QueryTimePeriod getAdvisoryQueryPeriod() {
        return new PeriodicDataSourcesOperations.QueryTimePeriod(getChangedSinceTimestamp(), getChangedUntilTimestamp());
    }

    public long getIncludeVulnerabilitiesChangedSinceTimestamp() {
        return TimeUtils.parseTimeFromInput(includeVulnerabilitiesChangedSince);
    }

    public boolean hasInitialInventoryContextName() {
        return StringUtils.hasText(initialInventoryContextName) && !initialInventoryContextName.equals("none") && !initialInventoryContextName.equals("null");
    }

    public Map parseReferenceInventories() {
        final List inventoryFiles = new ArrayList<>();

        for (File referenceInventory : referenceInventories) {
            final int amountInventoryFilesBefore = inventoryFiles.size();
            if (referenceInventory.isDirectory()) {
                inventoryFiles.addAll(FileUtils.listFiles(referenceInventory, new IOFileFilter() {
                    @Override
                    public boolean accept(File file) {
                        return isInventoryFile(file);
                    }

                    @Override
                    public boolean accept(File dir, String name) {
                        return isInventoryFile(new File(dir, name));
                    }
                }, null));
            } else {
                inventoryFiles.add(referenceInventory);
            }

            if (inventoryFiles.size() == amountInventoryFilesBefore) {
                LOG.warn("Reference inventory [{}] does not contain any inventory files", referenceInventory.getAbsolutePath());
            }
        }

        final Map parsed = parseReferenceInventories(inventoryFiles);
        parsed.putAll(referenceInventoriesInventories);

        return parsed;
    }

    private Map parseReferenceInventories(List files) {
        final Map inventories = new LinkedHashMap<>();

        for (File referenceInventory : files) {
            if (!referenceInventory.exists()) {
                throw new RuntimeException("Reference inventory does not exist [" + referenceInventory.getAbsolutePath() + "]");
            }

            if (isInventoryFile(referenceInventory)) {
                try {
                    LOG.info("Parsing reference inventory [{}]", referenceInventory.getAbsolutePath());
                    inventories.put(
                            referenceInventory.getName().replace(".xls", "").replace(".xlsx", ""),
                            new InventoryReader().readInventory(referenceInventory)
                    );
                } catch (IOException e) {
                    throw new RuntimeException("Failed to read reference inventory [" + referenceInventory.getAbsolutePath() + "]", e);
                }
            } else {
                LOG.warn("Skipping reference inventory [{}] as it is not an Excel file", referenceInventory.getAbsolutePath());
            }
        }

        return inventories;
    }

    private boolean isInventoryFile(File file) {
        return file.getName().endsWith(".xls") || file.getName().endsWith(".xlsx");
    }

    @Override
    public LinkedHashMap getProperties() {
        final LinkedHashMap configuration = super.getProperties();

        configuration.put("advisoryProviders", advisoryProviders);
        configuration.put("changedSince", changedSince);
        configuration.put("changedSinceTimestamp", getChangedSinceTimestamp());
        configuration.put("changedUntil", changedUntil);
        configuration.put("changedUntilTimestamp", getChangedUntilTimestamp());
        configuration.put("includeVulnerabilitiesChangedSince", includeVulnerabilitiesChangedSince);
        configuration.put("includeVulnerabilitiesChangedSinceTimestamp", getIncludeVulnerabilitiesChangedSinceTimestamp());
        configuration.put("filterUnaffected", filterUnaffected);
        configuration.put("filterUnclassified", filterUnclassified);
        configuration.put("filterUnaffectedVulnerabilities", filterUnaffectedVulnerabilities);
        configuration.put("filterVulnerabilitiesWithoutSpecifiedAdvisory", filterVulnerabilitiesWithoutSpecifiedAdvisory);
        configuration.put("vulnerabilityAdvisoryFilter", vulnerabilityAdvisoryFilter);
        configuration.put("includeAdvisoryTypes", includeAdvisoryTypes);
        configuration.put("referenceInventories", referenceInventories.stream().map(File::getAbsolutePath).collect(Collectors.toList()));
        configuration.put("initialInventoryContextName", initialInventoryContextName);

        return configuration;
    }

    @Override
    public void setProperties(LinkedHashMap properties) {
        super.setProperties(properties);

        super.loadJsonArrayProperty(properties, "advisoryProviders", this::setAdvisoryProviders);
        super.loadStringProperty(properties, "changedSince", this::setChangedSince);
        super.loadStringProperty(properties, "changedUntil", this::setChangedUntil);
        super.loadStringProperty(properties, "includeVulnerabilitiesChangedSince", this::setIncludeVulnerabilitiesChangedSince);
        super.loadBooleanProperty(properties, "filterUnaffected", this::setFilterUnaffected);
        super.loadBooleanProperty(properties, "filterUnclassified", this::setFilterUnclassified);
        super.loadBooleanProperty(properties, "filterUnaffectedVulnerabilities", this::setFilterUnaffectedVulnerabilities);
        super.loadBooleanProperty(properties, "filterVulnerabilitiesWithoutSpecifiedAdvisory", this::setFilterVulnerabilitiesWithoutSpecifiedAdvisory);
        super.loadJsonArrayProperty(properties, "vulnerabilityAdvisoryFilter", this::setVulnerabilityAdvisoryFilter);
        super.loadListProperty(properties, "includeAdvisoryTypes", String::valueOf, this::setIncludeAdvisoryTypes);
        super.loadStringProperty(properties, "initialInventoryContextName", this::setInitialInventoryContextName);
    }

    @Override
    protected void collectMisconfigurations(List misconfigurations) {
        if (advisoryProviders == null || advisoryProviders.isEmpty()) {
            misconfigurations.add(new ProcessMisconfiguration("advisoryProviders", "Advisory providers are not specified, use 'all' to include all providers"));
        }

        if (StringUtils.isEmpty(changedSince)) {
            misconfigurations.add(new ProcessMisconfiguration("changedSince", "Changed since is not specified, use a string like '3 months' or '2023-11-08'"));
        } else if (getChangedSinceTimestamp() == 0) {
            misconfigurations.add(new ProcessMisconfiguration("changedSince", "Changed since is in an invalid format, use a string like '3 months' or '2023-11-08'"));
        }

        if (includeAdvisoryTypes == null || includeAdvisoryTypes.isEmpty()) {
            misconfigurations.add(new ProcessMisconfiguration("includeAdvisoryTypes", "Include advisory types is not specified, use 'all' to include all types"));
        }
    }
}