All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • com.metaeffekt.mirror.contents.vulnerability.Vulnerability Maven / Gradle / Ivy

    The newest version!
    /*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.contents.vulnerability;

import com.metaeffekt.artifact.analysis.utils.CustomCollectors;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.InventoryAttribute;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.keywords.KeywordSet;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.score.VulnerabilityPriorityCalculator;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.vulnerabilitystatus.VulnerabilityStatus;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.vulnerabilitystatus.VulnerabilityStatusConverter;
import com.metaeffekt.mirror.contents.advisory.AdvisoryEntry;
import com.metaeffekt.mirror.contents.base.CvssConditionAttributes;
import com.metaeffekt.mirror.contents.base.MatchableDetailsAmbDataClass;
import com.metaeffekt.mirror.contents.base.Reference;
import com.metaeffekt.mirror.contents.epss.EpssData;
import com.metaeffekt.mirror.contents.kev.KevData;
import com.metaeffekt.mirror.contents.store.AdvisoryTypeIdentifier;
import com.metaeffekt.mirror.contents.store.OtherTypeStore;
import com.metaeffekt.mirror.contents.store.VulnerabilityTypeIdentifier;
import com.metaeffekt.mirror.contents.store.VulnerabilityTypeStore;
import com.metaeffekt.mirror.query.VulnerabilityIndexQuery;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.json.JSONArray;
import org.json.JSONObject;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.metaeffekt.core.inventory.processor.model.VulnerabilityMetaData;
import org.metaeffekt.core.inventory.processor.report.configuration.CentralSecurityPolicyConfiguration;
import org.metaeffekt.core.security.cvss.CvssSource;
import org.metaeffekt.core.security.cvss.CvssVector;
import org.metaeffekt.core.security.cvss.KnownCvssEntities;
import org.metaeffekt.core.security.cvss.processor.CvssSelectionResult;
import org.metaeffekt.core.security.cvss.processor.CvssSelectionResult.CvssScoreVersionSelectionPolicy;
import org.metaeffekt.core.security.cvss.processor.CvssSelector;
import org.metaeffekt.core.security.cvss.processor.CvssVectorSet;
import org.metaeffekt.core.security.cvss.v2.Cvss2;
import org.metaeffekt.core.security.cvss.v3.Cvss3P1;
import org.metaeffekt.core.security.cvss.v4P0.Cvss4P0;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import us.springett.parsers.cpe.Cpe;
import us.springett.parsers.cpe.exceptions.CpeValidationException;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

public class Vulnerability extends MatchableDetailsAmbDataClass {

    private static final Logger LOG = LoggerFactory.getLogger(Vulnerability.class);

    private final static Set CONVERSION_KEYS_AMB = new HashSet(MatchableDetailsAmbDataClass.CONVERSION_KEYS_AMB) {{
        addAll(Arrays.asList(
                VulnerabilityMetaData.Attribute.NAME.getKey(),
                VulnerabilityMetaData.Attribute.SOURCE.getKey(),
                VulnerabilityMetaData.Attribute.SOURCE_IMPLEMENTATION.getKey(),
                InventoryAttribute.DESCRIPTION.getKey(),
                VulnerabilityMetaData.Attribute.URL.getKey(),
                InventoryAttribute.VULNERABILITY_UPDATED_DATE_TIMESTAMP.getKey(),
                InventoryAttribute.VULNERABILITY_CREATED_DATE_TIMESTAMP.getKey(),
                VulnerabilityMetaData.Attribute.REFERENCES.getKey(),
                VulnerabilityMetaData.Attribute.WEAKNESS.getKey(),
                InventoryAttribute.TAGS.getKey(),
                InventoryAttribute.VULNERABILITY_STATUS.getKey()
        ));
    }};

    private final static Set CONVERSION_KEYS_MAP = new HashSet(MatchableDetailsAmbDataClass.CONVERSION_KEYS_MAP) {{
        addAll(Arrays.asList(
                "name", "description", "notes", "url", "cvssVectors", "createDate", "updateDate", "cwe",
                "vulnerable_software", "references", "tags", "vulnerabilityStatus",
                "cvss", "kevData", "epssData"
        ));
    }};

    /**
     * Stores what provider this vulnerability originates from.
    
     * Must be defined on all vulnerability instances.
     */
    protected VulnerabilityTypeIdentifier sourceIdentifier;

    @Getter
    @Setter
    private String description;
    @Getter
    @Setter
    private String notes;
    @Setter
    private String url;

    @Getter
    @Setter
    protected Date createDate;

    @Getter
    @Setter
    protected Date updateDate;

    private final CvssVectorSet cvssVectors = new CvssVectorSet();
    private CvssSelectionResult cvssSelectionResult;

    @Getter
    private final Set references = new LinkedHashSet<>();
    @Getter
    private final Set cwes = new LinkedHashSet<>();

    @Getter
    @Setter
    private EpssData epssData;

    @Getter
    private final Set vulnerableSoftwareConfigurations = new HashSet<>();

    @Getter
    private final Set securityAdvisories = new LinkedHashSet<>();

    private KevData kevData;

    @Getter
    private final Set tags = new LinkedHashSet<>();

    @Getter
    @Setter
    private VulnerabilityStatus vulnerabilityStatus;

    public final static Comparator UPDATE_CREATE_TIME_COMPARATOR = Comparator
            .comparing(Vulnerability::getUpdateDate)
            .thenComparing(Vulnerability::getCreateDate);

    public Vulnerability() {
    }

    public Vulnerability(String id) {
        super.setId(id);
    }

    public void setSourceIdentifier(VulnerabilityTypeIdentifier source) {
        if (source == null) {
            throw new IllegalArgumentException("Advisory source must not be null");
        }
        if (LOG.isDebugEnabled() && source != this.sourceIdentifier) {
            if (this.sourceIdentifier != null) {
                LOG.warn("Explicitly assigned source differs from originally assigned [{}] --> [{}]",
                        this.sourceIdentifier.toExtendedString(), source.toExtendedString());
            }
        }
        this.sourceIdentifier = source;
    }

    @Override
    public VulnerabilityTypeIdentifier getSourceIdentifier() {
        return sourceIdentifier;
    }

    public String getUrl() {
        if (url != null) {
            return url;
        } else {
            if (id.startsWith("CVE-")) {
                return "https://nvd.nist.gov/vuln/detail/" + id;
            } else {
                return null;
            }
        }
    }

    public boolean isCreatedAfter(Date date) {
        return createDate != null && createDate.after(date);
    }

    public boolean isCreatedBefore(Date date) {
        return createDate != null && createDate.before(date);
    }

    public boolean isUpdatedAfter(Date date) {
        return updateDate != null && updateDate.after(date);
    }

    public boolean isUpdatedBefore(Date date) {
        return updateDate != null && updateDate.before(date);
    }

    public boolean hasBeenUpdatedSince(long millis) {
        return (updateDate != null && updateDate.getTime() > millis) || (createDate != null && createDate.getTime() > millis);
    }

    public void addReference(Reference reference) {
        this.references.add(reference);
    }

    public void addReferences(Collection references) {
        this.references.addAll(references);
    }

    public void addCwe(String cwe) {
        if (cwe != null) {
            this.cwes.add(cwe);
        }
    }

    public void addCwe(String... cwe) {
        addCwes(Arrays.asList(cwe));
    }

    private void addCwes(Collection cwes) {
        cwes.forEach(this::addCwe);
    }

    public void addVulnerableSoftware(VulnerableSoftwareVersionRangeCpe vulnerableSoftware) {
        final VulnerableSoftwareTreeNode node = new VulnerableSoftwareTreeNode("OR");
        node.addNode(vulnerableSoftware);
        this.vulnerableSoftwareConfigurations.add(node);
    }

    public void addVulnerableSoftwares(Collection vulnerableSoftware) {
        vulnerableSoftware.forEach(this::addVulnerableSoftware);
    }

    public void addVulnerableSoftwareTreeNode(VulnerableSoftwareTreeNode vulnerableSoftware) {
        this.vulnerableSoftwareConfigurations.add(vulnerableSoftware);
    }

    public void addVulnerableSoftwaresTreeNodes(Collection vulnerableSoftware) {
        vulnerableSoftware.forEach(this::addVulnerableSoftwareTreeNode);
    }

    public boolean cpeFlatMatchesVulnerableSoftware(Cpe cpe) {
        return vulnerableSoftwareConfigurations.stream()
                .anyMatch(configuration -> configuration.isAffectedFlat(cpe));
    }

    public VulnerableSoftwareVersionRangeCpe getCpeFlatMatchedVulnerableSoftware(Cpe cpe) {
        return vulnerableSoftwareConfigurations.stream()
                .map(configuration -> configuration.getFlatAffectedNode(cpe))
                .filter(Objects::nonNull)
                .findFirst()
                .orElse(null);
    }

    public Optional optVulnerabilityStatus() {
        return Optional.ofNullable(vulnerabilityStatus);
    }

    public VulnerabilityStatus getOrCreateNewVulnerabilityStatus() {
        if (vulnerabilityStatus == null) {
            vulnerabilityStatus = new VulnerabilityStatus();
        }
        return vulnerabilityStatus;
    }

    public void addTag(String tag) {
        this.tags.add(tag);
    }

    public void addTags(Collection tags) {
        this.tags.addAll(tags);
    }

    public boolean hasTag(String tag) {
        return this.tags.contains(tag);
    }

    /* SECURITY ADVISORIES */

    public Map, Set> deepCopyReferencedSecurityAdvisories() {
        final Map, Set> copy = new HashMap<>();
        synchronized (this.referencedSecurityAdvisories) {
            for (Map.Entry, Set> referencedEntry : referencedSecurityAdvisories.entrySet()) {
                copy.put(referencedEntry.getKey(), new HashSet<>(referencedEntry.getValue()));
            }
        }
        return copy;
    }

    public void addSecurityAdvisory(AdvisoryEntry advisoryEntry) {
        this.securityAdvisories.add(advisoryEntry);
        this.addReferencedSecurityAdvisory(advisoryEntry);
    }

    public void removeSecurityAdvisory(AdvisoryEntry advisoryEntry) {
        this.securityAdvisories.remove(advisoryEntry);
        this.removeReferencedSecurityAdvisory(advisoryEntry);
    }

    public Set getRelatedAdvisors(AdvisoryTypeIdentifier type) {
        return getRelatedAdvisors(type, this.securityAdvisories);
    }

    public static Set getRelatedAdvisors(AdvisoryTypeIdentifier type, Collection advisoryProviders) {
        if (type == null) {
            return Collections.emptySet();
        }

        return advisoryProviders.stream()
                .filter(advisory -> advisory.getSourceIdentifier() == type)
                .collect(Collectors.toSet());
    }

    public  List getRelatedAdvisors(AdvisoryTypeIdentifier type, Class typeClass) {
        if (type == null || typeClass == null) {
            return Collections.emptyList();
        }

        return getRelatedAdvisors(type).stream()
                .filter(typeClass::isInstance)
                .map(typeClass::cast)
                .collect(Collectors.toList());
    }

    public Set> getRelatedAdvisorsTypes() {
        return securityAdvisories.stream()
                .map(AdvisoryEntry::getSourceIdentifier)
                .collect(Collectors.toSet());
    }

    public void setKevData(KevData kevData) {
        this.kevData = kevData;
    }

    public KevData getKevData() {
        return kevData;
    }

    /* CVSS */

    public CvssVectorSet getCvssVectors() {
        return this.cvssVectors;
    }

    /**
     * This method will only return a meaningful result if the instance has been created via the
     * {@link com.metaeffekt.mirror.contents.base.VulnerabilityContextInventory} class or if the
     * {@link #securityAdvisories} and {@link #affectedArtifacts} have manually been filled appropriately.
     * 
    
     * Effective in this context means that the {@link CvssVector}s from the {@link #securityAdvisories} are
     * added to the {@link #cvssVectors} if the {@link CvssVector#getApplicabilityCondition()} is met.
     * To determine whether this is the case, several factors like MSRC information from the affected artifacts are
     * taken into account.
     * 
    
     * Consider using {@link #selectEffectiveCvssVectors(CvssSelector, CvssSelector, List)} to pre-calculate the effective
     * {@link CvssVector}s if you expect to call this method multiple times to reduce the overhead of the calculation.
     *
     * @return a {@link CvssVectorSet} containing the effective {@link CvssVector}s for this vulnerability.
     */
    public CvssVectorSet calculateEffectiveCvssVectors() {
        return calculateEffectiveCvssVectors(this.cvssVectors);
    }

    public CvssVectorSet calculateEffectiveCvssVectors(CvssVectorSet cvssVectors) {
        final CvssVectorSet effectiveCvssVectors = new CvssVectorSet();

        // add all cvss vectors from the vulnerability itself
        effectiveCvssVectors.addAllCvssVectors(cvssVectors);

        // add all cvss vectors from the security advisories that are applicable
        for (AdvisoryEntry advisoryEntry : this.securityAdvisories) {
            for (CvssVector sourcedCvssVector : advisoryEntry.getCvssVectors().getCvssVectors()) {
                if (isCvssVectorApplicable(sourcedCvssVector.getApplicabilityCondition())) {
                    effectiveCvssVectors.addCvssVector(sourcedCvssVector);
                }
            }
        }

        return effectiveCvssVectors;
    }

    protected boolean isCvssVectorApplicable(JSONObject applicabilityCondition) {
        if (applicabilityCondition == null || applicabilityCondition.isEmpty()) {
            return true;
        }

        final Object findMsProductIdsObj = applicabilityCondition.opt(CvssConditionAttributes.MATCHES_ON_MS_PRODUCT_ID);
        if (findMsProductIdsObj instanceof JSONArray) {
            final List findMsProductIds = ((JSONArray) findMsProductIdsObj).toList().stream().map(String::valueOf).collect(Collectors.toList());
            boolean foundMatchingArtifact = false;

            for (Artifact artifact : this.getAffectedArtifactsByDefaultKey()) {
                final String msProductId = artifact.get(InventoryAttribute.MS_PRODUCT_ID);
                if (msProductId == null) {
                    continue;
                }
                final List artifactMsProductIds = Arrays.asList(msProductId.split(", "));
                if (artifactMsProductIds.stream().anyMatch(findMsProductIds::contains)) {
                    foundMatchingArtifact = true;
                    break;
                }
            }

            if (!foundMatchingArtifact) {
                return false;
            }
        }

        return true;
    }

    public CvssSelectionResult selectEffectiveCvssVectors(CvssVectorSet cvssVectorSet, CvssSelector baseSelector, CvssSelector effectiveSelector, List versionSelectionPolicy) {
        return new CvssSelectionResult(
                cvssVectorSet,
                baseSelector, effectiveSelector,
                versionSelectionPolicy
        );
    }

    public CvssSelectionResult selectEffectiveCvssVectors(CvssVectorSet cvssVectorSet, CentralSecurityPolicyConfiguration config) {
        return selectEffectiveCvssVectors(
                cvssVectorSet,
                config.getInitialCvssSelector(), config.getContextCvssSelector(),
                config.getCvssVersionSelectionPolicy()
        );
    }

    public void selectEffectiveCvssVectors(CvssSelector baseSelector, CvssSelector effectiveSelector, List versionSelectionPolicy) {
        this.cvssSelectionResult = selectEffectiveCvssVectors(
                this.calculateEffectiveCvssVectors(),
                baseSelector, effectiveSelector,
                versionSelectionPolicy
        );
    }

    public void selectEffectiveCvssVectors(CentralSecurityPolicyConfiguration config) {
        selectEffectiveCvssVectors(
                config.getInitialCvssSelector(), config.getContextCvssSelector(),
                config.getCvssVersionSelectionPolicy()
        );
    }

    public boolean isCvssSelectionResultAvailable() {
        return this.cvssSelectionResult != null;
    }

    public CvssSelectionResult getCvssSelectionResult() {
        if (!isCvssSelectionResultAvailable()) {
            throw new IllegalStateException("No cvss selection result available. Please call selectEffectiveCvssVectors() first, or use the getCvssSelectionResult(CentralSecurityPolicyConfiguration) method to select the effective cvss vectors on the fly.");
        }
        return cvssSelectionResult;
    }

    public CvssSelectionResult getCvssSelectionResult(CentralSecurityPolicyConfiguration config) {
        if (!isCvssSelectionResultAvailable()) {
            if (config == null) {
                throw new IllegalStateException("No cvss selection result available and passed CentralSecurityPolicyConfiguration is null, cannot select cvss vectors on the fly.");
            }
            selectEffectiveCvssVectors(config);
        }
        return cvssSelectionResult;
    }

    public void clearCvssSelectionResult() {
        this.cvssSelectionResult = null;
    }

    public void mapCvssSelectionResult(Function mapper) {
        if (this.cvssSelectionResult == null) {
            throw new IllegalStateException("No cvss selection result available. Please call selectEffectiveCvssVectors() first, or use the getCvssSelectionResult(CentralSecurityPolicyConfiguration) method to select the effective cvss vectors on the fly.");
        }
        this.cvssSelectionResult = mapper.apply(this.cvssSelectionResult);
    }

    public List parseKeywords() {
        return KeywordSet.fromVulnerability(this);
    }

    public VulnerabilityPriorityCalculator.PriorityScoreResult calculatePriorityScore(CentralSecurityPolicyConfiguration config) {
        return new VulnerabilityPriorityCalculator().contribute(this).calculatePriorityScore(config);
    }

    /* CLEANING DATA */

    public void clearNonTransferableStatusDetails() {
        this.setAdditionalAttribute(VulnerabilityMetaData.Attribute.STATUS, null);
        this.setAdditionalAttribute(VulnerabilityMetaData.Attribute.RATIONALE, null);
        this.setAdditionalAttribute(VulnerabilityMetaData.Attribute.RISK, null);

        this.setAdditionalAttribute(InventoryAttribute.STATUS_ACCEPTED, null);
        this.setAdditionalAttribute(InventoryAttribute.STATUS_REPORTED, null);
        this.setAdditionalAttribute(InventoryAttribute.STATUS_HISTORY, null);
        this.setAdditionalAttribute(InventoryAttribute.STATUS_TITLE, null);

        this.setAdditionalAttribute(InventoryAttribute.REVIEWED_ADVISORIES, null);
    }

    /* TYPE CONVERSION METHODS */

    @Override
    public VulnerabilityMetaData constructBaseModel() {
        return new VulnerabilityMetaData();
    }

    @Override
    public Vulnerability constructDataClass() {
        return new Vulnerability();
    }

    @Override
    protected Set conversionKeysAmb() {
        return CONVERSION_KEYS_AMB;
    }

    @Override
    protected Set conversionKeysMap() {
        return CONVERSION_KEYS_MAP;
    }

    public static Vulnerability fromVulnerabilityMetaData(VulnerabilityMetaData vmd) {
        if (vmd == null) {
            return null;
        }
        return new Vulnerability()
                .performAction(v -> v.appendFromBaseModel(vmd));
    }

    public static Vulnerability fromInputMap(Map map) {
        if (map == null) {
            return null;
        }
        return new Vulnerability()
                .performAction(v -> v.appendFromMap(map));
    }

    public static Vulnerability fromJson(JSONObject json) {
        if (json == null) {
            return null;
        }
        return fromInputMap(json.toMap());
    }

    public static Vulnerability fromDocument(Document document) {
        if (document == null) {
            return null;
        }
        return new Vulnerability()
                .performAction(v -> v.appendFromDocument(document));
    }

    @Override
    public void appendFromBaseModel(VulnerabilityMetaData vmd) {
        super.appendFromBaseModel(vmd);

        this.setId(vmd.get(VulnerabilityMetaData.Attribute.NAME));

        final String vulnerabilitySource = vmd.get(VulnerabilityMetaData.Attribute.SOURCE);
        final String vulnerabilitySourceImplementation = vmd.get(VulnerabilityMetaData.Attribute.SOURCE_IMPLEMENTATION);
        if (StringUtils.hasText(vulnerabilitySource) || StringUtils.hasText(vulnerabilitySourceImplementation)) {
            this.setSourceIdentifier(VulnerabilityTypeStore.get().fromNameAndImplementation(vulnerabilitySource, vulnerabilitySourceImplementation));
        } else {
            VulnerabilityTypeStore.get().inferSourceIdentifierFromIdIfAbsent(this);
        }

        this.setDescription(vmd.get(InventoryAttribute.DESCRIPTION.getKey()));
        this.setUrl(vmd.get(VulnerabilityMetaData.Attribute.URL));

        CvssSource.fromMultipleColumnHeaderStrings(vmd.getAttributes()).forEach((header, source) -> {
            if (StringUtils.hasText(vmd.get(header))) {
                this.cvssVectors.addCvssVector(source, vmd.get(header));
            }
        });

        if (StringUtils.hasText(vmd.get(InventoryAttribute.VULNERABILITY_UPDATED_DATE_TIMESTAMP.getKey()))) {
            this.setUpdateDate(new Date(Long.parseLong(vmd.get(InventoryAttribute.VULNERABILITY_UPDATED_DATE_TIMESTAMP.getKey()))));
        }
        if (StringUtils.hasText(vmd.get(InventoryAttribute.VULNERABILITY_CREATED_DATE_TIMESTAMP.getKey()))) {
            this.setCreateDate(new Date(Long.parseLong(vmd.get(InventoryAttribute.VULNERABILITY_CREATED_DATE_TIMESTAMP.getKey()))));
        }

        if (StringUtils.hasText(vmd.get(VulnerabilityMetaData.Attribute.REFERENCES))) {
            final String referencesString = vmd.get(VulnerabilityMetaData.Attribute.REFERENCES);

            if (referencesString.startsWith("[")) {
                final List references = Reference.fromJsonArray(new JSONArray(referencesString));
                this.addReferences(references);
            } else {
                for (String ref : referencesString.split(", ?")) {
                    final Matcher matcher = Reference.REFERENCE_STRING_PATTERN.matcher(ref);
                    if (matcher.matches()) {
                        this.addReference(Reference.fromTitleAndUrl(matcher.group(1) + "(" + matcher.group(3) + ")", matcher.group(2)));
                    } else {
                        this.addReference(Reference.fromUrl(ref));
                    }
                }
            }
        }

        if (vmd.get(InventoryAttribute.KEV_DATA) != null) {
            this.setKevData(KevData.fromJson(new JSONObject(vmd.get(InventoryAttribute.KEV_DATA))));
        }

        if (StringUtils.hasText(vmd.get(VulnerabilityMetaData.Attribute.WEAKNESS))) {
            this.addCwes(Arrays.asList(vmd.get(VulnerabilityMetaData.Attribute.WEAKNESS).split(", ?")));
        }

        if (vmd.get(InventoryAttribute.EPSS_DATA) != null) {
            this.setEpssData(EpssData.fromJson(new JSONObject(vmd.get(InventoryAttribute.EPSS_DATA))));
        }

        final String tagsString = vmd.get(InventoryAttribute.TAGS.getKey());
        if (StringUtils.hasText(tagsString)) {
            this.addTags(Arrays.asList(tagsString.split(", ")));
        }

        if (StringUtils.hasText(vmd.get(InventoryAttribute.VULNERABILITY_STATUS))) {
            this.setVulnerabilityStatus(VulnerabilityStatusConverter.fromJson(new JSONObject(vmd.get(InventoryAttribute.VULNERABILITY_STATUS))));
        }
    }

    @Override
    public void appendToBaseModel(VulnerabilityMetaData vmd) {
        super.appendToBaseModel(vmd);

        if (this.url != null) {
            vmd.set(VulnerabilityMetaData.Attribute.URL, this.url);
        } else if (StringUtils.hasText(this.id) && this.id.startsWith("CVE-")) {
            vmd.set(VulnerabilityMetaData.Attribute.URL, "https://nvd.nist.gov/vuln/detail/" + this.id);
        } else {
            vmd.set(VulnerabilityMetaData.Attribute.URL, null);
        }

        if (!this.cwes.isEmpty()) {
            vmd.set(VulnerabilityMetaData.Attribute.WEAKNESS, String.join(", ", this.cwes));
        } else {
            vmd.set(VulnerabilityMetaData.Attribute.WEAKNESS, null);
        }

        if (this.epssData != null) {
            vmd.set(InventoryAttribute.EPSS_DATA, this.epssData.toJson().toString());
        } else {
            vmd.set(InventoryAttribute.EPSS_DATA, null);
        }

        for (CvssVector entry : this.cvssVectors.getCvssVectors()) {
            if (entry.getCvssSource() == null) {
                LOG.warn("Using NVD-CNA-NVD for cvss vector [{}] for vulnerability [{}] as it has no source.", entry, this.id);
                vmd.set(new CvssSource(KnownCvssEntities.NVD, CvssSource.CvssIssuingEntityRole.CNA, KnownCvssEntities.NVD, entry.getClass()).toColumnHeaderString(), entry.toString());
            } else {
                vmd.set(entry.getCvssSource().toColumnHeaderString(), entry.toString());
            }
        }

        vmd.set(VulnerabilityMetaData.Attribute.SCORE_CONTEXT_OVERALL, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_INITIAL_OVERALL, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_INITIAL_OVERALL_SEVERITY, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_CONTEXT_OVERALL_SEVERITY, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_BASE, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_EXPLOITABILITY, null);
        vmd.set(VulnerabilityMetaData.Attribute.SCORE_IMPACT, null);

        if (this.isCvssSelectionResultAvailable()) {
            final CvssVector selectedContextCvss = this.cvssSelectionResult.getSelectedContextCvss();
            final CvssVector selectedInitialCvss = this.cvssSelectionResult.getSelectedInitialCvss();
            final CvssVector contextOrInitial = this.cvssSelectionResult.getSelectedContextIfAvailableOtherwiseInitial();

            if (selectedInitialCvss != null) {
                vmd.set(VulnerabilityMetaData.Attribute.SCORE_INITIAL_OVERALL, String.valueOf(selectedInitialCvss.getBakedScores().getOverallScore()));
                vmd.set(VulnerabilityMetaData.Attribute.SCORE_INITIAL_SELECTION, String.valueOf(selectedInitialCvss.toJson()));
            }
            if (selectedContextCvss != null) {
                vmd.set(VulnerabilityMetaData.Attribute.SCORE_CONTEXT_OVERALL, String.valueOf(selectedContextCvss.getBakedScores().getOverallScore()));
                vmd.set(VulnerabilityMetaData.Attribute.SCORE_CONTEXT_SELECTION, String.valueOf(selectedContextCvss.toJson()));
            }
            if (contextOrInitial != null) {
                if (!Double.isNaN(contextOrInitial.getBakedScores().getBaseScore())) {
                    vmd.set(VulnerabilityMetaData.Attribute.SCORE_BASE, String.valueOf(contextOrInitial.getBakedScores().getBaseScore()));
                }
                if (!Double.isNaN(contextOrInitial.getBakedScores().getExploitabilityScore())) {
                    vmd.set(VulnerabilityMetaData.Attribute.SCORE_EXPLOITABILITY, String.valueOf(contextOrInitial.getBakedScores().getExploitabilityScore()));
                }
                if (!Double.isNaN(contextOrInitial.getBakedScores().getImpactScore())) {
                    vmd.set(VulnerabilityMetaData.Attribute.SCORE_IMPACT, String.valueOf(contextOrInitial.getBakedScores().getImpactScore()));
                }
            }
        }

        if (this.kevData != null) {
            vmd.set(InventoryAttribute.KEV_DATA, kevData.toJson().toString());
        } else {
            vmd.set(InventoryAttribute.KEV_DATA, null);
        }

        if (this.createDate != null) {
            vmd.set(InventoryAttribute.VULNERABILITY_CREATED_DATE_TIMESTAMP.getKey(), Long.toString(this.createDate.getTime()));
            vmd.set(InventoryAttribute.VULNERABILITY_CREATED_DATE_FORMATTED.getKey(), TimeUtils.formatNormalizedDate(this.createDate));
        } else {
            vmd.set(InventoryAttribute.VULNERABILITY_CREATED_DATE_TIMESTAMP.getKey(), null);
        }
        if (this.updateDate != null) {
            vmd.set(InventoryAttribute.VULNERABILITY_UPDATED_DATE_TIMESTAMP.getKey(), Long.toString(this.updateDate.getTime()));
            vmd.set(InventoryAttribute.VULNERABILITY_UPDATED_DATE_FORMATTED.getKey(), TimeUtils.formatNormalizedDate(this.updateDate));
        } else {
            vmd.set(InventoryAttribute.VULNERABILITY_UPDATED_DATE_TIMESTAMP.getKey(), null);
        }

        if (!this.references.isEmpty()) {
            final List mergedReferences = Reference.mergeReferences(this.references, Reference.fromJsonArray((vmd.get(VulnerabilityMetaData.Attribute.REFERENCES))));
            vmd.set(VulnerabilityMetaData.Attribute.REFERENCES, mergedReferences.stream().map(Reference::toJson).collect(CustomCollectors.toJsonArray()).toString());
        } else {
            vmd.set(VulnerabilityMetaData.Attribute.REFERENCES, null);
        }

        if (this.description != null) {
            vmd.set(InventoryAttribute.DESCRIPTION.getKey(), this.description);
        } else {
            vmd.set(InventoryAttribute.DESCRIPTION.getKey(), null);
        }

        if (!this.tags.isEmpty()) {
            vmd.set(InventoryAttribute.TAGS.getKey(), String.join(", ", this.tags));
        } else {
            vmd.set(InventoryAttribute.TAGS.getKey(), null);
        }

        if (this.vulnerabilityStatus != null) {
            vmd.set(InventoryAttribute.VULNERABILITY_STATUS, this.vulnerabilityStatus.toJson().toString());
        } else {
            vmd.set(InventoryAttribute.VULNERABILITY_STATUS, null);
        }
    }

    @Override
    public void appendFromDataClass(Vulnerability dataClass) {
        super.appendFromDataClass(dataClass);

        if (StringUtils.hasText(dataClass.getDescription())) {
            this.setDescription(dataClass.getDescription());
        }
        if (StringUtils.hasText(dataClass.getNotes())) {
            this.setNotes(dataClass.getNotes());
        }
        if (StringUtils.hasText(dataClass.getUrl())) {
            this.setUrl(dataClass.getUrl());
        }

        if (dataClass.getCreateDate() != null) {
            this.setCreateDate(dataClass.getCreateDate());
        }
        if (dataClass.getUpdateDate() != null) {
            this.setUpdateDate(dataClass.getUpdateDate());
        }

        this.cvssVectors.addAllCvssVectors(dataClass.getCvssVectors());

        this.addReferences(dataClass.getReferences());
        this.addCwes(dataClass.getCwes());

        this.setKevData(dataClass.getKevData());

        this.setEpssData(dataClass.getEpssData());

        dataClass.getSecurityAdvisories().forEach(this::addSecurityAdvisory);

        if (dataClass.getVulnerabilityStatus() != null) {
            this.setVulnerabilityStatus(dataClass.getVulnerabilityStatus());
        }

        this.addTags(dataClass.getTags());
    }

    @Override
    public void appendFromMap(Map input) {
        super.appendFromMap(input);

        this.setId(getStringOrNullFromMap(input, "name"));

        final String source = (String) input.getOrDefault("source", null);
        final String sourceImplementation = (String) input.getOrDefault("sourceImplementation", null);
        if (source != null || sourceImplementation != null) {
            this.setSourceIdentifier(VulnerabilityTypeStore.get().fromNameAndImplementation(source, sourceImplementation));
        }

        this.setDescription(getStringOrNullFromMap(input, "description"));
        this.setNotes(getStringOrNullFromMap(input, "notes"));
        this.setUrl(getStringOrNullFromMap(input, "url"));

        if (input.get("cvss") != null) {
            final JSONArray cvssVectorsJson;
            if (input.get("cvss") instanceof String) {
                final String cvssVectors = (String) input.get("cvss");
                cvssVectorsJson = new JSONArray(cvssVectors);
            } else if (input.get("cvss") instanceof JSONArray) {
                cvssVectorsJson = (JSONArray) input.get("cvss");
            } else if (input.get("cvss") instanceof Collection) {
                cvssVectorsJson = new JSONArray((Collection) input.get("cvss"));
            } else {
                throw new RuntimeException("Unable to parse cvss vectors from input map: " + input.get("cvss"));
            }
            this.cvssVectors.addAllCvssVectors(CvssVectorSet.fromJson(cvssVectorsJson));
        }

        this.setCreateDate(ObjectUtils.firstNonNull(getDateOrNullFromMap(input, "createDate"), getDateOrNullFromMap(input, "publishedDate")));
        this.setUpdateDate(ObjectUtils.firstNonNull(getDateOrNullFromMap(input, "updateDate"), getDateOrNullFromMap(input, "lastModifiedDate")));


        for (String cwe : ((Collection) input.getOrDefault("cwe", Collections.EMPTY_SET))) {
            this.addCwe(cwe);
        }


        if (input.get("vulnerable_software") != null) {
            try {
                JSONArray array = new JSONArray(((ArrayList) input.get("vulnerable_software")).toArray(new Object[]{}));
                VulnerableSoftwareTreeNode.fromJson(array).forEach(this::addVulnerableSoftwareTreeNode);
            } catch (CpeValidationException e) {
                throw new RuntimeException("Unable to parse CPE on vulnerable software whilst parsing vulnerability from input map", e);
            }
        }

        final Object ref = input.get("references");
        if (ref instanceof ArrayList) {
            for (Object reference : (ArrayList) ref) {
                if (reference instanceof Map) {
                    this.addReference(Reference.fromMap((Map) reference));
                } else {
                    LOG.warn("Reference in custom vulnerability [{}] is not of type Map: {}", this.getId(), reference);
                }
            }
        }

        if (input.get("tags") != null) {
            this.addTags((Collection) input.get("tags"));
        }

        if (input.get("vulnerabilityStatus") != null) {
            this.setVulnerabilityStatus(VulnerabilityStatusConverter.fromJson(new JSONObject((Map) input.get("vulnerabilityStatus"))));
        }

        if (input.get("kevData") != null) {
            this.setKevData(KevData.fromJson(new JSONObject((Map) input.get("kevData"))));
        }

        if (input.get("epssData") != null) {
            this.setEpssData(EpssData.fromJson(new JSONObject((Map) input.get("epssData"))));
        }
    }

    @Override
    public void appendToJson(JSONObject json) {
        super.appendToJson(json);

        json.put("name", getId());
        json.put("description", getDescription());
        json.put("notes", getNotes());
        json.put("url", getUrl());

        json.put("cvss", this.cvssVectors.toJson());

        json.put("createDate", getCreateDate() != null ? getCreateDate().getTime() : null);
        json.put("updateDate", getUpdateDate() != null ? getUpdateDate().getTime() : null);

        json.put("cwe", getCwes());
        json.put("vulnerable_software", getVulnerableSoftwareConfigurations().stream().map(VulnerableSoftwareTreeNode::toJson).collect(CustomCollectors.toJsonArray()));
        json.put("references", getReferences().stream().map(Reference::toJson).collect(CustomCollectors.toJsonArray()));

        json.put("tags", getTags());
        json.put("vulnerabilityStatus", getVulnerabilityStatus() != null ? getVulnerabilityStatus().toJson() : null);

        if (this.kevData != null) {
            json.put("kevData", this.kevData.toJson());
        }
        if (this.epssData != null) {
            json.put("epssData", this.epssData.toJson());
        }
    }

    @Override
    public void appendFromDocument(Document document) {
        super.appendFromDocument(document);

        this.setId(document.get("name"));
        this.setDescription(document.get("description"));
        this.setNotes(document.get("notes"));
        this.setUrl(document.get("url"));

        // legacy format with individual fields
        if (document.get("cvssV2") != null) {
            this.cvssVectors.addCvssVector(new CvssSource(KnownCvssEntities.NVD, Cvss2.class), document.get("cvssV2"));
        }
        if (document.get("cvssV3") != null) {
            this.cvssVectors.addCvssVector(new CvssSource(KnownCvssEntities.NVD, Cvss3P1.class), document.get("cvssV3"));
        }
        if (document.get("cvssV4") != null) {
            this.cvssVectors.addCvssVector(new CvssSource(KnownCvssEntities.NVD, Cvss4P0.class), document.get("cvssV4"));
        }

        // new cvss format with single field
        if (document.get("cvssVectors") != null) {
            final String cvssVectors = document.get("cvssVectors");
            final JSONArray cvssVectorsJson = new JSONArray(cvssVectors);
            this.cvssVectors.addAllCvssVectors(CvssVectorSet.fromJson(cvssVectorsJson));
        }

        this.setCreateDate(getDateOrNullFromDocument(document, "createDate"));
        this.setUpdateDate(getDateOrNullFromDocument(document, "updateDate"));

        if (document.get("cwe") != null) {
            Arrays.stream(document.get("cwe").split(", ?")).forEach(this::addCwe);
        }

        if (document.get("vulnerable_software") != null) {
            try {
                VulnerableSoftwareTreeNode.fromJson(new JSONArray(document.get("vulnerable_software"))).forEach(this::addVulnerableSoftwareTreeNode);
            } catch (CpeValidationException e) {
                throw new RuntimeException("Unable to parse CPE on vulnerable software whilst parsing vulnerability from document", e);
            }
        }

        Reference.fromJsonArray(new JSONArray(document.get("references"))).forEach(this::addReference);
    }

    @Override
    public void appendToDocument(Document doc) {
        super.appendToDocument(doc);

        addToDocumentAsTextFieldIfNotEmpty(doc, "name", getId());
        addToDocumentAsTextFieldIfNotEmpty(doc, "description", getDescription());
        addToDocumentAsTextFieldIfNotEmpty(doc, "notes", getNotes());
        addToDocumentAsTextFieldIfNotEmpty(doc, "url", getUrl());

        // only new format with single field
        if (!this.cvssVectors.isEmpty()) {
            final JSONArray cvssVectors = this.cvssVectors.toJson();
            doc.add(new TextField("cvssVectors", cvssVectors.toString(), Field.Store.YES));
        }

        addToDocumentAsTextFieldIfNotEmpty(doc, "createDate", getCreateDate() != null ? "" + getCreateDate().getTime() : null);
        addToDocumentAsTextFieldIfNotEmpty(doc, "updateDate", getUpdateDate() != null ? "" + getUpdateDate().getTime() : null);

        doc.add(new TextField("cwe", String.join(",", getCwes()), Field.Store.YES));
        doc.add(new TextField("references", getReferences().stream().map(Reference::toJson).collect(CustomCollectors.toJsonArray()).toString(), Field.Store.YES));

        doc.add(new TextField("vulnerable_software", getVulnerableSoftwareConfigurations().stream().map(VulnerableSoftwareTreeNode::toJson).collect(CustomCollectors.toJsonArray()).toString(), Field.Store.YES));
        final String allVendorProducts = getVulnerableSoftwareConfigurations().stream()
                .map(VulnerableSoftwareTreeNode::getAllCpes)
                .flatMap(Collection::stream)
                .map(cpe -> cpe.getCpe().getVendor() + ":" + cpe.getCpe().getProduct())
                .collect(Collectors.joining(" "));
        doc.add(new TextField("vulnerable_software_vp", allVendorProducts, Field.Store.YES));
    }

    private static Date getDateOrNullFromDocument(Document document, String date) {
        if (document.get(date) != null) {
            return new Date(Long.parseLong(document.get(date)));
        }
        return null;
    }

    protected void addToDocumentAsTextFieldIfNotEmpty(Document doc, String fieldName, String value) {
        if (StringUtils.hasText(value)) {
            doc.add(new TextField(fieldName, value, Field.Store.YES));
        }
    }

    @Deprecated
    public static Vulnerability fromVulnerabilityMetaData(VulnerabilityMetaData vmd, VulnerabilityIndexQuery vulnerabilityIndex) {
        final Vulnerability vulnerability = fromVulnerabilityMetaData(vmd);

        vulnerabilityIndex.findVulnerabilityByName(vulnerability.getId()).ifPresent(officialVulnerability -> {
            vulnerability.addVulnerableSoftwaresTreeNodes(officialVulnerability.getVulnerableSoftwareConfigurations());
        });

        return vulnerability;
    }

    /**
     * Converts a JSON object representing a CVE item from the NVD mirror into a Vulnerability object.
    
     * This method will only work for the legacy NVD mirror format (version 1.0), which will be deprecated in the
     * future.
     *
     * @param cveItem the JSON object representing the CVE item
     * @return a Vulnerability object converted from the JSON data
     * @deprecated The NVD plans to retire the remaining legacy data feeds as well as all 1.0 APIs on December 15th. Change Timeline
     */
    @Deprecated
    public static Vulnerability fromNvdMirrorCveItem1P0(JSONObject cveItem) {
        final Vulnerability vulnerability = new Vulnerability();

        final JSONObject cveInformation = cveItem.has("cve") ? cveItem.getJSONObject("cve") : cveItem;

        if (!cveInformation.getString("data_type").equals("CVE")) {
            LOG.warn("Data type should be [CVE] in CVE item, but was [{}]", cveInformation.getString("data_type"));
        }

        vulnerability.setId(cveInformation.getJSONObject("CVE_data_meta").getString("ID"));
        vulnerability.setNotes("Assigner: " + cveInformation.getJSONObject("CVE_data_meta").getString("ASSIGNER"));

        vulnerability.addDataSource(VulnerabilityTypeStore.CVE);
        vulnerability.addDataSource(OtherTypeStore.NVD);

        // the description might contain multiple descriptions in different languages, try finding the 'en' description
        final JSONArray descriptions = cveInformation.getJSONObject("description").getJSONArray("description_data");
        String description = null;
        for (int i = 0; i < descriptions.length(); i++) {
            final JSONObject descriptionData = descriptions.getJSONObject(i);

            if (descriptionData.getString("lang").equals("en")) {
                description = descriptionData.getString("value");
                break;
            } else if (description == null) {
                description = descriptionData.getString("value");
            }
        }
        if (StringUtils.hasText(description)) {
            vulnerability.setDescription(description);
        }

        // dates using publishedDate, lastModifiedDate
        if (StringUtils.hasText(cveItem.optString("publishedDate"))) {
            vulnerability.setCreateDate(TimeUtils.tryParse(cveItem.getString("publishedDate")));
        }
        if (StringUtils.hasText(cveItem.optString("lastModifiedDate"))) {
            vulnerability.setUpdateDate(TimeUtils.tryParse(cveItem.getString("lastModifiedDate")));
        }


        final List references = new ArrayList<>();
        final JSONArray referencesData = cveInformation.getJSONObject("references").getJSONArray("reference_data");
        for (int i = 0; i < referencesData.length(); i++) {
            final JSONObject referenceData = referencesData.getJSONObject(i);
            final Reference reference = Reference.fromTitleAndUrl(
                    referenceData.getString("name"),
                    referenceData.getString("url")
            );

            referenceData.getJSONArray("tags").forEach(tag -> reference.addTag(String.valueOf(tag)));
            reference.addTag(referenceData.getString("refsource"));

            references.add(reference);
        }
        vulnerability.addReferences(references);


        final Set cwes = new HashSet<>();
        final JSONArray problemtypeData = cveInformation.getJSONObject("problemtype").getJSONArray("problemtype_data");
        for (int i = 0; i < problemtypeData.length(); i++) {
            final JSONArray descriptionData = problemtypeData.getJSONObject(i).getJSONArray("description");

            for (int j = 0; j < descriptionData.length(); j++) {
                final JSONObject descriptionDataItem = descriptionData.getJSONObject(j);
                if (!isNoInfoOtherCwe(descriptionDataItem.getString("value"))) {
                    cwes.add(descriptionDataItem.getString("value"));
                }
            }
        }
        vulnerability.addCwes(cwes);


        final JSONObject impact = cveItem.getJSONObject("impact");
        if (impact.has("baseMetricV2")) {
            final Cvss2 cvss2 = new Cvss2(impact.getJSONObject("baseMetricV2").getJSONObject("cvssV2").getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, Cvss2.class));
            vulnerability.cvssVectors.addCvssVector(cvss2);
        }
        if (impact.has("baseMetricV3")) {
            final Cvss3P1 cvss3 = new Cvss3P1(impact.getJSONObject("baseMetricV3").getJSONObject("cvssV3").getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, Cvss3P1.class));
            vulnerability.cvssVectors.addCvssVector(cvss3);
        }
        if (impact.has("baseMetricV4")) { // does, and will not exist, just for completeness
            final Cvss4P0 cvssV4 = new Cvss4P0(impact.getJSONObject("baseMetricV4").getJSONObject("cvssV4").getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, Cvss4P0.class));
        }


        // get the vulnerable software configurations
        final Set vulnerableSoftware = new HashSet<>();
        final JSONArray affectedConfigurations = cveItem.getJSONObject("configurations").getJSONArray("nodes");

        for (int i = 0; i < affectedConfigurations.length(); i++) {
            final JSONObject affectedConfigurationNode = affectedConfigurations.getJSONObject(i);
            try {
                final VulnerableSoftwareTreeNode vulnerableSoftwareTree = VulnerableSoftwareTreeNode.fromJson(affectedConfigurationNode);
                vulnerableSoftware.add(vulnerableSoftwareTree);
            } catch (CpeValidationException e) {
                throw new RuntimeException("Invalid CPE URI on [" + vulnerability.getId() + "] in software configuration: " + affectedConfigurationNode, e);
            }
        }
        vulnerability.addVulnerableSoftwaresTreeNodes(vulnerableSoftware);


        return vulnerability;
    }

    /**
     * Converts a JSON object representing a CVE item from the NVD mirror into a Vulnerability object.
    
     * This method will only work for the new NVD mirror format.
     *
     * @param cveItem the JSON object representing the CVE item
     * @return a Vulnerability object converted from the JSON data
     */
    public static Vulnerability fromNvdMirrorCveItem2P0(JSONObject cveItem) {
        final Vulnerability vulnerability = new Vulnerability();

        // keys: sourceIdentifier, references, configurations, weaknesses, id, published, lastModified, metrics, vulnStatus, descriptions

        vulnerability.setId(cveItem.getString("id"));
        vulnerability.addDataSource(VulnerabilityTypeStore.CVE);
        vulnerability.addDataSource(OtherTypeStore.NVD);


        // the description might contain multiple descriptions in different languages, try finding the 'en' description
        final JSONArray descriptions = cveItem.getJSONArray("descriptions");
        String description = null;
        for (int i = 0; i < descriptions.length(); i++) {
            final JSONObject descriptionData = descriptions.getJSONObject(i);

            if (descriptionData.getString("lang").equals("en")) {
                description = descriptionData.getString("value");
                break;
            } else if (description == null) {
                description = descriptionData.getString("value");
            }
        }
        if (StringUtils.hasText(description)) {
            vulnerability.setDescription(description);
        }


        // dates using published, lastModified
        if (StringUtils.hasText(cveItem.optString("published"))) {
            vulnerability.setCreateDate(TimeUtils.tryParse(cveItem.getString("published")));
        }
        if (StringUtils.hasText(cveItem.optString("lastModified"))) {
            vulnerability.setUpdateDate(TimeUtils.tryParse(cveItem.getString("lastModified")));
        }


        final List references = new ArrayList<>();
        final JSONArray referencesData = cveItem.getJSONArray("references");
        for (int i = 0; i < referencesData.length(); i++) {
            final JSONObject referenceData = referencesData.getJSONObject(i);
            final Reference reference = Reference.fromTitleAndUrl(
                    referenceData.getString("source"),
                    referenceData.getString("url")
            );

            final JSONArray tags = referenceData.optJSONArray("tags");
            if (tags != null) {
                tags.forEach(tag -> reference.addTag(String.valueOf(tag)));
            }
            reference.addTag(referenceData.getString("source"));

            references.add(reference);
        }
        vulnerability.addReferences(references);


        final JSONArray weaknesses = cveItem.optJSONArray("weaknesses");
        if (weaknesses != null) {
            final List cwes = new ArrayList<>();

            for (int i = 0; i < weaknesses.length(); i++) {
                final JSONObject weakness = weaknesses.getJSONObject(i);
                final JSONArray descriptionData = weakness.getJSONArray("description");
                for (int j = 0; j < descriptionData.length(); j++) {
                    final JSONObject descriptionObject = descriptionData.getJSONObject(j);
                    if (!isNoInfoOtherCwe(descriptionObject.getString("value"))) {
                        cwes.add(descriptionObject.getString("value"));
                        break;
                    }
                }
            }

            vulnerability.addCwes(cwes);
        }


        final JSONObject metrics = cveItem.optJSONObject("metrics");

        /*
        example:
        "cvssMetricV2": [
          {
            "source": "[email protected]",
            "type": "Primary",
            "cvssData": {
              "version": "2.0",
              "vectorString": "AV:N\/AC:M\/Au:N\/C:P\/I:P\/A:P",
            },
          }
        ]
        */
        if (metrics != null) {
            final JSONArray cvssMetricV2 = metrics.optJSONArray("cvssMetricV2");
            if (cvssMetricV2 != null) {
                for (int i = 0; i < cvssMetricV2.length(); i++) {
                    final JSONObject cvssMetricV2Object = cvssMetricV2.getJSONObject(i);

                    final String source = cvssMetricV2Object.getString("source");
                    final CvssSource.CvssEntity sourceEntity = source == null ? KnownCvssEntities.NVD : KnownCvssEntities.findByNameOrMailOrCreateNew(source);

                    final JSONObject cvssData = cvssMetricV2Object.getJSONObject("cvssData");
                    final Cvss2 cvssV2 = new Cvss2(cvssData.getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, CvssSource.CvssIssuingEntityRole.CNA, sourceEntity, Cvss2.class));

                    vulnerability.cvssVectors.addCvssVector(cvssV2);
                }
            }

            final JSONArray cvssMetricV3 = ObjectUtils.firstNonNull(
                    metrics.optJSONArray("cvssMetricV30"),
                    metrics.optJSONArray("cvssMetricV31")
            );
            if (cvssMetricV3 != null) {
                for (int i = 0; i < cvssMetricV3.length(); i++) {
                    final JSONObject cvssMetricV3Object = cvssMetricV3.getJSONObject(i);

                    final String source = cvssMetricV3Object.getString("source");
                    final CvssSource.CvssEntity sourceEntity = source == null ? KnownCvssEntities.NVD : KnownCvssEntities.findByNameOrMailOrCreateNew(source);

                    final JSONObject cvssData = cvssMetricV3Object.getJSONObject("cvssData");
                    final Cvss3P1 cvssV3 = new Cvss3P1(cvssData.getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, CvssSource.CvssIssuingEntityRole.CNA, sourceEntity, Cvss3P1.class));

                    vulnerability.cvssVectors.addCvssVector(cvssV3);
                }
            }

            // TODO: AEAA-356: check what key the NVD uses for CVSS 4.0; format does not yet specify this
            final JSONArray cvssMetricV4 = ObjectUtils.firstNonNull(
                    metrics.optJSONArray("cvssMetricV40"),
                    metrics.optJSONArray("cvssMetricV4")
            );
            if (cvssMetricV4 != null) {
                for (int i = 0; i < cvssMetricV4.length(); i++) {
                    final JSONObject cvssMetricV4Object = cvssMetricV4.getJSONObject(i);

                    final String source = cvssMetricV4Object.getString("source");
                    final CvssSource.CvssEntity sourceEntity = source == null ? KnownCvssEntities.NVD : KnownCvssEntities.findByNameOrMailOrCreateNew(source);

                    final JSONObject cvssData = cvssMetricV4Object.getJSONObject("cvssData");
                    final Cvss4P0 cvssV4 = new Cvss4P0(cvssData.getString("vectorString"), new CvssSource(KnownCvssEntities.NVD, CvssSource.CvssIssuingEntityRole.CNA, sourceEntity, Cvss4P0.class));

                    vulnerability.cvssVectors.addCvssVector(cvssV4);
                }
            }

            if (metrics.keySet().stream().anyMatch(key -> !key.equals("cvssMetricV2") && !key.equals("cvssMetricV30") && !key.equals("cvssMetricV31") && !key.equals("cvssMetricV40") && !key.equals("cvssMetricV4"))) {
                LOG.warn("Unknown CVSS metric version: {}", metrics.keySet());
            }
        }


        if (cveItem.has("configurations")) {
            final Set vulnerableSoftware = new HashSet<>();
            final JSONArray affectedConfigurations = cveItem.getJSONArray("configurations");

            for (int i = 0; i < affectedConfigurations.length(); i++) {
                final JSONArray nodes = affectedConfigurations.getJSONObject(i).getJSONArray("nodes");

                for (int j = 0; j < nodes.length(); j++) {
                    final JSONObject configurationNode = nodes.getJSONObject(j);
                    try {
                        final VulnerableSoftwareTreeNode vulnerableSoftwareTree = VulnerableSoftwareTreeNode.fromJson(configurationNode);
                        vulnerableSoftware.add(vulnerableSoftwareTree);
                    } catch (CpeValidationException e) {
                        throw new RuntimeException("Invalid CPE URI on [" + vulnerability.getId() + "] in software configuration: " + configurationNode, e);
                    }
                }
            }
            vulnerability.addVulnerableSoftwaresTreeNodes(vulnerableSoftware);
        }

        return vulnerability;
    }

    /**
     * Reads a custom vulnerability file or directory and converts the data into a list of Vulnerability objects.
     * This method supports YAML and JSON file formats.
    
     * If a directory is specified, all files in the directory will be read and parsed recursively.
     *
     * @param file the file or directory to read
     * @return a list of Vulnerability objects parsed from the file or directory
     */
    public static List fromCustomVulnerabilityFileOrDir(File file) {
        final List vulnerabilities = new ArrayList<>();

        if (!file.exists()) {
            LOG.warn("Vulnerability file does not exist: {}", file.getAbsolutePath());
            return vulnerabilities;
        }

        if (file.isDirectory()) {
            for (File subFile : file.listFiles()) {
                vulnerabilities.addAll(fromCustomVulnerabilityFileOrDir(subFile));
            }

        } else if (file.getName().endsWith(".yaml")) {
            try {
                final Object loaded = new Yaml().load(Files.newInputStream(file.toPath()));

                if (loaded instanceof List) {
                    final List loadedVulnerabilities = (List) loaded;

                    for (Object loadedVulnerability : loadedVulnerabilities) {
                        if (loadedVulnerability instanceof Map) {
                            vulnerabilities.add(Vulnerability.fromInputMap((Map) loadedVulnerability));
                        }
                    }
                } else if (loaded instanceof Map) {
                    vulnerabilities.add(Vulnerability.fromInputMap((Map) loaded));
                }
            } catch (Exception e) {
                LOG.error("Failed to parse YAML vulnerability file: " + file, e);
            }
        } else if (file.getName().endsWith(".json")) {
            try {
                final String lines = String.join("", FileUtils.readLines(file, StandardCharsets.UTF_8));

                if (lines.startsWith("[")) {
                    final JSONArray array = new JSONArray(lines);

                    for (int i = 0; i < array.length(); i++) {
                        vulnerabilities.add(Vulnerability.fromInputMap(array.getJSONObject(i).toMap()));
                    }
                } else {
                    vulnerabilities.add(Vulnerability.fromInputMap(new JSONObject(lines).toMap()));
                }
            } catch (Exception e) {
                LOG.error("Failed to parse JSON vulnerability file: " + file, e);
            }
        } else {
            LOG.info("Skipping file during vulnerability parsing: {}", file.getAbsolutePath());
        }

        LOG.info("Parsed [{}] vulnerabilit{} from {}", vulnerabilities.size(), vulnerabilities.size() == 1 ? "y" : "ies", file.getAbsolutePath());

        return vulnerabilities;
    }

    private static boolean isNoInfoOtherCwe(String value) {
        value = value.toLowerCase();
        return value.equals("nvd-cwe-noinfo") || value.equals("nvd-cwe-other");
    }

    private static String getStringOrNullFromMap(Map map, String key) {
        Object value = map.getOrDefault(key, null);
        if (value == null) {
            return null;
        }
        return String.valueOf(value);
    }

    private static Date getDateOrNullFromMap(Map map, String key) {
        Object value = map.getOrDefault(key, null);
        if (value == null) {
            return null;
        }
        return TimeUtils.tryParse(value.toString());
    }

    @Override
    public String toString() {
        return getId();
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (o == null || getClass() != o.getClass()) {
            return false;
        }
        Vulnerability that = (Vulnerability) o;
        return Objects.equals(id, that.id);
    }

    @Override
    public int hashCode() {
        return Objects.hash(id);
    }

    /* STATIC UTILITIES */

    public final static Comparator COMPARE_BY_NAME = Comparator.comparing(Vulnerability::getId);

    public static Map> groupVulnerabilitiesByArtifact(Collection vulnerabilities) {
        final Map> groupedVulnerabilities = new HashMap<>();

        for (Vulnerability vulnerability : vulnerabilities) {
            for (Artifact artifact : vulnerability.getAffectedArtifactsByDefaultKey()) {
                groupedVulnerabilities.computeIfAbsent(artifact, k -> new ArrayList<>()).add(vulnerability);
            }
        }

        return groupedVulnerabilities;
    }
}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy