• Home
• com.metaeffekt.artifact.analysis
• ae-artifact-analysis
• 0.132.0
• source code
• NvdCveApiIndex.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.metaeffekt.mirror.index.nvd.NvdCveApiIndex Maven / Gradle / Ivy

/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.index.nvd;

import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import com.metaeffekt.mirror.download.nvd.NvdCveApiDownload;
import com.metaeffekt.mirror.index.Index;
import org.apache.lucene.document.Document;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 
The NVD CVE index is CVE-centric and contains all published CVEs from the NVD with their details, such as descriptions, references and matching information.
 * It is used in many steps of the enrichment process, whenever information on CVEs is required.
 * It is mainly used to match component product CPEs into CVEs as the base vulnerability information.
 * 
Every file contained in the download directory is parsed and processed in the same way. An entry in the parsed JSON
 * Array has these fields: sourceIdentifier, references, configurations, weaknesses, id, published,
 * lastModified, metrics, vulnStatus, descriptions.
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
Mapping of JSON content to Vulnerability fields
JSON
Vulnerability
descriptions > multiple lang/value --> find en or fallback to any
description
published
create date
lastModified
update date
references > multiple source & url & tags
references
weaknesses > multiple description > multiple value
cwe
metrics > cvssMetricV2 > cvssData > vectorString
cvss v2
metrics > cvssMetricV30, cvssMetricV31 > cvssData > vectorString
cvss v3
metrics > cvssMetricV40 > cvssData > vectorString
cvss v4
configurations > nodes (parse nodes)
vulnerable software
 */
@MirrorMetadata(directoryName = "nvd-cve", mavenPropertyName = "nvdVulnerabilityIndex")
public class NvdCveApiIndex extends Index {

    private final static Logger LOG = LoggerFactory.getLogger(NvdCveApiIndex.class);

    public NvdCveApiIndex(File baseMirrorDirectory) {
        super(baseMirrorDirectory, NvdCveApiIndex.class, Collections.singletonList(NvdCveApiDownload.class), Collections.emptyList());
    }

    @Override
    protected Map createIndexDocuments() {
        final Map documents = new ConcurrentHashMap<>();

        final File[] files = super.requiredDownloads[0].listFiles(f -> f.isFile() && f.getName().endsWith(".json"));

        if (files == null) {
            throw new RuntimeException("Unable to list files in " + super.requiredDownloads[0]);
        }

        for (File file : files) {
            super.executor.submit(() -> documents.putAll(processFile(file)));
        }

        super.executor.setSize(6);
        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for indexing to complete.", e);
        }

        return documents;
    }

    private Map processFile(File file) {
        final Map documents = new HashMap<>();

        LOG.info("Processing file {}", file.getAbsolutePath());

        final JSONArray json;
        try {
            json = new JSONArray(String.join("", FileUtils.readLines(file, StandardCharsets.UTF_8)));
        } catch (IOException e) {
            throw new RuntimeException("Failed to read yearly NVD document from " + file.getAbsolutePath(), e);
        }

        for (int i = 0; i < json.length(); i++) {
            final JSONObject cve = json.getJSONObject(i);

            final Vulnerability vulnerability = Vulnerability.fromNvdMirrorCveItem2P0(cve);

            final Document document = vulnerability.toDocument();
            documents.put(vulnerability.getId(), document);
        }

        return documents;
    }
}