com.metaeffekt.mirror.index.nvd.NvdVulnerabilityIndex Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.index.nvd;

import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.contents.vulnerability.Vulnerability;
import com.metaeffekt.mirror.download.nvd.NvdDownload;
import com.metaeffekt.mirror.index.Index;
import org.apache.lucene.document.Document;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

@Deprecated
@MirrorMetadata(directoryName = "nvd-cve", mavenPropertyName = "nvdLegacyVulnerabilityIndex", deprecated = true)
public class NvdVulnerabilityIndex extends Index {

    private final static Logger LOG = LoggerFactory.getLogger(NvdVulnerabilityIndex.class);

    private final static String DATA_VERSION = "4.0";

    public NvdVulnerabilityIndex(File baseMirrorDirectory) {
        super(baseMirrorDirectory, NvdVulnerabilityIndex.class, Collections.singletonList(NvdDownload.class), Collections.emptyList());
    }

    @Override
    protected Map createIndexDocuments() {
        final Map documents = new ConcurrentHashMap<>();

        final File[] files = super.requiredDownloads[0].listFiles(f -> f.isFile() && f.getName().startsWith("nvd-") && f.getName().endsWith(".json"));

        if (files == null) {
            throw new RuntimeException("Unable to list files in " + super.requiredDownloads[0]);
        }

        for (File file : files) {
            super.executor.submit(() -> documents.putAll(processFile(file)));
        }

        super.executor.setSize(6);
        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for indexing to complete.", e);
        }

        return documents;
    }

    private Map processFile(File file) {
        final Map documents = new HashMap<>();

        LOG.info("Processing file {}", file.getAbsolutePath());

        final JSONObject json;
        try {
            json = new JSONObject(String.join("", FileUtils.readLines(file, StandardCharsets.UTF_8)));
        } catch (IOException e) {
            throw new RuntimeException("Failed to read yearly NVD document from " + file.getAbsolutePath(), e);
        }

        // keys: CVE_Items, CVE_data_type, CVE_data_format, CVE_data_timestamp, CVE_data_numberOfCVEs, CVE_data_version

        if (!json.optString("CVE_data_version", "UNSET").equals(DATA_VERSION)) {
            LOG.warn("Data version might not be compatible with indexing process: expected [{}] but was [{}] on {}", DATA_VERSION, json.optString("CVE_data_version", "UNSET"), file.getAbsolutePath());
        }

        final JSONArray cveItems = json.getJSONArray("CVE_Items");

        for (int i = 0; i < cveItems.length(); i++) {
            final JSONObject cveItem = cveItems.getJSONObject(i);

            final Vulnerability vulnerability = Vulnerability.fromNvdMirrorCveItem1P0(cveItem);

            final Document document = vulnerability.toDocument();
            documents.put(vulnerability.getId(), document);
        }

        return documents;
    }
}