com.metaeffekt.mirror.index.other.KevIndex Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
package com.metaeffekt.mirror.index.other;

import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.contents.kev.KevData;
import com.metaeffekt.mirror.download.documentation.DocRelevantMethods;
import com.metaeffekt.mirror.download.other.CisaKevDownload;
import com.metaeffekt.mirror.index.Index;
import org.apache.lucene.document.Document;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * This index processes Known Exploited Vulnerabilities (KEV) from CISA, which are provided in JSON format.
 * Each file contains multiple entries describing vulnerabilities, which are mapped to a document format using Apache Lucene for indexing.
 * It is used to add priority information to existing vulnerabilities.
 *
 * JSON Structure:
 * {
 *   "vulnerabilities": [
 *     {
 *       "cveID": "CVE-2022-12345",
 *       "vendorProject": "VendorX",
 *       "product": "ProductY",
 *       "vulnerabilityName": "Example Vulnerability",
 *       "requiredAction": "Apply patch",
 *       "notes": "None",
 *       "shortDescription": "This is a sample description.",
 *       "dateAdded": "2022-07-14",
 *       "dueDate": "2022-09-14",
 *       "knownRansomwareCampaignUse": "Known"
 *     }
 *   ]
 * }
 *
 * The KEV data files are processed as follows:
 * 
 * All JSON files are recursively read from the specified directory.
 * Each JSON file is parsed, and all entries in the "vulnerabilities" array are processed.
 * Each entry is converted into a Document object for indexing.
 * 
 */
@MirrorMetadata(directoryName = "kev", mavenPropertyName = "kevIndex")
public class KevIndex extends Index {

    private final static Logger LOG = LoggerFactory.getLogger(EolIndex.class);

    public KevIndex(File baseMirrorDirectory) {
        super(baseMirrorDirectory, KevIndex.class, Collections.singletonList(CisaKevDownload.class), Collections.emptyList());
    }

    @Override
    @DocRelevantMethods({"KevData#fromCisaKev"})
    protected Map createIndexDocuments() {
        final Map documents = new ConcurrentHashMap<>();

        final Collection files = super.getAllFilesRecursively(super.requiredDownloads[0]);

        for (File file : files) {
            if (!file.getName().endsWith(".json")) {
                continue;
            }
            LOG.info("Processing file: {}", file.getName());

            try {
                final String contents = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
                final JSONObject parsedContents = new JSONObject(contents);
                final JSONArray kevEntries = parsedContents.getJSONArray("vulnerabilities");
                final List kevEntriesDocs = fromJson(kevEntries);


                for (Document document : kevEntriesDocs) {
                    String uldid = document.get("vulnerability");
                    if (documents.containsKey(uldid)) {
                        LOG.warn("Duplicate entry found, skipping: {}", document);
                    } else {
                        documents.put(uldid, document);
                    }
                }
            } catch (IOException e) {
                throw new RuntimeException("Failed to read file: " + file.getAbsolutePath(), e);
            }
        }

        return documents;
    }

    private List fromJson(JSONArray jsonArray) {
        final List documents = new ArrayList<>();

        for (int i = 0; i < jsonArray.length(); i++) {
            KevData data = KevData.fromCisaKev(jsonArray.getJSONObject(i));
            documents.add(data.toDocument());
        }

        return documents;
    }
}