com.metaeffekt.mirror.index.advisor.MsrcKbChainIndex Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.index.advisor;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.contents.advisory.MsrcAdvisorEntry;
import com.metaeffekt.mirror.contents.msrcdata.MsrcProduct;
import com.metaeffekt.mirror.contents.msrcdata.MsrcRemediation;
import com.metaeffekt.mirror.contents.msrcdata.MsrcSupersedeNode;
import com.metaeffekt.mirror.download.advisor.MsrcManualCsvDownload;
import com.metaeffekt.mirror.download.advisor.MsrcSecurityGuideDownload;
import com.metaeffekt.mirror.download.documentation.DocRelevantMethods;
import com.metaeffekt.mirror.index.Index;
import com.metaeffekt.mirror.query.MsrcAdvisorIndexQuery;
import com.metaeffekt.mirror.query.MsrcProductIndexQuery;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.lucene.document.Document;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
@MirrorMetadata(directoryName = "msrc-kb-chains", mavenPropertyName = "msrcKbChainIndex")
public class MsrcKbChainIndex extends Index {
private final static Logger LOG = LoggerFactory.getLogger(MsrcKbChainIndex.class);
private final List additionalCsvFiles = new ArrayList<>();
public MsrcKbChainIndex(File baseMirrorDirectory) {
super(baseMirrorDirectory, MsrcKbChainIndex.class,
Collections.singletonList(MsrcSecurityGuideDownload.class), Arrays.asList(MsrcAdvisorIndex.class, MsrcProductIndex.class),
Collections.singletonList(MsrcManualCsvDownload.class), Collections.emptyList());
}
public MsrcKbChainIndex addAdditionalCsvFile(File additionalCsvFile) {
additionalCsvFiles.add(additionalCsvFile);
return this;
}
@Override
@DocRelevantMethods({"MsrcKbChainIndex#createMsrcApiNodes", "MsrcKbChainIndex#createMsrcUpdateGuideNodes", "MsrcKbChainIndex#createMsrcJsonDownloadNodes"})
protected Map createIndexDocuments() {
final MsrcAdvisorIndexQuery advisorQuery = new MsrcAdvisorIndexQuery(getRequiredIndex(MsrcAdvisorIndex.class));
final MsrcProductIndexQuery productQuery = new MsrcProductIndexQuery(getRequiredIndex(MsrcProductIndex.class));
final List allMsrcUpdateGuideCsvFiles = getAllMsrcUpdateGuideCsvFiles();
final List allMsrcUpdateGuideJsonFiles = getAllMsrcUpdateGuideJsonFiles();
LOG.info("");
LOG.info("Parsing KB nodes from:");
LOG.info(" - the MSRC Api Advisor Mirror");
if (!allMsrcUpdateGuideCsvFiles.isEmpty()) {
// total entries as of 2023-02-28: 114781
LOG.info(" - [{}] MSRC Security Update Guide CSV files", allMsrcUpdateGuideCsvFiles.size());
}
if (!allMsrcUpdateGuideJsonFiles.isEmpty()) {
// total entries as of 2023-07-11: 127517
LOG.info(" - [{}] MSRC Security Update Guide JSON files", allMsrcUpdateGuideJsonFiles.size());
}
LOG.info("");
LOG.info("- - - - - - - -");
// from the MSRC Api Advisor Mirror
final List msrcApiNodes = createMsrcApiNodes(advisorQuery);
// from the MSRC Security Update Guide CSV files
final Map> msrcUpdateGuideNodes = createMsrcUpdateGuideNodes(productQuery, allMsrcUpdateGuideCsvFiles);
// from the MSRC Security Update Guide JSON files
final Map> msrcUpdateGuideJsonNodes = createMsrcUpdateGuideJsonNodes(productQuery, allMsrcUpdateGuideJsonFiles);
// combine the lists into a single list
final List parsedNodes = new ArrayList<>(msrcApiNodes);
for (Map.Entry> fileEntry : msrcUpdateGuideNodes.entrySet()) {
parsedNodes.addAll(fileEntry.getValue());
}
for (Map.Entry> fileEntry : msrcUpdateGuideJsonNodes.entrySet()) {
parsedNodes.addAll(fileEntry.getValue());
}
// merge the nodes together
final Map uniqueNodes = MsrcSupersedeNode.mergeNodes(Collections.singleton(parsedNodes));
LOG.info("Deduplicated parsed KB entries [{} --> {}]", parsedNodes.size(), uniqueNodes.size());
// create documents
final Map documents = new HashMap<>();
for (Map.Entry entry : uniqueNodes.entrySet()) {
documents.put(entry.getKey(), entry.getValue().toDocument());
}
return documents;
}
private List getAllMsrcUpdateGuideCsvFiles() {
if (!super.optionalDownloads[0].exists()) {
return Collections.emptyList();
}
final Collection files = FileUtils.listFiles(super.optionalDownloads[0], new String[]{"csv"}, true);
for (File additionalCsvFile : additionalCsvFiles) {
files.addAll(FileUtils.listFiles(additionalCsvFile, new String[]{"csv"}, true));
}
return files.stream()
.sorted(Comparator.comparing(File::getName))
.collect(Collectors.toList());
}
private List getAllMsrcUpdateGuideJsonFiles() {
if (!super.requiredDownloads[0].exists()) {
return Collections.emptyList();
}
return FileUtils.listFiles(super.requiredDownloads[0], new String[]{"json"}, true).stream()
.sorted(Comparator.comparing(File::getName))
.collect(Collectors.toList());
}
private List createMsrcApiNodes(MsrcAdvisorIndexQuery advisorQuery) {
final Map nodes = new HashMap<>();
LOG.info("Querying MSRC API for KB entries");
for (MsrcAdvisorEntry entry : advisorQuery.findAll()) {
final String vulnerabilityId = entry.getId().replace("MSRC-", "");
for (MsrcRemediation msRemediation : entry.getMsRemediations()) {
final String description = msRemediation.getDescription();
if (isKbIdentifier(description)) {
final MsrcSupersedeNode node = nodes.computeIfAbsent(description, MsrcSupersedeNode::new);
final Set affectedProductIds = msRemediation.getAffectedProductIds();
final String supercedence = msRemediation.getSupercedence();
final List supersedence = extractSupersedeIdentifiers(supercedence);
for (String affectedProductId : affectedProductIds) {
node.addAffectsVulnerability(affectedProductId, vulnerabilityId);
for (String supersededKbId : supersedence) {
final MsrcSupersedeNode supersededNode = nodes.computeIfAbsent(supersededKbId, MsrcSupersedeNode::new);
node.addSupersedes(affectedProductId, supersededNode);
supersededNode.addSupersededBy(affectedProductId, node);
}
}
}
}
}
final Map normalized = MsrcSupersedeNode.mergeNodes(Collections.singletonList(nodes.values()));
if (normalized.size() != nodes.size()) {
LOG.info("Found [{}] --> [{}] KB entries", nodes.size(), normalized.size());
} else {
LOG.info("Found [{}] KB entries", nodes.size());
}
LOG.info("- - - - - - - -");
return new ArrayList<>(normalized.values());
}
private Map> createMsrcUpdateGuideNodes(MsrcProductIndexQuery productQuery, List files) {
final Map> msrcUpdateGuideNodes = new LinkedHashMap<>();
if (files.isEmpty()) {
return msrcUpdateGuideNodes;
}
for (File msrcUpdateGuideDownloadCsvFile : files) {
super.executor.submit(() -> {
try {
final List msrcCsvDownloadNodes = createMsrcCsvDownloadNodes(productQuery, msrcUpdateGuideDownloadCsvFile);
msrcUpdateGuideNodes.put(msrcUpdateGuideDownloadCsvFile, msrcCsvDownloadNodes);
} catch (IOException e) {
LOG.error("Failed to read MSRC update guide CSV file: " + msrcUpdateGuideDownloadCsvFile, e);
throw new RuntimeException("Failed to read MSRC update guide CSV file: " + msrcUpdateGuideDownloadCsvFile, e);
} catch (Exception e) {
LOG.error("Failed to parse MSRC update guide CSV file: " + msrcUpdateGuideDownloadCsvFile, e);
throw new RuntimeException("Failed to parse MSRC update guide CSV file: " + msrcUpdateGuideDownloadCsvFile, e);
}
});
}
super.executor.setSize(16);
super.executor.start();
try {
super.executor.join();
} catch (InterruptedException e) {
throw new RuntimeException("Failed to wait for indexing to complete.", e);
}
LOG.info("- - - - - - - -");
return msrcUpdateGuideNodes;
}
private Map> createMsrcUpdateGuideJsonNodes(MsrcProductIndexQuery productQuery, List files) {
final Map> msrcUpdateGuideNodes = new LinkedHashMap<>();
if (files.isEmpty()) {
return msrcUpdateGuideNodes;
}
for (File msrcUpdateGuideDownloadJsonFile : files) {
super.executor.submit(() -> {
try {
final List msrcJsonDownloadNodes = createMsrcJsonDownloadNodes(productQuery, msrcUpdateGuideDownloadJsonFile);
msrcUpdateGuideNodes.put(msrcUpdateGuideDownloadJsonFile, msrcJsonDownloadNodes);
} catch (IOException e) {
LOG.error("Failed to read MSRC update guide JSON file: " + msrcUpdateGuideDownloadJsonFile, e);
throw new RuntimeException("Failed to read MSRC update guide JSON file: " + msrcUpdateGuideDownloadJsonFile, e);
} catch (Exception e) {
LOG.error("Failed to parse MSRC update guide JSON file: " + msrcUpdateGuideDownloadJsonFile, e);
throw new RuntimeException("Failed to parse MSRC update guide JSON file: " + msrcUpdateGuideDownloadJsonFile, e);
}
});
}
super.executor.setSize(16);
super.executor.start();
try {
super.executor.join();
} catch (InterruptedException e) {
throw new RuntimeException("Failed to wait for indexing to complete.", e);
}
LOG.info("- - - - - - - -");
return msrcUpdateGuideNodes;
}
/**
* Example CSV entries:
*
* Release Date,Product,Platform,Impact,Max Severity,Article,Article,Download,Download,Details,Details
* "Dec 16, 2022",Microsoft Edge (Chromium-based),,,,Release Notes,https://docs.microsoft.com/en-us/DeployEdge/microsoft-edge-relnotes-security,Security Update,,CVE-2022-4436,https://msrc.microsoft.com//update-guide/vulnerability/CVE-2022-4436
* "Dec 13, 2022",PowerShell 7.2,,Remote Code Execution,Important,Release Notes,https://github.com/PowerShell/Announcements/issues/37,Security Update,https://github.com/PowerShell/Announcements/issues/37,CVE-2022-41089,https://msrc.microsoft.com//update-guide/vulnerability/CVE-2022-41089
* "Dec 13, 2022",Raw Image Extension,Windows 10 Version 1607 for x64-based Systems,Remote Code Execution,Important,Release Notes,https://support.microsoft.com/en-us/account-billing/get-updates-for-apps-and-games-in-microsoft-store-a1fe19c0-532d-ec47-7035-d1c5a1dd464f,Security Update,,CVE-2022-44687,https://msrc.microsoft.com//update-guide/vulnerability/CVE-2022-44687
*
*
* @param productQuery the product query to use for product name normalization and product ID lookup
* @param file the CSV file to parse
* @return the list of nodes created from the CSV file
* @throws IOException if the file could not be read
*/
private List createMsrcCsvDownloadNodes(MsrcProductIndexQuery productQuery, File file) throws IOException {
if (file.isDirectory() || !file.isFile() || !file.getName().endsWith(".csv")) {
LOG.warn("File is not a CSV file, but a directory: " + file.getAbsolutePath());
return Collections.emptyList();
}
LOG.info("Parsing CSV file from [{}]", file.getAbsolutePath());
final List> csvData = parseCsvFile(file);
LOG.info("Parsing CSV file with [{}] entries", csvData.size());
final Map nodes = new HashMap<>();
final Set unknownProducts = new HashSet<>();
for (Map entry : csvData) {
final String cveId = entry.get("Details");
final String article = entry.get("Article");
final String productName = entry.get("Product");
final String effectiveProduct = resolveEffectiveProduct(productQuery, productName, unknownProducts, cveId);
if (StringUtils.hasText(cveId) && StringUtils.hasText(article)) {
if (article.matches("\\d{6,8}")) {
for (String product : effectiveProduct.split(",")) {
nodes.computeIfAbsent(article, MsrcSupersedeNode::new).addAffectsVulnerability(product, cveId);
}
}
}
}
LOG.info("Parsed [{}] KB entries", nodes.size());
return new ArrayList<>(MsrcSupersedeNode.mergeNodes(Collections.singletonList(nodes.values())).values());
}
/**
* Example:
*
* {
* "productFamily": "Windows",
* "productFamilyId": 100000010,
* "severity": "Important",
* "temporalScore": "6.5",
* "product": "Windows 10 Version 1809 for 32-bit Systems",
* "productId": 11568,
* "releaseDate": "2023-01-10T08:00:00Z",
* "impactId": 100000001,
* "impact": "Denial of Service",
* "issuingCna": "Microsoft",
* "platformId": 0,
* "baseScore": "7.5",
* "kbArticles": [
* {
* "rebootRequired": "Yes",
* "articleName": "5022286",
* "knownIssuesName": "5022286",
* "affectedBinaries": [],
* "knownIssuesUrl": "https://support.microsoft.com/help/5022286",
* "downloadUrl": "https://catalog.update.microsoft.com/v7/site/Search.aspx?q=KB5022286",
* "downloadName": "Security Update",
* "articleUrl": "https://support.microsoft.com/help/5022286",
* "fixedBuildNumber": "10.0.17763.3887",
* "supercedence": "5021237",
* "ordinal": 0
* }
* ],
* "initialReleaseDate": "2023-01-10T08:00:00Z",
* "cveNumber": "CVE-2023-21527",
* "isMariner": false,
* "productVersion": "10.0.0",
* "architectureId": 0,
* "id": "00000000-0000-0000-302d-00006e4d7c04",
* "releaseNumber": "2023-Jan",
* "severityId": 100000001,
* "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H/E:U/RL:O/RC:C"
* }
*
*
* @param productQuery the product query to use for product name normalization and product ID lookup
* @param file the JSON file to parse
* @return the list of nodes created from the JSON file
* @throws IOException if the file could not be read
*/
private List createMsrcJsonDownloadNodes(MsrcProductIndexQuery productQuery, File file) throws IOException {
if (file.isDirectory() || !file.isFile() || !file.getName().endsWith(".json")) {
LOG.warn("File is not a JSON file, but a directory: " + file.getAbsolutePath());
return Collections.emptyList();
}
LOG.info("Parsing JSON file from [{}]", file.getAbsolutePath());
final JSONArray jsonArray = new JSONArray(FileUtils.readFileToString(file, StandardCharsets.UTF_8));
LOG.info("Parsing JSON file with [{}] entries", jsonArray.length());
final Map nodes = new HashMap<>();
final Set unknownProducts = new HashSet<>();
for (int i = 0; i < jsonArray.length(); i++) {
final JSONObject entry = jsonArray.getJSONObject(i);
final String cveId = entry.getString("cveNumber");
final JSONArray entryArticles = entry.getJSONArray("kbArticles");
final String productName = entry.getString("product");
final String effectiveProduct = resolveEffectiveProduct(productQuery, productName, unknownProducts, cveId);
for (int j = 0; j < entryArticles.length(); j++) {
final JSONObject currentArticle = entryArticles.getJSONObject(j);
final String article = currentArticle.optString("articleName", null);
final String articleUrl = ObjectUtils.firstNonNull(currentArticle.optString("articleUrl", null), currentArticle.optString("knownIssuesUrl", null));
final String downloadUrl = currentArticle.optString("downloadUrl", null);
if (StringUtils.hasText(cveId) && StringUtils.hasText(article)) {
if (article.matches("\\d{6,8}")) {
for (String product : effectiveProduct.split(",")) {
nodes.computeIfAbsent(article, a -> new MsrcSupersedeNode(a, articleUrl, downloadUrl))
.addAffectsVulnerability(product, cveId);
}
}
}
}
}
LOG.info("Parsed [{}] KB entries", nodes.size());
return new ArrayList<>(MsrcSupersedeNode.mergeNodes(Collections.singletonList(nodes.values())).values());
}
private String resolveEffectiveProduct(MsrcProductIndexQuery productQuery, String productName, Set unknownProducts, String cveId) {
final String effectiveProduct;
if (StringUtils.hasText(productName)) {
final MsrcProduct productByOriginalName = productQuery.findProductByName(productName);
if (productByOriginalName != null) {
effectiveProduct = productByOriginalName.getId();
} else {
final List products = productQuery.findProductByNameFuzzyIfNoExactMatch(productName);
if (products.isEmpty()) {
if (unknownProducts.add(productName)) {
LOG.info("Product not found: {}", productName);
}
effectiveProduct = "unknown";
} else {
effectiveProduct = products.stream().map(MsrcProduct::getId).collect(Collectors.joining(","));
}
}
} else {
effectiveProduct = "unknown";
LOG.warn("No product name found for [{}]", cveId);
}
return effectiveProduct;
}
private List> parseCsvFile(File file) throws IOException {
final CSVParser parser = CSVParser.parse(file, Charset.defaultCharset(), CSVFormat.DEFAULT.withFirstRecordAsHeader().withAllowDuplicateHeaderNames());
final List> csvData = new ArrayList<>();
for (CSVRecord record : parser) {
final Map entry = new HashMap<>();
int i = 0;
for (String header : record.getParser().getHeaderNames()) {
final String value = record.get(i++);
if (entry.containsKey(header)) {
entry.put(header + " (" + (entry.keySet().stream().filter(k -> k.startsWith(header)).count() + 1) + ")", value);
} else {
entry.put(header, value);
}
}
csvData.add(entry);
}
return csvData;
}
private List extractSupersedeIdentifiers(String identifiers) {
if (StringUtils.isEmpty(identifiers)) {
return Collections.emptyList();
}
return Arrays.stream(identifiers.split("([,;]|
) ?"))
.filter(StringUtils::hasText)
.map(kb -> kb.replace("\n", ""))
.map(kb -> kb.replace(" ", ""))
.filter(this::isKbIdentifier)
.collect(Collectors.toList());
}
private boolean isKbIdentifier(String id) {
return StringUtils.hasText(id) && id.matches("\\d{6,8}");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy