com.metaeffekt.mirror.index.nvd.NvdCpeApiIndex Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.index.nvd;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.vulnerability.CommonEnumerationUtil;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.download.documentation.DocRelevantMethods;
import com.metaeffekt.mirror.download.nvd.NvdCpeApiDownload;
import com.metaeffekt.mirror.index.Index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.springett.parsers.cpe.Cpe;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* This mirror contains all CPEs including all versions and their metadata that are known to the NVD.
* It is mainly used in the CPE URI Derivation algorithm that attempts to match CPE to inventory components, and in the vulnerability timeline generator, as displayed in the Vulnerability Assessment Dashboard.
* All entries from the API have been normalized to match the CPE Dictionary format in the according downloader step, which is why all CPE entries can be parsed in the same way.
* All entries have at least these fields: cpeName, cpeNameId, deprecated, titles, refs, lastModified and created
*
* Mapping of JSON content to Cpe fields
*
*
* JSON/XML
* data structure
*
*
*
*
* cpeNamecpe23Uri
*
* split CPE URI into parts
* part, vendor, product, version, update, edition, language, sw_edition, target_sw, target_hw, other
*
* cpeNameIdnvdId
*
* deprecateddeprecated (bool)
*
* titles > multiple titlestitles (JSON Array)
*
* references > multiple refsrefs (JSON Array)
*
*
* The NVD ID, titles, references and deprecated information can be queried later, where the titles and refs will be
* properly parsed as language-specific titles and reference instances.
*/
@MirrorMetadata(directoryName = "cpe-dict", mavenPropertyName = "nvdCpeIndex")
public class NvdCpeApiIndex extends Index {
private final static Logger LOG = LoggerFactory.getLogger(NvdCpeApiIndex.class);
public NvdCpeApiIndex(File baseMirrorDirectory) {
super(baseMirrorDirectory, NvdCpeApiIndex.class, Collections.singletonList(NvdCpeApiDownload.class), Collections.emptyList());
}
@Override
@DocRelevantMethods({"NvdCpeApiIndex#processFile"})
protected Map createIndexDocuments() {
final Map documents = new HashMap<>();
final File[] files = super.requiredDownloads[0].listFiles(f -> f.isFile() && f.getName().endsWith(".json"));
if (files == null) {
throw new RuntimeException("Unable to list files in " + super.requiredDownloads[0]);
}
for (File file : files) {
super.executor.submit(() -> {
synchronized (documents) {
documents.putAll(processFile(file));
}
});
}
super.executor.setSize(6);
super.executor.start();
try {
super.executor.join();
} catch (InterruptedException e) {
throw new RuntimeException("Failed to wait for indexing to complete.", e);
}
return documents;
}
private Map processFile(File file) {
final Map documents = new HashMap<>();
LOG.info("Processing file {}", file.getAbsolutePath());
final JSONArray json;
try {
json = new JSONArray(String.join("", FileUtils.readLines(file, StandardCharsets.UTF_8)));
} catch (IOException e) {
throw new RuntimeException("Failed to read yearly NVD document from " + file.getAbsolutePath(), e);
}
LOG.info("Processing [{}] entries for file {}", json.length(), file.getName());
for (int i = 0; i < json.length(); i++) {
final JSONObject entry = json.getJSONObject(i);
final String cpeName = entry.getString("cpeName");
final Document document = CommonEnumerationUtil.parseCpe(cpeName)
.map(NvdCpeApiIndex::createDocumentFromCpe)
.orElse(null);
if (document == null) {
throw new RuntimeException("Failed to parse CPE " + cpeName);
}
final String cpeNameId = entry.getString("cpeNameId");
final boolean deprecated = entry.getBoolean("deprecated");
final JSONArray titles = entry.optJSONArray("titles");
final JSONArray references = entry.optJSONArray("refs");
final String lastModified = entry.getString("lastModified");
final String created = entry.getString("created");
document.add(new TextField("nvdId", cpeNameId, Field.Store.YES));
document.add(new StringField("deprecated", Boolean.toString(deprecated), Field.Store.YES));
if (titles != null) document.add(new TextField("titles", titles.toString(), Field.Store.YES));
if (references != null) document.add(new TextField("references", references.toString(), Field.Store.YES));
document.add(new StringField("lastModified", lastModified, Field.Store.YES));
document.add(new StringField("created", created, Field.Store.YES));
if (documents.containsKey(cpeNameId)) {
LOG.warn("Duplicate CPE name id [{}]", cpeNameId);
}
documents.put(cpeNameId, document);
}
return documents;
}
private static Document createDocumentFromCpe(Cpe cpe) {
final Document doc = new Document();
doc.add(new StringField("part", cpe.getPart().getAbbreviation(), Field.Store.YES));
doc.add(new TextField("vendor", cpe.getVendor(), Field.Store.YES));
doc.add(new TextField("product", cpe.getProduct(), Field.Store.YES));
doc.add(new TextField("version", cpe.getVersion(), Field.Store.YES));
doc.add(new TextField("update", cpe.getUpdate(), Field.Store.YES));
doc.add(new TextField("edition", cpe.getEdition(), Field.Store.YES));
doc.add(new TextField("language", cpe.getLanguage(), Field.Store.YES));
doc.add(new TextField("sw_edition", cpe.getSwEdition(), Field.Store.YES));
doc.add(new TextField("target_sw", cpe.getTargetSw(), Field.Store.YES));
doc.add(new TextField("target_hw", cpe.getTargetHw(), Field.Store.YES));
doc.add(new TextField("other", cpe.getOther(), Field.Store.YES));
doc.add(new TextField("cpe23Uri", cpe.toCpe23FS(), Field.Store.YES));
return doc;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy