com.metaeffekt.artifact.enrichment.vulnerability.CpeDerivationUtilities Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.enrichment.vulnerability;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.metaeffekt.artifact.analysis.dashboard.Dashboard;
import com.metaeffekt.artifact.analysis.utils.CustomCollectors;
import com.metaeffekt.artifact.analysis.utils.LazySupplier;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.vulnerability.CommonEnumerationUtil;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.InventoryAttribute;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.warnings.InventoryWarningEntry;
import com.metaeffekt.artifact.analysis.vulnerability.enrichment.warnings.InventoryWarnings;
import com.metaeffekt.artifact.enrichment.configurations.CpeDerivationEnrichmentConfiguration;
import com.metaeffekt.mirror.query.NvdCpeApiIndexQuery;
import com.metaeffekt.mirror.query.NvdCpeApiVendorProductIndexQuery;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.json.JSONObject;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.metaeffekt.core.inventory.processor.model.Constants;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import us.springett.parsers.cpe.Cpe;
import us.springett.parsers.cpe.values.Part;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
public class CpeDerivationUtilities {
/**
* Set of unspecific product strings.
* Many product identifiers do not contribute to any meaningful result, which is why they are being ignored.
*/
private final static Set UNSPECIFIC_PRODUCTS;
static {
try {
UNSPECIFIC_PRODUCTS = Dashboard.readResourceAsStringList(CpeDerivationUtilities.class, "enrichment/cpe-derivation/unspecific-products.txt").stream()
.filter(StringUtils::hasText)
.filter(str -> !str.startsWith("#"))
.collect(Collectors.toSet());
} catch (IOException e) {
throw new RuntimeException("Failed to load unspecific products for CPE derivation from classpath resource [enrichment/cpe-derivation/unspecific-products.txt]", e);
}
}
private final LazySupplier cpeDictionary;
private final LazySupplier cpeDictionaryVendorProduct;
@Setter
@Getter
private CpeDerivationEnrichmentConfiguration configuration = new CpeDerivationEnrichmentConfiguration();
private Set topVpTerms = new HashSet<>();
private long topVpTermsHash = 0;
public CpeDerivationUtilities(File baseMirrorDirectory) {
cpeDictionary = new LazySupplier<>(() -> new NvdCpeApiIndexQuery(baseMirrorDirectory));
cpeDictionaryVendorProduct = new LazySupplier<>(() -> new NvdCpeApiVendorProductIndexQuery(baseMirrorDirectory));
}
public void deriveCpeUris(Artifact artifact) {
deriveCpeUris(null, artifact);
}
public void deriveCpeUris(Inventory inventory, Artifact artifact) {
synchronized (topVpTerms) {
if (topVpTerms.isEmpty() || topVpTermsHash != configuration.getRequireSecondaryIndicationTermsLimiters().hashCode()) {
this.topVpTerms = this.cpeDictionaryVendorProduct.get().findTopVpTerms(configuration.getRequireSecondaryIndicationTermsLimiters()).keySet();
this.topVpTermsHash = configuration.getRequireSecondaryIndicationTermsLimiters().hashCode();
// log.info("Top VP terms: [count: {}] {}", topVpTerms.size(), topVpTerms);
}
}
// only if the artifact is a hardware CPE, it may match "cpe:/h:..." hardware CPEs
final boolean isHardware = artifact.isHardware();
// collect aliases representing the artifact (vendor or product)
final List aliases = deriveArtifactAliases(artifact);
// find vendor/product pairs to the aliases
final List matchedVendorProducts = matchAliasesToVendorProducts(aliases);
// derive CPE URIs from vendor/product pairs
final Set derivedCpeUris = findVersionSpecificCpeUrisFromVendorProducts(matchedVendorProducts, isHardware,
cpes -> { // handler for when the max correlated CPE per artifact limit is reached
log.warn("Max correlated CPE URIs limit reached on [{} : {}], limiting to [{}]", artifact.getId(), artifact.getComponent(), configuration.getMaxCorrelatedCpePerArtifact());
if (inventory != null) {
addInventoryWarningAboutTooManyCpeUris(inventory, artifact, cpes);
}
}
);
// for deterministic processing (anticipating limits) the CPEs are ordered
final List sortedDerivedCpeUris = derivedCpeUris.stream().sorted(Cpe::compareTo).collect(Collectors.toList());
final String finalCpeString = sortedDerivedCpeUris.stream()
.limit(configuration.getMaxCorrelatedCpePerArtifact()) // reduce the resulting list to maxCorrelatedCPE
.map(CommonEnumerationUtil::toCpe22UriOrFallbackToCpe23FS)
.collect(Collectors.joining(", "));
if (!finalCpeString.isEmpty()) {
artifact.set(InventoryAttribute.DERIVED_CPE_URIS.getKey(), finalCpeString);
} else {
artifact.set(InventoryAttribute.DERIVED_CPE_URIS.getKey(), null);
}
if (this.configuration.isAddDetailedMatchingInformation()) {
artifact.set(InventoryAttribute.DERIVED_CPE_URIS_MATCHING_DETAILS.getKey(), matchedVendorProducts.stream().map(AliasMatchingResultsVendorProducts::toJson).collect(CustomCollectors.toJsonArray()).toString());
} else {
artifact.set(InventoryAttribute.DERIVED_CPE_URIS_MATCHING_DETAILS.getKey(), null);
}
}
private void addInventoryWarningAboutTooManyCpeUris(Inventory inventory, Artifact artifact, Collection sortedDerivedCpeUris) {
if (sortedDerivedCpeUris.size() > configuration.getMaxCorrelatedCpePerArtifact()) {
log.warn("Max correlated CPE URIs limit reached, reducing [{}] CPEs to [{}]", sortedDerivedCpeUris.size(), configuration.getMaxCorrelatedCpePerArtifact());
if (inventory != null) {
new InventoryWarnings(inventory).addArtifactWarning(new InventoryWarningEntry<>(artifact,
String.format("Max correlated CPE URIs limit reached, reducing [%d] CPEs to [%d]", sortedDerivedCpeUris.size(), configuration.getMaxCorrelatedCpePerArtifact()),
"CPE URI derivation"
));
}
}
}
public List deriveArtifactAliases(Artifact artifact) {
final Set aliases = new HashSet<>();
for (Map.Entry entry : fetchArtifactValues(artifact, Arrays.asList(Artifact.Attribute.COMPONENT.getKey(), Artifact.Attribute.GROUPID.getKey(), Artifact.Attribute.ID.getKey(), Constants.KEY_ORGANIZATION)).entrySet()) {
final String key = entry.getKey();
final String value = entry.getValue();
aliases.addAll(Alias.toAliases(deriveProductAliases(preprocessProduct(value)), key, value));
aliases.add(new Alias(value, key, value));
}
aliases.addAll(Alias.toAliases(deriveProductAliases(preprocessProduct(artifact.getArtifactId())), "artifact-id", artifact.getArtifactId()));
aliases.addAll(Alias.toAliases(deriveAliasesFromPurl(artifact.get(InventoryAttribute.PURL.getKey())), InventoryAttribute.PURL.getKey(), artifact.get(InventoryAttribute.PURL.getKey())));
aliases.addAll(Alias.toAliases(deriveAliasesFromPurl(artifact.get(InventoryAttribute.DT_PURL_FINDINGS.getKey())), InventoryAttribute.DT_PURL_FINDINGS.getKey(), artifact.get(InventoryAttribute.DT_PURL_FINDINGS.getKey())));
aliases.removeIf(this::isInvalidQueryAlias);
synchronized (topVpTerms) {
for (Alias alias : aliases) {
if (topVpTerms.contains(alias.getAlias()) || topVpTerms.contains(alias.getAlias().toLowerCase())) {
alias.setRequireOtherIndication(true);
}
}
}
return aliases.stream().sorted().collect(Collectors.toList());
}
private Map fetchArtifactValues(Artifact artifact, List keys) {
final Map values = new HashMap<>();
for (String key : keys) {
final String value = artifact.get(key);
if (StringUtils.hasText(value)) {
values.put(key, value);
}
}
return values;
}
/**
* A query alias may not be:
*
* - empty
* - be numeric (finds all CPE that contain a version in the vendor/product field)
* - contain a backslash, as this breaks the index query
*
*
* @param a the alias to check
* @return true if the alias is invalid based on the above criteria
*/
private boolean isInvalidQueryAlias(Alias a) {
final String alias = a.getAlias();
if (StringUtils.isEmpty(alias)) return true;
if (org.apache.commons.lang3.StringUtils.isNumeric(alias)) return true;
if (alias.contains("\\")) return true;
return false;
}
protected Set deriveAliasesFromPurl(String purl) {
final Set aliases = new HashSet<>();
if (StringUtils.isEmpty(purl)) {
return aliases;
}
// scheme:type/namespace/name@version?qualifiers#subpath
try {
final PackageURL packageURL = new PackageURL(purl);
if (StringUtils.hasText(packageURL.getName())) {
aliases.add(packageURL.getName());
}
} catch (MalformedPackageURLException e) {
log.warn("Could not parse PURL [{}]", purl);
}
return aliases;
}
protected Set deriveProductAliases(String product) {
if (product == null) return Collections.emptySet();
Set productAliases = new HashSet<>();
productAliases.add(product);
productAliases.add(product.replace('-', '_'));
productAliases.add(product.replace('_', '-'));
productAliases.add(product.replace('_', ' '));
productAliases.add(product.replace("_", ""));
productAliases.add(product.replace("-", ""));
productAliases.add(product.replace(' ', '_'));
productAliases.add(product.replace(' ', '-'));
productAliases.add(product.replace(" ", ""));
// remove jar/dll/x64 and lib/libs separately
productAliases.addAll(productAliases.stream().map(productAlias -> productAlias.replaceAll("[ _.-]?(x64|dll|jar)", "")).collect(Collectors.toSet()));
productAliases.addAll(productAliases.stream().map(productAlias -> productAlias.replaceAll("[ _.-]?libs?", "")).collect(Collectors.toSet()));
// reduce multiple underscore/space/dash following each other to one
productAliases.addAll(productAliases.stream().map(productAlias -> productAlias.replaceAll("([ _.-]){2,}", "$1")).collect(Collectors.toSet()));
// remove numbers from end
productAliases.addAll(productAliases.stream().map(productAlias -> productAlias.replaceAll("(\\d|[ _.-])+$", "")).collect(Collectors.toSet()));
// remove underscore/space/dash/dot from end
productAliases.addAll(productAliases.stream().map(productAlias -> productAlias.replaceAll("[ _.-]*$", "")).collect(Collectors.toSet()));
int dashIndex = product.indexOf("-");
if (dashIndex > 3) {
productAliases.addAll(deriveProductAliases(product.substring(0, dashIndex)));
}
int doubleUnderscoreIndex = product.indexOf("__");
if (doubleUnderscoreIndex > 3) {
productAliases.addAll(deriveProductAliases(product.substring(0, doubleUnderscoreIndex)));
}
return productAliases;
}
protected String preprocessProduct(String product) {
if (product == null) return null;
product = product.trim().toLowerCase();
String result = product.replace("-base", "");
result = result.replace("-minimal", "");
result = result.replace(".class", "");
result = result.replace("\\$[0-9]*.class", "");
// artifact ids may already include a version number
result = result.replaceAll("-", "_");
result = result.replaceAll("\\d*\\.[\\d._]+\\d+", ""); // remove version numbers in the form of d.dd or d.d.d (with at least two digits)
result = result.replaceAll("\\[0-9\\._]*", "");
result = result.replaceAll("_\\.v[\\d._]?.*", "");
result = result.replaceAll("\\.", "_");
while (result.endsWith(".") || result.endsWith("_") || result.endsWith("-")) {
result = result.substring(0, result.length() - 1);
}
return result.trim();
}
/**
* Attempts to match aliases to vendor products using a fallback strategy:
* It applies each vendor-product matching function to the given set of aliases until a non-empty mapping is found.
*
* Once a non-empty mapping is obtained, the loop breaks and the result is returned.
*/
private List matchAliasesToVendorProducts(Collection aliases) {
for (Function, List> function : VENDOR_PRODUCT_MATCHING_FUNCTIONS_FALLBACK_ORDER) {
final List matchedVendorProducts = function.apply(aliases);
if (!matchedVendorProducts.isEmpty()) {
return matchedVendorProducts;
}
}
return Collections.emptyList();
}
@Data
public static class AliasMatchingResultsVendorProducts {
private final String vendor;
private final Set products;
public AliasMatchingResultsVendorProducts(String vendor) {
this.vendor = vendor;
this.products = new LinkedHashSet<>();
}
public Set getRawProducts() {
return this.products.stream().map(AliasMatchingResultsProduct::getProduct).collect(Collectors.toSet());
}
public AliasMatchingResultsProduct addProduct(String product) {
return this.products.stream()
.filter(p -> p.getProduct().equals(product))
.findFirst()
.orElseGet(() -> {
final AliasMatchingResultsProduct newProduct = new AliasMatchingResultsProduct(product);
this.products.add(newProduct);
return newProduct;
});
}
public static AliasMatchingResultsVendorProducts addVendor(List vendorProducts, String vendor) {
return vendorProducts.stream()
.filter(vp -> vp.getVendor().equals(vendor))
.findFirst()
.orElseGet(() -> {
final AliasMatchingResultsVendorProducts newVendorProducts = new AliasMatchingResultsVendorProducts(vendor);
vendorProducts.add(newVendorProducts);
return newVendorProducts;
});
}
public JSONObject toJson() {
final JSONObject json = new JSONObject().put("vendor", vendor);
final JSONObject productsJson = new JSONObject();
for (AliasMatchingResultsProduct product : products) {
productsJson.put(product.getProduct(), product.toJson());
}
json.put("products", productsJson);
return json;
}
}
@Data
public static class AliasMatchingResultsProduct {
private final String product;
private final List> aliases;
public AliasMatchingResultsProduct(String product) {
this.product = product;
this.aliases = new ArrayList<>();
}
public void addAlias(Alias alias, int method) {
if (this.aliases.stream().noneMatch(p -> p.getKey().equals(alias))) {
this.aliases.add(Pair.of(alias, method));
}
}
public JSONObject toJson() {
return new JSONObject()
.put("aliases", aliases.stream().map(p -> new JSONObject()
.put("alias", p.getKey().getAlias())
.put("source", new JSONObject()
.put("value", p.getKey().getSourceValue())
.put("attribute", p.getKey().getSourceAttribute())
).put("requireOtherIndication", p.getKey().isRequireOtherIndication())
.put("method", p.getValue())
).collect(Collectors.toList()));
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AliasMatchingResultsProduct that = (AliasMatchingResultsProduct) o;
return Objects.equals(product, that.product);
}
@Override
public int hashCode() {
return Objects.hashCode(product);
}
}
private final List, List>> VENDOR_PRODUCT_MATCHING_FUNCTIONS_FALLBACK_ORDER = Arrays.asList(
// select all vendor / product pairs that match one of the aliases
this::matchVendorProductsFromAliases,
// as fallback match either or, but ignore the unspecific
this::matchProductsFromAliasesIgnoreUnderscoreRemoveUnspecific,
// as fallback match vendors and products fuzzy, but ignore the unspecific
this::matchVendorProductsFuzzyFromAliases
);
protected List matchVendorProductsFromAliases(Collection aliases) {
final List vendorProductPairs = new ArrayList<>();
final NvdCpeApiVendorProductIndexQuery cpeDictionaryVendorProduct = this.cpeDictionaryVendorProduct.get();
final AliasRequireOtherMatcher aliasRequireOtherMatcher = new AliasRequireOtherMatcher();
for (Alias candidate : aliases) {
final String alias = candidate.getAlias();
final Set vendorsForProduct = cpeDictionaryVendorProduct.findVendorsForProduct(alias);
final Set productsForVendor = cpeDictionaryVendorProduct.findProductsForVendor(alias);
if (vendorsForProduct.isEmpty() && productsForVendor.isEmpty()) {
continue;
}
if (!aliasRequireOtherMatcher.isMatching(candidate)) continue;
for (String vendor : vendorsForProduct) {
AliasMatchingResultsVendorProducts.addVendor(vendorProductPairs, vendor).addProduct(alias).addAlias(candidate, 0);
}
for (String product : productsForVendor) {
AliasMatchingResultsVendorProducts.addVendor(vendorProductPairs, alias).addProduct(product).addAlias(candidate, 0);
}
}
return vendorProductPairs;
}
protected List matchProductsFromAliasesIgnoreUnderscoreRemoveUnspecific(Collection aliases) {
final List vendorProductPairs = new ArrayList<>();
final NvdCpeApiVendorProductIndexQuery cpeDictionaryVendorProduct = this.cpeDictionaryVendorProduct.get();
final AliasRequireOtherMatcher aliasRequireOtherMatcher = new AliasRequireOtherMatcher();
for (Alias candidate : aliases) {
final String alias = candidate.getAlias();
for (Map.Entry> productVendors : cpeDictionaryVendorProduct.getProductVendorsMap().entrySet()) {
final String product = productVendors.getKey();
if (UNSPECIFIC_PRODUCTS.contains(product)) {
continue;
}
// alias must either be equal to or contain the product name
if (!alias.equalsIgnoreCase(product) &&
!alias.startsWith(product + "_") &&
!alias.contains("_" + product + "_") &&
!alias.endsWith("_" + product)) {
continue;
}
for (String vendor : productVendors.getValue()) {
if (UNSPECIFIC_PRODUCTS.contains(vendor)) {
continue;
}
// alias must either be equal to or contain the vendor name
if (!alias.equalsIgnoreCase(vendor) &&
!alias.startsWith(vendor + "_") &&
!alias.contains("_" + vendor + "_") &&
!alias.endsWith("_" + vendor)) {
continue;
}
if (!aliasRequireOtherMatcher.isMatching(candidate)) continue;
AliasMatchingResultsVendorProducts.addVendor(vendorProductPairs, vendor).addProduct(product).addAlias(candidate, 1);
}
}
}
return vendorProductPairs;
}
protected List matchVendorProductsFuzzyFromAliases(Collection aliases) {
final List vendorProductPairs = new ArrayList<>();
final NvdCpeApiVendorProductIndexQuery cpeDictionaryVendorProduct = this.cpeDictionaryVendorProduct.get();
final AliasRequireOtherMatcher aliasRequireOtherMatcher = new AliasRequireOtherMatcher();
for (Alias candidate : aliases) {
final String alias = candidate.getAlias();
if (alias.trim().length() <= 4) {
continue;
}
for (String vendor : this.findFirstVendorContainsAlias(cpeDictionaryVendorProduct, alias)) {
if (UNSPECIFIC_PRODUCTS.contains(vendor)) continue;
for (String product : cpeDictionaryVendorProduct.findProductsForVendor(vendor)) {
if (UNSPECIFIC_PRODUCTS.contains(product)) continue;
if (!aliasRequireOtherMatcher.isMatching(candidate)) continue;
AliasMatchingResultsVendorProducts.addVendor(vendorProductPairs, vendor).addProduct(product).addAlias(candidate, 2);
}
}
for (String product : this.findFirstProductContainsAlias(cpeDictionaryVendorProduct, alias)) {
if (UNSPECIFIC_PRODUCTS.contains(product)) continue;
for (String vendor : cpeDictionaryVendorProduct.findVendorsForProduct(product)) {
if (UNSPECIFIC_PRODUCTS.contains(vendor)) continue;
if (!aliasRequireOtherMatcher.isMatching(candidate)) continue;
AliasMatchingResultsVendorProducts.addVendor(vendorProductPairs, vendor).addProduct(product).addAlias(candidate, 2);
}
}
}
// if there are more than 10 found, limit them to 10.
// to find which ones to remove:
// - the one with the least aliases
// - the one with the largest distance between the alias and the vendor/product
final int maxVendorProductPairs = 10;
final List reducedVendors = new ArrayList<>();
while (true) {
// check how many need to be removed (if any)
final int totalProducts = vendorProductPairs.stream().mapToInt(vp -> vp.getProducts().size()).sum();
final int attemptRemovalCount = totalProducts - maxVendorProductPairs;
if (attemptRemovalCount <= 0) break;
// find all those with the smallest amount of aliases, prioritize those first, since they might yield the most relevant results
final Pair> minAliasProducts = fuzzyFromAliasGetMinCountResults(vendorProductPairs, reducedVendors);
if (StringUtils.isEmpty(minAliasProducts.getKey())) {
break;
}
reducedVendors.add(minAliasProducts.getKey());
// find the distance between the aliases and the vendors/products
final Map distances = fuzzyFromAliasFindProductDistancesToAliasesLevenshtein(minAliasProducts.getValue(), vendorProductPairs);
// sort by levenshtein distance, descending so that the largest distance is removed first
final List orderedRemovalPriority = distances.entrySet().stream()
.sorted((o1, o2) -> Integer.compare(o2.getValue(), o1.getValue()))
.limit(attemptRemovalCount)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
if (orderedRemovalPriority.isEmpty()) {
log.warn("Could not find a vendor/product pair to remove, even though the limit was reached");
break;
}
final AliasMatchingResultsVendorProducts removeFrom = vendorProductPairs.stream()
.filter(vp -> vp.getVendor().equals(minAliasProducts.getKey()))
.findFirst().get();
for (AliasMatchingResultsProduct toRemoveProduct : orderedRemovalPriority) {
removeFrom.getProducts().removeIf(p -> p.getProduct().equals(toRemoveProduct.getProduct()));
log.debug("Removed vendor/product pair [{} : {}] due to alias limit", removeFrom.getVendor(), toRemoveProduct.getProduct());
}
}
return vendorProductPairs;
}
/**
* Retrieves a list of products with the smallest number of aliases from the provided vendor-product pairs.
*
* This method iterates through each {@link AliasMatchingResultsVendorProducts} in the given list,
* examines each {@link AliasMatchingResultsProduct} within them, and identifies the products that have
* the minimum count of aliases. If multiple products share the same minimal alias count, all such
* products are included in the returned list.
*
* @param vendorProductPairs the list of vendor-product pairs to process
* @return a list of {@link AliasMatchingResultsProduct} with the smallest number of aliases.
* Returns an empty list if no products are found.
*/
private static Pair> fuzzyFromAliasGetMinCountResults(List vendorProductPairs, List exclude) {
int minAliases = Integer.MAX_VALUE;
List minAliasProducts = new ArrayList<>();
String vendor = null;
for (AliasMatchingResultsVendorProducts pair : vendorProductPairs) {
if (exclude.contains(pair.getVendor())) {
continue;
}
for (AliasMatchingResultsProduct product : pair.getProducts()) {
int aliasCount = product.getAliases().size();
if (aliasCount < minAliases) {
minAliases = aliasCount;
minAliasProducts.clear();
minAliasProducts.add(product);
vendor = pair.getVendor();
} else if (aliasCount == minAliases) {
minAliasProducts.add(product);
}
}
}
if (minAliases == Integer.MAX_VALUE) {
minAliasProducts = new ArrayList<>();
}
return Pair.of(vendor, minAliasProducts);
}
private static Map fuzzyFromAliasFindProductDistancesToAliasesLevenshtein(List minAliasProducts, List vendorProductPairs) {
// if there are multiple, find the one with the largest distance between the alias and the vendor/product (calculate both, then min)
// use the distance function: 'min(levenshteinDistance(alias, vendor), levenshteinDistance(alias, product1), )' via LevenshteinDistance.getDefaultInstance()
final Map distances = new HashMap<>();
for (AliasMatchingResultsProduct product : minAliasProducts) {
int minDistance = Integer.MAX_VALUE;
final String lookupVendor = vendorProductPairs.stream()
.filter(vp -> vp.getProducts().contains(product))
.map(AliasMatchingResultsVendorProducts::getVendor)
.findFirst().orElse(null);
if (lookupVendor != null) {
minDistance = Math.min(minDistance, LevenshteinDistance.getDefaultInstance().apply(lookupVendor, product.getProduct()));
}
for (Pair alias : product.getAliases()) {
final int distance = LevenshteinDistance.getDefaultInstance().apply(alias.getKey().getAlias(), product.getProduct());
if (distance < minDistance) {
minDistance = distance;
}
}
distances.put(product, minDistance);
}
return distances;
}
private Set findFirstVendorContainsAlias(NvdCpeApiVendorProductIndexQuery cpeDictionaryVendorProduct, String alias) {
return cpeDictionaryVendorProduct.findVendorsFuzzy(alias);
}
private Set findFirstProductContainsAlias(NvdCpeApiVendorProductIndexQuery cpeDictionaryVendorProduct, String alias) {
return cpeDictionaryVendorProduct.findProductsFuzzy(alias);
}
private Set findVersionSpecificCpeUrisFromVendorProducts(List vendorProductPairs, boolean isComponentHardware, Consumer> maxCorrelatedCpePerArtifactReachedCallback) {
final Set derivedCpeUris = new HashSet<>();
final NvdCpeApiIndexQuery cpeDictionary = this.cpeDictionary.get();
// add CPEs (version specific)
for (AliasMatchingResultsVendorProducts vendorProducts : vendorProductPairs) {
final String vendor = vendorProducts.getVendor();
for (String product : vendorProducts.getRawProducts()) {
// find all CPEs from the vendor/product pair
final List cpeByVendorProduct = cpeDictionary.findCpeByVendorProduct(vendor, product);
if (cpeByVendorProduct.isEmpty()) {
log.warn("No CPEs found for vendor/product pair [{} : {}]", vendor, product);
continue;
}
cpeByVendorProduct.stream()
// there are four combinations of hardware checks:
// | hardware | not hardware
// cpe h | allow | disallow
// cpe a/o | allow | allow
// matching application/os CPEs on a hardware component is allowed, as there are some CPEs registered as cpe:/a:, even though they are hardware (cpe:/a:yamaha:router).
// only matching hardware CPEs on a non-hardware component is disallowed, since we can be sure that this will be a false positive.
// in general, there are far more relevant a/o CPEs, so it is far more likely to have a false positive with an additional hardware CPE on a non-hardware component.
.filter(cpe -> hardwareCheck(isComponentHardware, cpe))
.map(CommonEnumerationUtil::keepOnlyPartVendorProduct)
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(derivedCpeUris::add);
if (derivedCpeUris.size() >= configuration.getMaxCorrelatedCpePerArtifact()) {
maxCorrelatedCpePerArtifactReachedCallback.accept(derivedCpeUris);
return derivedCpeUris;
}
}
}
return derivedCpeUris;
}
private static boolean hardwareCheck(boolean isComponentHardware, Cpe cpe) {
return cpe.getPart() != Part.HARDWARE_DEVICE || isComponentHardware;
}
@Data
@EqualsAndHashCode(exclude = {"requireOtherIndication", "sourceAttribute", "sourceValue"})
public static class Alias implements Comparable {
private final String alias;
private final String sourceAttribute;
private final String sourceValue;
private boolean requireOtherIndication = false;
private JSONObject toJson() {
return new JSONObject()
.put("alias", alias)
.put("sourceAttribute", sourceAttribute)
.put("sourceValue", sourceValue)
.put("requireOtherIndication", requireOtherIndication);
}
@Override
public String toString() {
return toJson().toString();
}
@Override
public int compareTo(Alias o) {
if (o == null) return 1;
if (this.alias == null) return -1;
if (o.alias == null) return 1;
return this.alias.compareTo(o.alias);
}
public static Set toAliases(Collection aliases, String sourceAttribute, String sourceValue) {
return aliases.stream().map(alias -> new Alias(alias, sourceAttribute, sourceValue)).collect(Collectors.toSet());
}
}
@Data
protected static class AliasRequireOtherMatcher {
private Alias matchingAlias;
public boolean isMatching(Alias candidate) {
if (candidate.isRequireOtherIndication()) {
if (matchingAlias == null) {
matchingAlias = candidate;
return false;
} else if (matchingAlias == candidate) {
return false;
}
return true;
} else {
return true;
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy