com.metaeffekt.mirror.query.NvdCpeApiVendorProductIndexQuery Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.query;
import com.metaeffekt.mirror.index.IndexSearch;
import com.metaeffekt.mirror.index.nvd.NvdCpeApiVendorProductIndex;
import lombok.Data;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.*;
import java.util.stream.Collectors;
public class NvdCpeApiVendorProductIndexQuery extends IndexQuery {
private final static Logger LOG = LoggerFactory.getLogger(NvdCpeApiVendorProductIndexQuery.class);
/**
* Maps vendor names to product names.
*/
private final Map> vendorProductsMap = new HashMap<>();
/**
* Maps product names to vendor names.
*/
private final Map> productVendorsMap = new HashMap<>();
public NvdCpeApiVendorProductIndexQuery(File baseMirrorDirectory) {
super(baseMirrorDirectory, NvdCpeApiVendorProductIndex.class);
}
public NvdCpeApiVendorProductIndexQuery(NvdCpeApiVendorProductIndex index) {
super(index);
}
public boolean vendorExists(String vendor) {
return !super.index.findDocuments(new IndexSearch().fieldEquals("vendor", vendor).fieldEquals("type", "vp")).isEmpty();
}
public boolean productExists(String product) {
return !super.index.findDocuments(new IndexSearch().fieldEquals("product", product).fieldEquals("type", "pv")).isEmpty();
}
public Set findVendorsFuzzy(String vendor) {
return super.index.findDocuments(new IndexSearch().fieldContains("vendor", vendor).fieldEquals("type", "vp")).stream()
.map(d -> d.get("vendor"))
.collect(Collectors.toSet());
}
public Set findProductsFuzzy(String product) {
return super.index.findDocuments(new IndexSearch().fieldContains("product", product).fieldEquals("type", "pv")).stream()
.map(d -> d.get("product"))
.collect(Collectors.toSet());
}
public Set findVendorsForProduct(String product) {
return getProductVendorsMap().getOrDefault(product, Collections.emptySet());
}
public Set findProductsForVendor(String vendor) {
return getVendorProductsMap().getOrDefault(vendor, Collections.emptySet());
}
public Map> getVendorProductsMap() {
synchronized (vendorProductsMap) {
if (vendorProductsMap.isEmpty()) {
LOG.debug("Building Vendor Products map for the first time");
super.index.findAndProcessAllDocuments(document -> {
if (document.get("type").equals("vp")) {
vendorProductsMap.computeIfAbsent(document.get("vendor"), k -> new HashSet<>()).addAll(Arrays.asList(document.get("product").split(", ")));
}
});
}
}
return vendorProductsMap;
}
public Map> getProductVendorsMap() {
synchronized (productVendorsMap) {
if (productVendorsMap.isEmpty()) {
LOG.debug("Building Product Vendors map for the first time");
super.index.findAndProcessAllDocuments(document -> {
if (document.get("type").equals("pv")) {
productVendorsMap.computeIfAbsent(document.get("product"), k -> new HashSet<>()).addAll(Arrays.asList(document.get("vendor").split(", ")));
}
});
}
}
return productVendorsMap;
}
// top terms detection
public Map findVendorTerms() {
return findTopTermsInternal(this.getVendorProductsMap());
}
public Map findProductTerms() {
return findTopTermsInternal(this.getProductVendorsMap());
}
public LinkedHashMap findTopVpTerms(TermsLimiter... termsLimiter) {
final Map vendorCount = this.findVendorTerms();
final Map productCount = this.findProductTerms();
final Map vpCount = new HashMap<>();
vpCount.putAll(vendorCount);
productCount.forEach((product, count) -> vpCount.merge(product, count, Integer::sum));
return TermsLimiter.limitMap(vpCount, termsLimiter);
}
public LinkedHashMap findTopVpTerms(List termsLimiters) {
return findTopVpTerms(termsLimiters.toArray(new TermsLimiter[0]));
}
private Map findTopTermsInternal(Map> data) {
final List>> dataBySize = data.entrySet().stream()
.sorted(Comparator.comparingInt(e -> ((Map.Entry>) e).getValue().size()).reversed())
.collect(Collectors.toList());
final Map keywordCount = new HashMap<>();
for (Map.Entry> entry : dataBySize) {
keywordCount.put(entry.getKey(), entry.getValue().size());
}
return keywordCount;
}
public interface TermsLimiter {
LinkedHashMap limitMap(Map map);
JSONObject toJson();
default JSONObject toBaseJson() {
return new JSONObject()
.put("type", this.getClass().getSimpleName());
}
static LinkedHashMap limitMap(Map map, TermsLimiter... limiter) {
LinkedHashMap result = new LinkedHashMap<>(map);
for (TermsLimiter l : limiter) {
result = l.limitMap(result);
}
return result;
}
static List fromJson(JSONArray inputArray) {
final List result = new ArrayList<>();
for (int i = 0; i < inputArray.length(); i++) {
final JSONObject input = inputArray.getJSONObject(i);
final String type = input.getString("type");
switch (type) {
case "TopPTermsLimiter":
result.add(TopPTermsLimiter.fromJson(input));
break;
case "TopNTermsLimiter":
result.add(TopNTermsLimiter.fromJson(input));
break;
case "MinTermsLimiter":
result.add(MinTermsLimiter.fromJson(input));
break;
case "MaxTermsLimiter":
result.add(MaxTermsLimiter.fromJson(input));
break;
default:
throw new IllegalArgumentException("Unknown terms limiter type: " + type);
}
}
return result;
}
}
@Data
public static class TopPTermsLimiter implements TermsLimiter {
private final double topP;
@Override
public LinkedHashMap limitMap(Map map) {
final int topN = (int) Math.ceil(map.size() * topP);
return map.entrySet().stream()
.sorted(Comparator.comparingInt(e -> ((Map.Entry) e).getValue()).reversed())
.limit(topN)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
}
@Override
public JSONObject toJson() {
return this.toBaseJson()
.put("topP", topP);
}
public static TopPTermsLimiter fromJson(JSONObject input) {
return new TopPTermsLimiter(input.getDouble("topP"));
}
}
@Data
public static class TopNTermsLimiter implements TermsLimiter {
private final int topN;
@Override
public LinkedHashMap limitMap(Map map) {
return map.entrySet().stream()
.sorted(Comparator.comparingInt(e -> ((Map.Entry) e).getValue()).reversed())
.limit(topN)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
}
@Override
public JSONObject toJson() {
return this.toBaseJson()
.put("topN", topN);
}
public static TopNTermsLimiter fromJson(JSONObject input) {
return new TopNTermsLimiter(input.getInt("topN"));
}
}
@Data
public static class MinTermsLimiter implements TermsLimiter {
private final int minTerms;
@Override
public LinkedHashMap limitMap(Map map) {
return map.entrySet().stream()
.filter(e -> e.getValue() >= minTerms)
.sorted(Comparator.comparingInt(e -> ((Map.Entry) e).getValue()).reversed())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
}
@Override
public JSONObject toJson() {
return this.toBaseJson()
.put("minTerms", minTerms);
}
public static MinTermsLimiter fromJson(JSONObject input) {
return new MinTermsLimiter(input.getInt("minTerms"));
}
}
@Data
public static class MaxTermsLimiter implements TermsLimiter {
private final int maxTerms;
@Override
public LinkedHashMap limitMap(Map map) {
return map.entrySet().stream()
.sorted(Comparator.comparingInt(e -> ((Map.Entry) e).getValue()).reversed())
.limit(maxTerms)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
}
@Override
public JSONObject toJson() {
return this.toBaseJson()
.put("maxTerms", maxTerms);
}
public static MaxTermsLimiter fromJson(JSONObject input) {
return new MaxTermsLimiter(input.getInt("maxTerms"));
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy