com.metaeffekt.mirror.plugin.DataMirrorMojo Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.plugin;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.advisor.*;
import com.metaeffekt.mirror.download.nvd.CpeDictionaryDownload;
import com.metaeffekt.mirror.download.nvd.NvdCpeApiDownload;
import com.metaeffekt.mirror.download.nvd.NvdCveApiDownload;
import com.metaeffekt.mirror.download.nvd.NvdDownload;
import com.metaeffekt.mirror.download.other.EolDownload;
import com.metaeffekt.mirror.download.other.CisaKevDownload;
import com.metaeffekt.mirror.download.other.EpssDownload;
import com.metaeffekt.mirror.index.Index;
import com.metaeffekt.mirror.index.advisor.*;
import com.metaeffekt.mirror.index.nvd.*;
import com.metaeffekt.mirror.index.other.EolIndex;
import com.metaeffekt.mirror.index.other.KevIndex;
import com.metaeffekt.mirror.index.other.EpssIndex;
import com.metaeffekt.mirror.initializer.*;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import java.io.File;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Mojo(name = "data-mirror", defaultPhase = LifecyclePhase.PROCESS_RESOURCES)
public class DataMirrorMojo extends AbstractMojo {
@Parameter(required = true)
private File mirrorDirectory;
@Parameter
private String proxyScheme, proxyHost, proxyUsername, proxyPassword;
@Parameter
private Integer proxyPort;
@Parameter
private DownloadInitializer msrcDownload;
@Deprecated
@Parameter
private DownloadInitializer msrcCsvDownload;
@Parameter
private DownloadInitializer msrcSecurityUpdateGuideDownload;
/**
* Deprecated: use {@link #nvdCpeDownload} instead.
*/
@Parameter
@Deprecated
private DownloadInitializer cpeDictionaryDownload;
@Parameter
private NvdApiDownloadInitializer nvdCpeDownload;
@Parameter
private DownloadInitializer certSeiDownload;
@Parameter
private DownloadInitializer certFrDownload;
@Parameter
private DownloadInitializer certEuDownload;
@Parameter
private DownloadInitializer cisaKevDownload;
@Parameter
private DownloadInitializer epssDownload;
@Parameter
private DownloadInitializer eolDownload;
/**
* Deprecated: use {@link #nvdCveDownload} instead.
*/
@Parameter
@Deprecated
private DownloadInitializer nvdLegacyDownload;
@Parameter
private NvdApiDownloadInitializer nvdCveDownload;
@Parameter
private GitDownloadInitializer githubAdvisorDownload;
@Parameter
private IndexInitializer certSeiAdvisorIndex;
@Parameter
private IndexInitializer certEuAdvisorIndex;
@Parameter
private IndexInitializer certFrAdvisorIndex;
/**
* Deprecated: use {@link #nvdCpeIndex} instead.
*/
@Parameter
@Deprecated
private IndexInitializer cpeDictionaryIndex;
/**
* Deprecated: use {@link #nvdCpeVendorProductIndex} instead.
*/
@Parameter
@Deprecated
private IndexInitializer cpeDictionaryVendorProductIndex;
@Parameter
private IndexInitializer nvdCpeIndex;
@Parameter
private IndexInitializer nvdCpeVendorProductIndex;
@Parameter
private IndexInitializer msrcProductIndex;
@Parameter
private IndexInitializer msrcAdvisorIndex;
@Parameter
private IndexInitializer msrcKbChainIndex;
/**
* Deprecated: use {@link #nvdVulnerabilityIndex} instead.
*/
@Parameter
@Deprecated
private IndexInitializer nvdLegacyVulnerabilityIndex;
@Parameter
private IndexInitializer nvdVulnerabilityIndex;
@Parameter
private IndexInitializer githubAdvisorIndex;
@Parameter
private IndexInitializer kevIndex;
@Parameter
private IndexInitializer epssIndex;
@Parameter
private IndexInitializer eolIndex;
@Parameter(defaultValue = "${project}", required = true, readonly = true)
private MavenProject project;
public MavenProject getProject() {
return project;
}
@Override
public void execute() throws MojoFailureException {
getLog().info("Starting mirroring process");
final Map downloadResults = processDownload();
final Map indexResults = processIndex();
logResults(downloadResults, indexResults);
if (hasProcessFailed(downloadResults, indexResults)) {
throw new MojoFailureException("Mirroring process completed with errors - see log for details");
}
getLog().info("Mirroring process completed successfully");
}
private Map processDownload() {
final Map mirrorResults = new LinkedHashMap<>();
final List downloads = buildDownloaders();
for (Download download : downloads) {
// getLog().info("");
// getLog().info("Downloading [" + Download.getDirectoryIdentifier(download.getClass()) + "]");
final MirrorResult result = mirrorResults.computeIfAbsent(download, d -> new MirrorResult());
try {
download.performDownloadIfRequired();
// getLog().info("Completed download [" + Download.getDirectoryIdentifier(download.getClass()) + "]");
} catch (Exception e) {
result.exception = e;
getLog().error("Failed on download " + download.getDownloadIntoDirectory().getName(), e);
} finally {
result.stop();
getLog().info("");
}
}
return mirrorResults;
}
private Map processIndex() {
final Map mirrorResults = new LinkedHashMap<>();
final List indexes = buildIndexes();
for (Index index : indexes) {
// getLog().info("");
// getLog().info("Indexing [" + Index.getDirectoryIdentifier(index.getClass()) + "]");
final MirrorResult result = mirrorResults.computeIfAbsent(index, d -> new MirrorResult());
try {
index.createIndexIfRequired();
// getLog().info("Completed indexing [" + Index.getDirectoryIdentifier(index.getClass()) + "]");
} catch (Exception e) {
result.exception = e;
getLog().error("Failed on download " + index.getClass().getSimpleName(), e);
} finally {
result.stop();
getLog().info("");
}
}
return mirrorResults;
}
private List buildDownloaders() {
final List downloads = new ArrayList<>();
if (isActive(msrcDownload)) {
applyProxySettings(msrcDownload);
final MsrcDownload download = new MsrcDownload(mirrorDirectory);
msrcDownload.apply(download);
downloads.add(download);
}
if (isActive(msrcCsvDownload)) {
applyProxySettings(msrcCsvDownload);
final MsrcManualCsvDownload download = new MsrcManualCsvDownload(mirrorDirectory);
msrcCsvDownload.apply(download);
downloads.add(download);
}
if (isActive(msrcSecurityUpdateGuideDownload)) {
applyProxySettings(msrcSecurityUpdateGuideDownload);
final MsrcSecurityGuideDownload download = new MsrcSecurityGuideDownload(mirrorDirectory);
msrcSecurityUpdateGuideDownload.apply(download);
downloads.add(download);
}
if (isActive(cpeDictionaryDownload)) {
applyProxySettings(cpeDictionaryDownload);
final CpeDictionaryDownload download = new CpeDictionaryDownload(mirrorDirectory);
cpeDictionaryDownload.apply(download);
downloads.add(download);
}
if (isActive(nvdCpeDownload)) {
applyProxySettings(nvdCpeDownload);
final NvdCpeApiDownload download = new NvdCpeApiDownload(mirrorDirectory);
nvdCpeDownload.apply(download);
downloads.add(download);
}
if (isActive(certSeiDownload)) {
applyProxySettings(certSeiDownload);
final CertSeiDownload download = new CertSeiDownload(mirrorDirectory);
certSeiDownload.apply(download);
downloads.add(download);
}
if (isActive(certFrDownload)) {
applyProxySettings(certFrDownload);
final CertFrDownload download = new CertFrDownload(mirrorDirectory);
certFrDownload.apply(download);
downloads.add(download);
}
if (isActive(certEuDownload)) {
applyProxySettings(certEuDownload);
final CertEuDownload download = new CertEuDownload(mirrorDirectory);
certEuDownload.apply(download);
downloads.add(download);
}
if (isActive(cisaKevDownload)) {
applyProxySettings(cisaKevDownload);
final CisaKevDownload download = new CisaKevDownload(mirrorDirectory);
cisaKevDownload.apply(download);
downloads.add(download);
}
if (isActive(epssDownload)) {
applyProxySettings(epssDownload);
final EpssDownload download = new EpssDownload(mirrorDirectory);
epssDownload.apply(download);
downloads.add(download);
}
if (isActive(eolDownload)) {
applyProxySettings(eolDownload);
final EolDownload download = new EolDownload(mirrorDirectory);
eolDownload.apply(download);
downloads.add(download);
}
if (isActive(githubAdvisorDownload)) {
applyProxySettings(githubAdvisorDownload);
final GhsaDownload download = new GhsaDownload(mirrorDirectory);
githubAdvisorDownload.apply(download);
downloads.add(download);
}
if (isActive(nvdLegacyDownload)) {
applyProxySettings(nvdLegacyDownload);
final NvdDownload download = new NvdDownload(mirrorDirectory);
nvdLegacyDownload.apply(download);
downloads.add(download);
}
if (isActive(nvdCveDownload)) {
applyProxySettings(nvdCveDownload);
final NvdCveApiDownload download = new NvdCveApiDownload(mirrorDirectory);
nvdCveDownload.apply(download);
downloads.add(download);
}
if (!downloads.isEmpty()) {
getLog().info("Downloads:");
for (Download download : downloads) {
getLog().info(" - " + Download.getDirectoryIdentifier(download.getClass()));
}
getLog().info("");
}
return downloads;
}
private List buildIndexes() {
final List indexes = new ArrayList<>();
if (isActive(certSeiAdvisorIndex)) {
final CertSeiAdvisorIndex index = new CertSeiAdvisorIndex(mirrorDirectory);
certSeiAdvisorIndex.apply(index);
indexes.add(index);
}
if (isActive(certFrAdvisorIndex)) {
final CertFrAdvisorIndex index = new CertFrAdvisorIndex(mirrorDirectory);
certFrAdvisorIndex.apply(index);
indexes.add(index);
}
if (isActive(certEuAdvisorIndex)) {
final CertEuAdvisorIndex index = new CertEuAdvisorIndex(mirrorDirectory);
certEuAdvisorIndex.apply(index);
indexes.add(index);
}
if (isActive(cpeDictionaryIndex)) {
final CpeDictionaryIndex index = new CpeDictionaryIndex(mirrorDirectory);
cpeDictionaryIndex.apply(index);
indexes.add(index);
}
if (isActive(cpeDictionaryVendorProductIndex)) {
final CpeDictionaryVendorProductIndex index = new CpeDictionaryVendorProductIndex(mirrorDirectory);
cpeDictionaryVendorProductIndex.apply(index);
indexes.add(index);
}
if (isActive(nvdCpeIndex)) {
final NvdCpeApiIndex index = new NvdCpeApiIndex(mirrorDirectory);
nvdCpeIndex.apply(index);
indexes.add(index);
}
if (isActive(nvdCpeVendorProductIndex)) {
final NvdCpeApiVendorProductIndex index = new NvdCpeApiVendorProductIndex(mirrorDirectory);
nvdCpeVendorProductIndex.apply(index);
indexes.add(index);
}
if (isActive(msrcProductIndex)) {
final MsrcProductIndex index = new MsrcProductIndex(mirrorDirectory);
msrcProductIndex.apply(index);
indexes.add(index);
}
if (isActive(msrcAdvisorIndex)) {
final MsrcAdvisorIndex index = new MsrcAdvisorIndex(mirrorDirectory);
msrcAdvisorIndex.apply(index);
indexes.add(index);
}
if (isActive(msrcKbChainIndex)) {
final MsrcKbChainIndex index = new MsrcKbChainIndex(mirrorDirectory);
msrcKbChainIndex.apply(index);
indexes.add(index);
}
if (isActive(githubAdvisorIndex)) {
final GhsaAdvisorIndex index = new GhsaAdvisorIndex(mirrorDirectory);
githubAdvisorIndex.apply(index);
indexes.add(index);
}
if (isActive(kevIndex)) {
final KevIndex index = new KevIndex(mirrorDirectory);
kevIndex.apply(index);
indexes.add(index);
}
if (isActive(epssIndex)) {
final EpssIndex index = new EpssIndex(mirrorDirectory);
epssIndex.apply(index);
indexes.add(index);
}
if (isActive(eolIndex)) {
final EolIndex index = new EolIndex(mirrorDirectory);
eolIndex.apply(index);
indexes.add(index);
}
if (isActive(nvdLegacyVulnerabilityIndex)) {
final NvdVulnerabilityIndex index = new NvdVulnerabilityIndex(mirrorDirectory);
nvdLegacyVulnerabilityIndex.apply(index);
indexes.add(index);
}
if (isActive(nvdVulnerabilityIndex)) {
final NvdCveApiIndex index = new NvdCveApiIndex(mirrorDirectory);
nvdVulnerabilityIndex.apply(index);
indexes.add(index);
}
if (!indexes.isEmpty()) {
getLog().info("Indexes:");
for (Index index : indexes) {
getLog().info(" - " + Index.getDirectoryIdentifier(index.getClass()));
}
getLog().info("");
}
return indexes;
}
/**
* Applies the global proxy settings to a specified {@link DownloadInitializer} instance.
* The settings are applied only if none of the fields in the initializer are already set;
* if any of the fields in the initializer are set, no changes are made.
*
* @param initializer the {@link DownloadInitializer} instance to which the proxy settings should be applied
*/
private void applyProxySettings(DownloadInitializer initializer) {
if (initializer.scheme != null || initializer.host != null || initializer.port != null
|| initializer.username != null || initializer.password != null) {
getLog().info("Downloader using initializer [" + initializer.getClass().getSimpleName() + "] already has proxy settings specified, ignoring global proxy settings");
return;
}
if (isValidProxySetting(proxyScheme)) initializer.scheme = proxyScheme;
if (isValidProxySetting(proxyHost)) initializer.host = proxyHost;
if (isValidProxySetting(proxyPort)) initializer.port = proxyPort;
if (isValidProxySetting(proxyUsername)) initializer.username = proxyUsername;
if (isValidProxySetting(proxyPassword)) initializer.password = proxyPassword;
}
/**
* Determines whether the specified value is a valid proxy setting.
*
* To be considered a valid proxy setting, the following conditions must be met:
*
* - The value should not be null.
* - If the value is a string, it must not be:
*
* - an empty string,
* - "null" (ignoring case),
* - "none" (ignoring case).
*
*
* - If the value is an integer, it must not be -1.
*
* If the value is of any other datatype, then it is always considered as invalid proxy setting.
*
* @param value the value to be checked.
* @return true if the value is a valid proxy setting according to the rules defined above, false otherwise.
*/
private boolean isValidProxySetting(Object value) {
if (value == null) return false;
if (value instanceof String) {
final String str = (String) value;
if (StringUtils.isEmpty(str)) return false;
if (str.equalsIgnoreCase("null")) return false;
if (str.equalsIgnoreCase("none")) return false;
} else if (value instanceof Integer) {
final Integer i = (Integer) value;
if (i == -1) return false;
} else {
return false;
}
return true;
}
private boolean isActive(MirrorInitializer initializer) {
return initializer != null && initializer.active;
}
private boolean hasProcessFailed(Map downloadResults, Map indexResults) {
for (MirrorResult result : downloadResults.values()) {
if (result.hasFailed()) {
return true;
}
}
for (MirrorResult result : indexResults.values()) {
if (result.hasFailed()) {
return true;
}
}
return false;
}
private void logResults(Map downloadResults, Map indexResults) {
getLog().info("Mirror results:");
if (!downloadResults.isEmpty()) {
getLog().info("Download:");
for (Map.Entry entry : downloadResults.entrySet()) {
entry.getValue().logSummary(Download.getDirectoryIdentifier(entry.getKey().getClass()));
}
getLog().info("");
}
if (!indexResults.isEmpty()) {
getLog().info("Index:");
for (Map.Entry entry : indexResults.entrySet()) {
entry.getValue().logSummary(Index.getDirectoryIdentifier(entry.getKey().getClass()));
}
getLog().info("");
}
if (!downloadResults.isEmpty()) {
for (Map.Entry entry : downloadResults.entrySet()) {
if (entry.getValue().hasFailed()) {
getLog().error(entry.getValue().exception);
getLog().error("");
}
}
}
if (!indexResults.isEmpty()) {
for (Map.Entry entry : indexResults.entrySet()) {
if (entry.getValue().hasFailed()) {
getLog().error(entry.getValue().exception);
getLog().error("");
}
}
}
}
private class MirrorResult {
public Exception exception;
private final long start = System.currentTimeMillis();
private long duration = -1;
public void stop() {
duration = System.currentTimeMillis() - start;
}
public String getDuration() {
if (duration >= 0) {
if (duration < 1000) {
return String.format("%4s ms", duration);
} else if (duration < 100000) {
return String.format("%5.1f s", duration / 1000.0);
} else {
return String.format("%5.0f s", duration / 1000.0);
}
} else {
return "unknown";
}
}
public boolean hasFailed() {
return exception != null;
}
public void logSummary(String name) {
if (hasFailed()) {
getLog().error("- FAILED: [" + getDuration() + "] " + name + ": " + exception.getMessage());
} else {
getLog().info("- SUCCESS: [" + getDuration() + "] " + name);
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy