com.metaeffekt.mirror.download.Download Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.download;
import com.metaeffekt.artifact.analysis.utils.BuildProperties;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.GitAccess;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.mirror.Mirror;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.concurrency.ScheduledDelayedThreadPoolExecutor;
import com.metaeffekt.mirror.download.advisor.CertFrDownload;
import com.metaeffekt.mirror.download.advisor.CertSeiDownload;
import com.metaeffekt.mirror.download.advisor.MsrcDownload;
import com.metaeffekt.mirror.download.nvd.CpeDictionaryDownload;
import com.metaeffekt.mirror.download.nvd.NvdCpeApiDownload;
import com.metaeffekt.mirror.download.nvd.NvdCveApiDownload;
import com.metaeffekt.mirror.download.nvd.NvdDownload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
public abstract class Download extends Mirror {
private final static Logger LOG = LoggerFactory.getLogger(Download.class);
protected final Map resourceLocations = new LinkedHashMap<>();
protected final WebAccess downloader = new WebAccess();
protected final GitAccess git = new GitAccess();
protected final File downloadIntoDirectory;
protected long maxAgeBeforeRecheck = 24 * 60 * 60 * 1000; // 1 day
protected long maxAgeBeforeReset = 4L * 7 * 24 * 60 * 60 * 1000; // 4 weeks
protected int lockFileTimeout = 10 * 60 * 1000; // 10 minutes
protected final ScheduledDelayedThreadPoolExecutor executor = new ScheduledDelayedThreadPoolExecutor(16, 0);
public Download(File baseMirrorDirectory, Class downloadClass) {
super(baseMirrorDirectory, Download.getDirectoryIdentifier(downloadClass));
this.downloadIntoDirectory = new File(new File(super.baseMirrorDirectory, "download"), super.mirrorIdentifier);
}
public void setMaxAgeBeforeRecheck(long maxAgeBeforeRecheck) {
this.maxAgeBeforeRecheck = maxAgeBeforeRecheck;
}
public void setMaxAgeBeforeReset(long maxAgeBeforeReset) {
this.maxAgeBeforeReset = maxAgeBeforeReset;
}
public void setLockFileTimeout(int lockFileTimeout) {
this.lockFileTimeout = lockFileTimeout;
}
public void setDownloaderProxyCredentials(String scheme, String host, int port, String username, String password) {
downloader.setDownloaderProxyCredentials(scheme, host, port, username, password);
git.setProxyConfig(scheme, host, port, username, password);
}
public File getDownloadIntoDirectory() {
return downloadIntoDirectory;
}
public void setRemoteResourceLocation(ResourceLocation location, String url) {
this.resourceLocations.put(location, url);
}
public abstract void setRemoteResourceLocation(String location, String url);
public String getRemoteResourceLocation(ResourceLocation location) {
return this.resourceLocations.getOrDefault(location, location.getDefault());
}
public URL getRemoteResourceLocationUrl(ResourceLocation location, Object... args) {
final String requestUrlString = String.format(getRemoteResourceLocation(location), args);
try {
return new URL(requestUrlString);
} catch (MalformedURLException e) {
throw new RuntimeException("Request URL is malformed in download: " + requestUrlString, e);
}
}
/**
* A method that is overwritten by a superclass only if it provides an internal download.
* An internal download is defined as a process that will construct some amount of files (archives, JSON, ...)
* that are to be hosted on a server, used for redistribution to allow other downloader processes to download them.
*
* Essentially, this should serve as a redistribution mechanism to support older formats of downloads when new ones
* have been released.
*/
public void performInternalDownload() {
throw new UnsupportedOperationException("Internal download is not supported for " + this.getClass().getSimpleName() + " download");
}
public void performDownloadIfRequired() {
boolean outcome = false;
try {
super.logTitle("");
super.waitForFileUnlockIfLocked(downloadIntoDirectory, lockFileTimeout);
super.lockFile(downloadIntoDirectory);
if (isResetRequired()) {
clearDownload();
setLastResetToNow();
}
if (isDownloadRequired()) {
performDownload();
setLastUpdatedToNow();
} else {
LOG.info("Download is already up to date: {}", downloadIntoDirectory);
}
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.DOWNLOAD_FAILED_FLAG.getKey(), "false");
outcome = true;
} catch (Exception e) {
try {
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.DOWNLOAD_FAILED_FLAG.getKey(), "true");
} catch (Exception ignored) {
}
throw new RuntimeException("Unable to update download " + this.getClass().getSimpleName() + " in " + downloadIntoDirectory + "\n" + e.getMessage(), e);
} finally {
super.unlockFile(downloadIntoDirectory);
if (outcome) {
super.logTitle("Done: ");
} else {
super.logTitle("FAILED: ");
}
}
}
protected boolean isDownloadRequired() {
this.setLastCheckedToNow();
final boolean downloadFailed = hasLastDownloadFailed();
if (downloadFailed) {
LOG.info("Download failed previously, update is required");
return true;
}
final long directoryLastModified = getDownloadDirectoryLastModified();
if (isUpdatedAgeOlderThan(directoryLastModified, maxAgeBeforeRecheck)) {
LOG.info("Last update is further back than [{}], update is required", TimeUtils.formatTimeDiff(maxAgeBeforeRecheck));
return true;
}
if (additionalIsDownloadRequired()) {
LOG.info("Datasets have changed, update is required");
return true;
}
return false;
}
public boolean hasLastDownloadFailed() {
return propertyFiles.getBoolean(downloadIntoDirectory, "info", InfoFileAttributes.DOWNLOAD_FAILED_FLAG.getKey())
.orElse(false);
}
protected long getDownloadDirectoryLastModified() {
return propertyFiles.getLong(this.downloadIntoDirectory, "info", InfoFileAttributes.LAST_UPDATED.getKey())
.orElse(0L);
}
protected boolean isResetRequired() {
final long directoryLastReset = propertyFiles.getLong(this.downloadIntoDirectory, "info", InfoFileAttributes.LAST_RESET.getKey())
.orElse(0L);
if (directoryLastReset == 0) {
setLastResetToNow();
return false;
}
return isUpdatedAgeOlderThan(directoryLastReset, maxAgeBeforeReset);
}
private void setLastUpdatedToNow() {
final long now = TimeUtils.utcNow();
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.LAST_UPDATED.getKey(), now);
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.LAST_UPDATED_FORMATTED.getKey(), new Date(now));
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.MIRROR_VERSION.getKey(), BuildProperties.getProjectVersion());
LOG.info("Set last updated to [{}] in {}", new Date(now), downloadIntoDirectory);
}
private void setLastCheckedToNow() {
final long now = TimeUtils.utcNow();
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.LAST_CHECKED.getKey(), now);
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.LAST_CHECKED_FORMATTED.getKey(), new Date(now));
LOG.info("Set last updated to [{}] in {}", new Date(now), downloadIntoDirectory);
}
private void setLastResetToNow() {
final long now = TimeUtils.utcNow();
super.propertyFiles.set(downloadIntoDirectory, "info", InfoFileAttributes.LAST_RESET.getKey(), now);
}
public void clearDownload() {
LOG.info("Clearing download in {}", this.downloadIntoDirectory.getAbsolutePath());
propertyFiles.flushCachedAePropertyFiles();
FileUtils.deleteDir(this.downloadIntoDirectory);
additionalClearDownload();
LOG.info("Cleared download in {}", this.downloadIntoDirectory.getAbsolutePath());
}
protected void additionalClearDownload() {
}
protected abstract void performDownload();
protected abstract boolean additionalIsDownloadRequired();
public static String getDirectoryIdentifier(Class download) {
if (download.isAnnotationPresent(MirrorMetadata.class)) {
return download.getAnnotation(MirrorMetadata.class).directoryName();
}
LOG.warn("Index {} does not have a directory name annotation", download.getSimpleName());
// FIXME: Remove these legacy mappings
if (NvdCveApiDownload.class.equals(download)) {
return "nvd";
} else if (NvdDownload.class.equals(download)) {
return "nvd-legacy-feed";
} else if (MsrcDownload.class.equals(download)) {
return "msrc";
} else if (CpeDictionaryDownload.class.equals(download)) {
return "cpe-dict-legacy-feed";
} else if (NvdCpeApiDownload.class.equals(download)) {
return "cpe-dict";
} else if (CertSeiDownload.class.equals(download)) {
return "certsei";
} else if (CertFrDownload.class.equals(download)) {
return "certfr";
}
throw new RuntimeException("Unknown download class: " + download);
}
public static Download getInstance(Class clazz, File baseMirrorDirectory) {
try {
return clazz.getConstructor(File.class)
.newInstance(baseMirrorDirectory);
} catch (Exception e) {
throw new RuntimeException("Unable to create download class: " + e);
}
}
}