com.metaeffekt.mirror.download.advisor.MsrcDownload Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.download.advisor;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.metaeffekt.mirror.download.advisor.MsrcDownload.ResourceLocationMsrc.MSRC_CVRF_BASE_URL;
import static com.metaeffekt.mirror.download.advisor.MsrcDownload.ResourceLocationMsrc.MSRC_UPDATES_URL;
/**
* References:
*
* - MSRC Homepage: Microsoft Security Response Center
* - MSRC CVRF API: Microsoft Security Updates API
*
* The MSRC (Microsoft Security Response Center) provides a REST-API for their Common Vulnerability Reporting Framework (CVRF) v3.0.
* This API can be used to download all advisories published by the MSRC. The documents provided are security advisories
* that can be used to 1. identify vulnerabilities using microsoft product ids and 2. to provide information on how to
* mitigate these vulnerabilities.
* The MSRC splits their data into three sources, this downloader accesses one of the two sources that can be queried automatically.
* Individual advisory entries are each stored in a separate XML file, to which their relative paths are all listed in a single data feed.
* In order to perform a download for this particular source, the following steps are executed:
*
* - All documents available are contained within the XML document available via
* https://api.msrc.microsoft.com/cvrf/v3.0/updates. This document
* contains multiple
CvrfUrl tags, that are collected into a list. The MSRC structures their advisors into one
* document per month, so each URL will lead to a different monthly XML file.
* - The downloader checks whether it already knows these documents/whether they are up-to-date.
* - All unknown/incomplete
CvrfUrl URLs are downloaded.
*
* These files are stored in one directory per year (starting 2016), with paths in the form of [YEAR]/[YEAR]-[MONTH].xml.
* .
* ├── 2016
* │ ├── 2016-Apr.xml
* │ ├── 2016-Aug.xml
* ...
* └── 2023
* └── 2023-Jan.xml
*
*/
@MirrorMetadata(directoryName = "msrc", mavenPropertyName = "msrcDownload")
public class MsrcDownload extends Download {
private final static Logger LOG = LoggerFactory.getLogger(MsrcDownload.class);
public MsrcDownload(File baseMirrorDirectory) {
super(baseMirrorDirectory, MsrcDownload.class);
}
@Override
protected void performDownload() {
final Map changedMonths = getChangedMonths();
LOG.info("Months with changed/missing contents: {}", changedMonths);
for (Map.Entry entry : changedMonths.entrySet()) {
final String monthId = entry.getKey();
final String currentReleaseDate = entry.getValue();
super.executor.submit(() -> {
final String year = monthId.substring(0, 4);
final File downloadDestinationParentDirectory = new File(super.downloadIntoDirectory, year);
final File downloadDestinationFile = new File(downloadDestinationParentDirectory, monthId + ".xml");
if (!downloadDestinationParentDirectory.exists()) {
downloadDestinationParentDirectory.mkdirs();
}
if (downloadDestinationFile.exists()) {
downloadDestinationFile.delete();
}
final URL requestUrl = getRemoteResourceLocationUrl(MSRC_CVRF_BASE_URL, monthId);
super.downloader.fetchResponseBodyFromUrlToFile(requestUrl, downloadDestinationFile);
super.propertyFiles.set(super.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "month-latest-" + monthId, currentReleaseDate);
});
}
super.executor.start();
try {
super.executor.join();
} catch (InterruptedException e) {
throw new RuntimeException("Failed to wait for download threads to finish.", e);
}
}
private Map getChangedMonths() {
final Map changedMonths = Collections.synchronizedMap(new HashMap<>());
final URL requestUrl = getRemoteResourceLocationUrl(MSRC_UPDATES_URL);
final List updateLines = super.downloader.fetchResponseBodyFromUrlAsList(requestUrl);
final JSONObject updateJson = new JSONObject(String.join("", updateLines));
final JSONArray monthlyUpdateData = updateJson.getJSONArray("value");
for (int i = 0; i < monthlyUpdateData.length(); i++) {
final JSONObject monthlyUpdate = monthlyUpdateData.getJSONObject(i);
final String monthTitle = monthlyUpdate.getString("DocumentTitle");
final String currentReleaseDate = monthlyUpdate.getString("CurrentReleaseDate");
final String monthId = extractOrDeriveMonthId(monthlyUpdate);
final String year = monthId.substring(0, 4);
final File monthFile = new File(super.downloadIntoDirectory, year + "/" + monthId + ".xml");
final String lastCurrentReleaseDate = super.propertyFiles.getString(super.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "month-latest-" + monthId)
.orElse("");
if (!monthFile.exists()) {
LOG.info("Month [{}: {}] is missing", monthId, monthTitle);
changedMonths.put(monthId, currentReleaseDate);
} else if (!lastCurrentReleaseDate.equals(currentReleaseDate)) {
LOG.info("Month [{}: {}] is out-of-date: [{}] -> [{}]", monthId, monthTitle, lastCurrentReleaseDate, currentReleaseDate);
changedMonths.put(monthId, currentReleaseDate);
}
}
return changedMonths;
}
private String extractOrDeriveMonthId(JSONObject monthlyUpdate) {
final String documentMonthId = monthlyUpdate.getString("ID");
final String monthId;
if (String.valueOf(documentMonthId).length() < 4) {
final String initialReleaseDate = monthlyUpdate.getString("InitialReleaseDate");
LOG.error("Invalid monthId provided by MS update document [{}] attempt to restore ID using InitialReleaseDate: [{}]", documentMonthId, initialReleaseDate);
final String[] split = initialReleaseDate.split("-"); // e.g. 2022-10-03T07:00:00Z to 2022-Oct
final String month = new String[]{"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}[Integer.parseInt(split[1]) - 1];
monthId = split[0] + "-" + month;
LOG.info("Derived monthId: [{}]", monthId);
} else {
monthId = documentMonthId;
}
return monthId;
}
@Override
protected boolean additionalIsDownloadRequired() {
final Map changedMonths = getChangedMonths();
if (!changedMonths.isEmpty()) {
LOG.info("Months with changed/missing contents, update required: {}", changedMonths);
return true;
}
return false;
}
@Override
public void setRemoteResourceLocation(String location, String url) {
super.setRemoteResourceLocation(ResourceLocationMsrc.valueOf(location), url);
}
public enum ResourceLocationMsrc implements ResourceLocation {
/**
* List of all monthly documents. Contains last update date used to determine whether an update is required.
* When viewed in a browser, the content will be displayed as XML, but when accessed via the API, the content
* will be returned as JSON.
*/
MSRC_UPDATES_URL("https://api.msrc.microsoft.com/cvrf/v3.0/updates"),
/**
* https://api.msrc.microsoft.com/cvrf/v3.0/swagger/v3/swagger.json.
* REST-API for the Microsoft Security Response Center (MSRC) Common Vulnerability Reporting Framework (CVRF) v2.0.
*
* %s CVRF document ID (yyyy-mmm) (example: 2021-dec)
*/
MSRC_CVRF_BASE_URL("https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/%s");
private final String defaultValue;
ResourceLocationMsrc(String defaultValue) {
this.defaultValue = defaultValue;
}
@Override
public String getDefault() {
return this.defaultValue;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy