com.metaeffekt.mirror.download.advisor.MsrcSecurityGuideDownload Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.mirror.download.advisor;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.Retry;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import com.metaeffekt.mirror.download.documentation.DocRelevantMethods;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static com.metaeffekt.mirror.download.advisor.MsrcSecurityGuideDownload.ResourceLocationMsrcSecurityUpdateGuide.AFFECTED_PRODUCTS_BASE_URL;
/**
* References:
*
* - Update Guide/CSV download: https://msrc.microsoft.com/update-guide
*
* As mentioned in the MSRC Download, the MSRC splits its data into three separate data sources.
* This particular data source used to only be available via manual CSV file downloads, but since then, a way was found to perform these requests automatically.
* Another thing to investigate is this new API (as of 2024-09-07) they made available for use:
* https://api.msrc.microsoft.com/cvrf/v3.0/swagger/v3/swagger.json
* and whether it can replace this existing one once more.
*/
@MirrorMetadata(directoryName = "msrc-security-guide", mavenPropertyName = "msrcSecurityGuideDownload")
public class MsrcSecurityGuideDownload extends Download {
private final static Logger LOG = LoggerFactory.getLogger(MsrcSecurityGuideDownload.class);
private final static String START_DATE = "2016-01-01T00:00:00+01:00";
private final static int WRITE_TO_FILE_THRESHOLD = 20;
public MsrcSecurityGuideDownload(File baseMirrorDirectory) {
super(baseMirrorDirectory, MsrcSecurityGuideDownload.class);
}
@Override
@DocRelevantMethods({"MsrcSecurityGuideDownload#downloadAllFromTo", "MsrcSecurityGuideDownload#downloadSecurityGuideByDateRangeAndOffset"})
public void performDownload() {
final String startDate = determineStartDate();
final String endDate = getCurrentTimestampForRequest();
downloadAllFromTo(startDate, endDate);
}
private void downloadAllFromTo(String startDate, String endDate) {
int offset = 0;
int totalEntriesCount = 0;
int currentEntriesCount = 0;
int currentRequestsSinceLastWriteToFile = 0;
long startTime = System.currentTimeMillis();
final Map> entriesToBeWrittenToFile = new ConcurrentHashMap<>();
super.executor.setDelay(0);
do {
try {
final JSONObject response = downloadSecurityGuideByDateRangeAndOffset(startDate, endDate, offset);
if (isResponseEmpty(response)) {
LOG.info("No more entries found. Stopping download.");
break;
}
totalEntriesCount = response.getInt("@odata.count");
currentEntriesCount = response.getJSONArray("value").length();
offset += currentEntriesCount;
currentRequestsSinceLastWriteToFile++;
sortEntriesIntoYears(response, entriesToBeWrittenToFile);
if (currentRequestsSinceLastWriteToFile >= WRITE_TO_FILE_THRESHOLD) {
final long currentTime = System.currentTimeMillis();
final long elapsedTime = currentTime - startTime;
final long expectedRemainingTime = (elapsedTime / offset) * (totalEntriesCount - offset);
LOG.info("Downloaded [{} / {}] entries, expected time remaining [{}]", offset, totalEntriesCount, TimeUtils.formatTimeDiff(expectedRemainingTime));
super.executor.submit(() -> {
try {
writeEntriesToFile(entriesToBeWrittenToFile);
} catch (IOException e) {
throw new RuntimeException("Failed to write entries to file", e);
}
});
super.executor.start();
currentRequestsSinceLastWriteToFile = 0;
}
} catch (IOException e) {
LOG.error("Error while downloading security guide.", e);
break;
}
} while (offset < totalEntriesCount);
try {
super.executor.join();
} catch (InterruptedException e) {
throw new RuntimeException("Failed to join executor.", e);
}
try {
writeEntriesToFile(entriesToBeWrittenToFile);
} catch (IOException e) {
LOG.error("Error while writing entries to file.", e);
}
try {
setPreviousTotalEntriesCount(getTotalAvailableEntriesCount());
} catch (IOException e) {
setPreviousTotalEntriesCount(totalEntriesCount);
}
setLastMirrorDate(endDate);
}
private void writeEntriesToFile(Map> entriesToBeWrittenToFile) throws IOException {
// parse entries from file, merge with new entries, write back to file
// check for duplicate unique ids
final Map> entriesToBeWrittenToFileCopy;
synchronized (entriesToBeWrittenToFile) {
if (entriesToBeWrittenToFile.isEmpty()) {
return;
}
entriesToBeWrittenToFileCopy = new HashMap<>(entriesToBeWrittenToFile);
entriesToBeWrittenToFile.clear();
}
for (Map.Entry> entry : entriesToBeWrittenToFileCopy.entrySet()) {
final String year = entry.getKey();
final List entries = entry.getValue();
final File yearFile = new File(super.downloadIntoDirectory, year + ".json");
final JSONArray yearFileContent;
if (yearFile.exists()) {
yearFileContent = new JSONArray(FileUtils.readFileToString(yearFile, StandardCharsets.UTF_8));
} else {
yearFileContent = new JSONArray();
}
for (JSONObject newEntry : entries) {
final String newEntryId = newEntry.getString("id");
boolean entryAlreadyExists = false;
for (int i = 0; i < yearFileContent.length(); i++) {
final JSONObject existingEntry = yearFileContent.getJSONObject(i);
final String existingEntryId = existingEntry.getString("id");
if (newEntryId.equals(existingEntryId)) {
entryAlreadyExists = true;
break;
}
}
if (!entryAlreadyExists) {
yearFileContent.put(newEntry);
}
}
FileUtils.writeStringToFile(yearFile, yearFileContent.toString(), StandardCharsets.UTF_8);
}
}
private void sortEntriesIntoYears(JSONObject response, Map> entriesByYear) {
final JSONArray entries = response.getJSONArray("value");
synchronized (entriesByYear) {
for (int i = 0; i < entries.length(); i++) {
final JSONObject entry = entries.getJSONObject(i);
final String releaseDate = entry.getString("releaseDate"); // 2023-06-29T22:02:55Z
final String year = releaseDate.substring(0, 4);
entriesByYear.computeIfAbsent(year, k -> new ArrayList<>()).add(entry);
}
}
}
private String determineStartDate() {
final String lastMirrorDate = getLastMirrorDate();
final String startDate;
if (StringUtils.hasText(lastMirrorDate)) {
startDate = lastMirrorDate;
} else {
startDate = START_DATE;
}
return startDate;
}
private JSONObject downloadSecurityGuideByDateRangeAndOffset(String startDate, String endDate, int offset) throws IOException {
return new Retry<>(() -> {
final Map getParameters = new HashMap<>();
getParameters.put("$orderBy", "releaseDate asc");
getParameters.put("$filter", "(releaseDate gt " + startDate + ") and (releaseDate lt " + endDate + ")");
getParameters.put("$skip", String.valueOf(offset));
final String getRequestUrl = super.downloader.buildGetRequest(getRemoteResourceLocation(AFFECTED_PRODUCTS_BASE_URL), getParameters);
final List response = super.downloader.fetchResponseBodyFromUrlAsList(new URL(getRequestUrl));
return new JSONObject(String.join("", response));
})
.onException(Throwable.class)
.withValidator(json -> { // must not be {"error":{"code":"400","message":"..."}}
if (json.has("error")) {
final JSONObject error = json.getJSONObject("error");
final String code = error.getString("code");
final String message = error.getString("message");
LOG.error("Error while downloading security guide: [{}] {}", code, message);
return false;
}
return true;
})
.retryCount(5)
.withDelay(1000 * 5)
.run();
}
private boolean isResponseEmpty(JSONObject response) {
return response.getJSONArray("value").isEmpty();
}
private String getCurrentTimestampForRequest() {
final OffsetDateTime now = OffsetDateTime.now(ZoneId.of("Europe/Paris")); // or: ZoneOffset.of("+01:00")
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX");
return now.format(formatter);
}
private void setPreviousTotalEntriesCount(int count) {
super.propertyFiles.set(this.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "entries-count", String.valueOf(count));
}
private int getPreviousTotalEntriesCount() {
return Integer.parseInt(super.propertyFiles.getString(this.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "entries-count").orElse("0"));
}
private void setLastMirrorDate(String date) {
super.propertyFiles.set(this.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "last-mirror-date", date);
}
private String getLastMirrorDate() {
return super.propertyFiles.getString(this.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "last-mirror-date").orElse("");
}
private int getTotalAvailableEntriesCount() throws IOException {
final JSONObject response = downloadSecurityGuideByDateRangeAndOffset(START_DATE, getCurrentTimestampForRequest(), 999999999);
return response.getInt("@odata.count");
}
@Override
protected boolean isResetRequired() {
return false;
}
@Override
protected boolean additionalIsDownloadRequired() {
try {
final int totalEntriesCount = getTotalAvailableEntriesCount();
final int previousTotalEntriesCount = getPreviousTotalEntriesCount();
if (previousTotalEntriesCount != totalEntriesCount) {
LOG.info("Total entries count changed from [{}] to [{}], download is required", previousTotalEntriesCount, totalEntriesCount);
return true;
} else {
return false;
}
} catch (IOException e) {
return true;
}
}
@Override
public void setRemoteResourceLocation(String location, String url) {
super.setRemoteResourceLocation(ResourceLocationMsrcSecurityUpdateGuide.valueOf(location), url);
}
public enum ResourceLocationMsrcSecurityUpdateGuide implements ResourceLocation {
/**
* Endpoint that https://msrc.microsoft.com/update-guide/
* uses to construct the xlsx and csv files locally.
* Has parameters that are filled by the code, without format strings here.
*/
AFFECTED_PRODUCTS_BASE_URL("https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct");
private final String defaultValue;
ResourceLocationMsrcSecurityUpdateGuide(String defaultValue) {
this.defaultValue = defaultValue;
}
@Override
public String getDefault() {
return this.defaultValue;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy