com.metaeffekt.mirror.download.advisor.MsrcDownload Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.download.advisor;

import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.metaeffekt.mirror.download.advisor.MsrcDownload.ResourceLocationMsrc.MSRC_CVRF_BASE_URL;
import static com.metaeffekt.mirror.download.advisor.MsrcDownload.ResourceLocationMsrc.MSRC_UPDATES_URL;

/**
 * References:
 * 
 * MSRC Homepage: Microsoft Security Response Center
 * MSRC CVRF API: Microsoft Security Updates API
 * 
 * The MSRC (Microsoft Security Response Center) provides a REST-API for their Common Vulnerability Reporting Framework (CVRF) v3.0.
 * This API can be used to download all advisories published by the MSRC. The documents provided are security advisories
 * that can be used to 1. identify vulnerabilities using microsoft product ids and 2. to provide information on how to
 * mitigate these vulnerabilities.
 * The MSRC splits their data into three sources, this downloader accesses one of the two sources that can be queried automatically.
 * Individual advisory entries are each stored in a separate XML file, to which their relative paths are all listed in a single data feed.
 * In order to perform a download for this particular source, the following steps are executed:
 * 
 * All documents available are contained within the XML document available via
 * https://api.msrc.microsoft.com/cvrf/v3.0/updates. This document
 * contains multiple CvrfUrl tags, that are collected into a list. The MSRC structures their advisors into one
 * document per month, so each URL will lead to a different monthly XML file.
 * The downloader checks whether it already knows these documents/whether they are up-to-date.
 * All unknown/incomplete CvrfUrl URLs are downloaded.
 * 
 * These files are stored in one directory per year (starting 2016), with paths in the form of [YEAR]/[YEAR]-[MONTH].xml.
 * .
 * ├── 2016
 * │   ├── 2016-Apr.xml
 * │   ├── 2016-Aug.xml
 * ...
 * └── 2023
 *     └── 2023-Jan.xml
 * 
 */
@MirrorMetadata(directoryName = "msrc", mavenPropertyName = "msrcDownload")
public class MsrcDownload extends Download {

    private final static Logger LOG = LoggerFactory.getLogger(MsrcDownload.class);

    public MsrcDownload(File baseMirrorDirectory) {
        super(baseMirrorDirectory, MsrcDownload.class);
    }

    @Override
    protected void performDownload() {
        final Map changedMonths = getChangedMonths();
        LOG.info("Months with changed/missing contents: {}", changedMonths);

        for (Map.Entry entry : changedMonths.entrySet()) {
            final String monthId = entry.getKey();
            final String currentReleaseDate = entry.getValue();

            super.executor.submit(() -> {
                final String year = monthId.substring(0, 4);
                final File downloadDestinationParentDirectory = new File(super.downloadIntoDirectory, year);
                final File downloadDestinationFile = new File(downloadDestinationParentDirectory, monthId + ".xml");

                if (!downloadDestinationParentDirectory.exists()) {
                    downloadDestinationParentDirectory.mkdirs();
                }

                if (downloadDestinationFile.exists()) {
                    downloadDestinationFile.delete();
                }

                final URL requestUrl = getRemoteResourceLocationUrl(MSRC_CVRF_BASE_URL, monthId);

                super.downloader.fetchResponseBodyFromUrlToFile(requestUrl, downloadDestinationFile);

                super.propertyFiles.set(super.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "month-latest-" + monthId, currentReleaseDate);
            });
        }

        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for download threads to finish.", e);
        }
    }

    private Map getChangedMonths() {
        final Map changedMonths = Collections.synchronizedMap(new HashMap<>());

        final URL requestUrl = getRemoteResourceLocationUrl(MSRC_UPDATES_URL);

        final List updateLines = super.downloader.fetchResponseBodyFromUrlAsList(requestUrl);
        final JSONObject updateJson = new JSONObject(String.join("", updateLines));

        final JSONArray monthlyUpdateData = updateJson.getJSONArray("value");

        for (int i = 0; i < monthlyUpdateData.length(); i++) {
            final JSONObject monthlyUpdate = monthlyUpdateData.getJSONObject(i);

            final String monthTitle = monthlyUpdate.getString("DocumentTitle");
            final String currentReleaseDate = monthlyUpdate.getString("CurrentReleaseDate");

            final String monthId = extractOrDeriveMonthId(monthlyUpdate);

            final String year = monthId.substring(0, 4);
            final File monthFile = new File(super.downloadIntoDirectory, year + "/" + monthId + ".xml");

            final String lastCurrentReleaseDate = super.propertyFiles.getString(super.downloadIntoDirectory, "info", InfoFileAttributes.MSRC_PREFIX.getKey() + "month-latest-" + monthId)
                    .orElse("");

            if (!monthFile.exists()) {
                LOG.info("Month [{}: {}] is missing", monthId, monthTitle);
                changedMonths.put(monthId, currentReleaseDate);
            } else if (!lastCurrentReleaseDate.equals(currentReleaseDate)) {
                LOG.info("Month [{}: {}] is out-of-date: [{}] -> [{}]", monthId, monthTitle, lastCurrentReleaseDate, currentReleaseDate);
                changedMonths.put(monthId, currentReleaseDate);
            }
        }

        return changedMonths;
    }

    private String extractOrDeriveMonthId(JSONObject monthlyUpdate) {
        final String documentMonthId = monthlyUpdate.getString("ID");
        final String monthId;

        if (String.valueOf(documentMonthId).length() < 4) {
            final String initialReleaseDate = monthlyUpdate.getString("InitialReleaseDate");
            LOG.error("Invalid monthId provided by MS update document [{}] attempt to restore ID using InitialReleaseDate: [{}]", documentMonthId, initialReleaseDate);

            final String[] split = initialReleaseDate.split("-"); // e.g. 2022-10-03T07:00:00Z to 2022-Oct
            final String month = new String[]{"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}[Integer.parseInt(split[1]) - 1];
            monthId = split[0] + "-" + month;

            LOG.info("Derived monthId: [{}]", monthId);
        } else {
            monthId = documentMonthId;
        }

        return monthId;
    }

    @Override
    protected boolean additionalIsDownloadRequired() {
        final Map changedMonths = getChangedMonths();

        if (!changedMonths.isEmpty()) {
            LOG.info("Months with changed/missing contents, update required: {}", changedMonths);
            return true;
        }

        return false;
    }

    @Override
    public void setRemoteResourceLocation(String location, String url) {
        super.setRemoteResourceLocation(ResourceLocationMsrc.valueOf(location), url);
    }

    public enum ResourceLocationMsrc implements ResourceLocation {
        /**
         * List of all monthly documents. Contains last update date used to determine whether an update is required.

         * When viewed in a browser, the content will be displayed as XML, but when accessed via the API, the content
         * will be returned as JSON.
         */
        MSRC_UPDATES_URL("https://api.msrc.microsoft.com/cvrf/v3.0/updates"),
        /**
         * https://api.msrc.microsoft.com/cvrf/v3.0/swagger/v3/swagger.json.
         * REST-API for the Microsoft Security Response Center (MSRC) Common Vulnerability Reporting Framework (CVRF) v2.0.
         * 
         *     %s CVRF document ID (yyyy-mmm) (example: 2021-dec)
         * 
         */
        MSRC_CVRF_BASE_URL("https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/%s");

        private final String defaultValue;

        ResourceLocationMsrc(String defaultValue) {
            this.defaultValue = defaultValue;
        }

        @Override
        public String getDefault() {
            return this.defaultValue;
        }
    }
}