com.metaeffekt.mirror.download.nvd.NvdDownload Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.download.nvd;

import com.metaeffekt.artifact.analysis.utils.ArchiveUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.Retry;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.net.URL;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
 * Uses the deprecated 1.x NVD JSON feeds to download the NVD data.

 * These data feeds are no longer available and can no longer be used. More information can be found at
 * https://nvd.nist.gov/general/news/api-20-announcements.
 */
@Deprecated
@MirrorMetadata(directoryName = "nvd", mavenPropertyName = "nvdLegacyDownload", deprecated = true)
public class NvdDownload extends Download {

    private final static Logger LOG = LoggerFactory.getLogger(NvdDownload.class);

    private final List availableArchiveYears;

    private Map cachedChangedYearlyArchives;
    private final static int META_FILE_YEAR_PLACEHOLDER = 1;

    public NvdDownload(File baseMirrorDirectory) {
        super(baseMirrorDirectory, NvdDownload.class);

        availableArchiveYears = Collections.unmodifiableList(IntStream.range(2002, Calendar.getInstance().get(Calendar.YEAR) + 1)
                .boxed()
                .collect(Collectors.toList()));
    }

    @Override
    protected boolean additionalIsDownloadRequired() {
        final Map changedArchiveYears = fetchChangedArchiveYears();

        if (cachedChangedYearlyArchives == null) {
            cachedChangedYearlyArchives = changedArchiveYears;
        }

        return changedArchiveYears.size() > 0;
    }

    @Override
    protected void performDownload() {
        final Map changedArchiveYears;

        if (cachedChangedYearlyArchives == null) {
            changedArchiveYears = fetchChangedArchiveYears();
        } else {
            changedArchiveYears = cachedChangedYearlyArchives;
            cachedChangedYearlyArchives = null;
        }

        LOG.info("Starting downloads for years {}", changedArchiveYears.keySet());

        for (Map.Entry changedYearEntry : changedArchiveYears.entrySet()) {
            super.executor.submit(() ->
                    new Retry<>(() -> {
                        final Integer year = changedYearEntry.getKey();
                        final String lastModifiedDate = changedYearEntry.getValue();

                        final URL requestUrl;
                        final File archiveFile;
                        final File targetFile;
                        final String propertyName;

                        if (year == META_FILE_YEAR_PLACEHOLDER) {
                            requestUrl = getRemoteResourceLocationUrl(ResourceLocationNvd.CVE_MODIFIED_URL);
                            archiveFile = new File(super.downloadIntoDirectory, "nvd-modified.json.gz");
                            targetFile = new File(super.downloadIntoDirectory, "nvd-modified.json");
                            propertyName = InfoFileAttributes.NVD_PREFIX.getKey() + "last-modified-meta";
                        } else {
                            requestUrl = getRemoteResourceLocationUrl(ResourceLocationNvd.CVE_YEAR_BASE_URL, year);
                            archiveFile = new File(super.downloadIntoDirectory, "nvd-" + year + ".json.gz");
                            targetFile = new File(super.downloadIntoDirectory, "nvd-" + year + ".json");
                            propertyName = InfoFileAttributes.NVD_PREFIX.getKey() + "last-modified-" + year;
                        }

                        if (targetFile.exists()) {
                            targetFile.delete();
                        }
                        if (archiveFile.exists()) {
                            archiveFile.delete();
                        }

                        super.downloader.fetchResponseBodyFromUrlToFile(requestUrl, archiveFile);

                        ArchiveUtils.unpackIfPossible(archiveFile, super.downloadIntoDirectory, new ArrayList<>());
                        archiveFile.delete();

                        super.propertyFiles.set(super.downloadIntoDirectory, "info", propertyName, lastModifiedDate);
                    }).onException(Exception.class)
                            .retryCount(5)
                            .run()
            );
        }

        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for download threads to finish.", e);
        }
    }

    private Map fetchChangedArchiveYears() {
        final Map changedYearsWithLastModifiedDate = new HashMap<>();

        for (Integer year : availableArchiveYears) {
            super.executor.submit(() -> {
                final String previousLastModifiedDate = super.propertyFiles.getString(super.downloadIntoDirectory, "info", InfoFileAttributes.NVD_PREFIX.getKey() + "last-modified-" + year)
                        .orElse("");

                final URL requestUrl = getRemoteResourceLocationUrl(ResourceLocationNvd.CVE_YEAR_META_URL, year);
                final String currentLastModifiedDate = fetchLastModifiedDateFromMetaFile(requestUrl);

                final File targetFile = new File(super.downloadIntoDirectory, "nvd-" + year + ".json");

                if (currentLastModifiedDate.length() == 0) {
                    LOG.warn("Meta file did not contain lastModifiedDate for year [{}], download required", year);
                    changedYearsWithLastModifiedDate.put(year, currentLastModifiedDate);
                } else if (!targetFile.exists() || previousLastModifiedDate.length() == 0) {
                    LOG.info("Archive year [{}] does not yet exist, download required", year);
                    changedYearsWithLastModifiedDate.put(year, currentLastModifiedDate);
                } else if (!previousLastModifiedDate.equals(currentLastModifiedDate)) {
                    LOG.info("Archive year [{}] is not up-to-date, download required [{}] -> [{}]", year, previousLastModifiedDate, currentLastModifiedDate);
                    changedYearsWithLastModifiedDate.put(year, currentLastModifiedDate);
                }
            });
        }

        super.executor.submit(() -> {
            final String previousLastModifiedDate = super.propertyFiles.getString(super.downloadIntoDirectory, "info", InfoFileAttributes.NVD_PREFIX.getKey() + "last-modified-meta")
                    .orElse("");

            final URL requestUrl = getRemoteResourceLocationUrl(ResourceLocationNvd.CVE_MODIFIED_META_URL);
            final String currentLastModifiedDate = fetchLastModifiedDateFromMetaFile(requestUrl);

            final File targetFile = new File(super.downloadIntoDirectory, "nvd-modified.json");

            if (currentLastModifiedDate.length() == 0) {
                LOG.warn("Meta file did not contain lastModifiedDate for meta, download required");
                changedYearsWithLastModifiedDate.put(META_FILE_YEAR_PLACEHOLDER, currentLastModifiedDate);
            } else if (!targetFile.exists() || previousLastModifiedDate.length() == 0) {
                LOG.info("Meta archive does not yet exist, download required");
                changedYearsWithLastModifiedDate.put(META_FILE_YEAR_PLACEHOLDER, currentLastModifiedDate);
            } else if (!previousLastModifiedDate.equals(currentLastModifiedDate)) {
                LOG.info("Meta archive is not up-to-date, download required [{}] -> [{}]", previousLastModifiedDate, currentLastModifiedDate);
                changedYearsWithLastModifiedDate.put(META_FILE_YEAR_PLACEHOLDER, currentLastModifiedDate);
            }
        });

        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for download threads to finish.", e);
        }

        return changedYearsWithLastModifiedDate;
    }

    private String fetchLastModifiedDateFromMetaFile(URL requestUrl) {
        final AtomicReference lastModifiedDate = new AtomicReference<>();

        new Retry<>(() -> {
            final List metaProperties = super.downloader.fetchResponseBodyFromUrlAsList(requestUrl);

            lastModifiedDate.set(metaProperties.stream()
                    .filter(e -> e.contains(":"))
                    .map(e -> e.split(":", 2))
                    .filter(e -> e[0].equals("lastModifiedDate"))
                    .map(e -> e[1])
                    .findFirst()
                    .orElse(""));

        }).onException(Exception.class)
                .withValidator((result) -> lastModifiedDate.get() != null && lastModifiedDate.get().length() > 0)
                .retryCount(5)
                .run();

        return lastModifiedDate.get();
    }

    @Override
    public void setRemoteResourceLocation(String location, String url) {
        super.setRemoteResourceLocation(ResourceLocationNvd.valueOf(location), url);
    }

    public enum ResourceLocationNvd implements ResourceLocation {
        /**
         * The URL to retrieve the recently modified and added CVE entries (last 8 days) using the JSON data feeds.

         * modified feeds are updated every two hours.
         */
        CVE_MODIFIED_URL("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"),
        /**
         * Meta-information on the last modified and added CVE entries, such as the last modified date.
         */
        CVE_MODIFIED_META_URL("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.meta"),
        /**
         * The URL to get a specific yearly CVE list using the JSON data feeds.
         * 
         *     %d Year (example: 2022)
         * 
         */
        CVE_YEAR_BASE_URL("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"),
        /**
         * Meta-information on the specific year files, such as the last modified date.

         * Content is of style: key:value
         * 
         *     %d Year (example: 2022)
         * 
         */
        CVE_YEAR_META_URL("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.meta");

        private final String defaultValue;

        ResourceLocationNvd(String defaultValue) {
            this.defaultValue = defaultValue;
        }

        @Override
        public String getDefault() {
            return this.defaultValue;
        }
    }
}