com.metaeffekt.mirror.download.other.EpssDownload Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.download.other;

import com.metaeffekt.artifact.analysis.utils.ArchiveUtils;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import lombok.extern.slf4j.Slf4j;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Optional;

/**
 * References:
 * 
 * EPSS data provider: https://www.first.org/epss/data_stats
 * 
 * The EPSS (Exploit Prediction Scoring System) scores are available as a single CSV file, compressed using GZIP format, which is retrieved from the official EPSS data provider.
 */
@Slf4j
@MirrorMetadata(directoryName = "epss", mavenPropertyName = "epssDownload")
public class EpssDownload extends Download {
    public EpssDownload(File baseMirrorDirectory) {
        super(baseMirrorDirectory, EpssDownload.class);
    }

    @Override
    protected void performDownload() {
        final URL epssScoreDownloadUrl = getRemoteResourceLocationUrl(ResourceLocationEpss.EPSS_ALL_SCORES_CSV);
        final File downloadIntoFile = getDownloadIntoFile();
        super.downloader.fetchResponseBodyFromUrlToFile(epssScoreDownloadUrl, downloadIntoFile);

        final ArrayList unpackIssues = new ArrayList<>();
        if (!ArchiveUtils.unpackIfPossible(downloadIntoFile, super.getDownloadIntoDirectory(), unpackIssues)) {
            log.error("Issue(s) during unpacking of EPSS scores archive from: {}", downloadIntoFile);
            unpackIssues.forEach(issue -> log.error(" - {}", issue));

            throw new IllegalStateException("Issue(s) during unpacking of EPSS scores archive from: " + downloadIntoFile + "\n - " + String.join("\n - ", unpackIssues));
        }

        super.propertyFiles.set(super.getDownloadIntoDirectory(), "info", InfoFileAttributes.EPSS_PREFIX.getKey() + "resource-size", String.valueOf(downloadIntoFile.length()));

        try {
            FileUtils.delete(downloadIntoFile);
        } catch (IOException e) {
            throw new IllegalStateException("Could not delete downloaded EPSS scores archive: " + downloadIntoFile, e);
        }
    }

    private File getDownloadIntoFile() {
        return new File(super.getDownloadIntoDirectory(), "epss_scores-current.csv.gz");
    }

    @Override
    protected boolean additionalIsDownloadRequired() {
        final URL epssScoreDownloadUrl = getRemoteResourceLocationUrl(ResourceLocationEpss.EPSS_ALL_SCORES_CSV);
        final long fetchedFileSize = super.downloader.fetchFileSizeFromUrl(epssScoreDownloadUrl, false);

        if (fetchedFileSize < 0) {
            log.info("Skipping check for EPSS mirror requirement: EPSS scores file size at {} could not be fetched without downloading entire file.", epssScoreDownloadUrl);
            return false;
        }

        final Optional presentSize = super.propertyFiles.getLong(super.getDownloadIntoDirectory(), "info", InfoFileAttributes.EPSS_PREFIX.getKey() + "resource-size");
        if (!presentSize.isPresent()) {
            log.info("EPSS mirror is required: EPSS scores file size at {} is not present. New size: {}", epssScoreDownloadUrl, fetchedFileSize);
            super.propertyFiles.set(super.getDownloadIntoDirectory(), "info", InfoFileAttributes.EPSS_PREFIX.getKey() + "resource-size", String.valueOf(fetchedFileSize));
            return true;
        }

        final boolean isNotEqual = fetchedFileSize != presentSize.get();

        if (isNotEqual) {
            log.info("EPSS mirror is required: EPSS scores file size at {} has changed. Old size: {}, new size: {}", epssScoreDownloadUrl, presentSize.get(), fetchedFileSize);
            super.propertyFiles.set(super.getDownloadIntoDirectory(), "info", InfoFileAttributes.EPSS_PREFIX.getKey() + "resource-size", String.valueOf(fetchedFileSize));
        }

        return isNotEqual;
    }

    @Override
    public void setRemoteResourceLocation(String location, String url) {
        super.setRemoteResourceLocation(ResourceLocationEpss.valueOf(location), url);
    }

    /**
     * The ResourceLocationEOL enum defines the URLs for accessing the EPSS data feeds.
     * https://www.first.org/epss/data_stats
     */
    public enum ResourceLocationEpss implements ResourceLocation {
        /**
         * URL for fetching the current CSV file with the EPSS scores.

         * Data is of format:
         *          * #model_version:v2023.03.01,score_date:2024-04-17T00:00:00+0000
         * cve,epss,percentile
         * CVE-1999-0001,0.00383,0.72764
         * CVE-1999-0002,0.02080,0.88906
         * CVE-1999-0003,0.04409,0.92276
         * CVE-1999-0004,0.00917,0.82643
         * CVE-1999-0005,0.91963,0.98891
         * 
         * Default value: https://epss.cyentia.com/epss_scores-current.csv.gz
         */
        EPSS_ALL_SCORES_CSV("https://epss.cyentia.com/epss_scores-current.csv.gz"),
        /**
         * If you would like historical data for EPSS, you can directly access daily files by entering the target date into the URL in this format:
         * https://epss.cyentia.com/epss_scores-YYYY-mm-dd.csv.gz, where YYYY is the four digit year, mm is the two digit month and dd is a two digit day.
         * For example, if you want to pull February 1st, 2023, you would retrieve https://epss.cyentia.com/epss_scores-2023-02-01.csv.gz.
         * 

         *     %s - Year (YYYY)
         *     %s - Month (mm)
         *     %s - Day (dd)
         * 
         */
        EPSS_SPECIFIC_DATE_CSV("https://epss.cyentia.com/epss_scores-%s-%s-%s.csv.gz");

        private final String defaultValue;

        ResourceLocationEpss(String defaultValue) {
            this.defaultValue = defaultValue;
        }

        @Override
        public String getDefault() {
            return this.defaultValue;
        }
    }
}