com.metaeffekt.mirror.download.nvd.CpeDictionaryDownload Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.download.nvd;

import com.metaeffekt.artifact.analysis.utils.ArchiveUtils;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.ResourceLocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;

import static com.metaeffekt.mirror.download.nvd.CpeDictionaryDownload.ResourceLocationCpeDictionary.CPE_DICTIONARY_URL;
import static com.metaeffekt.mirror.download.nvd.CpeDictionaryDownload.ResourceLocationCpeDictionary.CPE_MATCH_URL;

/**
 * Uses the deprecated 1.x NVD JSON feeds to download the NVD data.

 * These data feeds are no longer available and can no longer be used. More information can be found at
 * https://nvd.nist.gov/general/news/api-20-announcements.
 */
@Deprecated
@MirrorMetadata(directoryName = "cpe-dict", mavenPropertyName = "cpeDictionaryDownload", deprecated = true)
public class CpeDictionaryDownload extends Download {

    private final static Logger LOG = LoggerFactory.getLogger(CpeDictionaryDownload.class);

    public CpeDictionaryDownload(File baseMirrorDirectory) {
        super(baseMirrorDirectory, CpeDictionaryDownload.class);
    }

    @Override
    protected void performDownload() {
        super.executor.submit(() -> {
            try {
                fetchZippedFile(CPE_DICTIONARY_URL, "cpe-dict.xml", "official-cpe-dictionary_v2.2.xml");
            } catch (IOException e) {
                throw new RuntimeException("Unable to download and extract CPE Dictionary from " + CPE_DICTIONARY_URL, e);
            }
        });

        super.executor.submit(() -> {
            try {
                fetchZippedFile(CPE_MATCH_URL, "cpe-match.json", "nvdcpematch-1.0.json");
            } catch (IOException e) {
                throw new RuntimeException("Unable to download and extract CPE Match from " + CPE_MATCH_URL, e);
            }
        });

        super.executor.start();
        try {
            super.executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for download threads to finish.", e);
        }
    }

    private void fetchZippedFile(ResourceLocationCpeDictionary cpeDictionaryUrl, String downloadToFilename, String downloadFilename) throws IOException {
        final URL requestUrl = getRemoteResourceLocationUrl(cpeDictionaryUrl);
        final File downloadDestinationFile = new File(super.downloadIntoDirectory, downloadToFilename + ".zip");

        super.downloader.fetchResponseBodyFromUrlToFile(requestUrl, downloadDestinationFile);
        final long fileSize = downloadDestinationFile.length();

        final File unzipDirectory = new File(super.downloadIntoDirectory, downloadToFilename + "_unzipped");
        if (unzipDirectory.exists()) {
            FileUtils.deleteDir(unzipDirectory);
        }

        ArchiveUtils.unpackIfPossible(downloadDestinationFile, unzipDirectory, new ArrayList<>());

        downloadDestinationFile.delete();

        final File destinationFile = new File(super.downloadIntoDirectory, downloadToFilename);
        if (destinationFile.exists()) {
            destinationFile.delete();
        }

        final File unzippedFile = new File(unzipDirectory, downloadFilename);
        FileUtils.moveFile(unzippedFile, destinationFile);

        if (unzipDirectory.exists()) {
            FileUtils.deleteDir(unzipDirectory);
        }

        if (downloadToFilename.contains("cpe-dict")) {
            super.propertyFiles.set(super.downloadIntoDirectory, "info", InfoFileAttributes.CPE_DICTIONARY_PREFIX.getKey() + "cpe-dict-size", fileSize);
        } else if (downloadToFilename.contains("cpe-match")) {
            super.propertyFiles.set(super.downloadIntoDirectory, "info", InfoFileAttributes.CPE_DICTIONARY_PREFIX.getKey() + "cpe-match-size", fileSize);
        }
    }

    @Override
    protected boolean additionalIsDownloadRequired() {
        final File unzipFileCpeDictionary = new File(super.downloadIntoDirectory, "cpe-dict.xml");
        final File unzipFileCpeMatch = new File(super.downloadIntoDirectory, "cpe-match.json");

        if (!unzipFileCpeDictionary.exists() || !unzipFileCpeMatch.exists()) {
            LOG.info("CPE Dictionary and/or CPE Match files are not present, download required");
            return true;
        }

        long lastCpeDictSize = super.propertyFiles.getLong(super.downloadIntoDirectory, "info", InfoFileAttributes.CPE_DICTIONARY_PREFIX.getKey() + "cpe-dict-size")
                .orElse(0L);
        if (lastCpeDictSize == 0) {
            LOG.info("CPE Dictionary size has not been set yet, download required");
            return true;
        } else {
            final URL cpeDictRequestUrl = getRemoteResourceLocationUrl(CPE_DICTIONARY_URL);
            long currentCpeDictSize = super.downloader.fetchFileSizeFromUrl(cpeDictRequestUrl);

            if (lastCpeDictSize != currentCpeDictSize) {
                LOG.info("CPE Dictionary size has changed. Old size: [{}] New size: [{}], download required", lastCpeDictSize, currentCpeDictSize);
                return true;
            }
        }

        long lastCpeMatchSize = super.propertyFiles.getLong(super.downloadIntoDirectory, "info", InfoFileAttributes.CPE_DICTIONARY_PREFIX.getKey() + "cpe-match-size")
                .orElse(0L);
        if (lastCpeMatchSize == 0) {
            LOG.info("CPE Match size has not been set yet, download required");
            return true;
        } else {
            final URL cpeMatchRequestUrl = getRemoteResourceLocationUrl(CPE_MATCH_URL);
            long currentCpeMatchSize = super.downloader.fetchFileSizeFromUrl(cpeMatchRequestUrl);

            if (lastCpeMatchSize != currentCpeMatchSize) {
                LOG.info("CPE Match size has changed. Old size: [{}] New size: [{}], download required", lastCpeMatchSize, currentCpeMatchSize);
                return true;
            }
        }

        return false;
    }

    @Override
    public void setRemoteResourceLocation(String location, String url) {
        super.setRemoteResourceLocation(ResourceLocationCpeDictionary.valueOf(location), url);
    }

    public enum ResourceLocationCpeDictionary implements ResourceLocation {
        /**
         * The official CPE dictionary: https://nvd.nist.gov/products/cpe
         */
        CPE_DICTIONARY_URL("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.zip"),
        /**
         * The CPE match data feed contains additional information on the specific CPE metadata:
         * https://nvd.nist.gov/vuln/data-feeds#cpeMatch
         */
        CPE_MATCH_URL("https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip");

        private final String defaultValue;

        ResourceLocationCpeDictionary(String defaultValue) {
            this.defaultValue = defaultValue;
        }

        @Override
        public String getDefault() {
            return this.defaultValue;
        }
    }
}