• Home
• com.metaeffekt.artifact.analysis
• ae-artifact-analysis
• 0.132.0
• source code
• GhsaAdvisorIndex.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.metaeffekt.mirror.index.advisor.GhsaAdvisorIndex Maven / Gradle / Ivy

/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror.index.advisor;

import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import com.metaeffekt.mirror.contents.advisory.GhsaAdvisorEntry;
import com.metaeffekt.mirror.download.advisor.GhsaDownload;
import com.metaeffekt.mirror.download.documentation.DocRelevantMethods;
import com.metaeffekt.mirror.index.Index;
import org.apache.lucene.document.Document;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 
The GitHub Security Advisories are available as JSON files. Each file contains a single advisory, which is identified
 * by its GHSA ID. The ID is also used as the filename.
 * 
A speciality with this data source is that it provides matching information, containing the ecosystem, package name and
 * version range for each affected package. This information is stored in the affected field, which is an array of
 * objects.
 * 
"affected": [
 *   {
 *     "package": {
 *       "ecosystem": "Packagist",
 *       "name": "francoisjacquet/rosariosis"
 *     },
 *     "ranges": [ {
 *       "type": "ECOSYSTEM",
 *       "events": [
 *         { "introduced": "0" },
 *         { "fixed": "8.9.3" }
 *       ]
 *     } ]
 *   }
 * ]
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
Mapping of JSON content to GhsaAdvisorEntry fields
JSON
GhsaAdvisorEntry
id
id
published
createDate
modified
updateDate
references
references
aliases & (database_specific > cwe_ids)
referenceIds
summary
summary
description
description
database_specific > severity
severity
database_specific > githubReviewed
githubReviewed
database_specific > githubReviewedAt
githubReviewedAt
database_specific > nvdPublishedAt
nvdPublishedAt
affected > each package and ranges
vulnerableSoftwares
severity > where type == CVSS_V3
cvss3
severity > where type == CVSS_V2
cvss2
 */
@MirrorMetadata(directoryName = "github-advisory-database", mavenPropertyName = "githubAdvisorIndex")
public class GhsaAdvisorIndex extends Index {

    private final static Logger LOG = LoggerFactory.getLogger(GhsaAdvisorIndex.class);

    public GhsaAdvisorIndex(File baseMirrorDirectory) {
        super(baseMirrorDirectory, GhsaAdvisorIndex.class, Collections.singletonList(GhsaDownload.class), Collections.emptyList());
    }

    @Override
    @DocRelevantMethods({"GhsaAdvisorEntry#fromGitRepoJson"})
    protected Map createIndexDocuments() {
        final Map documents = new ConcurrentHashMap<>();

        LOG.info("Searching for files in repository, this may take a while {}", super.requiredDownloads[0].getAbsolutePath());
        final Collection files = super.getAllFilesRecursively(super.requiredDownloads[0]);
        LOG.info("Found [{}] files in repository", files.size());

        for (File file : files) {
            if (!file.getName().endsWith(".json")) {
                continue;
            }

            super.executor.submit(() -> {
                try {
                    final String contents = FileUtils.readFileToString(file, StandardCharsets.UTF_8);

                    final GhsaAdvisorEntry parsedEntry;
                    try {
                        parsedEntry = GhsaAdvisorEntry.fromGitRepoJson(new JSONObject(contents));
                    } catch (Exception e) {
                        LOG.error("Unable to parse file content during indexing: {}\n{}", file.getAbsolutePath(), contents, e);
                        throw e;
                    }

                    if (documents.containsKey(parsedEntry.getId())) {
                        LOG.warn("Duplicate entry found, skipping: {}", parsedEntry.toJson());
                    } else {
                        documents.put(parsedEntry.getId(), parsedEntry.toDocument());
                    }
                } catch (IOException e) {
                    throw new RuntimeException("Unable to read file contents during indexing: " + file.getAbsolutePath(), e);
                } catch (Exception e) {
                    throw new RuntimeException("Unable to parse file content during indexing: " + file.getAbsolutePath(), e);
                }
            });
        }

        executor.setSize(16);
        executor.start();
        try {
            executor.join();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to wait for all vulnerability sheets to be created", e);
        }

        return documents;
    }
}