• Home
• org.ossreviewtoolkit
• model
• 37.0.0
• source code

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

toolkit.model.37.0.0.source-code.ScannerRun.kt Maven / Gradle / Ivy

/*
 * Copyright (C) 2017 The ORT Project Authors (see )
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 * License-Filename: LICENSE
 */

package org.ossreviewtoolkit.model

import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.databind.annotation.JsonSerialize

import java.io.File
import java.time.Instant

import kotlin.time.measureTimedValue

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.model.config.ScannerConfiguration
import org.ossreviewtoolkit.model.utils.FileListSortedSetConverter
import org.ossreviewtoolkit.model.utils.ProvenanceResolutionResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScanResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScannersMapConverter
import org.ossreviewtoolkit.model.utils.getKnownProvenancesWithoutVcsPath
import org.ossreviewtoolkit.model.utils.mergeScanResultsByScanner
import org.ossreviewtoolkit.model.utils.prependPath
import org.ossreviewtoolkit.model.utils.vcsPath
import org.ossreviewtoolkit.utils.common.getDuplicates
import org.ossreviewtoolkit.utils.ort.Environment

/**
 * The summary of a single run of the scanner.
 */
data class ScannerRun(
    /**
     * The [Instant] the scanner was started.
     */
    val startTime: Instant,

    /**
     * The [Instant] the scanner has finished.
     */
    val endTime: Instant,

    /**
     * The [Environment] in which the scanner was executed.
     */
    val environment: Environment,

    /**
     * The [ScannerConfiguration] used for this run.
     */
    val config: ScannerConfiguration,

    /**
     * The results of the provenance resolution for all projects and packages.
     */
    @JsonSerialize(converter = ProvenanceResolutionResultSortedSetConverter::class)
    val provenances: Set,

    /**
     * The scan results for each resolved provenance.
     */
    @JsonSerialize(converter = ScanResultSortedSetConverter::class)
    val scanResults: Set,

    /**
     * The names of the scanners which have been used to scan the package.
     */
    @JsonSerialize(converter = ScannersMapConverter::class)
    val scanners: Map>,

    /**
     * The list of files for each resolved provenance.
     */
    @JsonSerialize(converter = FileListSortedSetConverter::class)
    val files: Set
) {
    companion object {
        /**
         * A constant for a [ScannerRun] where all properties are empty.
         */
        @JvmField
        val EMPTY = ScannerRun(
            startTime = Instant.EPOCH,
            endTime = Instant.EPOCH,
            environment = Environment(),
            config = ScannerConfiguration(),
            provenances = emptySet(),
            scanResults = emptySet(),
            files = emptySet(),
            scanners = emptyMap()
        )
    }

    init {
        scanResults.forEach { scanResult ->
            require(scanResult.provenance is KnownProvenance) {
                "Found a scan result with an unknown provenance, which is not allowed."
            }

            (scanResult.provenance as? RepositoryProvenance)?.let { repositoryProvenance ->
                require(repositoryProvenance.vcsInfo.path.isEmpty()) {
                    "Found a scan result with a non-empty VCS path, which is not allowed."
                }

                require(repositoryProvenance.vcsInfo.revision == repositoryProvenance.resolvedRevision) {
                    "The revision and resolved revision of a scan result are not equal, which is not allowed."
                }
            }
        }

        provenances.getDuplicates { it.id }.keys.let { idsForDuplicateProvenanceResolutionResults ->
            require(idsForDuplicateProvenanceResolutionResults.isEmpty()) {
                "Found multiple provenance resolution results for the following ids: " +
                    "${idsForDuplicateProvenanceResolutionResults.joinToString { it.toCoordinates() }}."
            }
        }

        val scannedProvenances = scanResults.mapTo(mutableSetOf()) { it.provenance }
        val resolvedProvenances = provenances.flatMapTo(mutableSetOf()) {
            it.getKnownProvenancesWithoutVcsPath().values
        }

        (scannedProvenances - resolvedProvenances).let {
            require(it.isEmpty()) {
                "Found scan results which do not correspond to any resolved provenances, which is not allowed: \n" +
                    it.toYaml()
            }
        }

        val fileListProvenances = files.mapTo(mutableSetOf()) { it.provenance }
        (fileListProvenances - resolvedProvenances).let {
            require(it.isEmpty()) {
                "Found a file lists which do not correspond to any resolved provenances, which is not allowed: \n" +
                    it.toYaml()
            }
        }

        files.forEach { fileList ->
            (fileList.provenance as? RepositoryProvenance)?.let {
                require(it.vcsInfo.path.isEmpty()) {
                    "Found a file list with a non-empty VCS path, which is not allowed."
                }

                require(it.vcsInfo.revision == it.resolvedRevision) {
                    "The revision and resolved revision of a file list are not equal, which is not allowed."
                }
            }
        }
    }

    private val provenancesById: Map by lazy {
        provenances.associateBy { it.id }
    }

    private val scanResultsByProvenance: Map> by lazy {
        scanResults.groupBy { it.provenance as KnownProvenance }
    }

    private val scanResultsById: Map> by lazy {
        logger.debug { "Merging scan results..." }

        val (result, duration) = measureTimedValue {
            provenances.map { it.id }.associateWith { id -> getMergedResultsForId(id) }
        }

        logger.debug { "Merging scan results took $duration." }

        result
    }

    private val fileListByProvenance: Map by lazy {
        files.associateBy { it.provenance }
    }

    private val fileListById: Map by lazy {
        provenances.mapNotNull {
            getMergedFileListForId(it.id)?.let { fileList ->
                it.id to fileList
            }
        }.toMap()
    }

    /**
     * Return all scan results related to [id] with the internal sub-repository scan results merged into the root
     * repository scan results. ScanResults for different scanners are not merged, so that the output contains exactly
     * one scan result per scanner. In case of any provenance resolution issue, a fake scan result just containing the
     * issue is returned.
     */
    private fun getMergedResultsForId(id: Identifier): List {
        val resolutionResult = provenancesById.getValue(id).apply {
            if (issues.isNotEmpty()) {
                return listOf(scanResultForProvenanceResolutionIssues(packageProvenance, issues))
            }
        }

        @Suppress("UnsafeCallOnNullableType")
        val packageProvenance = resolutionResult.packageProvenance!!

        val scanResultsByPath = resolutionResult.getKnownProvenancesWithoutVcsPath().mapValues { (_, provenance) ->
            scanResultsByProvenance[provenance].orEmpty()
        }.mapValues { (_, scanResults) ->
            scanResults.filter { it.scanner.name in scanners[id].orEmpty() }
        }

        // TODO: Handle the case of incomplete scan results (per scanner), e.g. propagate an issue.
        return mergeScanResultsByScanner(scanResultsByPath, packageProvenance).map { scanResult ->
            scanResult.filterByPath(packageProvenance.vcsPath).filterByIgnorePatterns(config.ignorePatterns)
        }.map { scanResult ->
            // The VCS revision of scan result is equal to the resolved revision. So, use the package provenance
            // to re-align the VCS revision with the package's metadata.
            scanResult.copy(summary = scanResult.summary.addIssue(resolutionResult.nestedProvenanceResolutionIssue))
        }
    }

    private fun getMergedFileListForId(id: Identifier): FileList? {
        val resolutionResult = provenancesById[id]?.takeIf {
            it.packageProvenanceResolutionIssue == null && it.nestedProvenanceResolutionIssue == null
        } ?: return null

        @Suppress("UnsafeCallOnNullableType")
        val packageProvenance = resolutionResult.packageProvenance!!

        val fileListsByPath = resolutionResult.getKnownProvenancesWithoutVcsPath().mapValues { (_, provenance) ->
            // If there was an issue creating at least one file list, then return null instead of an incomplete file
            // list.
            fileListByProvenance[provenance] ?: return null
        }

        return mergeFileLists(fileListsByPath)
            .filterByVcsPath(packageProvenance.vcsPath)
            .copy(provenance = packageProvenance)
    }

    @JsonIgnore
    fun getAllScanResults(): Map> = scanResultsById

    fun getScanResults(id: Identifier): List = scanResultsById[id].orEmpty()

    @JsonIgnore
    fun getAllFileLists(): Map = fileListById

    fun getFileList(id: Identifier): FileList? = fileListById[id]

    @JsonIgnore
    fun getAllIssues(): Map> =
        scanResultsById.mapValues { (_, scanResults) ->
            scanResults.flatMapTo(mutableSetOf()) { it.summary.issues }
        }
}

private fun scanResultForProvenanceResolutionIssues(packageProvenance: KnownProvenance?, issues: List) =
    ScanResult(
        provenance = packageProvenance ?: UnknownProvenance,
        scanner = ScannerDetails(name = "ProvenanceResolver", version = "", configuration = ""),
        summary = ScanSummary.EMPTY.copy(issues = issues)
    )

private fun ScanSummary.addIssue(issue: Issue?): ScanSummary =
    if (issue == null) this else copy(issues = (issues + issue).distinct())

private fun mergeFileLists(fileListByPath: Map): FileList {
    val provenance = requireNotNull(fileListByPath[""]) {
        "There must be a file list associated with the root path."
    }.provenance

    val files = fileListByPath.flatMapTo(mutableSetOf()) { (path, fileList) ->
        fileList.files.map { fileEntry ->
            fileEntry.copy(path = fileEntry.path.prependPath(path))
        }
    }

    return FileList(provenance, files)
}

private fun FileList.filterByVcsPath(path: String): FileList {
    if (path.isBlank()) return this

    require(provenance is RepositoryProvenance) {
        "Expected a repository provenance but got a ${provenance.javaClass.simpleName}."
    }

    val provenance = provenance.copy(vcsInfo = provenance.vcsInfo.copy(path = path))

    // Do not keep files outside the VCS path in contrast to ScanSummary.filterByVcsPath().
    val files = files.filterTo(mutableSetOf()) { fileEntry ->
        File(fileEntry.path).startsWith(path)
    }

    return FileList(provenance, files)
}

val ProvenanceResolutionResult.issues: List
    get() = listOfNotNull(packageProvenanceResolutionIssue, nestedProvenanceResolutionIssue)