com.metaeffekt.artifact.analysis.scancode.ScanCodeSupport Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.artifact.analysis.scancode;

import com.metaeffekt.artifact.analysis.model.DefaultPropertyProvider;
import com.metaeffekt.artifact.analysis.model.PropertyProvider;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.PropertyUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import org.json.JSONException;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.Properties;
import java.util.TreeSet;

import static com.metaeffekt.artifact.analysis.metascan.Constants.*;

public class ScanCodeSupport {

    private static final Logger LOG = LoggerFactory.getLogger(ScanCodeSupport.class);
    private static final String COPYRIGHT_AUTHOR_SEPARATOR = String.format("|%n");

    private final ScanCodeClient scancodeClient;

    private final PropertyProvider propertyProvider;

    public ScanCodeSupport(PropertyProvider propertyProvider) {
        this.propertyProvider = propertyProvider == null ? new DefaultPropertyProvider() : propertyProvider;
        scancodeClient = createClient();
    }

    protected ScanCodeClient createClient() {
        if (propertyProvider.isProperty("analyze.scan.scancode.clientType", "service", null)) {
            String endpointUrl = propertyProvider.getProperty("scancode.service.endpointUrl", "http://localhost:8000");
            ScanCodeService service = ScanCodeService.create(endpointUrl);
            ScanCodeDefaultBackend backend = new ScanCodeDefaultBackend(service, ScanCodeService.getErrorConverter(endpointUrl));
            return new ScanCodeHttpClient(backend, new WaitForeverStrategy(10));
        } else {
            return new ScanCodeProcessClient(propertyProvider.getProperty("analyze.scancode.threads", "2"));
        }
    }

    public boolean execute(Artifact artifact, File analysisDir) throws JSONException {
        final String sourceFolderName = analysisDir.getName();
        final File targetFolder = new File(analysisDir.getParentFile(), sourceFolderName + "-analysis");
        final File intermediateFolder = new File(analysisDir.getParentFile(), sourceFolderName + "-intermediate");

        return execute(artifact, intermediateFolder, targetFolder);
    }

    public boolean execute(Artifact artifact, File intermediateDir, File resultDir) throws JSONException {

        final String filename = artifact.getId().replace("/", "_");
        final File scanCodeResultPropertiesFile = new File(resultDir, filename + "_scancode.properties");
        final File scanCodeResultFile = new File(resultDir, filename + "_scancode.json");
        resultDir.mkdirs();

        boolean scancodeEnabled = isProperty("analyze.scan.scancode.enabled", "true", "false");
        boolean overwriteParsedResult = isProperty("analyze.scan.scancode.parse.overwrite", "true", "false");
        boolean overwriteScancodeResult = isProperty("analyze.scan.scancode.overwrite", "true", "false");

        // use a reference timestamp to determine whether a new scan is required
        long overwriteResultsOlderThan = Long.parseLong(getProperty("analyze.scan.scancode.overwrite.timestamp", "0"));

        boolean outdatedResult = !scanCodeResultFile.exists() || scanCodeResultFile.lastModified() < overwriteResultsOlderThan;

        final boolean overwrite = outdatedResult || overwriteScancodeResult;

        // overwriteParsedResult is predetermined if overwriteScancodeResult is true
        if (overwrite) overwriteParsedResult = true;

        // in case the result file exist and overwrite is not set we simply apply the content of result properties file
        // to the artifact without further parsing
        if (!overwriteParsedResult && scanCodeResultPropertiesFile.exists()) {
            Properties p = PropertyUtils.loadProperties(scanCodeResultPropertiesFile);
            applyToArtifact(artifact, p);
            return false;
        }

        // all further processing imply that scancode is enabled
        if (!scancodeEnabled) {
            return false;
        }

        String includes = getProperty("analyze.metascan.license.includes", "**/*");
        includes = getProperty("analyze.scancode.license.includes", includes);
        final String[] scanIncludes = includes.split(",");

        String excludes = getProperty("analyze.metascan.license.excludes", "**/.git/**/*");
        excludes = getProperty("analyze.scancode.license.excludes", excludes);
        final String[] scanExcludes = excludes.split(",");

        // use existing scanCodeResultAsString if exists
        String scanCodeResultAsString = null;
        if (!overwrite && scanCodeResultFile.exists()) {
            scanCodeResultAsString = testFile(scanCodeResultFile, scanIncludes, scanExcludes);
        }

        // run scan tool if enabled and scanCodeResultAsString is available yet (implies overwrite)
        if (StringUtils.isEmpty(scanCodeResultAsString)) {
            scanCodeResultAsString = runScanCode(intermediateDir, scanCodeResultFile);
        }

        if (scanCodeResultFile.exists() && StringUtils.notEmpty(scanCodeResultAsString)) {
            parseResults(artifact, intermediateDir, scanCodeResultAsString, scanIncludes, scanExcludes, scanCodeResultPropertiesFile);
        }

        return true;
    }

    private static String testFile(File scanCodeResultFile, String[] scanIncludes, String[] scanExcludes) {
        String scanCodeResultAsString;
        try {
            scanCodeResultAsString = FileUtils.readFileToString(scanCodeResultFile, FileUtils.ENCODING_UTF_8);
            if (StringUtils.hasText(scanCodeResultAsString)) {
                TreeSet licenseList = new TreeSet<>();
                TreeSet copyrightList = new TreeSet<>();
                TreeSet authorList = new TreeSet<>();
                ScanCodeParser.parseScanCodeResult(scanCodeResultAsString, licenseList, copyrightList,
                        authorList, scanIncludes, scanExcludes);
            }
        } catch (IOException e) {
            LOG.warn("Failure parsing existing scancode result: {}", e.getMessage());
            scanCodeResultAsString = null;
        }
        if (scanCodeResultAsString == null || scanCodeResultAsString.trim().isEmpty()) {
            scanCodeResultAsString = null;
        }
        return scanCodeResultAsString;
    }

    private String runScanCode(File intermediateDir, File scanCodeResultFile) {
        LOG.info("  Running scancode on folder {}...", intermediateDir.getAbsolutePath());
        String scanCodeResultAsString;

        try {
            scancodeClient.scan(intermediateDir.getAbsolutePath(), scanCodeResultFile.getAbsolutePath());
        } catch (IOException e) {
            LOG.error("Failure executing scancode. {}", e.getMessage(), e);
        }

        // read the just produced file
        if (scanCodeResultFile.exists()) {
            try {
                scanCodeResultAsString = FileUtils.readFileToString(scanCodeResultFile, "UTF-8");
            } catch (IOException e) {
                throw new IllegalStateException("ScanCode execution was not successful. Cannot read file.");
            }
        } else {
            throw new IllegalStateException("ScanCode execution was not successful. No file was produced. Expected: " + scanCodeResultFile);
        }
        LOG.info("  Running ScanCode on folder {} completed.", intermediateDir);
        return scanCodeResultAsString;
    }

    private void parseResults(Artifact artifact, File intermediateDir, String scanCodeResultAsString, String[] scanIncludes, String[] scanExcludes, File scanCodeResultPropertiesFile) {
        LOG.info("  Parsing ScanCode results for folder {}...", intermediateDir);

        // Collect the results in a properties instance
        Properties resultProperties = new Properties();

        // hint: we don't care about multiple mentions and order
        TreeSet licenseList = new TreeSet<>();
        TreeSet copyrightList = new TreeSet<>();
        TreeSet authorList = new TreeSet<>();

        ScanCodeParser.parseScanCodeResult(scanCodeResultAsString, licenseList, copyrightList, authorList, scanIncludes, scanExcludes);
        resultProperties.setProperty("detected.licenses", licenseList.toString());

        // separate copyrights by |\n
        String copyrightListString = String.join(COPYRIGHT_AUTHOR_SEPARATOR, copyrightList);
        resultProperties.setProperty("detected.copyrights", copyrightListString);

        // separate authors by |\n
        String authorListString = String.join(COPYRIGHT_AUTHOR_SEPARATOR, authorList);
        resultProperties.setProperty("detected.authors", authorListString);

        // TODO: extract further attributes from ScanCode results
        // - license expressions

        LOG.debug("    ScanCode license list: {}", licenseList);
        LOG.debug("    ScanCode copyright list: {}", copyrightList);

        // apply to artifact
        applyToArtifact(artifact, resultProperties);

        PropertyUtils.saveProperties(scanCodeResultPropertiesFile, resultProperties);
        LOG.info("  Parsing ScanCode results on folder {} completed.", intermediateDir);
    }

    public boolean isProperty(String key, String testValue, String defaultValue) {
        return propertyProvider.isProperty(key, testValue, defaultValue);
    }

    public String getProperty(String key, String defaultValue) {
        return propertyProvider.getProperty(key, defaultValue);
    }

    protected void applyToArtifact(Artifact artifact, Properties p) {
        artifact.set(KEY_DERIVED_LICENSES_SCANCODE, p.getProperty("detected.licenses"));
        artifact.set(KEY_EXTRACTED_COPYRIGHTS_SCANCODE, p.getProperty("detected.copyrights"));
        artifact.set(KEY_EXTRACTED_AUTHORS_SCANCODE, p.getProperty("detected.authors"));
    }
}