com.metaeffekt.artifact.analysis.flow.DocumentFlow Maven / Gradle / Ivy

Go to download
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.artifact.analysis.flow;

import com.metaeffekt.artifact.analysis.flow.notice.GenerateNoticeFlow;
import com.metaeffekt.artifact.analysis.flow.notice.GenerateNoticeFlowParam;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.InventoryMergeUtils;
import com.metaeffekt.artifact.analysis.utils.InventoryUtils;
import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.workbench.LicenseAggregationUtils;
import com.metaeffekt.artifact.analysis.workbench.ProjectInventoryFilter;
import com.metaeffekt.flow.common.AbstractFlow;
import com.metaeffekt.resource.InventoryResource;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import org.metaeffekt.core.inventory.processor.model.InventoryInfo;
import org.metaeffekt.core.inventory.processor.reader.InventoryReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DocumentFlow extends AbstractFlow {

    private static final Logger LOG = LoggerFactory.getLogger(DocumentFlow.class);

    private static final String KEY_DOCUMENTATION_TEMPLATE = "workbench.documentation.template";
    private static final String KEY_DOCUMENTATION_TEMPLATE_INVENTORY_PATH_PATTERN = "workbench.documentation.template.%s.inventory.path";
    private static final String KEY_DOCUMENTATION_TEMPLATE_RESULTS_PATH_PATTERN = "workbench.documentation.template.%s.results.path";

    public DocumentFlowResult process(DocumentFlowParam documentFlowParam) throws IOException {
        // documentation template (variable and from workbench-control.properties)
        final String documentationTemplatePath = documentFlowParam.getPropertyProvider().getProperty(KEY_DOCUMENTATION_TEMPLATE, null);
        if (documentationTemplatePath == null) {
            LOG.info("Skipping documentation. No documentation template defined.");
            return null;
        }

        // control file
        final File documentControlFile = documentFlowParam.getDocumentControlFile();
        if (documentControlFile == null || !documentControlFile.exists()) {
            LOG.info("Skipping documentation. No file [{}] defined.", documentControlFile);
            return null;
        }

        // scanner result integration
        final File scannerOutputBaseDir =  documentFlowParam.getScannerOutputBaseDir();
        final String scannerOutputInventoryPattern = documentFlowParam.getScannerInventoryOutputPattern();

        // documentation settings
        final File documentOutputBaseDir = documentFlowParam.getDocumentOutputBaseDir();
        final File tmpBaseDir = documentFlowParam.getTmpBaseDir();
        final File documentationProjectBaseDir = new File(tmpBaseDir, "documentation");

        final File documentationTemplateBaseDir = new File(documentationTemplatePath);

        final Inventory documentControlInventory = new InventoryReader().readInventory(documentControlFile);

        for (InventoryInfo info : documentControlInventory.getInventoryInfo()) {
            // TODO: validate info for document creation

            final String documentId = info.getId();

            // initialize target project base dir
            final File targetProjectBaseDir = new File(documentationProjectBaseDir, documentId);

            final String profile = info.get("document.type");

            final String documentationTemplateDocumentPath = documentFlowParam.getPropertyProvider().
                    getProperty(String.format(KEY_DOCUMENTATION_TEMPLATE_RESULTS_PATH_PATTERN, profile), null);

            // copy resulting document/annex to output folder
            final File resultDocumentFile = new File(targetProjectBaseDir, documentationTemplateDocumentPath);
            final File targetDocumentBaseDir = new File(documentOutputBaseDir, documentId);

            // FIXME: currently we anticipate that a single document is created; we may have to expect a complete
            //   file structure (such as annex information)
            final String configuredResultFileName = info.get("document.filename.pdf");
            final String resultFileName = !StringUtils.isEmpty(configuredResultFileName) ? configuredResultFileName :
                    info.get("product.name") + "-" + info.get("product.version") + "-" + info.get("document.name") + ".pdf";
            final File targetDocumentFile = new File(targetDocumentBaseDir, resultFileName);

            if (targetDocumentFile.exists()) {
                continue;
            }

            List inventoryFiles = collectInventoryFiles(scannerOutputInventoryPattern, scannerOutputBaseDir, info);
            if (inventoryFiles.isEmpty()) {
                continue;
            }

            List commandParts = new ArrayList<>();
            commandParts.add("mvn");
            commandParts.add("clean");
            commandParts.add("package");

            // use document type to select profile; the maven project should support the profile to trigger different
            // document types for different purposes
            commandParts.add(String.format("-P %s", profile));

            // get template specific paths
            final String documentationTemplateInventoriesPath = documentFlowParam.getPropertyProvider().
                    getProperty(String.format(KEY_DOCUMENTATION_TEMPLATE_INVENTORY_PATH_PATTERN, profile), null);

            // propagate parameters on the command line
            for (String key : info.getAttributes()) {
                final String value = info.get(key);

                // in case no value is specified, do not insert any property (enable defaults)
                if (StringUtils.hasText(value)) {
                    commandParts.add(String.format("-D%s=%s", key, value));
                }
            }

            // configure reference inventory
            commandParts.add("-Dreference.inventory.dir=" + documentFlowParam.getReferenceInventoryDir().getAbsolutePath());
            commandParts.add("-Dreference.inventory.includes=" + documentFlowParam.getReferenceInventoryIncludes());

            final Inventory referenceInventory = InventoryUtils.readInventory(
                    documentFlowParam.getReferenceInventoryDir(), documentFlowParam.getReferenceInventoryIncludes());

            // create a clean copy of the template
            FileUtils.deleteDir(targetProjectBaseDir);
            FileUtils.copyDirectory(documentationTemplateBaseDir, targetProjectBaseDir);

            // the template project accepts multiple inventories copied into the target inventory path. The
            // inventories are then merged
            final File targetInventoryDir = new File(targetProjectBaseDir, documentationTemplateInventoriesPath);

            String documentName = targetDocumentFile.getName();
            int lastDotIndex = targetDocumentFile.getName().lastIndexOf(".");
            if (lastDotIndex > 0) {
                documentName = targetDocumentFile.getName().substring(0, lastDotIndex);
            }

            // merge multiple files into a single one
            final InventoryResource resource = InventoryResource.fromInventory(new Inventory());
            new InventoryMergeUtils().merge(inventoryFiles, resource.getInventory());

            // apply filter
            final ProjectInventoryFilter filter = new ProjectInventoryFilter();
            filter.setEnableUseDerivedLicense(true);

            // the curation is done to update the resulting inventory regarding curation data; the current conveyed
            // information may be outdated.
            filter.setEnableApplyCurationData(true);

            filter.process(resource.getInventory(), referenceInventory);

            // contribute assessment data
            ScanConsumer.transferBusinessCaseAssessmentData(referenceInventory, resource.getInventory());

            GenerateNoticeFlowParam generateNoticeFlowParam = GenerateNoticeFlowParam.builder()
                    .inventoryResource(resource)
                    .normalizationMetaData(documentFlowParam.getNormalizationMetaData())
                    .build();

            GenerateNoticeFlow generateNoticeFlow = new GenerateNoticeFlow();
            generateNoticeFlow.process(generateNoticeFlowParam);

            new AttachReportFlow().attachReport(resource);
            new FormatInventoryFlow().filterAndFormat(resource);

            // sync into documentation template; currently the core plugins don't behave well on 'xlsx bombs'.
            resource.sync(new File(targetInventoryDir, documentName + ".xlsx"));

            // aggregate licenses
            LicenseAggregationUtils.copyLicenses(resource.getInventory(), targetDocumentBaseDir,
                documentFlowParam.getNormalizationMetaData(), documentFlowParam.getLicenseTextProvider(), false);

            // sync into document folder to support curation activities
            resource.sync(new File(targetDocumentBaseDir, documentName + ".xlsx"));

            final File targetSpdxDocumentFile = new File(targetDocumentBaseDir, documentName + "-spdx.json");
            SpdxDocumentFlow.produceSpdxDocument(resource, documentName, targetSpdxDocumentFile,
                documentFlowParam.getNormalizationMetaData(), documentFlowParam.getLicenseTextProvider(), documentFlowParam.getPropertyProvider());

            // execute the command (represented by the collected command parts)
            ExecUtils.executeCommandAndWaitForProcessToTerminate(commandParts, null, targetProjectBaseDir);

            FileUtils.copyFile(resultDocumentFile, targetDocumentFile);
        }

        // FIXME-KKL: currently the DocumentFlowResult does not convey any attributes
        return new DocumentFlowResult();
    }

    protected List collectInventoryFiles(String inventoryPattern, File outputBaseDir, InventoryInfo info) {
        final String assets = info.get("Assets");

        // we scan the output directory for child folders
        final String[] strings = FileUtils.scanDirectoryForFolders(outputBaseDir, assets);

        // the patterns may have captured several output folders with different timestamps; we build a map for each asset
        final Map> assetWithoutTimestampToAssetMap = new HashMap<>();
        for (String assetInventoryName : strings) {
            String assetWithoutTimestamp = assetInventoryName.substring(0, assetInventoryName.lastIndexOf("-"));
            assetWithoutTimestampToAssetMap.computeIfAbsent(assetWithoutTimestamp, n -> new ArrayList<>()).add(assetInventoryName);
        }

        // now we iterate the timestamp-agnostic to collect the most recent inventory files matching the pattern
        final List inventoryFiles = new ArrayList<>();
        for (String assetWithoutTimestamp : assetWithoutTimestampToAssetMap.keySet()) {

            // order the list (latest is last)
            final List assetInventoriesInGroup = assetWithoutTimestampToAssetMap.get(assetWithoutTimestamp);
            assetInventoriesInGroup.sort(String.CASE_INSENSITIVE_ORDER);

            // get last item (latest)
            final String asset = assetInventoriesInGroup.get(assetInventoriesInGroup.size() - 1);

            // construct file for output
            final File scannerResultsDir = new File(outputBaseDir, asset);

            // isolate the result inventory
            File inventoryFile = FileUtils.findSingleFile(scannerResultsDir, inventoryPattern);
            if (inventoryFile != null) {
                inventoryFiles.add(inventoryFile);
            }
        }
        return inventoryFiles;
    }

}