com.metaeffekt.artifact.analysis.bom.spdx.SpdxExporter Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.bom.spdx;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.metaeffekt.artifact.analysis.bom.BomConstants;
import com.metaeffekt.artifact.analysis.bom.LicenseProcessor;
import com.metaeffekt.artifact.analysis.bom.spdx.facade.SpdxApiFacade;
import com.metaeffekt.artifact.analysis.bom.spdx.facade.SpdxJsonFilter;
import com.metaeffekt.artifact.analysis.bom.spdx.mapper.MappingUtils;
import com.metaeffekt.artifact.analysis.bom.spdx.mapper.SpdxPackageMapper;
import com.metaeffekt.artifact.analysis.bom.spdx.relationship.RelationshipGraph;
import com.metaeffekt.artifact.analysis.bom.spdx.relationship.RelationshipGraphEdge;
import com.metaeffekt.artifact.analysis.bom.spdx.relationship.RelationshipGraphNode;
import com.metaeffekt.artifact.analysis.utils.InventoryUtils;
import com.metaeffekt.artifact.terms.model.LicenseTextProvider;
import com.metaeffekt.artifact.terms.model.NormalizationMetaData;
import com.metaeffekt.artifact.terms.model.TermsMetaData;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.metaeffekt.core.inventory.processor.model.AssetMetaData;
import org.metaeffekt.core.inventory.processor.model.Inventory;
import org.spdx.jacksonstore.MultiFormatStore;
import org.spdx.library.InvalidSPDXAnalysisException;
import org.spdx.library.ModelCopyManager;
import org.spdx.library.SpdxConstants;
import org.spdx.library.model.*;
import org.spdx.library.model.license.AnyLicenseInfo;
import org.spdx.library.model.license.ListedLicenses;
import org.spdx.library.model.license.SpdxListedLicense;
import org.spdx.storage.IModelStore;
import org.spdx.storage.listedlicense.SpdxListedLicenseLocalStore;
import org.spdx.storage.simple.InMemSpdxStore;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
/**
* This class is responsible for exporting any given input inventory into an SPDX-document of any
* supported type. This is currently limited to .json format.
* To do this, the class heavily relies on the spdx java library to adhere to the SPDX standard, as well as a custom
* graph structure to accurately map the relationships between different assets and artifact.
*/
@Slf4j
public class SpdxExporter {
private static final String unspecificIdComment =
"This license identifier is marked \"unspecific\" and does not uniquely match a license of a specific version or variant.\n" +
"A license text can therefore not be asserted. Further analysis is required to determine the exact license.";
@Getter
private final Set referencedLicenses = new HashSet<>();
/**
* NormalizationMetaData, required for license string translation.
*/
private final NormalizationMetaData normalizationMetaData;
/**
* Provider of license texts, used to append license texts that spdx doesn't know about.
*/
private final LicenseTextProvider licenseTextProvider;
/**
* These attributes are approved for inclusion by ways of the key-value map in an annotation.
* No mapping is required for these, as the key-value map is considered a sufficiently good representation.
*/
private final Set approvedAttributes;
private final Map customLicenseMappings;
public SpdxExporter(File approvedAttributes, NormalizationMetaData normalizationMetaData,
LicenseTextProvider licenseTextProvider, File customLicenseMappings) {
if (approvedAttributes == null || approvedAttributes.length() == 0) {
this.approvedAttributes = null;
} else {
ObjectMapper objectMapper = new ObjectMapper();
try {
List approvedAttributesList = objectMapper.readValue(approvedAttributes, List.class);
this.approvedAttributes = new HashSet<>(approvedAttributesList);
} catch (IOException e) {
throw new RuntimeException("Failed to read approved attributes file: " + approvedAttributes, e);
}
}
if (customLicenseMappings == null || customLicenseMappings.length() == 0) {
this.customLicenseMappings = null;
} else {
ObjectMapper objectMapper = new ObjectMapper();
try {
this.customLicenseMappings = objectMapper.readValue(customLicenseMappings, Map.class);
} catch (IOException e) {
throw new RuntimeException("Failed to read custom license mappings file: " + customLicenseMappings, e);
}
}
this.normalizationMetaData = Objects.requireNonNull(normalizationMetaData);
this.licenseTextProvider = licenseTextProvider;
}
protected void ensureParentDirExists(File someFile) {
try {
if (FileUtils.createParentDirectories(someFile) == null) {
throw new IllegalArgumentException("Could not create parent directories for this file.");
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public void exportToSpdxDocument(Inventory inventory, File outFile, DocumentSpec documentSpec, boolean prettyPrint) {
ensureParentDirExists(Objects.requireNonNull(outFile));
MultiFormatStore.Format spdxFormat = getSpdxFormat(documentSpec, prettyPrint);
if (!documentSpec.isIncludeAssets()) {
InventoryUtils.removeAssetsAndReferences(inventory);
}
if (spdxFormat == null) {
throw new RuntimeException("Input format provided is not supported.");
}
try (final IModelStore baseStore = new InMemSpdxStore()) {
try (final MultiFormatStore modelStore = new MultiFormatStore(baseStore, spdxFormat)) {
createDocument(inventory, modelStore, documentSpec);
try (final OutputStream outputStream = Files.newOutputStream(outFile.toPath(),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING)) {
modelStore.serialize(documentSpec.getDocumentId(), outputStream);
log.info("Document created at: {}", outFile.getAbsolutePath());
}
}
} catch (Exception e) {
// this catch-all rethrow is required since the store's close() method throws "Exception"
throw new RuntimeException(e);
}
}
public MultiFormatStore.Format getSpdxFormat(DocumentSpec documentSpec, boolean prettyPrint) {
MultiFormatStore.Format spdxFormat = null;
if (documentSpec.getFormat() != null) {
if (documentSpec.getFormat().equals(BomConstants.Format.JSON)) {
if (prettyPrint) {
spdxFormat = MultiFormatStore.Format.JSON_PRETTY;
} else {
spdxFormat = MultiFormatStore.Format.JSON;
}
} else if (documentSpec.getFormat().equals(BomConstants.Format.XML)) {
spdxFormat = MultiFormatStore.Format.XML;
}
}
if (spdxFormat == null) {
spdxFormat = MultiFormatStore.Format.JSON; // Fallback if no format is provided.
}
return spdxFormat;
}
private void createDocument(Inventory inventory, MultiFormatStore modelStore, DocumentSpec documentSpec) {
SpdxDocument spdxDocument;
try {
spdxDocument = new SpdxDocument(modelStore, documentSpec.getDocumentId(), new ModelCopyManager(), true);
// Ensures that we use the locally cached licenses in [spdx-library]/resources/stdlicenses
// TODO: Check if this directory can be changed
// FIXME: this is not the idea; we would be stuck with the old local version
// ListedLicenses.initializeListedLicenses(new SpdxListedLicenseLocalStore());
// we could release our own version of this artifact; the artifact could contain all spdx license list versions
log.info("Created document.");
prepareCreationInformation(spdxDocument, documentSpec);
log.info("Added creation information");
prepareArtifactsAndAssets(spdxDocument, documentSpec, inventory);
log.info("Added all artifacts and assets listed to the document.");
prepareRelationships(spdxDocument, inventory);
log.info("Added relationships between artifacts to the document.");
addReferencedLicenses(spdxDocument, referencedLicenses);
log.info("Added referenced licenses listed to the document.");
} catch (InvalidSPDXAnalysisException e) {
throw new RuntimeException(e);
}
}
private void prepareCreationInformation(SpdxDocument spdxDocument, DocumentSpec documentSpec) {
List creators = new ArrayList<>();
if (StringUtils.isNotBlank(documentSpec.getOrganization())) {
creators.add("Organization: " + documentSpec.getOrganization());
}
if (StringUtils.isNotBlank(documentSpec.getPerson())) {
creators.add("Person: " + documentSpec.getPerson());
}
if (StringUtils.isNotBlank(documentSpec.getTool())) {
creators.add("Tool: " + documentSpec.getTool());
}
try {
// FIXME: licenseListVersion should be taken from universe
final String time = new SimpleDateFormat(SpdxConstants.SPDX_DATE_FORMAT).format(Calendar.getInstance().getTime());
spdxDocument.setCreationInfo(
spdxDocument.createCreationInfo(creators,time)
.setLicenseListVersion("3.25")
.setComment("Organization URL: " + documentSpec.getOrganizationUrl() + "\nBom iteration: " + documentSpec.getDocumentIteration()));
spdxDocument.setName(documentSpec.getDocumentName());
AnyLicenseInfo anyLicenseInfo = new SpdxListedLicense(spdxDocument.getModelStore(),
documentSpec.getDocumentId(), SpdxConstants.SPDX_DATA_LICENSE_ID, spdxDocument.getCopyManager(),
true);
spdxDocument.setDataLicense(anyLicenseInfo);
spdxDocument.setSpecVersion("SPDX-2.3");
} catch (InvalidSPDXAnalysisException e) {
throw new RuntimeException(e);
}
}
private void prepareArtifactsAndAssets(SpdxDocument spdxDocument, DocumentSpec documentSpec,
Inventory inventory) {
if (approvedAttributes == null || approvedAttributes.isEmpty()) {
log.warn("No additional approved attributes were provided. Only exporting predefined attributes from the inventory.");
}
final LicenseStringConverter licenseStringConverter = new LicenseStringConverter(normalizationMetaData, null);
final SpdxPackageMapper spdxPackageMapper = new SpdxPackageMapper(documentSpec, approvedAttributes,
spdxDocument, licenseStringConverter, customLicenseMappings);
try {
for (Artifact artifact : inventory.getArtifacts()) {
spdxPackageMapper.map(artifact);
}
List unmappedAssets = inventory.getAssetMetaData();
unmappedAssets.removeIf(MappingUtils.getAssetToArtifactMap(inventory)::containsKey);
for (AssetMetaData assetMetaData : unmappedAssets) {
spdxPackageMapper.map(assetMetaData);
}
referencedLicenses.addAll(spdxPackageMapper.getReferencedLicenses());
addReferencedLicenses(spdxDocument, referencedLicenses);
} catch (InvalidSPDXAnalysisException e) {
throw new RuntimeException("Failed to map assets/artifacts to spdx packages from inventory: " + inventory, e);
}
}
private void prepareRelationships(SpdxDocument spdxDocument, Inventory inventory) {
final RelationshipGraph relationshipGraph = new RelationshipGraph(inventory, MappingUtils.getAssetToArtifactMap(inventory));
for (RelationshipGraphEdge edge : relationshipGraph.getAllRelationships()) {
final List toNodes = edge.getToNodes().stream()
.map(RelationshipGraphNode::getId)
.collect(Collectors.toList());
SpdxApiFacade.addRelationshipsToDocument(spdxDocument,
edge.getFromNode().getId(), toNodes, edge.getRelationshipType());
}
}
// TODO: rework this method. a lot of its edge cases should be simplified either now or soon.
// e.g. no more fake tmd objects, separate handling of unknown "license names" etc.
protected void addReferencedLicenses(SpdxDocument spdxDocument, Collection referencedLicenses)
throws InvalidSPDXAnalysisException {
Map identifierToTextMap =
LicenseProcessor.getReferencedLicenseText(referencedLicenses, licenseTextProvider, normalizationMetaData);
if (identifierToTextMap != null) {
for (Map.Entry entry : identifierToTextMap.entrySet()) {
if (entry.getValue() == null) {
SpdxApiFacade.createExtractedLicenseInfo(entry.getKey(), null, unspecificIdComment, spdxDocument);
} else {
SpdxApiFacade.createExtractedLicenseInfo(entry.getKey(), entry.getValue(), null, spdxDocument);
}
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy