com.metaeffekt.artifact.analysis.bom.spdx.mapper.SpdxPackageMapper Maven / Gradle / Ivy
package com.metaeffekt.artifact.analysis.bom.spdx.mapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.metaeffekt.artifact.analysis.bom.BomConstants;
import com.metaeffekt.artifact.analysis.bom.LicenseProcessor;
import com.metaeffekt.artifact.analysis.bom.spdx.DocumentSpec;
import com.metaeffekt.artifact.analysis.bom.spdx.LicenseStringConverter;
import com.metaeffekt.artifact.analysis.bom.spdx.facade.SpdxApiFacade;
import com.metaeffekt.artifact.terms.model.TermsMetaData;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.metaeffekt.common.notice.model.ComponentDefinition;
import org.metaeffekt.common.notice.model.NoticeParameters;
import org.metaeffekt.core.inventory.processor.model.*;
import org.spdx.library.InvalidSPDXAnalysisException;
import org.spdx.library.SpdxConstants;
import org.spdx.library.model.*;
import org.spdx.library.model.enumerations.AnnotationType;
import org.spdx.library.model.enumerations.ChecksumAlgorithm;
import org.spdx.library.model.enumerations.Purpose;
import org.spdx.library.model.enumerations.ReferenceCategory;
import org.spdx.library.model.license.SpdxNoAssertionLicense;
import org.spdx.storage.IModelStore;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.*;
import static com.metaeffekt.artifact.analysis.metascan.Constants.KEY_BINARY_ARTIFACT_HASH_SHA256;
import static org.metaeffekt.core.inventory.processor.model.Constants.*;
@Slf4j
public class SpdxPackageMapper {
private final SpdxDocument spdxDocument;
private final DocumentSpec documentSpec;
private final LicenseStringConverter licenseStringConverter;
private final Map customLicenseMappings;
@Getter
private final HashSet referencedLicenses;
private final HashSet writtenAttributes;
private final Set approvedAttributes;
public SpdxPackageMapper(DocumentSpec documentSpec, Set approvedAttributes, SpdxDocument spdxDocument,
LicenseStringConverter licenseStringConverter, Map customLicenseMappings) {
this.spdxDocument = spdxDocument;
this.documentSpec = documentSpec;
this.licenseStringConverter = licenseStringConverter;
this.writtenAttributes = new HashSet<>();
this.referencedLicenses = new HashSet<>();
this.approvedAttributes = approvedAttributes;
this.customLicenseMappings = customLicenseMappings;
}
public SpdxPackage map(AbstractModelBase modelBase) throws InvalidSPDXAnalysisException {
SpdxPackage.SpdxPackageBuilder spdxPackageBuilder = null;
if (modelBase instanceof Artifact) {
spdxPackageBuilder = spdxDocument.createPackage(
spdxDocument.getModelStore().getNextId(IModelStore.IdType.SpdxId, documentSpec.getDocumentId()),
modelBase.get(Constants.KEY_ID),
new SpdxNoAssertionLicense(),
SpdxConstants.NOASSERTION_VALUE,
new SpdxNoAssertionLicense());
} else if (modelBase instanceof AssetMetaData) {
spdxPackageBuilder = spdxDocument.createPackage(
spdxDocument.getModelStore().getNextId(IModelStore.IdType.SpdxId, documentSpec.getDocumentId()),
modelBase.get(Constants.KEY_ASSET_ID),
new SpdxNoAssertionLicense(),
SpdxConstants.NOASSERTION_VALUE,
new SpdxNoAssertionLicense());
}
mapAdditionals(modelBase, spdxPackageBuilder);
mapPrimaryPackagePurpose(modelBase, spdxPackageBuilder);
mapUrls(modelBase, spdxPackageBuilder);
mapFileTypeInformation(modelBase, spdxPackageBuilder);
mapVersionInfo(modelBase, spdxPackageBuilder);
mapChecksums(modelBase, spdxPackageBuilder);
mapAuthors(modelBase, spdxPackageBuilder);
mapOverflowKeyValueAnnotation(modelBase, spdxPackageBuilder, approvedAttributes);
mapExternalRefs(modelBase, spdxPackageBuilder);
mapNotesAnnotation(modelBase, spdxPackageBuilder);
writtenAttributes.removeAll(writtenAttributes);
if (spdxPackageBuilder != null) {
SpdxPackage built = spdxPackageBuilder.build();
mapLicensesAndCopyrights(modelBase, referencedLicenses, built);
return built;
} else {
throw new RuntimeException("AbstractModelBase: " + modelBase + " is neither an asset or artifact.");
}
}
private void mapAdditionals(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
if (StringUtils.isNotBlank(modelBase.get("Deployable URL"))) {
spdxPackageBuilder.setDownloadLocation(modelBase.get("Deployable URL"));
} else {
spdxPackageBuilder.setDownloadLocation(SpdxConstants.NOASSERTION_VALUE);
}
if (StringUtils.isNotEmpty(modelBase.get(KEY_COMPONENT_SOURCE_TYPE))) {
spdxPackageBuilder.setSourceInfo(modelBase.get(KEY_COMPONENT_SOURCE_TYPE));
}
if (StringUtils.isNotEmpty(modelBase.get(Artifact.Attribute.PROJECTS))) {
spdxPackageBuilder.setPackageFileName(SpdxApiFacade.getFilePathFromProjectLocations(
modelBase.get(Artifact.Attribute.PROJECTS)));
}
if (StringUtils.isNotBlank(modelBase.get("Primary"))) {
spdxPackageBuilder.setComment("Primary");
}
}
private void mapPrimaryPackagePurpose(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
if (StringUtils.isBlank(modelBase.get(Constants.KEY_TYPE))) {
return;
}
ObjectMapper objectMapper = new ObjectMapper();
try {
InputStream inputStream = SpdxPackageMapper.class.getClassLoader().getResourceAsStream("sbom/typeMap.json");
JsonNode jsonNode = objectMapper.readTree(inputStream);
Iterator> iterator = jsonNode.fields();
while (iterator.hasNext()) {
Map.Entry entry = iterator.next();
List valuesList = new ArrayList<>();
if (entry.getValue().isArray()) {
entry.getValue().forEach(e -> valuesList.add(e.asText()));
}
if (valuesList.contains(modelBase.get(Constants.KEY_TYPE))) {
try {
spdxPackageBuilder.setPrimaryPurpose(Purpose.valueOf(entry.getKey()));
} catch (IllegalArgumentException e) {
spdxPackageBuilder.setPrimaryPurpose(Purpose.OTHER);
}
writtenAttributes.add(Constants.KEY_TYPE);
}
// TODO: Some Assets do not have their respective Types mapped correctly, for example composite ->
// container is probably not correct. (Check json file)
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private void mapUrls(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
final String url = modelBase.get(Constants.KEY_URL);
final String orgUrl = modelBase.get(KEY_ORGANIZATION_URL);
// FIXME: Transfer to inventory
final String sourceCodeUrl = modelBase.get("Source Code URL");
if (StringUtils.isNotEmpty(url)) {
try {
new URL(url);
spdxPackageBuilder.setHomepage(url);
writtenAttributes.add(Constants.KEY_URL);
} catch (MalformedURLException e) {
log.warn("The url [{}] contained in [{}], is not a valid URL.", url, getModelBaseId(modelBase));
}
} else if (StringUtils.isNotBlank(orgUrl)) {
try {
new URL(orgUrl);
spdxPackageBuilder.setHomepage(orgUrl);
writtenAttributes.add(Constants.KEY_ORGANIZATION_URL);
} catch (MalformedURLException e) {
log.warn("The url [{}] contained in [{}], is not a valid URL.", orgUrl, getModelBaseId(modelBase));
}
} else {
spdxPackageBuilder.setHomepage(SpdxConstants.NOASSERTION_VALUE);
}
// Fill supplier info
if (StringUtils.isNotBlank(sourceCodeUrl)) {
spdxPackageBuilder.setSourceInfo(sourceCodeUrl);
} else {
spdxPackageBuilder.setSourceInfo(SpdxConstants.NOASSERTION_VALUE);
}
}
private void mapFileTypeInformation(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
final String archive = modelBase.get(Constants.KEY_ARCHIVE);
final String structured = modelBase.get(Constants.KEY_STRUCTURED);
final String executable = modelBase.get(Constants.KEY_EXECUTABLE);
String comment = "";
if (StringUtils.isNotBlank(archive)) {
comment += "Archive: " + archive + "\n";
}
if (StringUtils.isNotBlank(structured)) {
comment += "Structured: " + structured + "\n";
}
if (StringUtils.isNotBlank(executable)) {
comment += "Executable: " + executable + "\n";
}
if (StringUtils.isNotBlank(comment)) {
spdxPackageBuilder.setComment(comment);
}
}
private void mapVersionInfo(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
if (StringUtils.isBlank(modelBase.get(Constants.KEY_VERSION))) {
spdxPackageBuilder.setVersionInfo(SpdxConstants.NOASSERTION_VALUE);
return;
}
spdxPackageBuilder.setVersionInfo(modelBase.get(Constants.KEY_VERSION));
writtenAttributes.add(Artifact.Attribute.VERSION.getKey());
}
private void mapChecksums(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder)
throws InvalidSPDXAnalysisException {
List checksums = new ArrayList<>();
Map> checksumMappings = new HashMap<>();
checksumMappings.put(ChecksumAlgorithm.MD5, Arrays.asList("Checksum", "Checksum (MD5)"));
checksumMappings.put(ChecksumAlgorithm.SHA1, Arrays.asList("Checksum (SHA-1)", KEY_HASH_SHA1));
checksumMappings.put(ChecksumAlgorithm.SHA256, Arrays.asList("Checksum (SHA-256)", "Hash (SHA-256)", KEY_BINARY_ARTIFACT_HASH_SHA256));
checksumMappings.put(ChecksumAlgorithm.SHA512, Arrays.asList("Checksum (SHA-512)", "HASH (SHA-512)"));
for (Map.Entry> entry : checksumMappings.entrySet()) {
ChecksumAlgorithm algorithm = entry.getKey();
List possibleKeys = entry.getValue();
String checksumValue = getFirstNonBlankValue(modelBase, possibleKeys);
if (StringUtils.isNotBlank(checksumValue)) {
checksums.add(spdxDocument.createChecksum(algorithm, checksumValue));
writtenAttributes.addAll(possibleKeys);
}
}
if (!checksums.isEmpty()) {
spdxPackageBuilder.setChecksums(checksums);
}
String digest = modelBase.get("Digest");
// TODO: varied hash support and conversion to spdx (as of writing, only sha256 will work)
if (StringUtils.isNotBlank(digest) && digest.startsWith("sha256:")) {
final Checksum sha256sum = spdxDocument.createChecksum(ChecksumAlgorithm.SHA256, digest.substring(7));
checksums.add(sha256sum);
spdxPackageBuilder.setChecksums(checksums);
}
}
private void mapAuthors(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder) {
String authorsKey = "Extracted Authors (ScanCode)";
String authors = modelBase.get(authorsKey);
if (StringUtils.isNotBlank(authors)) {
spdxPackageBuilder.setAttributionText(Arrays.asList(authors.split("\\|\n")));
}
writtenAttributes.add(authorsKey);
if (StringUtils.isNotBlank(modelBase.get("Supplier"))) {
spdxPackageBuilder.setSupplier("Organization: " + modelBase.get("Supplier"));
}
}
private void mapOverflowKeyValueAnnotation(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder, Collection approved)
throws InvalidSPDXAnalysisException {
Map validOverflowContent = new HashMap<>();
if (approved != null) {
// iterate keys that were approved for inclusion by key-value map
for (String key : approved) {
if (writtenAttributes.contains(key)) {
continue;
}
// get the value for this particular attribute
String value = modelBase.get(key);
// only add the attribute if the value is not empty
if (StringUtils.isNotBlank(value)) {
validOverflowContent.put(key, value);
}
// mark this attribute as "written"
writtenAttributes.add(key);
}
}
if (!validOverflowContent.isEmpty()) {
spdxPackageBuilder
.addAnnotation(MappingUtils.getOverflowAnnotation(spdxDocument, documentSpec, validOverflowContent));
}
}
private void mapExternalRefs(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder)
throws InvalidSPDXAnalysisException {
final String purl = modelBase.get(KEY_PURL);
if (StringUtils.isNotBlank(purl)) {
final ReferenceType referenceType =
new ReferenceType(new ReferenceType(SpdxConstants.SPDX_LISTED_REFERENCE_TYPES_PREFIX + "purl"));
final ExternalRef externalRef =
spdxDocument.createExternalRef(ReferenceCategory.PACKAGE_MANAGER, referenceType, purl, null);
spdxPackageBuilder.addExternalRef(externalRef);
}
if (StringUtils.isNotBlank(modelBase.get("Repository"))) {
spdxPackageBuilder.addExternalRef(
spdxDocument.createExternalRef(
ReferenceCategory.OTHER,
new ReferenceType(SpdxConstants.SPDX_LISTED_REFERENCE_TYPES_PREFIX + "url"),
modelBase.get("Repository"),
null
)
);
}
}
private void mapNotesAnnotation(AbstractModelBase modelBase, SpdxPackage.SpdxPackageBuilder spdxPackageBuilder)
throws InvalidSPDXAnalysisException {
NoticeParameters noticeParameters = LicenseProcessor.readNoticeParameters(modelBase);
if (noticeParameters == null) {
return;
}
String notesAnnotationText = buildNotesAnnotation(noticeParameters);
String annotator;
if (StringUtils.isNotBlank(documentSpec.getTool())) {
annotator = "Tool: " + documentSpec.getTool();
} else {
throw new RuntimeException("No tool set in DocumentSpec for this document. This is mandatory.");
}
if (StringUtils.isNotBlank(notesAnnotationText)) {
spdxPackageBuilder
.addAnnotation(spdxDocument.createAnnotation(annotator,
// storing additional data, so type is OTHER
AnnotationType.OTHER,
new SimpleDateFormat(SpdxConstants.SPDX_DATE_FORMAT).format(new Date()),
notesAnnotationText));
}
}
private String buildNotesAnnotation(NoticeParameters noticeParam) {
StringJoiner annotationTextsJoiner = new StringJoiner("\n\n");
ComponentDefinition component = noticeParam.getComponent();
if (component != null && StringUtils.isNotBlank(component.getNote()) && StringUtils.isNotBlank(component.getNote())) {
annotationTextsJoiner.add(
"The Component: " + component.getName() + " has notes: " + component.getNote());
}
if (noticeParam.getSubcomponents() != null) {
for (ComponentDefinition subcomp : noticeParam.getSubcomponents()) {
if (subcomp.getNote() != null && StringUtils.isNotBlank(subcomp.getName()) && StringUtils.isNotBlank(subcomp.getNote())) {
annotationTextsJoiner.add(
"The Component: " + subcomp.getName() + " has notes: " + subcomp.getNote());
}
}
}
return annotationTextsJoiner.toString();
}
private void mapLicensesAndCopyrights(AbstractModelBase modelBase, Set referencedLicenses, SpdxPackage built) throws InvalidSPDXAnalysisException {
final NoticeParameters noticeParameters = LicenseProcessor.readNoticeParameters(modelBase);
built.setLicenseConcluded(SpdxLicenseMapper.deriveConcludedLicense(noticeParameters, referencedLicenses,
spdxDocument, licenseStringConverter, customLicenseMappings));
built.setLicenseDeclared(SpdxLicenseMapper.deriveDeclaredLicense(noticeParameters, referencedLicenses,
spdxDocument, licenseStringConverter, customLicenseMappings));
built.setCopyrightText(SpdxLicenseMapper.deriveCopyrightText(noticeParameters));
// list all attributes that are "regarded" consumed
writtenAttributes.add(Artifact.Attribute.LICENSE.getKey());
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_DERIVED_NOTICE_PARAMETER);
// NOTE: attrWritten once meant "all relevant data has been consumed".
// notice param contains much more, but we mark it written anyway and consider it "good enough" for now.
writtenAttributes.add("Inherited Notice Parameter");
writtenAttributes.add("Inherited License");
// do not include licenses
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_DERIVED_LICENSES);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_BINARY_ARTIFACT_DERIVED_LICENSES);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_DESCRIPTOR_DERIVED_LICENSES);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_SOURCE_ARCHIVE_DERIVED_LICENSES);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_SOURCE_ARTIFACT_DERIVED_LICENSES);
// do not include markers
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_DERIVED_MARKERS);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_BINARY_ARTIFACT_DERIVED_MARKERS);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_SOURCE_ARTIFACT_DERIVED_MARKERS);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_SOURCE_ARCHIVE_DERIVED_MARKERS);
writtenAttributes.add(com.metaeffekt.artifact.analysis.metascan.Constants.KEY_DESCRIPTOR_DERIVED_MARKERS);
}
private String getFirstNonBlankValue(AbstractModelBase modelBase, List keys) {
for (String key : keys) {
String value = modelBase.get(key);
if (StringUtils.isNotBlank(value)) {
return value;
}
}
return null;
}
private String getModelBaseId(AbstractModelBase modelBase) {
if (modelBase instanceof Artifact) {
return modelBase.get(KEY_ID);
} else {
return modelBase.get(KEY_ASSET_ID);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy