com.metaeffekt.artifact.analysis.spdxbom.mapper.AbstractArtifactMapper Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.spdxbom.mapper;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.metaeffekt.artifact.analysis.metascan.Constants;
import com.metaeffekt.artifact.analysis.spdxbom.LicenseStringConverter;
import com.metaeffekt.artifact.analysis.spdxbom.config.StoredConfig;
import com.metaeffekt.artifact.analysis.spdxbom.context.SpdxDocumentContext;
import com.metaeffekt.artifact.analysis.spdxbom.facade.SpdxApiFacade;
import com.metaeffekt.artifact.analysis.spdxbom.mapper.exception.PurlGenerationFailedException;
import com.metaeffekt.artifact.analysis.utils.InventoryUtils;
import com.metaeffekt.artifact.terms.model.NormalizationMetaData;
import com.metaeffekt.artifact.terms.model.TermsMetaData;
import org.apache.commons.lang3.StringUtils;
import org.metaeffekt.common.notice.model.ComponentDefinition;
import org.metaeffekt.common.notice.model.NoticeParameters;
import org.metaeffekt.common.notice.model.UnassignedInformation;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spdx.library.InvalidSPDXAnalysisException;
import org.spdx.library.SpdxConstants;
import org.spdx.library.model.SpdxPackage;
import org.spdx.library.model.license.AnyLicenseInfo;
import org.spdx.library.model.license.InvalidLicenseStringException;
import java.util.*;
import java.util.stream.Collectors;
public abstract class AbstractArtifactMapper implements ArtifactMapper {
private static final Logger LOG = LoggerFactory.getLogger(AbstractArtifactMapper.class);
protected final LicenseStringConverter licenseStringConverter;
/**
* These attributes are approved for inclusion by ways of the key-value map. No mapping is required for these,
* as the key-value map is considered a sufficiently good representation.
*/
protected final Set keyValueApprovedAttributes;
public AbstractArtifactMapper(Map licenseStringAssessments,
Set keyValueApprovedAttributes,
NormalizationMetaData normalizationMetaData) {
// initialize license string converter
licenseStringConverter = new LicenseStringConverter(normalizationMetaData, licenseStringAssessments);
this.keyValueApprovedAttributes = keyValueApprovedAttributes == null ?
Collections.emptySet() :
Collections.unmodifiableSet(new HashSet<>(keyValueApprovedAttributes));
}
protected Map getAttributes(Artifact artifact) {
// pull all attributes from artifact
HashMap artifactAttributes = new HashMap<>();
for (String key : artifact.getAttributes()) {
artifactAttributes.put(key, artifact.get(key));
}
return artifactAttributes;
}
@Override
public StoredConfig getDefaultConfig() {
return null;
}
// FIXME: we need in analogy to ArtifactResolver a generic converter to produce purls
protected String getMavenPurlString(Artifact artifact, Set written) {
String purl;
try {
// before accessing the getArtifactId the artifactId must be computed
artifact.deriveArtifactId();
purl = new PackageURL(PackageURL.StandardTypes.MAVEN,
artifact.getGroupId(),
artifact.getArtifactId(), // this must be the artifactId not the artifact.id
artifact.getVersion(),
null,
null).toString();
} catch (MalformedPackageURLException e) {
throw new PurlGenerationFailedException(e);
}
written.add(Artifact.Attribute.GROUPID.getKey());
written.add(Artifact.Attribute.ID.getKey());
written.add(Artifact.Attribute.VERSION.getKey());
return purl;
}
protected String getNodejsPurlString(Artifact artifact, Set written) {
String purl;
try {
purl = new PackageURL(PackageURL.StandardTypes.NPM,
null,
// FIXME: safe removal of version part
artifact.getId().replace("-" + artifact.getVersion(), ""),
artifact.getVersion(),
null,
null).toString();
} catch (MalformedPackageURLException e) {
throw new PurlGenerationFailedException(e);
}
written.add(Artifact.Attribute.ID.getKey());
written.add(Artifact.Attribute.VERSION.getKey());
return purl;
}
protected String getGenericPurlString(Artifact artifact, Set written) {
String purl;
try {
purl = new PackageURL(PackageURL.StandardTypes.GENERIC,
null,
// FIXME: safe removal of version part
artifact.getId().replace("-" + artifact.getVersion(), ""),
artifact.getVersion(),
null,
null).toString();
} catch (MalformedPackageURLException e) {
throw new PurlGenerationFailedException(e);
}
written.add(Artifact.Attribute.ID.getKey());
written.add(Artifact.Attribute.VERSION.getKey());
return purl;
}
protected NoticeParameters readNoticeParameters(Artifact artifact) {
// get notice parameter from the artifact
String noticeParametersString = artifact.get(Constants.KEY_NOTICE_PARAMETER);
// FIXME-KKL: resolve issue
// the inherited notice parameter is curated; (could be also an exact match; since no inventory filtering is applied yet)
if (StringUtils.isBlank(noticeParametersString)) {
noticeParametersString = artifact.get("Inherited Notice Parameter");
}
// the derived notice parameter is automatically determined
if (StringUtils.isBlank(noticeParametersString)) {
noticeParametersString = artifact.get(Constants.KEY_DERIVED_NOTICE_PARAMETER);
}
if (StringUtils.isNotBlank(noticeParametersString)) {
// convert notice parameter string into object
try {
return NoticeParameters.readYaml(noticeParametersString);
} catch (RuntimeException e) {
LOG.warn("Unable to parse notice parameter:\n[{}]", noticeParametersString);
}
}
// fallback to other provided licenses/copyrights
final String associatedLicenses = deriveAssociatedLicenses(artifact);
final List copyrights = extractCopyrights(artifact);
if (StringUtils.isNotEmpty(associatedLicenses)) {
final ComponentDefinition componentDefinition = new ComponentDefinition();
componentDefinition.setAssociatedLicenses(InventoryUtils.tokenizeLicense(associatedLicenses, true, true));
final NoticeParameters noticeParameters = new NoticeParameters();
noticeParameters.setComponent(componentDefinition);
UnassignedInformation unassignedInformation = new UnassignedInformation();
unassignedInformation.setCopyrights(copyrights);
noticeParameters.setUnassignedInformation(unassignedInformation);
return noticeParameters;
} else if (!copyrights.isEmpty()) {
final NoticeParameters noticeParameters = new NoticeParameters();
UnassignedInformation unassignedInformation = new UnassignedInformation();
unassignedInformation.setCopyrights(copyrights);
noticeParameters.setUnassignedInformation(unassignedInformation);
return noticeParameters;
}
// unable to proceed with empty licenses / notice parameter
return null;
}
protected AnyLicenseInfo deriveConcludedLicense(NoticeParameters noticeParameters, Set referencedLicenses, SpdxDocumentContext spdxDocumentContext)
throws InvalidLicenseStringException {
if (noticeParameters != null) {
// collect effective licenses using the notice engine
List effectiveLicenses = aggregateEffectiveLicenses(noticeParameters);
if (effectiveLicenses.isEmpty() || effectiveLicenses.stream().allMatch(StringUtils::isBlank)) {
// no effective licenses could be extracted from the notice parameter
return SpdxApiFacade.parseLicenseString(SpdxConstants.NOASSERTION_VALUE, spdxDocumentContext);
}
return convertToLicenseInfo(spdxDocumentContext, referencedLicenses, effectiveLicenses);
}
return SpdxApiFacade.parseLicenseString(SpdxConstants.NOASSERTION_VALUE, spdxDocumentContext);
}
public List aggregateEffectiveLicenses(NoticeParameters noticeParameters) {
final Set aggregated = new HashSet<>();
if (noticeParameters.getComponent() != null) {
aggregated.addAll(aggregateEffectiveLicenses(noticeParameters.getComponent()));
}
if (noticeParameters.getSubcomponents() != null) {
for (ComponentDefinition componentDefinition : noticeParameters.getSubcomponents()) {
aggregated.addAll(aggregateEffectiveLicenses(componentDefinition));
}
}
return aggregated.stream().sorted(String::compareToIgnoreCase).collect(Collectors.toList());
}
private Collection aggregateEffectiveLicenses(ComponentDefinition component) {
if (component != null) {
if (component.getEffectiveLicenses() == null || component.getEffectiveLicenses().isEmpty()) {
final List associatedLicenses = component.getAssociatedLicenses();
return InventoryUtils.tokenizeLicense(InventoryUtils.deriveEffectiveLicenses(associatedLicenses), true, false);
} else {
return component.getEffectiveLicenses();
}
}
return Collections.emptyList();
}
private AnyLicenseInfo convertToLicenseInfo(SpdxDocumentContext spdxDocumentContext, Set referencedLicenses, List effectiveLicenses) throws InvalidLicenseStringException {
// create the spdx expression
List results = effectiveLicenses.stream()
.map(licenseStringConverter::licenseStringToSpdxExpression)
.collect(Collectors.toList());
final StringJoiner joiner = new StringJoiner(") AND (", "(", ")");
for (LicenseStringConverter.ToSpdxResult converterResult : results) {
// need to process intermediate results to collect
if (StringUtils.isNotBlank(converterResult.getExpression())) {
referencedLicenses.addAll(converterResult.getReferencedLicenses());
// FIXME: add support for "exception" TMD type.
// often we only find a license and some exception.
// matching them together can only work with tmd expressions, which now have "spdxExpression" fields.
// otherwise, exceptions are currently rejected by specialized checks in conversion logic and
// converted to licenseRefs so as to not crash spdx.
// see also https://github.com/spdx/spdx-spec/issues/153 but this may not help us with the core issue.
joiner.add(converterResult.getExpression());
}
}
return SpdxApiFacade.parseLicenseString(joiner.toString(), spdxDocumentContext);
}
private String deriveCopyrightText(NoticeParameters noticeParameters) {
if (noticeParameters != null) {
final List copyrightsList = noticeParameters.aggregateCopyrights();
if (!copyrightsList.isEmpty()) {
String copyrights = StringUtils.join(copyrightsList, "\n");
if (StringUtils.isNotBlank(copyrights)) {
return copyrights;
}
}
}
return SpdxConstants.NOASSERTION_VALUE;
}
protected AnyLicenseInfo deriveDeclaredLicense(NoticeParameters noticeParameters,
Set referencedLicenses, SpdxDocumentContext spdxDocumentContext) throws InvalidLicenseStringException {
if (noticeParameters != null) {
// collect associated licenses using the notice parameter
List associatedLicenses = noticeParameters.aggregateAssociatedLicenses();
if (associatedLicenses.isEmpty() || associatedLicenses.stream().allMatch(StringUtils::isBlank)) {
// no effective licenses could be extracted from the notice parameter
return SpdxApiFacade.parseLicenseString(SpdxConstants.NOASSERTION_VALUE, spdxDocumentContext);
}
return convertToLicenseInfo(spdxDocumentContext, referencedLicenses, associatedLicenses);
}
return SpdxApiFacade.parseLicenseString(SpdxConstants.NOASSERTION_VALUE, spdxDocumentContext);
}
private String deriveAssociatedLicenses(Artifact artifact) {
// getLicenses delivers the curated associated licenses
String associatedLicenses = artifact.getLicense();
// FIXME: currently a fallback strategy
// FIXME: merge binary and source license lists (via set union) to create the most complete possible list
// FIXME: consider approach via notice parameter for each level and merging the component pattern in the end
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get("Binary Artifact - Derived Licenses");
}
// fallback to detected licenses on binary
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get(Constants.KEY_DERIVED_LICENSES);
}
// fallback to source artifact
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get("Source Artifact - Derived Licenses");
}
// fallback on source archive
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get("Source Archive - Derived Licenses");
}
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get("Descriptor - Derived Licenses");
}
// fallback to package specified licenses (mapped)
if (StringUtils.isEmpty(associatedLicenses)) {
associatedLicenses = artifact.get("Specified Package License (mapped)");
}
return associatedLicenses;
}
protected void deriveLicensesAndCopyrights(Artifact artifact, SpdxDocumentContext spdxDocumentContext,
Set attributesWritten, Set referencedLicenses, SpdxPackage built) throws InvalidSPDXAnalysisException {
final NoticeParameters noticeParameters = readNoticeParameters(artifact);
built.setLicenseConcluded(
deriveConcludedLicense(noticeParameters, referencedLicenses, spdxDocumentContext)
);
built.setLicenseDeclared(
deriveDeclaredLicense(noticeParameters, referencedLicenses, spdxDocumentContext)
);
built.setCopyrightText(
deriveCopyrightText(noticeParameters)
);
// list all attributes that are "regarded" consumed
attributesWritten.add(Artifact.Attribute.LICENSE.getKey());
attributesWritten.add(Constants.KEY_DERIVED_NOTICE_PARAMETER);
// NOTE: attrWritten once meant "all relevant data has been consumed".
// notice param contains much more, but we mark it written anyway and consider it "good enough" for now.
attributesWritten.add("Inherited Notice Parameter");
attributesWritten.add("Inherited License");
// do not include licenses
attributesWritten.add(Constants.KEY_DERIVED_LICENSES);
attributesWritten.add(Constants.KEY_BINARY_ARTIFACT_DERIVED_LICENSES);
attributesWritten.add(Constants.KEY_DESCRIPTOR_DERIVED_LICENSES);
attributesWritten.add(Constants.KEY_SOURCE_ARCHIVE_DERIVED_LICENSES);
attributesWritten.add(Constants.KEY_SOURCE_ARTIFACT_DERIVED_LICENSES);
// do not include markers
attributesWritten.add(Constants.KEY_DERIVED_MARKERS);
attributesWritten.add(Constants.KEY_BINARY_ARTIFACT_DERIVED_MARKERS);
attributesWritten.add(Constants.KEY_SOURCE_ARTIFACT_DERIVED_MARKERS);
attributesWritten.add(Constants.KEY_SOURCE_ARCHIVE_DERIVED_MARKERS);
attributesWritten.add(Constants.KEY_DESCRIPTOR_DERIVED_MARKERS);
}
/**
* Attempts to extract copyright information from the artifact or returns "NOASSERTION".
*
* @return text containing copyright information or "NOASSERTION"
*/
protected List extractCopyrights(Artifact artifact) {
String copyrights = artifact.get(Constants.KEY_EXTRACTED_COPYRIGHTS_SCANCODE);
// FIXME: currently a fallback strategy
if (StringUtils.isEmpty(copyrights)) {
copyrights = artifact.get("Binary Artifact - Extracted Copyrights (ScanCode)");
}
if (StringUtils.isEmpty(copyrights)) {
copyrights = artifact.get("Source Artifact - Extracted Copyrights (ScanCode)");
}
if (StringUtils.isEmpty(copyrights)) {
copyrights = artifact.get("Source Archive - Extracted Copyrights (ScanCode)");
}
if (StringUtils.isEmpty(copyrights)) {
copyrights = artifact.get("Descriptor - Extracted Copyrights (ScanCode)");
}
if (!StringUtils.isEmpty(copyrights)) {
return InventoryUtils.filterCopyrightsAndAuthorsList(copyrights);
}
return Collections.emptyList();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy