org.metaeffekt.artifact.resolver.deb.DebArtifactResolver Maven / Gradle / Ivy
package org.metaeffekt.artifact.resolver.deb;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.metaeffekt.artifact.resolver.ArtifactResolver;
import org.metaeffekt.artifact.resolver.ResolverResult;
import org.metaeffekt.artifact.resolver.deb.index.packages.DebPackagesIndex;
import org.metaeffekt.artifact.resolver.deb.index.packages.parser.DebianPackagesEntry;
import org.metaeffekt.artifact.resolver.deb.index.packages.parser.PackagesEntryParsingException;
import org.metaeffekt.artifact.resolver.deb.ubuntu.UbuntuLaunchpadAdapter;
import org.metaeffekt.artifact.resolver.deb.ubuntu.UbuntuPoolAdapter;
import org.metaeffekt.artifact.resolver.download.WebAccess;
import org.metaeffekt.artifact.resolver.generic.utils.MarkerUtils;
import org.metaeffekt.artifact.resolver.model.*;
import org.metaeffekt.core.inventory.processor.model.Artifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Objects;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
public class DebArtifactResolver implements ArtifactResolver {
private static final Logger log = LoggerFactory.getLogger(DebArtifactResolver.class);
private static final Pattern controlFilePattern = Pattern.compile("[./]*control$");
private final UbuntuLaunchpadAdapter ubuntuLaunchpadAdapter;
private final UbuntuPoolAdapter ubuntuPoolAdapter;
private final DebPackagesIndex ubuntuIndex;
private final DebArtifactResolverConfig resolverConfig;
public DebArtifactResolver(DownloadLocation downloadLocation,
WebAccess webAccess,
DebArtifactResolverConfig resolverConfig) {
this.ubuntuLaunchpadAdapter = new UbuntuLaunchpadAdapter(downloadLocation, webAccess);
this.ubuntuIndex = new DebPackagesIndex(downloadLocation, webAccess, resolverConfig.getUbuntuIndexConfig());
this.resolverConfig = resolverConfig;
this.ubuntuPoolAdapter = new UbuntuPoolAdapter(downloadLocation,
webAccess,
resolverConfig.getUbuntuPoolAdapterConfig());
}
@Override
public ArtifactPartResolvers collectResolvers(Artifact artifact) {
final Collection parts = new HashSet<>();
try {
parts.addAll(getArtifactResolverPartsForPurl(artifact));
} catch (Exception e) {
log.warn("Unexpected exception while trying to get resolver for [{}]: [{}].",
artifact,
ExceptionUtils.getStackTrace(e));
}
return new ArtifactPartResolvers(parts);
}
/**
* Gets a source reference from parsing a deb's control file.
*
* We also perform some basic sanity checks with the previous binary reference and log on failure.
*
* @param debFile the deb file that we wish to parse
* @param binaryRef a binary reference to do checks against (version, maybe arch)
* @param diagnostic diagnostic string, reference to the artifact we are currently resolving
* @return the derived source reference
*/
private DebArtifactReference getSourceRefFromDebFile(File debFile,
DebArtifactReference binaryRef,
String diagnostic) {
try (final InputStream inputStream = Files.newInputStream(debFile.toPath());
final ArArchiveInputStream arInputStream = new ArArchiveInputStream(inputStream)) {
ArArchiveEntry arEntry;
while (true) {
arEntry = arInputStream.getNextEntry();
if (arEntry == null) {
break;
}
final InputStream decompInputStream;
if (arEntry.getName().equals("control.tar.gz")) {
decompInputStream = new GZIPInputStream(CloseShieldInputStream.wrap(arInputStream));
} else if (arEntry.getName().equals("control.tar.xz")) {
decompInputStream = new XZCompressorInputStream(CloseShieldInputStream.wrap(arInputStream));
} else {
continue;
}
try (final TarArchiveInputStream innerTarInputStream = new TarArchiveInputStream(decompInputStream)) {
TarArchiveEntry innerTarEntry;
while (true) {
innerTarEntry = innerTarInputStream.getNextEntry();
if (innerTarEntry == null) {
break;
}
if (!controlFilePattern.matcher(innerTarEntry.getName()).matches()) {
if (log.isTraceEnabled()) {
log.trace("Ignoring non-matching file [{}] inside control of [{}] for [{}].",
innerTarEntry.getName(),
debFile,
diagnostic);
}
continue;
}
// read directly and try to parse
final String controlFileContent = IOUtils.toString(innerTarInputStream, StandardCharsets.UTF_8);
DebianPackagesEntry parsedControl =
DebianPackagesEntry.parseFromSnippet(controlFileContent, diagnostic);
if (!Objects.equals(parsedControl.getVersion(), binaryRef.getVersion())) {
log.warn("Parsed control file from, binary ref mismatch field version: " +
"[{}] vs [{}] while resolving [{}].",
parsedControl.getVersion(),
binaryRef.getVersion(),
diagnostic);
}
if (!Objects.equals(parsedControl.getArchitecture(), binaryRef.getArchitecture())) {
log.warn("Parsed control file and binary ref mismatch in architecture field: " +
"[{}] vs [{}] while resolving [{}].",
parsedControl.getArchitecture(),
binaryRef.getArchitecture(),
diagnostic);
}
if (binaryRef.getNamespace() != DebArtifactReference.DebNamespace.UBUNTU) {
log.debug("Unexpected namespace [{}] (expected [{}]) in ref [{}] while resolving [{}].",
binaryRef.getNamespace().toString(),
DebArtifactReference.DebNamespace.UBUNTU,
binaryRef,
diagnostic);
}
final DebArtifactReference derivedSourceRef = new DebArtifactReference(
parsedControl.getSourcePackageName(),
parsedControl.getSourcePackageVersion(),
parsedControl.getArchitecture(),
binaryRef.getNamespace());
log.debug("Derived source ref [{}] from parsed [{}] from deb [{}] of [{}].",
derivedSourceRef,
parsedControl,
debFile.toPath(),
diagnostic);
return derivedSourceRef;
}
} catch (PackagesEntryParsingException e) {
log.warn("Failure to parse control file from deb [{}].",
debFile.toPath(),
e
);
throw new RuntimeException(e);
}
}
} catch (IOException e) {
log.warn("Exception while trying to read tar [{}] for [{}].", debFile.toPath(), diagnostic);
log.info("Invalidating marker after read failure for file [{}] while resolving [{}].",
debFile.toPath(),
diagnostic);
MarkerUtils.invalidateMarkerFor(debFile, diagnostic);
throw new RuntimeException(e);
}
log.debug("Failed derivation from deb [{}] for binary ref [{}] for [{}].",
debFile.toPath(),
binaryRef,
diagnostic);
return null;
}
/**
* Tries to convert binary package references to source package references.
* @param binaryPackageRef binary package reference to resolve
* @param diagnostic identifying the artifact we're currently resolving, used for logging
* @return returns a source package ref or null if resolving failed
*/
public DebArtifactReference getSourcePackageReference(DebArtifactReference binaryPackageRef, String diagnostic) {
final DebianPackagesEntry packagesEntry = ubuntuIndex.lookupNameVersionArch(binaryPackageRef.getName(),
binaryPackageRef.getVersion(),
binaryPackageRef.getArchitecture());
if (packagesEntry != null && packagesEntry.getSourcePackageName() != null) {
final DebArtifactReference ref = new DebArtifactReference(
packagesEntry.getSourcePackageName(),
packagesEntry.getSourcePackageVersion(),
packagesEntry.getArchitecture(),
binaryPackageRef.getNamespace());
log.trace("Using source ref [{}] derived from ubuntu 'Packages' file entry [{}] from binary ref [{}].",
ref, packagesEntry, binaryPackageRef);
return ref;
}
log.debug("Starting early download to get binary from ubuntu pool for binary ref [{}] for [{}].",
binaryPackageRef, diagnostic);
// attempt another lookup via binary from UbuntuPoolAdapter before falling back to direct
final ResolverResult binaryDebResolverResult = ubuntuPoolAdapter.getBinaryFromPool(binaryPackageRef, null);
final File binaryDeb = binaryDebResolverResult.getResolvedFile();
if (binaryDeb != null && binaryDeb.exists()) {
log.trace("Start reading binary deb [{}] for binary ref [{}] to get source ref for [{}].",
binaryDeb.toPath(), binaryPackageRef, diagnostic);
final DebArtifactReference sourceRef = getSourceRefFromDebFile(binaryDeb, binaryPackageRef, diagnostic);
log.debug("Using source ref [{}] derived from binary deb [{}] from binary ref [{}] of [{}].",
sourceRef, binaryDeb.getPath(), binaryPackageRef, diagnostic);
return sourceRef;
} else {
// could not resolve
log.debug("Could not resolve source reference for [{}] of [{}].", binaryPackageRef, diagnostic);
return null;
}
}
protected Collection getArtifactResolverParts(PackageURL purl, Artifact artifact) {
// create reference
if (!"deb".equals(purl.getType())) {
log.trace("Rejecting artifact [{}] by purl: purl not of type 'deb'", artifact.toString());
return Collections.emptyList();
}
final DebArtifactReference givenPackageRef = new DebArtifactReference(purl);
if (!givenPackageRef.isValid()) {
log.debug("The binary reference [{}] derived from parsed purl [{}] is invalid. Aborting.",
givenPackageRef,
purl);
return Collections.emptyList();
}
final Collection parts = new HashSet<>();
switch (givenPackageRef.getNamespace()) {
case UBUNTU:
final DebArtifactReference sourcePackageRef;
if ("source".equals(givenPackageRef.getArchitecture())) {
sourcePackageRef = givenPackageRef;
} else {
sourcePackageRef = getSourcePackageReference(givenPackageRef, artifact.toString());
}
parts.add(new ArtifactPartResolver(artifact, ArtifactPartType.BINARY_ARTIFACT,
() -> ubuntuPoolAdapter.getBinaryFromPool(givenPackageRef, sourcePackageRef),
rap -> enrich(rap))
);
if (sourcePackageRef != null) {
parts.add(new ArtifactPartResolver(artifact, ArtifactPartType.SOURCE_ARTIFACT,
() -> ubuntuLaunchpadAdapter.downloadSourceArtifact(sourcePackageRef),
rap -> enrich(rap))
);
} else {
if (resolverConfig != null && resolverConfig.isDesperateSourceResolve()) {
log.debug("Using desperate resolver using binary ref for sources of [{}]", artifact);
parts.add(new ArtifactPartResolver(artifact, ArtifactPartType.SOURCE_ARTIFACT,
() -> ubuntuLaunchpadAdapter.downloadSourceArtifact(givenPackageRef),
rap -> enrich(rap))
);
}
}
break;
case DEBIAN:
log.trace("Debian resolver not supported yet.");
break;
default:
log.debug("Type [{}] is not supported by this resolver.", givenPackageRef.getNamespace());
}
return parts;
}
public Collection getArtifactResolverPartsForPurl(Artifact artifact) {
final String purlString = artifact.get(Artifact.Attribute.PURL);
if (purlString == null) {
// refuse to look into the void
log.trace("Rejecting null purl from artifact [{}].", artifact);
return Collections.emptyList();
}
// read purl from artifact
final PackageURL purl;
try {
purl = new PackageURL(purlString);
} catch (MalformedPackageURLException e) {
log.info("Malformed purl trying to parse purl [{}] of artifact [{}].",
purlString,
artifact);
artifact.append(
"Errors",
String.format("[PURL [%s] malformed.]", purlString),
"\n"
);
return Collections.emptyList();
}
return getArtifactResolverParts(purl, artifact);
}
private Artifact enrich(ResolvedArtifactPart resolvedArtifactPart) {
final Artifact enrichedArtifact = new Artifact(resolvedArtifactPart.getOriginalArtifact());
final ArtifactPartType artifactPartType = resolvedArtifactPart.getArtifactPartType();
final File resolvedFile = resolvedArtifactPart.getResolvedFile();
if (resolvedFile != null) {
enrichedArtifact.set(artifactPartType.modulatePathAttribute(), resolvedFile.getAbsolutePath());
}
return enrichedArtifact;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy