org.metaeffekt.artifact.resolver.alpine.AlpineEnvironment Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.metaeffekt.artifact.resolver.alpine;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.client.Config;
import lombok.NonNull;
import org.apache.commons.io.file.PathUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.metaeffekt.artifact.resolver.ArtifactResolverConfig;
import org.metaeffekt.artifact.resolver.alpine.bodge.CurlDownloadReference;
import org.metaeffekt.artifact.resolver.alpine.bodge.StringPieceAssembler;
import org.metaeffekt.artifact.resolver.download.ProxyConfig;
import org.metaeffekt.artifact.resolver.manager.execenv.DownloadEnvironment;
import org.metaeffekt.artifact.resolver.manager.execenv.EnvironmentParameters;
import org.metaeffekt.artifact.resolver.manager.execenv.exception.EnvironmentInitializationFailure;
import org.metaeffekt.artifact.resolver.manager.execenv.exception.EnvironmentOperationFailure;
import org.metaeffekt.core.container.control.ExecutorUtils;
import org.metaeffekt.core.container.control.exception.CommandExecutionFailed;
import org.metaeffekt.core.container.control.kubernetesapi.KubernetesCommandExecutor;
import org.metaeffekt.core.container.control.kubernetesapi.KubernetesContainerCommandProcess;
import org.metaeffekt.core.container.control.kubernetesapi.NamespaceHandle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
/**
* An initialized and ready-to-run environment.
* Methods for this environment should run in an atomic manner, such that underlying pods can't be "yanked"
* while some stateful process is still in progress.
*/
public class AlpineEnvironment implements DownloadEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(AlpineEnvironment.class);
// TODO: make concurrent, maybe by only locking individual aports paths instead of synchronizing entry methods
/**
* Parameters used to init this environment.
*/
private final AlpineEnvironmentParameters params;
/**
* The underlying executor; interfaces with the pod / container etc.
*/
private final KubernetesCommandExecutor executor;
/**
* Maps known package names to their source dirs.
*
* This is useful since alpine has "subpackages" made from the same source package and build script directory,
* which differ in their name and content.
*
* This should be initialized once and then used from then on.
*/
private final Map pkgNameVerToAportsPath;
private boolean closed = false;
public AlpineEnvironment(Config kubeconfig,
@NonNull NamespaceHandle namespaceHandle,
@NonNull EnvironmentParameters params) throws Exception {
this(kubeconfig, namespaceHandle, (AlpineEnvironmentParameters) params);
}
public AlpineEnvironment(Config kubeconfig,
@NonNull NamespaceHandle namespaceHandle,
@NonNull AlpineEnvironmentParameters params) throws Exception {
this.params = params;
// initialize EnvVar list
final List envVars = new ArrayList<>();
// set proxy inside container if available
final ProxyConfig proxyConfig = params.getProxyConfig();
if (proxyConfig != null) {
final String proxyString = proxyConfig.getProxyString();
envVars.add(new EnvVar("HTTP_PROXY", proxyString, null));
envVars.add(new EnvVar("HTTPS_PROXY", proxyString, null));
envVars.add(new EnvVar("NO_PROXY", proxyConfig.getNonProxyHosts(), null));
}
this.executor = new KubernetesCommandExecutor(kubeconfig, namespaceHandle.getName(),
params.getImageIdentifier(), envVars);
// install required packages. SDK may include all needed packages, but also includes unneeded gcc.
// runInitCommand(executor, "apk", "add", "alpine-sdk");
runInitCommand(executor, "apk", "add", "git", "coreutils", "curl", "abuild", "tar");
// aports (build scripts) repo initialization
runInitCommand(
executor,
"git",
"clone",
// shallow clone to save time and resources
"--depth", "1",
"--branch",
params.getAlpineSourcesTag(),
"https://git.alpinelinux.org/aports",
// https://gitlab.alpinelinux.org/alpine/aports.git
"/aports"
);
this.pkgNameVerToAportsPath = Collections.unmodifiableMap(buildAlpinePackageIndexWithAbuild(executor));
}
private static void runInitCommand(@NonNull KubernetesCommandExecutor executor, @NonNull String... command)
throws EnvironmentInitializationFailure {
try {
LOG.trace("Running command [{}].", Arrays.toString(command));
final int exitValue;
try (KubernetesContainerCommandProcess process = executor.executeCommand(command)) {
process.waitFor(10, TimeUnit.MINUTES);
exitValue = process.exitValue();
if (exitValue != 0) {
LOG.error("stdout: [{}]", new String(process.getAllStdOut(), StandardCharsets.UTF_8));
LOG.error("stderr: [{}]", new String(process.getAllStdErr(), StandardCharsets.UTF_8));
}
}
if (exitValue != 0) {
LOG.error("Failed to execute init command [{}]: exit value [{}].", Arrays.toString(command), exitValue);
throw new EnvironmentInitializationFailure("Failed to execute init command: exit value " + exitValue);
}
} catch (Exception e) {
throw new EnvironmentInitializationFailure("Failed to run init command", e);
}
}
private static Map buildAlpinePackageIndexWithAbuild(KubernetesCommandExecutor executor)
throws EnvironmentInitializationFailure {
Map nameVerToPath = new HashMap<>();
// build a package index mapping a subpackage to its APKBUILD' directory in aports
try {
// prepare temporary directory for index
Path tempDir = Files.createTempDirectory("aecc-");
Path packagesListingPath = tempDir.resolve("subpackagesListing.txt");
// prepare an index by ourselves. this is because abuild listpkg only outputs (sub)pkgname-version pairs
// make a list of all non-testing APKBUILDs
// find /aports -type f -name 'APKBUILD' -print -o -type d -name \"testing\" -prune"
runInitCommand(
executor,
"sh", "-c",
"find /aports -type f -name 'APKBUILD' -print -o -type d -name \"testing\" -prune " +
"> /aports/apkbuildList.txt"
);
LOG.debug("Starting to generate subpackage names (this may take a while)...");
// get all subpackages that each apkbuild provides
// the format is separated by spaces like so: path subpkgname subpkgname subpkgname subpkgname...
runInitCommand(
executor,
"sh", "-c", "cat /aports/apkbuildList.txt | sed \"s/APKBUILD\\$//g\" | while read i; " +
"do cd \"$i\" ; printf \"%s \" \"$PWD\"; " +
"abuild -F listpkg | tr \"\\n\" \" \"; " +
"printf \"\\n\"; " +
"done " +
"> /aports/subpackagesListing.txt"
);
// transfer the tar to the host via api call
executor.downloadFile("/aports/subpackagesListing.txt", packagesListingPath);
if (!packagesListingPath.toFile().exists()) {
throw new EnvironmentInitializationFailure(
"Panic: subpackagesListing.txt could not be transferred.");
}
if (!Files.isRegularFile(packagesListingPath)) {
throw new EnvironmentInitializationFailure("Panic: subpackagesListing.txt was not a regular file.");
}
if (Files.size(packagesListingPath) < 1) {
throw new EnvironmentInitializationFailure(
"Panic: Transferred subpackagesListing.txt appears to be empty");
}
if (Files.readAllLines(packagesListingPath).size() < 512) {
LOG.error("subpackages listing file output: [{}]", Files.readAllLines(packagesListingPath));
throw new EnvironmentInitializationFailure("Panic: " +
"Less than 512 packages found. That can't be right.");
}
// just pray that alpine uses UTF_8
try (Stream lineStream = Files.lines(packagesListingPath, StandardCharsets.UTF_8)) {
// parse lines from index and build a java data structure
Map> pathToNameVerMap = lineStream.collect(
HashMap::new,
AlpineUtils::parseAbuildIndexLine,
HashMap::putAll
);
// reverse the map, checking for duplicates
for (Map.Entry> pathToNameVer : pathToNameVerMap.entrySet()) {
for (String nameVer : pathToNameVer.getValue()) {
String previousPath = nameVerToPath.putIfAbsent(nameVer, pathToNameVer.getKey());
if (previousPath != null) {
/*
if the previous value wasn't null, we are overwriting something
this means there must have been multiple directories looking to build the same subpackage.
we didn't think this edge-case would happen and don't know how to handle this ambiguity.
error.
*/
LOG.warn(
"Ambiguous name-ver [{}] occurs in: [{}], also in [{}]. Removing from resolver.",
nameVer,
previousPath,
pathToNameVer.getKey()
);
}
}
}
}
// delete temporary directory
PathUtils.deleteDirectory(tempDir);
} catch (IOException e) {
throw new EnvironmentInitializationFailure("Error while preparing APKBUILD index", e);
}
return nameVerToPath;
}
private void ensureOpen() throws IllegalStateException {
if (closed) {
throw new IllegalStateException("Environment has already been destroyed.");
}
}
/**
* Tries to escape single quotes using double quotes for the shell.
*/
private static String squote(String input) {
return "'" + input.replaceAll("'", "\"'\"") + "'";
}
/**
* Shellify helper for command execution. May have weird implementation-specific stuff. Look at the code!
*
* @param absolutePath path to cd into before executing command
* @param command raw command to execute inside the shell.
*/
private static String[] shellify(String absolutePath, @NonNull String command) {
return new String[]{"sh", "-c", "cd " + squote(absolutePath) + " && " + command};
}
private synchronized void abuildFetch(@NonNull String absolutePath) throws CommandExecutionFailed {
String[] fetchCommand = shellify(absolutePath, "abuild -F fetch");
// if 10 minutes aren't enough then there is either an error or the network is too damn slow
ExecutorUtils.demandSuccess(executor, fetchCommand, 10, TimeUnit.MINUTES);
}
private synchronized void abuildVerify(@NonNull String absolutePath) throws CommandExecutionFailed {
String[] verifyCommand = shellify(absolutePath, "abuild -F verify");
ExecutorUtils.demandSuccess(executor, verifyCommand, 5, TimeUnit.MINUTES);
}
private synchronized void abuildSrcpkg(@NonNull String absolutePath) throws CommandExecutionFailed {
// NOTE: abuild's "REPODEST" must be absolute (for output location via "-P")
String[] srcpkgCommand = shellify(absolutePath, "abuild -F -P " + squote(absolutePath + "/ae-out") + " srcpkg");
// packaging should not take very long at all. give it a tighter timeout.
ExecutorUtils.demandSuccess(executor, srcpkgCommand, 2, TimeUnit.MINUTES);
}
private synchronized void curlDownloadSource(@NonNull String absolutePath,
@NonNull String outputFilename,
@NonNull String url)
throws CommandExecutionFailed {
String [] curlDownloadCommand = shellify(absolutePath, "curl -L -o " + squote(outputFilename) + " " +
squote(url));
LOG.trace("Running Curl download [{}].", Arrays.toString(curlDownloadCommand));
ExecutorUtils.demandSuccess(executor, curlDownloadCommand, 1, TimeUnit.MINUTES);
LOG.trace("Custom curl download SUCCESS: [{}].", Arrays.toString(curlDownloadCommand));
}
private synchronized String getSrcpkgPath(@NonNull String absolutePath)
throws EnvironmentOperationFailure, CommandExecutionFailed {
// simply print the paths of all files in this directory. should only be the srcpkg. we'll check this.
String[] getPkgPathCommand = shellify(
"/",
"for file in " + squote(absolutePath) + "/ae-out/src/* ; do printf \"%s\\0\" \"$file\" ; done"
);
String stdOut;
try (KubernetesContainerCommandProcess process = executor.executeCommand(getPkgPathCommand)) {
process.waitFor(1, TimeUnit.MINUTES);
if (process.exitValue() != 0) {
LOG.error(
"Command execution of command [{}] failed with exit value [{}].",
Arrays.toString(getPkgPathCommand),
process.exitValue()
);
throw new CommandExecutionFailed("Command execution returned with non-zero exit code");
}
stdOut = new String(process.getAllStdOut(), StandardCharsets.UTF_8);
} catch (Exception e) {
throw new CommandExecutionFailed("Could not get srcpkg path because command failed", e);
}
// ensure that only one file has been output to this directory
if (stdOut.indexOf("\0") == stdOut.lastIndexOf("\0") && StringUtils.isNotBlank(stdOut)) {
// only one NUL char, so only one path was output. we can return this.
return stdOut.substring(0, stdOut.length() - 1);
} else {
// there might have been multiple. shouldn't happen but i don't want to crash runtime with this
LOG.error("Failing resolution: multiple files in srcpkg outdir of [{}], expected one.", absolutePath);
throw new EnvironmentOperationFailure("Multiple files found in srcpkg outdir of [{}], expected one.");
}
}
private synchronized String doDownloadAndSrcpkg(@NonNull String aportsPath)
throws CommandExecutionFailed, EnvironmentOperationFailure {
ensureOpen();
LOG.debug("Doing downloads for package at [{}]", aportsPath);
// run the apk fetch, then verify, then build source package and return where in the container it was stored.
abuildFetch(aportsPath);
abuildVerify(aportsPath);
abuildSrcpkg(aportsPath);
return getSrcpkgPath(aportsPath);
}
private synchronized void tryCustomDownload(@NonNull String aportsPath,
@NonNull AlpinePackageReference packageRef,
@NonNull AlpinePackageResolverConfig resolverConfig)
throws CommandExecutionFailed {
List downloads = resolverConfig.getExplicitSourceDownloads()
.get(packageRef.getPkgname());
if (downloads.isEmpty()) {
LOG.trace("Can't do anything to fix up missing sources for package ref [{}].", packageRef);
return;
}
for (CurlDownloadReference download : downloads) {
String outputFilename = StringPieceAssembler.createString(download.getOutputFilenamePieces(), packageRef);
String downloadUrl = StringPieceAssembler.createString(download.getUrlPieces(), packageRef);
if (outputFilename.contains("..")) {
LOG.warn(
"Potentially problematic filename [{}] with '..' generated for package ref [{}].",
outputFilename,
packageRef
);
}
LOG.info(
"Downloading manually specified sources for ref [{}] from [{}] to [{}]",
packageRef,
downloadUrl,
outputFilename
);
curlDownloadSource(aportsPath, outputFilename, downloadUrl);
}
}
/**
* Tries to download sources and build a source package for the given reference.
*
* @param reference describing the packge to be loaded
* @param outputPath where the finished srcpkg should be put
* @throws EnvironmentOperationFailure if the file could not be acquired
*/
@Override
public synchronized void getSourceArchive(@NonNull AlpinePackageReference reference,
@NonNull Path outputPath,
@NonNull ArtifactResolverConfig resolverConfig)
throws EnvironmentOperationFailure {
ensureOpen();
// sanity check version because we can, warning of there was a mismatch.
if (!params.getAlpineSourcesTag().equals("v" + reference.getAlpineVersion())) {
LOG.warn("Using alpine environment with aports tag [{}] with a reference that requested [{}].",
params.getAlpineSourcesTag(),
reference.getAlpineVersion()
);
}
// find the APKBUILD / aports directory (possibly via subpackage lookups)
String aportsPath = pkgNameVerToAportsPath.get(reference.getPkgname() + "-" + reference.getPkgver());
if (aportsPath == null) {
LOG.error("No alpine package found for ref [{}]", reference);
LOG.trace("Map was: [{}]", pkgNameVerToAportsPath);
throw new EnvironmentOperationFailure("Could not find aports path for given reference.");
}
String pathInContainer = null;
try {
pathInContainer = doDownloadAndSrcpkg(aportsPath);
} catch (Exception e) {
LOG.info("First download attempt for [{}] failed with exception: [{}]",
reference,
ExceptionUtils.getStackTrace(e));
}
try {
if (pathInContainer == null) {
LOG.info("Trying custom download for [{}].", reference);
tryCustomDownload(aportsPath, reference, resolverConfig.getAlpinePackageResolverConfig());
pathInContainer = doDownloadAndSrcpkg(aportsPath);
}
} catch (Exception e) {
LOG.warn("Second download attempt for [{}] failed with exception: [{}]",
reference,
ExceptionUtils.getStackTrace(e));
}
if (pathInContainer == null) {
throw new EnvironmentOperationFailure("Could not create source archive.");
}
try {
// transfer the file from the kubernetes pod to this machine
LOG.debug("Starting transfer from container's [{}] to host's [{}].", pathInContainer, outputPath);
executor.downloadFile(pathInContainer, outputPath);
} catch (Exception e) {
LOG.error("Failed to transfer the source archive for [{}].", reference);
throw new EnvironmentOperationFailure("Environment operation failed", e);
}
// some sanity check to see if the file really has been transferred successfully.
try {
if (!Files.isRegularFile(outputPath)) {
// transfer failed
LOG.error("Transferred file did not create a regular file.");
throw new EnvironmentOperationFailure("Mustn't happen: Transferred output was not a regular file.");
}
final long size = Files.size(outputPath);
if (size < 1) {
LOG.error("Transferred file [{}] appears to be empty (size [{}]).", outputPath, size);
throw new EnvironmentOperationFailure("Mustn't happen: transferred file appears to be empty.");
}
} catch (IOException e) {
throw new EnvironmentOperationFailure("Operation failed due to unexpected Exception.", e);
}
}
@Override
public synchronized void close() throws Exception {
if (!closed) {
closed = true;
executor.close();
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy