org.technologybrewery.habushu.PublishToPyPiRepoMojo Maven / Gradle / Ivy
Show all versions of habushu-maven-plugin Show documentation
package org.technologybrewery.habushu;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicates;
import io.github.itning.retry.RetryException;
import io.github.itning.retry.Retryer;
import io.github.itning.retry.RetryerBuilder;
import io.github.itning.retry.strategy.stop.StopStrategies;
import io.github.itning.retry.strategy.stop.StopStrategy;
import io.github.itning.retry.strategy.wait.WaitStrategies;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.technologybrewery.habushu.exec.PoetryCommandHelper;
/**
* Publishes the distribution archives generated by
* {@link BuildDeploymentArtifactsMojo} to the configured PyPI repository.
* {@link PublishToPyPiRepoMojo} leverages Poetry to support publishing to
* private PyPI repositories as well as the official PyPI repository.
*
* If publishing to a private PyPI repository, both {@link #pypiRepoId} and
* {@link #pypiRepoUrl} MUST be specified, and it is expected that the
* relevant username/password credentials are configured in a settings.xml
* {@literal } entry that has an {@literal } that aligns with the
* provided {@link #pypiRepoId}.
*
* If neither {@link #pypiRepoId} nor {@link #pypiRepoUrl} are provided, *OR*
* {@link #pypiRepoId} is set to {@code pypi} and {@link #pypiRepoUrl} is not
* provided, it is assumed that the archives will be published to the official
* PyPI repository. As described above, developers are expected to provide their
* PyPI credentials via a username/password entry in their settings.xml with an
* {@code } that matches {@code pypi}, or use the appropriate Poetry command
* to configure their PyPI credentials in an adhoc fashion (i.e.
* {@code poetry config pypi-token.pypi my-token}).
*
* If the POM version of the module being published is a SNAPSHOT, the Poetry
* package will be published to the configured PyPI repository as a Python
* developmental release. Developers may use
* {@link #snapshotNumberDateFormatPattern} to adjust the formatting of the
* numeric component of the published version.
*/
@Mojo(name = "publish-to-pypi-repo", defaultPhase = LifecyclePhase.DEPLOY)
public class PublishToPyPiRepoMojo extends AbstractHabushuMojo {
private static final String VERSION = "version";
/**
* {@link DateTimeFormatter} compliant pattern that configures the numeric
* portion of SNAPSHOT Poetry package versions that are published to the
* configured PyPI repository. By default, the version of SNAPSHOT published
* packages align with PEP-440 developmental releases and use a numeric
* component that corresponds to the number of seconds since the epoch. For
* example, if the POM version is {@code 1.2.3-SNAPSHOT}, the package may be
* published by default as {@code 1.2.3.dev1658238063}. If
* {@link #snapshotNumberDateFormatPattern} is provided, the numeric component
* will reflect the given date format pattern applied to the current build time.
* For example, if "YYYYMMddHHmm" is provided, {@code 1.2.3.dev202207191002} may
* be published.
*/
@Parameter(property = "habushu.snapshotNumberDateFormatPattern")
protected String snapshotNumberDateFormatPattern;
/**
* Skips the entire execution of the deploy phase and does *not* publish the
* Poetry package to the configured PyPI repository. This configuration may be
* useful when individual Habushu modules within a larger multi-module project
* hierarchy should *not* be published to PyPI, but it is still desirable to
* automate the project's release via the {@code maven-release-plugin}.
*/
@Parameter(property = "habushu.skipDeploy", defaultValue = "false")
protected boolean skipDeploy;
/**
* Allows tailoring of the path used for pushing to a PyPI repository for deployment. Some repositories, like
* Nexus or Artifactory, do not require an url path on top of the base repository url. Others, do (often using
* "legacy/"). This variable allows customization in a manner that does not impact the installation API for the
* same repository. Defaults to empty as the most common scenario when overriding the repository URL is to leverage
* one of the repositories mentioned above.
* Note: The property must be set equal to "" in order for Maven to not set the default value to null
*/
@Parameter(property = "habushu.pypiUploadSuffix", defaultValue = "")
protected String pypiUploadSuffix = "";
/**
* {{@link #pypiUploadSuffix repositoryUploadSuffix} contains critical information. The main difference is that
* this dev repository url path defaults to "legacy/" as the most common scenario when overriding the dev
* repository URL is to leverage test.pypi.org, which needs this configuration.
*/
@Parameter(property = "habushu.devRepositoryUrlUploadSuffix", defaultValue = "legacy/")
protected String devRepositoryUrlUploadSuffix;
/**
* Specifies the number of times a push to the configured PyPI repository will be attempted before stopping (inclusive
* of the initial attempt). While this defaults to three and is fully configurable, it can be set to zero to never
* retry or set to any negative number for unlimited retries. Unlimited retries will follow a fibonacci backoff
* interval.
*/
@Parameter(property = "habushu.pyPiPushRetries", defaultValue = "3")
protected int pypiPushRetries = 3;
/**
* A multiplier, in milliseconds, to apply to the retry wait time (e.g., round of fibonacci).
*/
@Parameter(property = "habushu.pypiPushRetryMultiplier", defaultValue = "15000")
protected long pypiPushRetryMultiplier;
/**
* Maximum time to wait for a retry, in minutes, even if the backoff algorithm is above this threshold.
*/
@Parameter(property = "habushu.pypiPushRetryMaxTimeout", defaultValue = "3")
protected long pypiPushRetryMaxTimeout;
@Override
public void doExecute() throws MojoExecutionException, MojoFailureException {
if (this.skipDeploy) {
getLog().info(String.format(
"Skipping deploy phase - package for %s will not be published to the configured PyPI repository",
this.project.getId()));
return;
}
PoetryCommandHelper poetryHelper = createPoetryCommandHelper();
String pomVersion = project.getVersion();
if (this.overridePackageVersion && isPomVersionSnapshot(pomVersion)) {
String currentPythonPackageVersion = poetryHelper.execute(Arrays.asList(VERSION, "-s"));
String snapshotVersionToPublish = getPythonPackageVersion(pomVersion, true,
snapshotNumberDateFormatPattern);
try {
getLog().info(
String.format("Setting version of Poetry package to publish to %s", snapshotVersionToPublish));
poetryHelper.executeAndLogOutput(Arrays.asList(VERSION, snapshotVersionToPublish));
publishPackage(poetryHelper, true);
} finally {
getLog().info(
String.format("Resetting Poetry package version back to %s", currentPythonPackageVersion));
poetryHelper.executeAndLogOutput(Arrays.asList(VERSION, currentPythonPackageVersion));
}
} else {
publishPackage(poetryHelper, false);
}
}
/**
* Helper method that encapsulates publishing the Poetry package to the
* configured PyPI repository.
*
* @param poetryHelper Poetry command helper that delegates publishing
* commands to Poetry.
* @param rebuildPackage whether to rebuild the package prior to publishing it.
* This is typically only required for SNAPSHOT packages
* where the version of the Poetry package may be
* dynamically set in
* {@link PublishToPyPiRepoMojo#execute()} and as a
* result, the artifacts that were built by previously
* executed build phase (i.e.
* {@link BuildDeploymentArtifactsMojo}) do not reference
* a version that aligns with the target version to be
* published.
* @throws MojoExecutionException
*/
protected void publishPackage(PoetryCommandHelper poetryHelper, boolean rebuildPackage)
throws MojoExecutionException {
boolean publishToDev = rebuildPackage && useDevRepository;
if (publishToDev) {
getLog().info("Publishing to dev repository (useDevRepository=true, dev version being published)");
}
String username = null;
String password = null;
String repoId = publishToDev ? devRepositoryId : pypiRepoId;
if (StringUtils.isNotEmpty(repoId)) {
username = findUsernameForServer(repoId);
password = findPasswordForServer(repoId);
}
String repoUrl = getRepositoryUrl(publishToDev);
if (StringUtils.isNotEmpty(repoUrl)) {
if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {
throw new MojoExecutionException(String.format(
"Please ensure that both and are provided for the with %s in your settings.xml configuration!",
repoId));
}
getLog().info(String.format("Adding repository configuration to poetry.toml for %s at %s", repoId,
repoUrl));
poetryHelper.execute(
Arrays.asList("config", "--local", String.format("repositories.%s", repoId), repoUrl));
}
List> publishToRepoWithCredsArgs = Collections.emptyList();
if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) {
publishToRepoWithCredsArgs = new ArrayList<>();
if (!PUBLIC_PYPI_REPO_ID.equals(repoId)) {
publishToRepoWithCredsArgs.add(new ImmutablePair<>("--repository", false));
publishToRepoWithCredsArgs.add(new ImmutablePair<>(repoId, false));
}
publishToRepoWithCredsArgs.add(new ImmutablePair<>("--username", false));
publishToRepoWithCredsArgs.add(new ImmutablePair<>(username, false));
publishToRepoWithCredsArgs.add(new ImmutablePair<>("--password", false));
publishToRepoWithCredsArgs.add(new ImmutablePair<>(password, true));
}
String publishCommand = rewriteLocalPathDepsInArchives ? "publish-rewrite-path-deps" : "publish";
getLog().info(String.format("Publishing archives to %s %s",
StringUtils.isNotEmpty(repoUrl) ? repoUrl : "official PyPI repository",
rewriteLocalPathDepsInArchives ? "with poetry-monorepo-dependency-plugin" : ""));
if (!publishToRepoWithCredsArgs.isEmpty()) {
publishToRepoWithCredsArgs.add(0, new ImmutablePair<>(publishCommand, false));
if (rebuildPackage) {
publishToRepoWithCredsArgs.add(1, new ImmutablePair<>("--build", false));
}
} else {
getLog().warn(String.format(
"PyPI repository credentials not specified in element in settings.xml with of %s",
PUBLIC_PYPI_REPO_ID));
getLog().warn(
"Please populate settings.xml with PyPI credentials or ensure that Poetry is manually " +
"configured with the correct PyPI credentials (i.e. poetry config pypi-token.pypi my-token)");
publishToRepoWithCredsArgs.add(new ImmutablePair<>(publishCommand, false));
if (rebuildPackage) {
publishToRepoWithCredsArgs.add(new ImmutablePair<>("--build", false));
}
}
invokePublish(poetryHelper, publishToRepoWithCredsArgs);
}
protected void invokePublish(PoetryCommandHelper poetryHelper, List> publishToRepoWithCredsArgs) {
Callable callable = getPyPiPushCallable(poetryHelper, publishToRepoWithCredsArgs);
Retryer retryer = getRetryer();
try {
Boolean result = retryer.call(callable);
if (Boolean.FALSE.equals(result)) {
throw new HabushuException("Push to PyPI repository failed!");
}
} catch (RetryException e) {
throw new HabushuException("Exceeded retry setting of: " + pypiPushRetries, e);
} catch (ExecutionException e) {
throw new HabushuException("Could not execute PyPI push!", e);
}
}
protected Callable getPyPiPushCallable(PoetryCommandHelper poetryHelper, List> publishToRepoWithCredsArgs) {
Callable callable = new Callable<>() {
private boolean firstAttempt = true;
public Boolean call() throws Exception {
if (!firstAttempt) {
publishToRepoWithCredsArgs.removeIf(arg -> "--build".equals(arg.getLeft()));
getLog().debug("Removing build command from retry due to the general issue error described in https://github.com/python-poetry/cleo/issues/351");
} else {
firstAttempt = false;
}
int result = poetryHelper.executeWithSensitiveArgsAndLogOutput(publishToRepoWithCredsArgs);
if (result != 0) {
getLog().warn("PyPI Publish process result code: " + result);
}
return result == 0;
}
};
return callable;
}
protected Retryer getRetryer() {
RetryerBuilder retryBuilder = RetryerBuilder.newBuilder();
if (pypiPushRetries != 0) {
StopStrategy stopStrategy = (pypiPushRetries < 0) ? StopStrategies.neverStop()
: StopStrategies.stopAfterAttempt(pypiPushRetries);
retryBuilder.retryIfResult(Predicates.equalTo(Boolean.FALSE))
.retryIfRuntimeException()
.withStopStrategy(stopStrategy)
.withWaitStrategy(WaitStrategies.fibonacciWait(pypiPushRetryMultiplier, pypiPushRetryMaxTimeout, TimeUnit.MINUTES));
}
return retryBuilder.build();
}
String getRepositoryUrl(boolean publishToDev) {
String repoUrl = addTrailingSlash(pypiRepoUrl);
if (publishToDev) {
repoUrl = addTrailingSlash(devRepositoryUrl) + addTrailingSlash(devRepositoryUrlUploadSuffix);
} else if(!StringUtils.isEmpty(pypiUploadSuffix)) {
repoUrl += addTrailingSlash(pypiUploadSuffix);
}
return repoUrl;
}
static String addTrailingSlash(String inputUrl) {
if (StringUtils.isNotBlank(inputUrl) && !StringUtils.endsWith(inputUrl, "/")) {
// PEP-0694 likes a trailing slash:
inputUrl += "/";
}
return inputUrl;
}
}