org.opensearch.gradle.DistributionDownloadPlugin Maven / Gradle / Ivy
Show all versions of build-tools Show documentation
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/
package org.opensearch.gradle;
import org.opensearch.gradle.OpenSearchDistribution.Platform;
import org.opensearch.gradle.OpenSearchDistribution.Type;
import org.opensearch.gradle.docker.DockerSupportPlugin;
import org.opensearch.gradle.docker.DockerSupportService;
import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
import org.opensearch.gradle.transform.UnzipTransform;
import org.opensearch.gradle.util.GradleUtils;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.dsl.DependencyHandler;
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
import org.gradle.api.internal.artifacts.ArtifactAttributes;
import org.gradle.api.provider.Provider;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
/**
* A plugin to manage getting and extracting distributions of OpenSearch.
*
* The plugin provides hooks to register custom distribution resolutions.
* This plugin resolves distributions from the OpenSearch downloads service if
* no registered resolution strategy can resolve to a distribution.
*/
public class DistributionDownloadPlugin implements Plugin {
static final String RESOLUTION_CONTAINER_NAME = "opensearch_distributions_resolutions";
private static final String CONTAINER_NAME = "opensearch_distributions";
private static final String FAKE_IVY_GROUP = "opensearch-distribution";
private static final String FAKE_SNAPSHOT_IVY_GROUP = "opensearch-distribution-snapshot";
private static final String DOWNLOAD_REPO_NAME = "opensearch-downloads";
private static final String SNAPSHOT_REPO_NAME = "opensearch-snapshots";
public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "opensearch_distro_extracted_";
// for downloading Elasticsearch OSS distributions to run BWC
private static final String FAKE_IVY_GROUP_ES = "elasticsearch-distribution";
private static final String DOWNLOAD_REPO_NAME_ES = "elasticsearch-downloads";
private static final String SNAPSHOT_REPO_NAME_ES = "elasticsearch-snapshots";
private static final String FAKE_SNAPSHOT_IVY_GROUP_ES = "elasticsearch-distribution-snapshot";
private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
private static final String SNAPSHOT_PATTERN_LAYOUT =
"/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
private static final String BUNDLE_PATTERN_LAYOUT =
"/ci/dbc/distribution-build-opensearch/[revision]/latest/linux/x64/tar/dist/opensearch/[module]-[revision](-[classifier]).[ext]";
private NamedDomainObjectContainer distributionsContainer;
private NamedDomainObjectContainer distributionsResolutionStrategiesContainer;
@Override
public void apply(Project project) {
project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
Provider dockerSupport = GradleUtils.getBuildService(
project.getGradle().getSharedServices(),
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
);
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
});
ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName());
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
});
setupResolutionsContainer(project);
setupDistributionContainer(project, dockerSupport);
setupDownloadServiceRepo(project);
project.afterEvaluate(this::setupDistributions);
}
private void setupDistributionContainer(Project project, Provider dockerSupport) {
distributionsContainer = project.container(OpenSearchDistribution.class, name -> {
Configuration fileConfiguration = project.getConfigurations().create("opensearch_distro_file_" + name);
Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);
extractedConfiguration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
return new OpenSearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
});
project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
}
private void setupResolutionsContainer(Project project) {
distributionsResolutionStrategiesContainer = project.container(DistributionResolution.class);
// We want this ordered in the same resolution strategies are added
distributionsResolutionStrategiesContainer.whenObjectAdded(
resolveDependencyNotation -> resolveDependencyNotation.setPriority(distributionsResolutionStrategiesContainer.size())
);
project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategiesContainer);
}
@SuppressWarnings("unchecked")
public static NamedDomainObjectContainer getContainer(Project project) {
return (NamedDomainObjectContainer) project.getExtensions().getByName(CONTAINER_NAME);
}
@SuppressWarnings("unchecked")
public static NamedDomainObjectContainer getRegistrationsContainer(Project project) {
return (NamedDomainObjectContainer) project.getExtensions().getByName(RESOLUTION_CONTAINER_NAME);
}
// pkg private for tests
void setupDistributions(Project project) {
for (OpenSearchDistribution distribution : distributionsContainer) {
distribution.finalizeValues();
DependencyHandler dependencies = project.getDependencies();
// for the distribution as a file, just depend on the artifact directly
DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
dependencies.add(distribution.configuration.getName(), distributionDependency.getDefaultNotation());
// no extraction allowed for rpm, deb or docker
if (distribution.getType().shouldExtract()) {
// The extracted configuration depends on the artifact directly but has
// an artifact transform registered to resolve it as an unpacked folder.
dependencies.add(distribution.getExtracted().getName(), distributionDependency.getExtractedNotation());
}
}
}
private DistributionDependency resolveDependencyNotation(Project p, OpenSearchDistribution distribution) {
return distributionsResolutionStrategiesContainer.stream()
.sorted(Comparator.comparingInt(DistributionResolution::getPriority))
.map(r -> r.getResolver().resolve(p, distribution))
.filter(d -> d != null)
.findFirst()
.orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
}
private static void addIvyRepo(Project project, String name, String url, String group, String... patternLayout) {
final List repos = Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
repo.setName(name);
repo.setUrl(url);
repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
repo.patternLayout(layout -> layout.artifact(pattern));
})).collect(Collectors.toList());
project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
exclusiveContentRepository.filter(config -> config.includeGroup(group));
exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()]));
});
}
private static void addIvyRepo2(Project project, String name, String url, String group) {
IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
repo.setName(name);
repo.setUrl(url);
repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
repo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/elasticsearch-oss-[revision](-[classifier]).[ext]"));
});
project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
exclusiveContentRepository.filter(config -> config.includeGroup(group));
exclusiveContentRepository.forRepositories(ivyRepo);
});
}
private static void setupDownloadServiceRepo(Project project) {
if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
return;
}
Object customDistributionUrl = project.findProperty("customDistributionUrl");
Object customDistributionDownloadType = project.findProperty("customDistributionDownloadType");
// distributionDownloadType is default min if is not specified; download the distribution from CI if is bundle
String distributionDownloadType = customDistributionDownloadType != null
&& customDistributionDownloadType.toString().equals("bundle") ? "bundle" : "min";
addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
if (customDistributionUrl != null) {
addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
return;
}
switch (distributionDownloadType) {
case "bundle":
addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://ci.opensearch.org", FAKE_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://ci.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
break;
case "min":
addIvyRepo(
project,
DOWNLOAD_REPO_NAME,
"https://artifacts.opensearch.org",
FAKE_IVY_GROUP,
"/releases" + RELEASE_PATTERN_LAYOUT,
"/release-candidates" + RELEASE_PATTERN_LAYOUT
);
addIvyRepo(
project,
SNAPSHOT_REPO_NAME,
"https://artifacts.opensearch.org",
FAKE_SNAPSHOT_IVY_GROUP,
SNAPSHOT_PATTERN_LAYOUT
);
break;
default:
throw new IllegalArgumentException("Unsupported property argument: " + distributionDownloadType);
}
}
/**
* Returns a dependency object representing the given distribution.
*
* The returned object is suitable to be passed to {@link DependencyHandler}.
* The concrete type of the object will be a set of maven coordinates as a {@link String}.
* Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
* coordinates that resolve to the Elastic download service through an ivy repository.
*/
private String dependencyNotation(OpenSearchDistribution distribution) {
Version distroVersion = Version.fromString(distribution.getVersion());
if (distribution.getType() == Type.INTEG_TEST_ZIP) {
if (distroVersion.onOrAfter("1.0.0")) {
return "org.opensearch.distribution.integ-test-zip:opensearch:" + distribution.getVersion() + "@zip";
} else {
return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
}
}
String extension = distribution.getType().toString();
String classifier = distroVersion.onOrAfter("1.0.0") ? ":x64" : ":x86_64";
if (distribution.getType() == Type.ARCHIVE) {
extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
if (distroVersion.onOrAfter("1.0.0")) {
switch (distribution.getArchitecture()) {
case ARM64:
classifier = ":" + distribution.getPlatform() + "-arm64";
break;
case X64:
classifier = ":" + distribution.getPlatform() + "-x64";
break;
case S390X:
classifier = ":" + distribution.getPlatform() + "-s390x";
break;
case PPC64LE:
classifier = ":" + distribution.getPlatform() + "-ppc64le";
break;
default:
throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
}
} else if (distroVersion.onOrAfter("7.0.0")) {
classifier = ":" + distribution.getPlatform() + "-x86_64";
} else {
classifier = "";
}
} else if (distribution.getType() == Type.DEB) {
if (distroVersion.onOrAfter("7.0.0")) {
classifier = ":amd64";
} else {
classifier = "";
}
} else if (distribution.getType() == Type.RPM && distroVersion.before("7.0.0")) {
classifier = "";
}
String group;
if (distroVersion.onOrAfter("1.0.0")) {
group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
return group + ":opensearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
} else {
group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP_ES : FAKE_IVY_GROUP_ES;
return group + ":elasticsearch-oss" + ":" + distribution.getVersion() + classifier + "@" + extension;
}
}
}