Please wait. This can take some minutes ...
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.solr.packagemanager.PackageManager Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.packagemanager;
import static org.apache.solr.packagemanager.PackageUtils.getMapper;
import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.packagemanager.SolrPackage.Command;
import org.apache.solr.packagemanager.SolrPackage.Manifest;
import org.apache.solr.packagemanager.SolrPackage.Plugin;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.util.SolrCLI;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Strings;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
/**
* Handles most of the management of packages that are already installed in Solr.
*/
public class PackageManager implements Closeable {
final String solrBaseUrl;
final HttpSolrClient solrClient;
final SolrZkClient zkClient;
private Map> packages = null;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public PackageManager(HttpSolrClient solrClient, String solrBaseUrl, String zkHost) {
this.solrBaseUrl = solrBaseUrl;
this.solrClient = solrClient;
this.zkClient = new SolrZkClient(zkHost, 30000);
log.info("Done initializing a zkClient instance...");
}
@Override
public void close() throws IOException {
if (zkClient != null) {
zkClient.close();
}
}
public List fetchInstalledPackageInstances() throws SolrException {
log.info("Getting packages from packages.json...");
List ret = new ArrayList();
packages = new HashMap>();
try {
Map packagesZnodeMap = null;
if (zkClient.exists(ZkStateReader.SOLR_PKGS_PATH, true) == true) {
packagesZnodeMap = (Map)getMapper().readValue(
new String(zkClient.getData(ZkStateReader.SOLR_PKGS_PATH, null, null, true), "UTF-8"), Map.class).get("packages");
for (Object packageName: packagesZnodeMap.keySet()) {
List pkg = (List)packagesZnodeMap.get(packageName);
for (Map pkgVersion: (List)pkg) {
Manifest manifest = PackageUtils.fetchManifest(solrClient, solrBaseUrl, pkgVersion.get("manifest").toString(), pkgVersion.get("manifestSHA512").toString());
List solrplugins = manifest.plugins;
SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null,
pkgVersion.get("version").toString(), manifest, solrplugins, manifest.parameterDefaults);
List list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList();
list.add(pkgInstance);
packages.put(packageName.toString(), list);
ret.add(pkgInstance);
}
}
}
} catch (Exception e) {
throw new SolrException(ErrorCode.BAD_REQUEST, e);
}
log.info("Got packages: "+ret);
return ret;
}
public Map getPackagesDeployed(String collection) {
Map packages = null;
try {
NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(),
solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER);
packages = (Map) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap());
} catch (PathNotFoundException ex) {
// Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
}
if (packages == null) return Collections.emptyMap();
Map ret = new HashMap<>();
for (String packageName: packages.keySet()) {
if (Strings.isNullOrEmpty(packageName) == false && // There can be an empty key, storing the version here
packages.get(packageName) != null) { // null means the package was undeployed from this package before
ret.put(packageName, getPackageInstance(packageName, packages.get(packageName)));
}
}
return ret;
}
private void ensureCollectionsExist(List collections) {
try {
List existingCollections = zkClient.getChildren("/collections", null, true);
Set nonExistent = new HashSet<>(collections);
nonExistent.removeAll(existingCollections);
if (nonExistent.isEmpty() == false) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection(s) doesn't exist: " + nonExistent.toString());
}
} catch (KeeperException | InterruptedException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to fetch list of collections from ZK.");
}
}
private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt,
List collections, String overrides[]) {
List previouslyDeployed = new ArrayList<>(); // collections where package is already deployed in
for (String collection: collections) {
SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name);
if (packageInstance.equals(deployedPackage)) {
if (!pegToLatest) {
PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection);
previouslyDeployed.add(collection);
continue;
}
} else {
if (deployedPackage != null && !isUpdate) {
PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter.");
previouslyDeployed.add(collection);
continue;
}
}
Map collectionParameterOverrides = getCollectionParameterOverrides(packageInstance, isUpdate, overrides, collection);
// Get package params
try {
boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class)
.getOrDefault("response", Collections.emptyMap())).containsKey("params");
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set",
Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides)))));
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
// Set the package version in the collection's parameters
try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
"{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}");
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
// If updating, refresh the package version for this to take effect
if (isUpdate || pegToLatest) {
try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageInstance.name + "\"}");
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
}
// If it is a fresh deploy on a collection, run setup commands all the plugins in the package
if (!isUpdate) {
Map systemParams = PackageUtils.map("collection", collection, "package-name", packageInstance.name, "package-version", packageInstance.version);
for (Plugin plugin: packageInstance.plugins) {
Command cmd = plugin.setupCommand;
if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
if ("POST".equalsIgnoreCase(cmd.method)) {
try {
String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
String path = PackageUtils.resolve(cmd.path, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Executing " + payload + " for path:" + path);
boolean shouldExecute = true;
if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin
PackageUtils.print(PackageUtils.YELLOW, "Execute this command (y/n): ");
String userInput = new Scanner(System.in, "UTF-8").next();
if (!"yes".equalsIgnoreCase(userInput) && !"y".equalsIgnoreCase(userInput)) {
shouldExecute = false;
PackageUtils.printRed("Skipping setup command for deploying (deployment verification may fail)."
+ " Please run this step manually or refer to package documentation.");
}
}
if (shouldExecute) {
SolrCLI.postJsonToSolr(solrClient, path, payload);
}
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for setup commands");
}
} else {
PackageUtils.printRed("There is no setup command to execute for plugin: " + plugin.name);
}
}
}
// Set the package version in the collection's parameters
try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
"{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}");
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
}
List deployedCollections = collections.stream().filter(c -> !previouslyDeployed.contains(c)).collect(Collectors.toList());
boolean success = true;
if (deployedCollections.isEmpty() == false) {
// Verify that package was successfully deployed
success = verify(packageInstance, deployedCollections);
if (success) {
PackageUtils.printGreen("Deployed on " + deployedCollections + " and verified package: " + packageInstance.name + ", version: " + packageInstance.version);
}
}
if (previouslyDeployed.isEmpty() == false) {
PackageUtils.printRed("Already Deployed on " + previouslyDeployed + ", package: " + packageInstance.name + ", version: " + packageInstance.version);
}
return previouslyDeployed.isEmpty() && success;
}
private Map getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate,
String[] overrides, String collection) {
Map collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap();
if (overrides != null) {
for (String override: overrides) {
collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]);
}
}
return collectionParameterOverrides;
}
@SuppressWarnings({"rawtypes", "unchecked"})
Map getPackageParams(String packageName, String collection) {
try {
return (Map)((Map)((Map)((Map)
PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class)
.get("response"))
.get("params"))
.get("packages")).get(packageName);
} catch (Exception ex) {
// This should be because there are no parameters. Be tolerant here.
return Collections.emptyMap();
}
}
/**
* Given a package and list of collections, verify if the package is installed
* in those collections. It uses the verify command of every plugin in the package (if defined).
*/
public boolean verify(SolrPackageInstance pkg, List collections) {
boolean success = true;
for (Plugin plugin: pkg.plugins) {
Command cmd = plugin.verifyCommand;
if (plugin.verifyCommand != null && !Strings.isNullOrEmpty(cmd.path)) {
for (String collection: collections) {
Map collectionParameterOverrides = getPackageParams(pkg.name, collection);
Map systemParams = PackageUtils.map("collection", collection, "package-name", pkg.name, "package-version", pkg.version);
String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Executing " + url + " for collection:" + collection);
if ("GET".equalsIgnoreCase(cmd.method)) {
String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url);
PackageUtils.printGreen(response);
String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration())
.read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams));
String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue);
if (!expectedValue.equals(actualValue)) {
PackageUtils.printRed("Failed to deploy plugin: " + plugin.name);
success = false;
}
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for verify commands");
}
}
}
}
return success;
}
/**
* Get the installed instance of a specific version of a package. If version is null, PackageUtils.LATEST or PackagePluginHolder.LATEST,
* then it returns the highest version available in the system for the package.
*/
public SolrPackageInstance getPackageInstance(String packageName, String version) {
fetchInstalledPackageInstances();
List versions = packages.get(packageName);
SolrPackageInstance latest = null;
if (versions != null && !versions.isEmpty()) {
latest = versions.get(0);
for (int i=0; i collectionParameterOverrides = getPackageParams(packageName, collection);
// Run the uninstall command for all plugins
Map systemParams = PackageUtils.map("collection", collection, "package-name", deployedPackage.name, "package-version", deployedPackage.version);
for (Plugin plugin: deployedPackage.plugins) {
Command cmd = plugin.uninstallCommand;
if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
if ("POST".equalsIgnoreCase(cmd.method)) {
try {
String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
String path = PackageUtils.resolve(cmd.path, deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Executing " + payload + " for path:" + path);
SolrCLI.postJsonToSolr(solrClient, path, payload);
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for uninstall commands");
}
} else {
PackageUtils.printRed("There is no uninstall command to execute for plugin: " + plugin.name);
}
}
// Set the package version in the collection's parameters
try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
"{set: {PKG_VERSIONS: {"+packageName+": null}}}"); // Is it better to "unset"? If so, build support in params API for "unset"
SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}");
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex);
}
// TODO: Also better to remove the package parameters PKG_VERSION etc.
}
}
/**
* Given a package, return a map of collections where this package is
* installed to the installed version (which can be {@link PackagePluginHolder#LATEST})
*/
public Map getDeployedCollections(String packageName) {
List allCollections;
try {
allCollections = zkClient.getChildren(ZkStateReader.COLLECTIONS_ZKNODE, null, true);
} catch (KeeperException | InterruptedException e) {
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e);
}
Map deployed = new HashMap();
for (String collection: allCollections) {
// Check package version installed
String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true");
String version = null;
try {
version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration())
.read("$['response'].['params'].['PKG_VERSIONS'].['"+packageName+"'])");
} catch (PathNotFoundException ex) {
// Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
}
if (version != null) {
deployed.put(collection, version);
}
}
return deployed;
}
}