org.apache.solr.packagemanager.PackageManager Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.packagemanager;

import static org.apache.solr.packagemanager.PackageUtils.getMapper;

import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.packagemanager.SolrPackage.Command;
import org.apache.solr.packagemanager.SolrPackage.Manifest;
import org.apache.solr.packagemanager.SolrPackage.Plugin;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.util.SolrCLI;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Strings;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;

/**
 * Handles most of the management of packages that are already installed in Solr.
 */
public class PackageManager implements Closeable {

  final String solrBaseUrl;
  final HttpSolrClient solrClient;
  final SolrZkClient zkClient;

  private Map> packages = null;

  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());


  public PackageManager(HttpSolrClient solrClient, String solrBaseUrl, String zkHost) {
    this.solrBaseUrl = solrBaseUrl;
    this.solrClient = solrClient;
    this.zkClient = new SolrZkClient(zkHost, 30000);
    log.info("Done initializing a zkClient instance...");
  }

  @Override
  public void close() throws IOException {
    if (zkClient != null) {
      zkClient.close();
    }
  }

  public List fetchInstalledPackageInstances() throws SolrException {
    log.info("Getting packages from packages.json...");
    List ret = new ArrayList();
    packages = new HashMap>();
    try {
      Map packagesZnodeMap = null;

      if (zkClient.exists(ZkStateReader.SOLR_PKGS_PATH, true) == true) {
        packagesZnodeMap = (Map)getMapper().readValue(
            new String(zkClient.getData(ZkStateReader.SOLR_PKGS_PATH, null, null, true), "UTF-8"), Map.class).get("packages");
        for (Object packageName: packagesZnodeMap.keySet()) {
          List pkg = (List)packagesZnodeMap.get(packageName);
          for (Map pkgVersion: (List)pkg) {
            Manifest manifest = PackageUtils.fetchManifest(solrClient, solrBaseUrl, pkgVersion.get("manifest").toString(), pkgVersion.get("manifestSHA512").toString());
            List solrplugins = manifest.plugins;
            SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null, 
                pkgVersion.get("version").toString(), manifest, solrplugins, manifest.parameterDefaults);
            List list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList();
            list.add(pkgInstance);
            packages.put(packageName.toString(), list);
            ret.add(pkgInstance);
          }
        }
      }
    } catch (Exception e) {
      throw new SolrException(ErrorCode.BAD_REQUEST, e);
    }
    log.info("Got packages: "+ret);
    return ret;
  }

  public Map getPackagesDeployed(String collection) {
    Map packages = null;
    try {
      NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(),
          solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER);
      packages = (Map) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap());
    } catch (PathNotFoundException ex) {
      // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
    }
    if (packages == null) return Collections.emptyMap();
    Map ret = new HashMap<>();
    for (String packageName: packages.keySet()) {
      if (Strings.isNullOrEmpty(packageName) == false && // There can be an empty key, storing the version here
          packages.get(packageName) != null) { // null means the package was undeployed from this package before
        ret.put(packageName, getPackageInstance(packageName, packages.get(packageName)));
      }
    }
    return ret;
  }

  private void ensureCollectionsExist(List collections) {
    try {
      List existingCollections = zkClient.getChildren("/collections", null, true);
      Set nonExistent = new HashSet<>(collections);
      nonExistent.removeAll(existingCollections);
      if (nonExistent.isEmpty() == false) {
          throw new SolrException(ErrorCode.BAD_REQUEST, "Collection(s) doesn't exist: " + nonExistent.toString());
      }
    } catch (KeeperException | InterruptedException e) {
      throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to fetch list of collections from ZK.");
    }
  }
  
  private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt,
      List collections, String overrides[]) {
    List previouslyDeployed =  new ArrayList<>(); // collections where package is already deployed in

    for (String collection: collections) {
      SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name);
      if (packageInstance.equals(deployedPackage)) {
        if (!pegToLatest) {
          PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection);
          previouslyDeployed.add(collection);
          continue;
        }
      } else {
        if (deployedPackage != null && !isUpdate) {
          PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter.");
          previouslyDeployed.add(collection);
          continue;
        }
      }

      Map collectionParameterOverrides = getCollectionParameterOverrides(packageInstance, isUpdate, overrides, collection);

      // Get package params
      try {
        boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class)
            .getOrDefault("response", Collections.emptyMap())).containsKey("params");
        SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
            getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set",
                Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides)))));
      } catch (Exception e) {
        throw new SolrException(ErrorCode.SERVER_ERROR, e);
      }

      // Set the package version in the collection's parameters
      try {
        SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
            "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}");
      } catch (Exception ex) {
        throw new SolrException(ErrorCode.SERVER_ERROR, ex);
      }

      // If updating, refresh the package version for this to take effect
      if (isUpdate || pegToLatest) {
        try {
          SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageInstance.name + "\"}");
        } catch (Exception ex) {
          throw new SolrException(ErrorCode.SERVER_ERROR, ex);
        }
      }

      // If it is a fresh deploy on a collection, run setup commands all the plugins in the package
      if (!isUpdate) {
        Map systemParams = PackageUtils.map("collection", collection, "package-name", packageInstance.name, "package-version", packageInstance.version);

        for (Plugin plugin: packageInstance.plugins) {
          Command cmd = plugin.setupCommand;
          if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
            if ("POST".equalsIgnoreCase(cmd.method)) {
              try {
                String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
                String path = PackageUtils.resolve(cmd.path, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
                PackageUtils.printGreen("Executing " + payload + " for path:" + path);
                boolean shouldExecute = true;
                if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin
                  PackageUtils.print(PackageUtils.YELLOW, "Execute this command (y/n): ");
                  String userInput = new Scanner(System.in, "UTF-8").next();
                  if (!"yes".equalsIgnoreCase(userInput) && !"y".equalsIgnoreCase(userInput)) {
                    shouldExecute = false;
                    PackageUtils.printRed("Skipping setup command for deploying (deployment verification may fail)."
                        + " Please run this step manually or refer to package documentation.");
                  }
                }
                if (shouldExecute) {
                  SolrCLI.postJsonToSolr(solrClient, path, payload);
                }
              } catch (Exception ex) {
                throw new SolrException(ErrorCode.SERVER_ERROR, ex);
              }
            } else {
              throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for setup commands");
            }
          } else {
            PackageUtils.printRed("There is no setup command to execute for plugin: " + plugin.name);
          }
        }
      }

      // Set the package version in the collection's parameters
      try {
        SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
            "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}");
      } catch (Exception ex) {
        throw new SolrException(ErrorCode.SERVER_ERROR, ex);
      }
    }

    List deployedCollections = collections.stream().filter(c -> !previouslyDeployed.contains(c)).collect(Collectors.toList());

    boolean success = true;
    if (deployedCollections.isEmpty() == false) {
      // Verify that package was successfully deployed
      success = verify(packageInstance, deployedCollections);
      if (success) {
        PackageUtils.printGreen("Deployed on " + deployedCollections + " and verified package: " + packageInstance.name + ", version: " + packageInstance.version);
      }
    }
    if (previouslyDeployed.isEmpty() == false) {
      PackageUtils.printRed("Already Deployed on " + previouslyDeployed + ", package: " + packageInstance.name + ", version: " + packageInstance.version);
    }
    return previouslyDeployed.isEmpty() && success;
  }

  private Map getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate,
      String[] overrides, String collection) {
    Map collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap();
    if (overrides != null) {
      for (String override: overrides) {
        collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]);
      }
    }
    return collectionParameterOverrides;
  }

  @SuppressWarnings({"rawtypes", "unchecked"})
  Map getPackageParams(String packageName, String collection) {
    try {
      return (Map)((Map)((Map)((Map)
          PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class)
          .get("response"))
          .get("params"))
          .get("packages")).get(packageName);
    } catch (Exception ex) {
      // This should be because there are no parameters. Be tolerant here.
      return Collections.emptyMap();
    }
  }

  /**
   * Given a package and list of collections, verify if the package is installed
   * in those collections. It uses the verify command of every plugin in the package (if defined).
   */
  public boolean verify(SolrPackageInstance pkg, List collections) {
    boolean success = true;
    for (Plugin plugin: pkg.plugins) {
      Command cmd = plugin.verifyCommand;
      if (plugin.verifyCommand != null && !Strings.isNullOrEmpty(cmd.path)) {
        for (String collection: collections) {
          Map collectionParameterOverrides = getPackageParams(pkg.name, collection);

          Map systemParams = PackageUtils.map("collection", collection, "package-name", pkg.name, "package-version", pkg.version);
          String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
          PackageUtils.printGreen("Executing " + url + " for collection:" + collection);

          if ("GET".equalsIgnoreCase(cmd.method)) {
            String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url);
            PackageUtils.printGreen(response);
            String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration())
                .read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams));
            String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
            PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue);
            if (!expectedValue.equals(actualValue)) {
              PackageUtils.printRed("Failed to deploy plugin: " + plugin.name);
              success = false;
            }
          } else {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for verify commands");
          }
        }
      }
    }
    return success;
  }

  /**
   * Get the installed instance of a specific version of a package. If version is null, PackageUtils.LATEST or PackagePluginHolder.LATEST,
   * then it returns the highest version available in the system for the package.
   */
  public SolrPackageInstance getPackageInstance(String packageName, String version) {
    fetchInstalledPackageInstances();
    List versions = packages.get(packageName);
    SolrPackageInstance latest = null;
    if (versions != null && !versions.isEmpty()) {
      latest = versions.get(0);
      for (int i=0; i collectionParameterOverrides = getPackageParams(packageName, collection);

      // Run the uninstall command for all plugins
      Map systemParams = PackageUtils.map("collection", collection, "package-name", deployedPackage.name, "package-version", deployedPackage.version);

      for (Plugin plugin: deployedPackage.plugins) {
        Command cmd = plugin.uninstallCommand;
        if (cmd != null && !Strings.isNullOrEmpty(cmd.method)) {
          if ("POST".equalsIgnoreCase(cmd.method)) {
            try {
              String payload = PackageUtils.resolve(getMapper().writeValueAsString(cmd.payload), deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
              String path = PackageUtils.resolve(cmd.path, deployedPackage.parameterDefaults, collectionParameterOverrides, systemParams);
              PackageUtils.printGreen("Executing " + payload + " for path:" + path);
              SolrCLI.postJsonToSolr(solrClient, path, payload);
            } catch (Exception ex) {
              throw new SolrException(ErrorCode.SERVER_ERROR, ex);
            }
          } else {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Non-POST method not supported for uninstall commands");
          }
        } else {
          PackageUtils.printRed("There is no uninstall command to execute for plugin: " + plugin.name);
        }
      }

      // Set the package version in the collection's parameters
      try {
        SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
            "{set: {PKG_VERSIONS: {"+packageName+": null}}}"); // Is it better to "unset"? If so, build support in params API for "unset"
        SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}");
      } catch (Exception ex) {
        throw new SolrException(ErrorCode.SERVER_ERROR, ex);
      }

      // TODO: Also better to remove the package parameters PKG_VERSION etc.
    }
  }

  /**
   * Given a package, return a map of collections where this package is
   * installed to the installed version (which can be {@link PackagePluginHolder#LATEST})
   */
  public Map getDeployedCollections(String packageName) {
    List allCollections;
    try {
      allCollections = zkClient.getChildren(ZkStateReader.COLLECTIONS_ZKNODE, null, true);
    } catch (KeeperException | InterruptedException e) {
      throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e);
    }
    Map deployed = new HashMap();
    for (String collection: allCollections) {
      // Check package version installed
      String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true");
      String version = null;
      try {
        version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration())
            .read("$['response'].['params'].['PKG_VERSIONS'].['"+packageName+"'])");
      } catch (PathNotFoundException ex) {
        // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
      }
      if (version != null) {
        deployed.put(collection, version);
      }
    }
    return deployed;
  }

}