org.elasticsearch.plugins.PluginsService Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.bootstrap.JarHell;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.transport.TransportSettings;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
public class PluginsService {
private static final Logger logger = LogManager.getLogger(PluginsService.class);
private final Settings settings;
private final Path configPath;
/**
* We keep around a list of plugins and modules
*/
private final List> plugins;
private final PluginsAndModules info;
public static final Setting> MANDATORY_SETTING =
Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
public List> getPluginSettings() {
return plugins.stream().flatMap(p -> p.v2().getSettings().stream()).collect(Collectors.toList());
}
public List getPluginSettingsFilter() {
return plugins.stream().flatMap(p -> p.v2().getSettingsFilter().stream()).collect(Collectors.toList());
}
/**
* Constructs a new PluginService
* @param settings The settings of the system
* @param modulesDirectory The directory modules exist in, or null if modules should not be loaded from the filesystem
* @param pluginsDirectory The directory plugins exist in, or null if plugins should not be loaded from the filesystem
* @param classpathPlugins Plugins that exist in the classpath which should be loaded
*/
public PluginsService(
Settings settings,
Path configPath,
Path modulesDirectory,
Path pluginsDirectory,
Collection> classpathPlugins
) {
this.settings = settings;
this.configPath = configPath;
List> pluginsLoaded = new ArrayList<>();
List pluginsList = new ArrayList<>();
// we need to build a List of plugins for checking mandatory plugins
final List pluginsNames = new ArrayList<>();
// first we load plugins that are on the classpath. this is for tests and transport clients
for (Class pluginClass : classpathPlugins) {
Plugin plugin = loadPlugin(pluginClass, settings, configPath);
PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", Version.CURRENT, "1.8",
pluginClass.getName(), Collections.emptyList(), false);
if (logger.isTraceEnabled()) {
logger.trace("plugin loaded from classpath [{}]", pluginInfo);
}
pluginsLoaded.add(new Tuple<>(pluginInfo, plugin));
pluginsList.add(pluginInfo);
pluginsNames.add(pluginInfo.getName());
}
Set seenBundles = new LinkedHashSet<>();
List modulesList = new ArrayList<>();
// load modules
if (modulesDirectory != null) {
try {
Set modules = getModuleBundles(modulesDirectory);
for (Bundle bundle : modules) {
modulesList.add(bundle.plugin);
}
seenBundles.addAll(modules);
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize modules", ex);
}
}
// now, find all the ones that are in plugins/
if (pluginsDirectory != null) {
try {
// TODO: remove this leniency, but tests bogusly rely on it
if (isAccessibleDirectory(pluginsDirectory, logger)) {
checkForFailedPluginRemovals(pluginsDirectory);
Set plugins = getPluginBundles(pluginsDirectory);
for (final Bundle bundle : plugins) {
pluginsList.add(bundle.plugin);
pluginsNames.add(bundle.plugin.getName());
}
seenBundles.addAll(plugins);
}
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize plugins", ex);
}
}
List> loaded = loadBundles(seenBundles);
pluginsLoaded.addAll(loaded);
this.info = new PluginsAndModules(pluginsList, modulesList);
this.plugins = Collections.unmodifiableList(pluginsLoaded);
// Checking expected plugins
List mandatoryPlugins = MANDATORY_SETTING.get(settings);
if (mandatoryPlugins.isEmpty() == false) {
Set missingPlugins = new HashSet<>();
for (String mandatoryPlugin : mandatoryPlugins) {
if (!pluginsNames.contains(mandatoryPlugin) && !missingPlugins.contains(mandatoryPlugin)) {
missingPlugins.add(mandatoryPlugin);
}
}
if (!missingPlugins.isEmpty()) {
final String message = String.format(
Locale.ROOT,
"missing mandatory plugins [%s], found plugins [%s]",
Strings.collectionToDelimitedString(missingPlugins, ", "),
Strings.collectionToDelimitedString(pluginsNames, ", "));
throw new IllegalStateException(message);
}
}
// we don't log jars in lib/ we really shouldn't log modules,
// but for now: just be transparent so we can debug any potential issues
logPluginInfo(info.getModuleInfos(), "module", logger);
logPluginInfo(info.getPluginInfos(), "plugin", logger);
}
private static void logPluginInfo(final List pluginInfos, final String type, final Logger logger) {
assert pluginInfos != null;
if (pluginInfos.isEmpty()) {
logger.info("no " + type + "s loaded");
} else {
for (final String name : pluginInfos.stream().map(PluginInfo::getName).sorted().collect(Collectors.toList())) {
logger.info("loaded " + type + " [" + name + "]");
}
}
}
public Settings updatedSettings() {
Map foundSettings = new HashMap<>();
final Map features = new TreeMap<>();
final Settings.Builder builder = Settings.builder();
for (Tuple plugin : plugins) {
Settings settings = plugin.v2().additionalSettings();
for (String setting : settings.keySet()) {
String oldPlugin = foundSettings.put(setting, plugin.v1().getName());
if (oldPlugin != null) {
throw new IllegalArgumentException("Cannot have additional setting [" + setting + "] " +
"in plugin [" + plugin.v1().getName() + "], already added in plugin [" + oldPlugin + "]");
}
}
builder.put(settings);
final Optional maybeFeature = plugin.v2().getFeature();
if (maybeFeature.isPresent()) {
final String feature = maybeFeature.get();
if (features.containsKey(feature)) {
final String message = String.format(
Locale.ROOT,
"duplicate feature [%s] in plugin [%s], already added in [%s]",
feature,
plugin.v1().getName(),
features.get(feature));
throw new IllegalArgumentException(message);
}
features.put(feature, plugin.v1().getName());
}
}
for (final String feature : features.keySet()) {
builder.put(TransportSettings.FEATURE_PREFIX + "." + feature, true);
}
return builder.put(this.settings).build();
}
public Collection createGuiceModules() {
List modules = new ArrayList<>();
for (Tuple plugin : plugins) {
modules.addAll(plugin.v2().createGuiceModules());
}
return modules;
}
public List> getExecutorBuilders(Settings settings) {
final ArrayList> builders = new ArrayList<>();
for (final Tuple plugin : plugins) {
builders.addAll(plugin.v2().getExecutorBuilders(settings));
}
return builders;
}
/** Returns all classes injected into guice by plugins which extend {@link LifecycleComponent}. */
public Collection> getGuiceServiceClasses() {
List> services = new ArrayList<>();
for (Tuple plugin : plugins) {
services.addAll(plugin.v2().getGuiceServiceClasses());
}
return services;
}
public void onIndexModule(IndexModule indexModule) {
for (Tuple plugin : plugins) {
plugin.v2().onIndexModule(indexModule);
}
}
/**
* Get information about plugins and modules
*/
public PluginsAndModules info() {
return info;
}
// a "bundle" is a group of jars in a single classloader
static class Bundle {
final PluginInfo plugin;
final Set urls;
Bundle(PluginInfo plugin, Path dir) throws IOException {
this.plugin = Objects.requireNonNull(plugin);
Set urls = new LinkedHashSet<>();
// gather urls for jar files
try (DirectoryStream jarStream = Files.newDirectoryStream(dir, "*.jar")) {
for (Path jar : jarStream) {
// normalize with toRealPath to get symlinks out of our hair
URL url = jar.toRealPath().toUri().toURL();
if (urls.add(url) == false) {
throw new IllegalStateException("duplicate codebase: " + url);
}
}
}
this.urls = Objects.requireNonNull(urls);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Bundle bundle = (Bundle) o;
return Objects.equals(plugin, bundle.plugin);
}
@Override
public int hashCode() {
return Objects.hash(plugin);
}
}
/**
* Extracts all installed plugin directories from the provided {@code rootPath}.
*
* @param rootPath the path where the plugins are installed
* @return a list of all plugin paths installed in the {@code rootPath}
* @throws IOException if an I/O exception occurred reading the directories
*/
public static List findPluginDirs(final Path rootPath) throws IOException {
final List plugins = new ArrayList<>();
final Set seen = new HashSet<>();
if (Files.exists(rootPath)) {
try (DirectoryStream stream = Files.newDirectoryStream(rootPath)) {
for (Path plugin : stream) {
if (FileSystemUtils.isDesktopServicesStore(plugin) ||
plugin.getFileName().toString().startsWith(".removing-")) {
continue;
}
if (seen.add(plugin.getFileName().toString()) == false) {
throw new IllegalStateException("duplicate plugin: " + plugin);
}
plugins.add(plugin);
}
}
}
return plugins;
}
/**
* Verify the given plugin is compatible with the current Elasticsearch installation.
*/
static void verifyCompatibility(PluginInfo info) {
if (info.getElasticsearchVersion().equals(Version.CURRENT) == false) {
throw new IllegalArgumentException("Plugin [" + info.getName() + "] was built for Elasticsearch version "
+ info.getElasticsearchVersion() + " but version " + Version.CURRENT + " is running");
}
JarHell.checkJavaVersion(info.getName(), info.getJavaVersion());
}
static void checkForFailedPluginRemovals(final Path pluginsDirectory) throws IOException {
/*
* Check for the existence of a marker file that indicates any plugins are in a garbage state from a failed attempt to remove the
* plugin.
*/
try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory, ".removing-*")) {
final Iterator iterator = stream.iterator();
if (iterator.hasNext()) {
final Path removing = iterator.next();
final String fileName = removing.getFileName().toString();
final String name = fileName.substring(1 + fileName.indexOf("-"));
final String message = String.format(
Locale.ROOT,
"found file [%s] from a failed attempt to remove the plugin [%s]; execute [elasticsearch-plugin remove %2$s]",
removing,
name);
throw new IllegalStateException(message);
}
}
}
/** Get bundles for plugins installed in the given modules directory. */
static Set getModuleBundles(Path modulesDirectory) throws IOException {
return findBundles(modulesDirectory, "module");
}
/** Get bundles for plugins installed in the given plugins directory. */
static Set getPluginBundles(final Path pluginsDirectory) throws IOException {
return findBundles(pluginsDirectory, "plugin");
}
// searches subdirectories under the given directory for plugin directories
private static Set findBundles(final Path directory, String type) throws IOException {
final Set bundles = new HashSet<>();
for (final Path plugin : findPluginDirs(directory)) {
final Bundle bundle = readPluginBundle(bundles, plugin, type);
bundles.add(bundle);
}
return bundles;
}
// get a bundle for a single plugin dir
private static Bundle readPluginBundle(final Set bundles, final Path plugin, String type) throws IOException {
LogManager.getLogger(PluginsService.class).trace("--- adding [{}] [{}]", type, plugin.toAbsolutePath());
final PluginInfo info;
try {
info = PluginInfo.readFromProperties(plugin);
} catch (final IOException e) {
throw new IllegalStateException("Could not load plugin descriptor for " + type +
" directory [" + plugin.getFileName() + "]", e);
}
final Bundle bundle = new Bundle(info, plugin);
if (bundles.add(bundle) == false) {
throw new IllegalStateException("duplicate " + type + ": " + info);
}
return bundle;
}
/**
* Return the given bundles, sorted in dependency loading order.
*
* This sort is stable, so that if two plugins do not have any interdependency,
* their relative order from iteration of the provided set will not change.
*
* @throws IllegalStateException if a dependency cycle is found
*/
// pkg private for tests
static List sortBundles(Set bundles) {
Map namedBundles = bundles.stream().collect(Collectors.toMap(b -> b.plugin.getName(), Function.identity()));
LinkedHashSet sortedBundles = new LinkedHashSet<>();
LinkedHashSet dependencyStack = new LinkedHashSet<>();
for (Bundle bundle : bundles) {
addSortedBundle(bundle, namedBundles, sortedBundles, dependencyStack);
}
return new ArrayList<>(sortedBundles);
}
// add the given bundle to the sorted bundles, first adding dependencies
private static void addSortedBundle(Bundle bundle, Map bundles, LinkedHashSet sortedBundles,
LinkedHashSet dependencyStack) {
String name = bundle.plugin.getName();
if (dependencyStack.contains(name)) {
StringBuilder msg = new StringBuilder("Cycle found in plugin dependencies: ");
dependencyStack.forEach(s -> {
msg.append(s);
msg.append(" -> ");
});
msg.append(name);
throw new IllegalStateException(msg.toString());
}
if (sortedBundles.contains(bundle)) {
// already added this plugin, via a dependency
return;
}
dependencyStack.add(name);
for (String dependency : bundle.plugin.getExtendedPlugins()) {
Bundle depBundle = bundles.get(dependency);
if (depBundle == null) {
throw new IllegalArgumentException("Missing plugin [" + dependency + "], dependency of [" + name + "]");
}
addSortedBundle(depBundle, bundles, sortedBundles, dependencyStack);
assert sortedBundles.contains(depBundle);
}
dependencyStack.remove(name);
sortedBundles.add(bundle);
}
private List> loadBundles(Set bundles) {
List> plugins = new ArrayList<>();
Map loaded = new HashMap<>();
Map> transitiveUrls = new HashMap<>();
List sortedBundles = sortBundles(bundles);
for (Bundle bundle : sortedBundles) {
checkBundleJarHell(JarHell.parseClassPath(), bundle, transitiveUrls);
final Plugin plugin = loadBundle(bundle, loaded);
plugins.add(new Tuple<>(bundle.plugin, plugin));
}
return Collections.unmodifiableList(plugins);
}
// jar-hell check the bundle against the parent classloader and extended plugins
// the plugin cli does it, but we do it again, in case lusers mess with jar files manually
static void checkBundleJarHell(Set classpath, Bundle bundle, Map> transitiveUrls) {
// invariant: any plugins this plugin bundle extends have already been added to transitiveUrls
List exts = bundle.plugin.getExtendedPlugins();
try {
final Logger logger = LogManager.getLogger(JarHell.class);
Set urls = new HashSet<>();
for (String extendedPlugin : exts) {
Set pluginUrls = transitiveUrls.get(extendedPlugin);
assert pluginUrls != null : "transitive urls should have already been set for " + extendedPlugin;
Set intersection = new HashSet<>(urls);
intersection.retainAll(pluginUrls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! extended plugins " + exts +
" have duplicate codebases with each other: " + intersection);
}
intersection = new HashSet<>(bundle.urls);
intersection.retainAll(pluginUrls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! duplicate codebases with extended plugin [" +
extendedPlugin + "]: " + intersection);
}
urls.addAll(pluginUrls);
JarHell.checkJarHell(urls, logger::debug); // check jarhell as we add each extended plugin's urls
}
urls.addAll(bundle.urls);
JarHell.checkJarHell(urls, logger::debug); // check jarhell of each extended plugin against this plugin
transitiveUrls.put(bundle.plugin.getName(), urls);
// check we don't have conflicting codebases with core
Set intersection = new HashSet<>(classpath);
intersection.retainAll(bundle.urls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! duplicate codebases between plugin and core: " + intersection);
}
// check we don't have conflicting classes
Set union = new HashSet<>(classpath);
union.addAll(bundle.urls);
JarHell.checkJarHell(union, logger::debug);
} catch (Exception e) {
throw new IllegalStateException("failed to load plugin " + bundle.plugin.getName() + " due to jar hell", e);
}
}
private Plugin loadBundle(Bundle bundle, Map loaded) {
String name = bundle.plugin.getName();
verifyCompatibility(bundle.plugin);
// collect loaders of extended plugins
List extendedLoaders = new ArrayList<>();
for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
Plugin extendedPlugin = loaded.get(extendedPluginName);
assert extendedPlugin != null;
if (ExtensiblePlugin.class.isInstance(extendedPlugin) == false) {
throw new IllegalStateException("Plugin [" + name + "] cannot extend non-extensible plugin [" + extendedPluginName + "]");
}
extendedLoaders.add(extendedPlugin.getClass().getClassLoader());
}
// create a child to load the plugin in this bundle
ClassLoader parentLoader = PluginLoaderIndirection.createLoader(getClass().getClassLoader(), extendedLoaders);
ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), parentLoader);
// reload SPI with any new services from the plugin
reloadLuceneSPI(loader);
for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
// note: already asserted above that extended plugins are loaded and extensible
ExtensiblePlugin.class.cast(loaded.get(extendedPluginName)).reloadSPI(loader);
}
Class pluginClass = loadPluginClass(bundle.plugin.getClassname(), loader);
Plugin plugin = loadPlugin(pluginClass, settings, configPath);
loaded.put(name, plugin);
return plugin;
}
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after the new classloader has been created to
* register the services for use.
*/
static void reloadLuceneSPI(ClassLoader loader) {
// do NOT change the order of these method calls!
// Codecs:
PostingsFormat.reloadPostingsFormats(loader);
DocValuesFormat.reloadDocValuesFormats(loader);
Codec.reloadCodecs(loader);
// Analysis:
CharFilterFactory.reloadCharFilters(loader);
TokenFilterFactory.reloadTokenFilters(loader);
TokenizerFactory.reloadTokenizers(loader);
}
private Class loadPluginClass(String className, ClassLoader loader) {
try {
return loader.loadClass(className).asSubclass(Plugin.class);
} catch (ClassNotFoundException e) {
throw new ElasticsearchException("Could not find plugin class [" + className + "]", e);
}
}
private Plugin loadPlugin(Class pluginClass, Settings settings, Path configPath) {
final Constructor[] constructors = pluginClass.getConstructors();
if (constructors.length == 0) {
throw new IllegalStateException("no public constructor for [" + pluginClass.getName() + "]");
}
if (constructors.length > 1) {
throw new IllegalStateException("no unique public constructor for [" + pluginClass.getName() + "]");
}
final Constructor constructor = constructors[0];
if (constructor.getParameterCount() > 2) {
throw new IllegalStateException(signatureMessage(pluginClass));
}
final Class[] parameterTypes = constructor.getParameterTypes();
try {
if (constructor.getParameterCount() == 2 && parameterTypes[0] == Settings.class && parameterTypes[1] == Path.class) {
return (Plugin)constructor.newInstance(settings, configPath);
} else if (constructor.getParameterCount() == 1 && parameterTypes[0] == Settings.class) {
return (Plugin)constructor.newInstance(settings);
} else if (constructor.getParameterCount() == 0) {
return (Plugin)constructor.newInstance();
} else {
throw new IllegalStateException(signatureMessage(pluginClass));
}
} catch (final ReflectiveOperationException e) {
throw new IllegalStateException("failed to load plugin class [" + pluginClass.getName() + "]", e);
}
}
private String signatureMessage(final Class clazz) {
return String.format(
Locale.ROOT,
"no public constructor of correct signature for [%s]; must be [%s], [%s], or [%s]",
clazz.getName(),
"(org.elasticsearch.common.settings.Settings,java.nio.file.Path)",
"(org.elasticsearch.common.settings.Settings)",
"()");
}
public List filterPlugins(Class type) {
return plugins.stream().filter(x -> type.isAssignableFrom(x.v2().getClass()))
.map(p -> ((T)p.v2())).collect(Collectors.toList());
}
}