org.elasticsearch.plugins.PluginsService Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.bootstrap.JarHell;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.threadpool.ExecutorBuilder;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
public class PluginsService extends AbstractComponent {
private final Path configPath;
/**
* We keep around a list of plugins and modules
*/
private final List> plugins;
private final PluginsAndModules info;
public static final Setting> MANDATORY_SETTING =
Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
public List> getPluginSettings() {
return plugins.stream().flatMap(p -> p.v2().getSettings().stream()).collect(Collectors.toList());
}
public List getPluginSettingsFilter() {
return plugins.stream().flatMap(p -> p.v2().getSettingsFilter().stream()).collect(Collectors.toList());
}
/**
* Constructs a new PluginService
* @param settings The settings of the system
* @param modulesDirectory The directory modules exist in, or null if modules should not be loaded from the filesystem
* @param pluginsDirectory The directory plugins exist in, or null if plugins should not be loaded from the filesystem
* @param classpathPlugins Plugins that exist in the classpath which should be loaded
*/
public PluginsService(Settings settings, Path configPath, Path modulesDirectory, Path pluginsDirectory, Collection> classpathPlugins) {
super(settings);
this.configPath = configPath;
List> pluginsLoaded = new ArrayList<>();
List pluginsList = new ArrayList<>();
// first we load plugins that are on the classpath. this is for tests and transport clients
for (Class pluginClass : classpathPlugins) {
Plugin plugin = loadPlugin(pluginClass, settings, configPath);
PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA",
pluginClass.getName(), Collections.emptyList(), false, false);
if (logger.isTraceEnabled()) {
logger.trace("plugin loaded from classpath [{}]", pluginInfo);
}
pluginsLoaded.add(new Tuple<>(pluginInfo, plugin));
pluginsList.add(pluginInfo);
}
Set seenBundles = new LinkedHashSet<>();
List modulesList = new ArrayList<>();
// load modules
if (modulesDirectory != null) {
try {
Set modules = getModuleBundles(modulesDirectory);
for (Bundle bundle : modules) {
modulesList.add(bundle.plugin);
}
seenBundles.addAll(modules);
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize modules", ex);
}
}
// now, find all the ones that are in plugins/
if (pluginsDirectory != null) {
try {
// TODO: remove this leniency, but tests bogusly rely on it
if (isAccessibleDirectory(pluginsDirectory, logger)) {
checkForFailedPluginRemovals(pluginsDirectory);
Set plugins = getPluginBundles(pluginsDirectory);
for (Bundle bundle : plugins) {
pluginsList.add(bundle.plugin);
}
seenBundles.addAll(plugins);
}
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize plugins", ex);
}
}
List> loaded = loadBundles(seenBundles);
pluginsLoaded.addAll(loaded);
this.info = new PluginsAndModules(pluginsList, modulesList);
this.plugins = Collections.unmodifiableList(pluginsLoaded);
// We need to build a List of plugins for checking mandatory plugins
Set pluginsNames = new HashSet<>();
for (Tuple tuple : this.plugins) {
pluginsNames.add(tuple.v1().getName());
}
// Checking expected plugins
List mandatoryPlugins = MANDATORY_SETTING.get(settings);
if (mandatoryPlugins.isEmpty() == false) {
Set missingPlugins = new HashSet<>();
for (String mandatoryPlugin : mandatoryPlugins) {
if (!pluginsNames.contains(mandatoryPlugin) && !missingPlugins.contains(mandatoryPlugin)) {
missingPlugins.add(mandatoryPlugin);
}
}
if (!missingPlugins.isEmpty()) {
throw new ElasticsearchException("Missing mandatory plugins [" + Strings.collectionToDelimitedString(missingPlugins, ", ") + "]");
}
}
// we don't log jars in lib/ we really shouldn't log modules,
// but for now: just be transparent so we can debug any potential issues
logPluginInfo(info.getModuleInfos(), "module", logger);
logPluginInfo(info.getPluginInfos(), "plugin", logger);
}
private static void logPluginInfo(final List pluginInfos, final String type, final Logger logger) {
assert pluginInfos != null;
if (pluginInfos.isEmpty()) {
logger.info("no " + type + "s loaded");
} else {
for (final String name : pluginInfos.stream().map(PluginInfo::getName).sorted().collect(Collectors.toList())) {
logger.info("loaded " + type + " [" + name + "]");
}
}
}
public Settings updatedSettings() {
Map foundSettings = new HashMap<>();
final Settings.Builder builder = Settings.builder();
for (Tuple plugin : plugins) {
Settings settings = plugin.v2().additionalSettings();
for (String setting : settings.keySet()) {
String oldPlugin = foundSettings.put(setting, plugin.v1().getName());
if (oldPlugin != null) {
throw new IllegalArgumentException("Cannot have additional setting [" + setting + "] " +
"in plugin [" + plugin.v1().getName() + "], already added in plugin [" + oldPlugin + "]");
}
}
builder.put(settings);
}
return builder.put(this.settings).build();
}
public Collection createGuiceModules() {
List modules = new ArrayList<>();
for (Tuple plugin : plugins) {
modules.addAll(plugin.v2().createGuiceModules());
}
return modules;
}
public List> getExecutorBuilders(Settings settings) {
final ArrayList> builders = new ArrayList<>();
for (final Tuple plugin : plugins) {
builders.addAll(plugin.v2().getExecutorBuilders(settings));
}
return builders;
}
/** Returns all classes injected into guice by plugins which extend {@link LifecycleComponent}. */
public Collection> getGuiceServiceClasses() {
List> services = new ArrayList<>();
for (Tuple plugin : plugins) {
services.addAll(plugin.v2().getGuiceServiceClasses());
}
return services;
}
public void onIndexModule(IndexModule indexModule) {
for (Tuple plugin : plugins) {
plugin.v2().onIndexModule(indexModule);
}
}
/**
* Get information about plugins and modules
*/
public PluginsAndModules info() {
return info;
}
// a "bundle" is a group of plugins in a single classloader
// really should be 1-1, but we are not so fortunate
static class Bundle {
final PluginInfo plugin;
final Set urls;
Bundle(PluginInfo plugin, Path dir) throws IOException {
this.plugin = Objects.requireNonNull(plugin);
Set urls = new LinkedHashSet<>();
// gather urls for jar files
try (DirectoryStream jarStream = Files.newDirectoryStream(dir, "*.jar")) {
for (Path jar : jarStream) {
// normalize with toRealPath to get symlinks out of our hair
URL url = jar.toRealPath().toUri().toURL();
if (urls.add(url) == false) {
throw new IllegalStateException("duplicate codebase: " + url);
}
}
}
this.urls = Objects.requireNonNull(urls);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Bundle bundle = (Bundle) o;
return Objects.equals(plugin, bundle.plugin);
}
@Override
public int hashCode() {
return Objects.hash(plugin);
}
}
// similar in impl to getPluginBundles, but DO NOT try to make them share code.
// we don't need to inherit all the leniency, and things are different enough.
static Set getModuleBundles(Path modulesDirectory) throws IOException {
// damn leniency
if (Files.notExists(modulesDirectory)) {
return Collections.emptySet();
}
Set bundles = new LinkedHashSet<>();
try (DirectoryStream stream = Files.newDirectoryStream(modulesDirectory)) {
for (Path module : stream) {
PluginInfo info = PluginInfo.readFromProperties(module);
if (bundles.add(new Bundle(info, module)) == false) {
throw new IllegalStateException("duplicate module: " + info);
}
}
}
return bundles;
}
static void checkForFailedPluginRemovals(final Path pluginsDirectory) throws IOException {
/*
* Check for the existence of a marker file that indicates any plugins are in a garbage state from a failed attempt to remove the
* plugin.
*/
try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory, ".removing-*")) {
final Iterator iterator = stream.iterator();
if (iterator.hasNext()) {
final Path removing = iterator.next();
final String fileName = removing.getFileName().toString();
final String name = fileName.substring(1 + fileName.indexOf("-"));
final String message = String.format(
Locale.ROOT,
"found file [%s] from a failed attempt to remove the plugin [%s]; execute [elasticsearch-plugin remove %2$s]",
removing,
name);
throw new IllegalStateException(message);
}
}
}
/**
* Get the plugin bundles from the specified directory.
*
* @param pluginsDirectory the directory
* @return the set of plugin bundles in the specified directory
* @throws IOException if an I/O exception occurs reading the plugin bundles
*/
static Set getPluginBundles(final Path pluginsDirectory) throws IOException {
return getPluginBundles(pluginsDirectory, true);
}
/**
* Get the plugin bundles from the specified directory. If {@code enforceVersion} is true, then the version in each plugin descriptor
* must match the current version.
*
* @param pluginsDirectory the directory
* @param enforceVersion whether or not to enforce the version when reading plugin descriptors
* @return the set of plugin bundles in the specified directory
* @throws IOException if an I/O exception occurs reading the plugin bundles
*/
static Set getPluginBundles(final Path pluginsDirectory, final boolean enforceVersion) throws IOException {
Logger logger = Loggers.getLogger(PluginsService.class);
Set bundles = new LinkedHashSet<>();
List infos = PluginInfo.extractAllPlugins(pluginsDirectory);
for (Path plugin : infos) {
logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
final PluginInfo info;
try {
info = PluginInfo.readFromProperties(plugin, enforceVersion);
} catch (IOException e) {
throw new IllegalStateException("Could not load plugin descriptor for existing plugin ["
+ plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
}
if (bundles.add(new Bundle(info, plugin)) == false) {
throw new IllegalStateException("duplicate plugin: " + info);
}
}
return bundles;
}
/**
* Return the given bundles, sorted in dependency loading order.
*
* This sort is stable, so that if two plugins do not have any interdependency,
* their relative order from iteration of the provided set will not change.
*
* @throws IllegalStateException if a dependency cycle is found
*/
// pkg private for tests
static List sortBundles(Set bundles) {
Map namedBundles = bundles.stream().collect(Collectors.toMap(b -> b.plugin.getName(), Function.identity()));
LinkedHashSet sortedBundles = new LinkedHashSet<>();
LinkedHashSet dependencyStack = new LinkedHashSet<>();
for (Bundle bundle : bundles) {
addSortedBundle(bundle, namedBundles, sortedBundles, dependencyStack);
}
return new ArrayList<>(sortedBundles);
}
// add the given bundle to the sorted bundles, first adding dependencies
private static void addSortedBundle(Bundle bundle, Map bundles, LinkedHashSet sortedBundles,
LinkedHashSet dependencyStack) {
String name = bundle.plugin.getName();
if (dependencyStack.contains(name)) {
StringBuilder msg = new StringBuilder("Cycle found in plugin dependencies: ");
dependencyStack.forEach(s -> {
msg.append(s);
msg.append(" -> ");
});
msg.append(name);
throw new IllegalStateException(msg.toString());
}
if (sortedBundles.contains(bundle)) {
// already added this plugin, via a dependency
return;
}
dependencyStack.add(name);
for (String dependency : bundle.plugin.getExtendedPlugins()) {
Bundle depBundle = bundles.get(dependency);
if (depBundle == null) {
throw new IllegalArgumentException("Missing plugin [" + dependency + "], dependency of [" + name + "]");
}
addSortedBundle(depBundle, bundles, sortedBundles, dependencyStack);
assert sortedBundles.contains(depBundle);
}
dependencyStack.remove(name);
sortedBundles.add(bundle);
}
private List> loadBundles(Set bundles) {
List> plugins = new ArrayList<>();
Map loaded = new HashMap<>();
Map> transitiveUrls = new HashMap<>();
List sortedBundles = sortBundles(bundles);
for (Bundle bundle : sortedBundles) {
checkBundleJarHell(bundle, transitiveUrls);
final Plugin plugin = loadBundle(bundle, loaded);
plugins.add(new Tuple<>(bundle.plugin, plugin));
}
return Collections.unmodifiableList(plugins);
}
// jar-hell check the bundle against the parent classloader and extended plugins
// the plugin cli does it, but we do it again, in case lusers mess with jar files manually
static void checkBundleJarHell(Bundle bundle, Map> transitiveUrls) {
// invariant: any plugins this plugin bundle extends have already been added to transitiveUrls
List exts = bundle.plugin.getExtendedPlugins();
try {
Set urls = new HashSet<>();
for (String extendedPlugin : exts) {
Set pluginUrls = transitiveUrls.get(extendedPlugin);
assert pluginUrls != null : "transitive urls should have already been set for " + extendedPlugin;
Set intersection = new HashSet<>(urls);
intersection.retainAll(pluginUrls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! extended plugins " + exts +
" have duplicate codebases with each other: " + intersection);
}
intersection = new HashSet<>(bundle.urls);
intersection.retainAll(pluginUrls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! duplicate codebases with extended plugin [" +
extendedPlugin + "]: " + intersection);
}
urls.addAll(pluginUrls);
JarHell.checkJarHell(urls); // check jarhell as we add each extended plugin's urls
}
urls.addAll(bundle.urls);
JarHell.checkJarHell(urls); // check jarhell of each extended plugin against this plugin
transitiveUrls.put(bundle.plugin.getName(), urls);
Set classpath = JarHell.parseClassPath();
// check we don't have conflicting codebases with core
Set intersection = new HashSet<>(classpath);
intersection.retainAll(bundle.urls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! duplicate codebases between plugin and core: " + intersection);
}
// check we don't have conflicting classes
Set union = new HashSet<>(classpath);
union.addAll(bundle.urls);
JarHell.checkJarHell(union);
} catch (Exception e) {
throw new IllegalStateException("failed to load plugin " + bundle.plugin.getName() + " due to jar hell", e);
}
}
private Plugin loadBundle(Bundle bundle, Map loaded) {
String name = bundle.plugin.getName();
// collect loaders of extended plugins
List extendedLoaders = new ArrayList<>();
for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
Plugin extendedPlugin = loaded.get(extendedPluginName);
assert extendedPlugin != null;
if (ExtensiblePlugin.class.isInstance(extendedPlugin) == false) {
throw new IllegalStateException("Plugin [" + name + "] cannot extend non-extensible plugin [" + extendedPluginName + "]");
}
extendedLoaders.add(extendedPlugin.getClass().getClassLoader());
}
// create a child to load the plugin in this bundle
ClassLoader parentLoader = PluginLoaderIndirection.createLoader(getClass().getClassLoader(), extendedLoaders);
ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), parentLoader);
// reload SPI with any new services from the plugin
reloadLuceneSPI(loader);
for (String extendedPluginName : bundle.plugin.getExtendedPlugins()) {
// note: already asserted above that extended plugins are loaded and extensible
ExtensiblePlugin.class.cast(loaded.get(extendedPluginName)).reloadSPI(loader);
}
Class pluginClass = loadPluginClass(bundle.plugin.getClassname(), loader);
Plugin plugin = loadPlugin(pluginClass, settings, configPath);
loaded.put(name, plugin);
return plugin;
}
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after the new classloader has been created to
* register the services for use.
*/
static void reloadLuceneSPI(ClassLoader loader) {
// do NOT change the order of these method calls!
// Codecs:
PostingsFormat.reloadPostingsFormats(loader);
DocValuesFormat.reloadDocValuesFormats(loader);
Codec.reloadCodecs(loader);
// Analysis:
CharFilterFactory.reloadCharFilters(loader);
TokenFilterFactory.reloadTokenFilters(loader);
TokenizerFactory.reloadTokenizers(loader);
}
private Class loadPluginClass(String className, ClassLoader loader) {
try {
return loader.loadClass(className).asSubclass(Plugin.class);
} catch (ClassNotFoundException e) {
throw new ElasticsearchException("Could not find plugin class [" + className + "]", e);
}
}
private Plugin loadPlugin(Class pluginClass, Settings settings, Path configPath) {
final Constructor[] constructors = pluginClass.getConstructors();
if (constructors.length == 0) {
throw new IllegalStateException("no public constructor for [" + pluginClass.getName() + "]");
}
if (constructors.length > 1) {
throw new IllegalStateException("no unique public constructor for [" + pluginClass.getName() + "]");
}
final Constructor constructor = constructors[0];
if (constructor.getParameterCount() > 2) {
throw new IllegalStateException(signatureMessage(pluginClass));
}
final Class[] parameterTypes = constructor.getParameterTypes();
try {
if (constructor.getParameterCount() == 2 && parameterTypes[0] == Settings.class && parameterTypes[1] == Path.class) {
return (Plugin)constructor.newInstance(settings, configPath);
} else if (constructor.getParameterCount() == 1 && parameterTypes[0] == Settings.class) {
return (Plugin)constructor.newInstance(settings);
} else if (constructor.getParameterCount() == 0) {
return (Plugin)constructor.newInstance();
} else {
throw new IllegalStateException(signatureMessage(pluginClass));
}
} catch (final ReflectiveOperationException e) {
throw new IllegalStateException("failed to load plugin class [" + pluginClass.getName() + "]", e);
}
}
private String signatureMessage(final Class clazz) {
return String.format(
Locale.ROOT,
"no public constructor of correct signature for [%s]; must be [%s], [%s], or [%s]",
clazz.getName(),
"(org.elasticsearch.common.settings.Settings,java.nio.file.Path)",
"(org.elasticsearch.common.settings.Settings)",
"()");
}
public List filterPlugins(Class type) {
return plugins.stream().filter(x -> type.isAssignableFrom(x.v2().getClass()))
.map(p -> ((T)p.v2())).collect(Collectors.toList());
}
}