org.elasticsearch.plugins.PluginsService Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.bootstrap.JarHell;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.threadpool.ExecutorBuilder;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
/**
*
*/
public class PluginsService extends AbstractComponent {
/**
* We keep around a list of plugins and modules
*/
private final List> plugins;
private final PluginsAndModules info;
public static final Setting> MANDATORY_SETTING =
Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
public List> getPluginSettings() {
return plugins.stream().flatMap(p -> p.v2().getSettings().stream()).collect(Collectors.toList());
}
public List getPluginSettingsFilter() {
return plugins.stream().flatMap(p -> p.v2().getSettingsFilter().stream()).collect(Collectors.toList());
}
/**
* Constructs a new PluginService
* @param settings The settings of the system
* @param modulesDirectory The directory modules exist in, or null if modules should not be loaded from the filesystem
* @param pluginsDirectory The directory plugins exist in, or null if plugins should not be loaded from the filesystem
* @param classpathPlugins Plugins that exist in the classpath which should be loaded
*/
public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDirectory, Collection> classpathPlugins) {
super(settings);
List> pluginsLoaded = new ArrayList<>();
List pluginsList = new ArrayList<>();
// first we load plugins that are on the classpath. this is for tests and transport clients
for (Class pluginClass : classpathPlugins) {
Plugin plugin = loadPlugin(pluginClass, settings);
PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", pluginClass.getName(), false);
if (logger.isTraceEnabled()) {
logger.trace("plugin loaded from classpath [{}]", pluginInfo);
}
pluginsLoaded.add(new Tuple<>(pluginInfo, plugin));
pluginsList.add(pluginInfo);
}
Set seenBundles = new LinkedHashSet<>();
List modulesList = new ArrayList<>();
// load modules
if (modulesDirectory != null) {
try {
Set modules = getModuleBundles(modulesDirectory);
for (Bundle bundle : modules) {
modulesList.add(bundle.plugin);
}
seenBundles.addAll(modules);
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize modules", ex);
}
}
// now, find all the ones that are in plugins/
if (pluginsDirectory != null) {
try {
Set plugins = getPluginBundles(pluginsDirectory);
for (Bundle bundle : plugins) {
pluginsList.add(bundle.plugin);
}
seenBundles.addAll(plugins);
} catch (IOException ex) {
throw new IllegalStateException("Unable to initialize plugins", ex);
}
}
List> loaded = loadBundles(seenBundles);
pluginsLoaded.addAll(loaded);
this.info = new PluginsAndModules(pluginsList, modulesList);
this.plugins = Collections.unmodifiableList(pluginsLoaded);
// We need to build a List of plugins for checking mandatory plugins
Set pluginsNames = new HashSet<>();
for (Tuple tuple : this.plugins) {
pluginsNames.add(tuple.v1().getName());
}
// Checking expected plugins
List mandatoryPlugins = MANDATORY_SETTING.get(settings);
if (mandatoryPlugins.isEmpty() == false) {
Set missingPlugins = new HashSet<>();
for (String mandatoryPlugin : mandatoryPlugins) {
if (!pluginsNames.contains(mandatoryPlugin) && !missingPlugins.contains(mandatoryPlugin)) {
missingPlugins.add(mandatoryPlugin);
}
}
if (!missingPlugins.isEmpty()) {
throw new ElasticsearchException("Missing mandatory plugins [" + Strings.collectionToDelimitedString(missingPlugins, ", ") + "]");
}
}
// we don't log jars in lib/ we really shouldn't log modules,
// but for now: just be transparent so we can debug any potential issues
logPluginInfo(info.getModuleInfos(), "module", logger);
logPluginInfo(info.getPluginInfos(), "plugin", logger);
}
private static void logPluginInfo(final List pluginInfos, final String type, final Logger logger) {
assert pluginInfos != null;
if (pluginInfos.isEmpty()) {
logger.info("no " + type + "s loaded");
} else {
for (final String name : pluginInfos.stream().map(PluginInfo::getName).sorted().collect(Collectors.toList())) {
logger.info("loaded " + type + " [" + name + "]");
}
}
}
public Settings updatedSettings() {
Map foundSettings = new HashMap<>();
final Settings.Builder builder = Settings.builder();
for (Tuple plugin : plugins) {
Settings settings = plugin.v2().additionalSettings();
for (String setting : settings.getAsMap().keySet()) {
String oldPlugin = foundSettings.put(setting, plugin.v1().getName());
if (oldPlugin != null) {
throw new IllegalArgumentException("Cannot have additional setting [" + setting + "] " +
"in plugin [" + plugin.v1().getName() + "], already added in plugin [" + oldPlugin + "]");
}
}
builder.put(settings);
}
return builder.put(this.settings).build();
}
public Collection createGuiceModules() {
List modules = new ArrayList<>();
for (Tuple plugin : plugins) {
modules.addAll(plugin.v2().createGuiceModules());
}
return modules;
}
public List> getExecutorBuilders(Settings settings) {
final ArrayList> builders = new ArrayList<>();
for (final Tuple plugin : plugins) {
builders.addAll(plugin.v2().getExecutorBuilders(settings));
}
return builders;
}
/** Returns all classes injected into guice by plugins which extend {@link LifecycleComponent}. */
public Collection> getGuiceServiceClasses() {
List> services = new ArrayList<>();
for (Tuple plugin : plugins) {
services.addAll(plugin.v2().getGuiceServiceClasses());
}
return services;
}
public void onIndexModule(IndexModule indexModule) {
for (Tuple plugin : plugins) {
plugin.v2().onIndexModule(indexModule);
}
}
/**
* Get information about plugins and modules
*/
public PluginsAndModules info() {
return info;
}
// a "bundle" is a group of plugins in a single classloader
// really should be 1-1, but we are not so fortunate
static class Bundle {
final PluginInfo plugin;
final Set urls;
Bundle(PluginInfo plugin, Set urls) {
this.plugin = Objects.requireNonNull(plugin);
this.urls = Objects.requireNonNull(urls);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Bundle bundle = (Bundle) o;
return Objects.equals(plugin, bundle.plugin);
}
@Override
public int hashCode() {
return Objects.hash(plugin);
}
}
// similar in impl to getPluginBundles, but DO NOT try to make them share code.
// we don't need to inherit all the leniency, and things are different enough.
static Set getModuleBundles(Path modulesDirectory) throws IOException {
// damn leniency
if (Files.notExists(modulesDirectory)) {
return Collections.emptySet();
}
Set bundles = new LinkedHashSet<>();
try (DirectoryStream stream = Files.newDirectoryStream(modulesDirectory)) {
for (Path module : stream) {
PluginInfo info = PluginInfo.readFromProperties(module);
Set urls = new LinkedHashSet<>();
// gather urls for jar files
try (DirectoryStream jarStream = Files.newDirectoryStream(module, "*.jar")) {
for (Path jar : jarStream) {
// normalize with toRealPath to get symlinks out of our hair
URL url = jar.toRealPath().toUri().toURL();
if (urls.add(url) == false) {
throw new IllegalStateException("duplicate codebase: " + url);
}
}
}
if (bundles.add(new Bundle(info, urls)) == false) {
throw new IllegalStateException("duplicate module: " + info);
}
}
}
return bundles;
}
static void checkForFailedPluginRemovals(final Path pluginsDirectory) throws IOException {
/*
* Check for the existence of a marker file that indicates any plugins are in a garbage state from a failed attempt to remove the
* plugin.
*/
try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory, ".removing-*")) {
final Iterator iterator = stream.iterator();
if (iterator.hasNext()) {
final Path removing = iterator.next();
final String fileName = removing.getFileName().toString();
final String name = fileName.substring(1 + fileName.indexOf("-"));
final String message = String.format(
Locale.ROOT,
"found file [%s] from a failed attempt to remove the plugin [%s]; execute [elasticsearch-plugin remove %2$s]",
removing,
name);
throw new IllegalStateException(message);
}
}
}
static Set getPluginBundles(Path pluginsDirectory) throws IOException {
Logger logger = Loggers.getLogger(PluginsService.class);
// TODO: remove this leniency, but tests bogusly rely on it
if (!isAccessibleDirectory(pluginsDirectory, logger)) {
return Collections.emptySet();
}
Set bundles = new LinkedHashSet<>();
checkForFailedPluginRemovals(pluginsDirectory);
try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) {
for (Path plugin : stream) {
if (FileSystemUtils.isDesktopServicesStore(plugin)) {
continue;
}
logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
final PluginInfo info;
try {
info = PluginInfo.readFromProperties(plugin);
} catch (IOException e) {
throw new IllegalStateException("Could not load plugin descriptor for existing plugin ["
+ plugin.getFileName() + "]. Was the plugin built before 2.0?", e);
}
Set urls = new LinkedHashSet<>();
try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
for (Path jar : jarStream) {
// normalize with toRealPath to get symlinks out of our hair
URL url = jar.toRealPath().toUri().toURL();
if (urls.add(url) == false) {
throw new IllegalStateException("duplicate codebase: " + url);
}
}
}
if (bundles.add(new Bundle(info, urls)) == false) {
throw new IllegalStateException("duplicate plugin: " + info);
}
}
}
return bundles;
}
private List> loadBundles(Set bundles) {
List> plugins = new ArrayList<>();
for (Bundle bundle : bundles) {
// jar-hell check the bundle against the parent classloader
// pluginmanager does it, but we do it again, in case lusers mess with jar files manually
try {
Set classpath = JarHell.parseClassPath();
// check we don't have conflicting codebases
Set intersection = new HashSet<>(classpath);
intersection.retainAll(bundle.urls);
if (intersection.isEmpty() == false) {
throw new IllegalStateException("jar hell! duplicate codebases between" +
" plugin and core: " + intersection);
}
// check we don't have conflicting classes
Set union = new HashSet<>(classpath);
union.addAll(bundle.urls);
JarHell.checkJarHell(union);
} catch (Exception e) {
throw new IllegalStateException("failed to load plugin " + bundle.plugin +
" due to jar hell", e);
}
// create a child to load the plugin in this bundle
ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]),
getClass().getClassLoader());
// reload lucene SPI with any new services from the plugin
reloadLuceneSPI(loader);
final Class pluginClass =
loadPluginClass(bundle.plugin.getClassname(), loader);
final Plugin plugin = loadPlugin(pluginClass, settings);
plugins.add(new Tuple<>(bundle.plugin, plugin));
}
return Collections.unmodifiableList(plugins);
}
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after the new classloader has been created to
* register the services for use.
*/
static void reloadLuceneSPI(ClassLoader loader) {
// do NOT change the order of these method calls!
// Codecs:
PostingsFormat.reloadPostingsFormats(loader);
DocValuesFormat.reloadDocValuesFormats(loader);
Codec.reloadCodecs(loader);
// Analysis:
CharFilterFactory.reloadCharFilters(loader);
TokenFilterFactory.reloadTokenFilters(loader);
TokenizerFactory.reloadTokenizers(loader);
}
private Class loadPluginClass(String className, ClassLoader loader) {
try {
return loader.loadClass(className).asSubclass(Plugin.class);
} catch (ClassNotFoundException e) {
throw new ElasticsearchException("Could not find plugin class [" + className + "]", e);
}
}
private Plugin loadPlugin(Class pluginClass, Settings settings) {
try {
try {
return pluginClass.getConstructor(Settings.class).newInstance(settings);
} catch (NoSuchMethodException e) {
try {
return pluginClass.getConstructor().newInstance();
} catch (NoSuchMethodException e1) {
throw new ElasticsearchException("No constructor for [" + pluginClass + "]. A plugin class must " +
"have either an empty default constructor or a single argument constructor accepting a " +
"Settings instance");
}
}
} catch (Exception e) {
throw new ElasticsearchException("Failed to load plugin class [" + pluginClass.getName() + "]", e);
}
}
public List filterPlugins(Class type) {
return plugins.stream().filter(x -> type.isAssignableFrom(x.v2().getClass()))
.map(p -> ((T)p.v2())).collect(Collectors.toList());
}
}