org.apache.drill.exec.store.PluginBootstrapLoaderImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.store;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import org.apache.drill.common.config.ConfigConstants;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.logical.StoragePlugins;
import org.apache.drill.exec.store.dfs.FileSystemConfig;
import org.apache.drill.exec.util.ActionOnFile;
import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import org.apache.drill.shaded.guava.com.google.common.io.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Loads the set of bootstrap plugin configurations for new systems.
* Also handles upgrades to new and existing systems. Each set is given
* by a Jackson-serialized JSON file with the list of plugin (configurations).
*
* The files are given via indirections in the config system to allow easier
* testing when "non-stock" configurations.
*
*
Bootstrap Files
*
* When a new system starts (boots), Drill populates the persistent store
* with a "starter" (bootstrap) set of storage plugin configurations as
* provided by each connector locator. The classic locator uses one or
* more bootstrap files, serialized to JSON, and visible at the root of the
* class path. Each plugin implementation, if packaged in its own project,
* can provide a bootstrap file ({@code bootstrap-storage-plugins.json}).
*
* The core plugins share a single file in the exec module which has a list of
* storage plugins used as default on fresh start up. Then we have contrib module
* where some plugins and formats reside. We want them to be included in default
* plugins as well during fresh start up which is done by reading each bootstrap
* file in the classpath.
*
*
Format Plugin Bootstrap
*
* All works fine for the plugins, since each plugin also has a
* {@code bootstrap-storage-plugins.json} loaded via this process. However, format
* plugins are defined inside storage plugins. This presents a problem when
* the format is defined outside of the core (such as in the contrib module). In
* this case, when new format was added, the developer had to modify the
* {@code bootstrap-storage-plugins.json} in exec module to ensure that the new format
* was included.
*
* The {@code bootstrap-format-plugins.json} solves this problem: it allows non-core
* modules to include formats during bootstrap. The file must still indicate which
* plugin (config) to modify, and must list the same format for each supported file
* system plugin such as {@code dfs} and {@ cp}. Despite this limitation, the additional
* file makes the non-core format plugins independent of the core files.
*
* The resulting bootstrap process is to 1) gather the storage plugins, and 2)
* merge any format bootstrap into these plugins.
*
*
Upgrade Files
*
* Bootstrap files populate a new system. There are times when a user upgrades
* their Drill install and we would like to push out configurations for newly
* added plugins. This is done via the optional upgrade file.
*
* If the system is new, configurations from the upgrade file replace
* (override) values from the bootstrap file. The net effect, bootstrap plus
* upgrades, are written into the persistent store.
*
* If the system is upgraded, then the current technique uses a highly
* unreliable system: an upgrade file exists. Any upgrades are applied on top
* of the user's stored values. The upgrade file is then deleted to avoid
* applying the same upgrades multiple times. Not idea, but it is what it
* is for now.
*
*
Format Plugins
*
* Earlier versions of this code provided support for format plugins. However,
* this code never worked. The original design of Drill has format plugins
* as a set of attributes of the file system plugin config. Additional work
* is needed to allow file system configs to share plugins. Such work is a
* good idea, so the ill-fated format plugin code still appears, but does
* nothing.
*
* Config Properties
*
*
* - {@code ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE}
* ({@code drill.exec.storage.bootstrap.storage})
* - The name of the bootstrap file, normally
* {@code bootstrap-storage-plugins.json}.
* - {@code ExecConstants.UPGRADE_STORAGE_PLUGINS_FILE}
* ({@code drill.exec.storage.upgrade.storage})
* - The name of the upgrade file, normally
* {@code storage-plugins-override.conf}. Unlike th bootstrap
* file, only one upgrade file can appear in the class path.
* The upgrade file is optional.
*
*/
public class PluginBootstrapLoaderImpl implements PluginBootstrapLoader {
private static final Logger logger = LoggerFactory.getLogger(PluginBootstrapLoaderImpl.class);
private final PluginRegistryContext context;
/**
* The file read the first time a Drillbit starts up, and deleted
* afterwards. A poor-man's way of handling upgrades. Will be
* non-null when an upgrade is needed, null thereafter until a
* new release.
*/
private URL pluginsOverrideFileUrl;
public PluginBootstrapLoaderImpl(PluginRegistryContext context) {
this.context = context;
}
@Override
public StoragePlugins bootstrapPlugins() throws IOException {
Map pluginURLMap = new HashMap<>();
StoragePlugins bootstrapPlugins = loadBootstrapPlugins(pluginURLMap);
// Upgrade the bootstrap plugins with any updates. Seems odd,
// but this is how Drill 1.17 works, so keeping the code.
// Uses the enabled status from the updates if different than
// the bootstrap version.
StoragePlugins updatedPlugins = updatedPlugins();
if (updatedPlugins != null) {
bootstrapPlugins.putAll(updatedPlugins);
}
applyFormatPlugins(bootstrapPlugins, pluginURLMap);
return bootstrapPlugins;
}
@VisibleForTesting
protected StoragePlugins loadBootstrapPlugins(Map pluginURLMap) throws IOException {
// bootstrap load the config since no plugins are stored.
String storageBootstrapFileName = context.config().getString(
ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
Set storageUrls = ClassPathScanner.forResource(storageBootstrapFileName, false);
if (storageUrls == null || storageUrls.isEmpty()) {
throw new IOException("Cannot find storage plugin boostrap file: " + storageBootstrapFileName);
}
logger.info("Loading the storage plugin configs from URLs {}.", storageUrls);
StoragePlugins bootstrapPlugins = new StoragePlugins();
for (URL url : storageUrls) {
try {
loadStoragePlugins(url, bootstrapPlugins, pluginURLMap);
} catch (IOException e) {
throw new IOException("Failed to load bootstrap plugins from " + url.toString(), e );
}
}
return bootstrapPlugins;
}
/**
* Given a set of bootstrap plugins, updated selected plugins with the list
* of format plugins. Since the plugins do not yet reside in the registry,
* they can be modified in place without making copies.
*
* Only one module (core) should have defined the file system plugin
* configs. Only one module should define each format plugin. As a result,
* order should not matter when applying the format configs.
* @throws IOException for load failures
*/
private void applyFormatPlugins(StoragePlugins bootstrapPlugins,
Map pluginURLMap) throws IOException {
String formatBootstrapFileName = context.config().getString(
ExecConstants.BOOTSTRAP_FORMAT_PLUGINS_FILE);
Set formatUrls = ClassPathScanner.forResource(formatBootstrapFileName, false);
if (formatUrls == null) {
return;
}
for (URL url : formatUrls) {
logger.info("Loading format plugin configs from {}.", url);
loadFormatPlugins(url, bootstrapPlugins, pluginURLMap);
}
}
/**
* Get the new storage plugins from the
* {@link ConfigConstants#STORAGE_PLUGINS_OVERRIDE_CONF} file if it exists,
* null otherwise
*/
@Override
public StoragePlugins updatedPlugins() {
String upgradeFileName = context.config().getString(
ExecConstants.UPGRADE_STORAGE_PLUGINS_FILE);
Set urlSet = ClassPathScanner.forResource(upgradeFileName, false);
if (urlSet.isEmpty()) {
logger.trace(
"The {} file is absent. Proceed without updating of the storage plugins configs",
upgradeFileName);
return null;
}
if (urlSet.size() != 1) {
throw DrillRuntimeException.create(
"More than one %s file is placed in Drill's classpath: %s",
upgradeFileName, urlSet);
}
pluginsOverrideFileUrl = urlSet.iterator().next();
try (InputStream is = pluginsOverrideFileUrl.openStream();) {
return context.hoconMapper().readValue(is, StoragePlugins.class);
} catch (IOException e) {
logger.error("Failures are obtained while loading file: '{}'. Proceeding without update.",
upgradeFileName, e);
return null;
}
}
@Override
public void onUpgrade() {
if (pluginsOverrideFileUrl == null) {
return;
}
String fileAction = context.config().getString(
ExecConstants.ACTION_ON_STORAGE_PLUGINS_OVERRIDE_FILE);
Optional actionOnFile = Arrays.stream(ActionOnFile.values())
.filter(action -> action.name().equalsIgnoreCase(fileAction))
.findFirst();
actionOnFile.ifPresent(action -> action.action(pluginsOverrideFileUrl));
// TODO: replace with ifPresentOrElse() once the project will be on Java9
if (!actionOnFile.isPresent()) {
logger.warn("Unknown value {} for {} boot option. Nothing will be done with file.",
fileAction, ExecConstants.ACTION_ON_STORAGE_PLUGINS_OVERRIDE_FILE);
}
}
/**
* Loads storage plugins from the given URL
*
* @param url
* URL to the storage plugins bootstrap file
* @param bootstrapPlugins
* a collection where the plugins should be loaded to
* @param pluginURLMap
* a map to store correspondence between storage plugins and
* bootstrap files in which they are defined. Used for logging
* @throws IOException
* if failed to retrieve a plugin from a bootstrap file
*/
private void loadStoragePlugins(URL url, StoragePlugins bootstrapPlugins,
Map pluginURLMap) throws IOException {
StoragePlugins plugins = getPluginsFromResource(url);
plugins.forEach(plugin -> {
StoragePluginConfig oldPluginConfig = bootstrapPlugins.putIfAbsent(plugin.getKey(), plugin.getValue());
if (oldPluginConfig != null) {
logger.warn("Duplicate plugin instance '[{}]' defined in [{}, {}], ignoring the later one.",
plugin.getKey(), pluginURLMap.get(plugin.getKey()), url);
} else {
pluginURLMap.put(plugin.getKey(), url);
}
});
}
/**
* Loads format plugins from the given URL and adds the formats to the
* specified storage plugins
*
* @param url
* URL to the format plugins bootstrap file
* @param bootstrapPlugins
* a collection with loaded storage plugins. New formats will be
* added to them
* @param pluginURLMap
* a map to store correspondence between storage plugins and
* bootstrap files in which they are defined. Used for logging
* @throws IOException
* if failed to retrieve a plugin from a bootstrap file
*/
private void loadFormatPlugins(URL url, StoragePlugins bootstrapPlugins,
Map pluginURLMap) throws IOException {
StoragePlugins plugins = getPluginsFromResource(url);
for (Entry sourceEntry : plugins) {
String pluginName = sourceEntry.getKey();
StoragePluginConfig sourcePlugin = sourceEntry.getValue();
if (!(sourcePlugin instanceof FileSystemConfig)) {
logger.warn("Formats are only supported by File System plugins. Source name '{}' is of type '{}'.",
pluginName, sourcePlugin.getClass().getName());
continue;
}
StoragePluginConfig targetPlugin = bootstrapPlugins.getConfig(pluginName);
if (targetPlugin == null) {
logger.warn("No boostrap storage plugin matches the name '{}'", pluginName);
continue;
}
if (!(targetPlugin instanceof FileSystemConfig)) {
logger.warn("Formats are only supported by File System plugins. Source name '{}' " +
"is of type '{}' but the bootstrap plugin of that name is of type '{}.",
pluginName, sourcePlugin.getClass().getName(),
targetPlugin.getClass().getName());
continue;
}
FileSystemConfig targetFsConfig = (FileSystemConfig) targetPlugin;
FileSystemConfig sourceFsConfig = (FileSystemConfig) sourcePlugin;
sourceFsConfig.getFormats().forEach((formatName, formatValue) -> {
FormatPluginConfig oldPluginConfig = targetFsConfig.getFormats().putIfAbsent(formatName, formatValue);
if (oldPluginConfig != null) {
logger.warn("Duplicate format instance '{}' defined in '{}' and '{}', ignoring the later one.",
formatName, pluginURLMap.get(pluginName), url);
}
});
}
}
private StoragePlugins getPluginsFromResource(URL resource) throws IOException {
String pluginsData = Resources.toString(resource, Charsets.UTF_8);
return context.hoconMapper().readValue(pluginsData, StoragePlugins.class);
}
}