org.apache.hadoop.hbase.coprocessor.CoprocessorHost Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hbase-server Show documentation
Show all versions of hbase-server Show documentation
Server functionality for HBase
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.coprocessor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.security.User;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
import org.apache.hadoop.hbase.util.SortedList;
/**
* Provides the common setup framework and runtime services for coprocessor
* invocation from HBase services.
* @param type of specific coprocessor this host will handle
* @param type of specific coprocessor environment this host requires.
* provides
*/
@InterfaceAudience.Private
public abstract class CoprocessorHost> {
public static final String REGION_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.region.classes";
public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.regionserver.classes";
public static final String USER_REGION_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.user.region.classes";
public static final String MASTER_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.master.classes";
public static final String WAL_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.wal.classes";
public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror";
public static final boolean DEFAULT_ABORT_ON_ERROR = true;
public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled";
public static final boolean DEFAULT_COPROCESSORS_ENABLED = true;
public static final String USER_COPROCESSORS_ENABLED_CONF_KEY =
"hbase.coprocessor.user.enabled";
public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true;
public static final String SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR =
"hbase.skip.load.duplicate.table.coprocessor";
public static final boolean DEFAULT_SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR = false;
private static final Logger LOG = LoggerFactory.getLogger(CoprocessorHost.class);
protected Abortable abortable;
/** Ordered set of loaded coprocessors with lock */
protected final SortedList coprocEnvironments =
new SortedList<>(new EnvironmentPriorityComparator());
protected Configuration conf;
// unique file prefix to use for local copies of jars when classloading
protected String pathPrefix;
protected AtomicInteger loadSequence = new AtomicInteger();
public CoprocessorHost(Abortable abortable) {
this.abortable = abortable;
this.pathPrefix = UUID.randomUUID().toString();
}
/**
* Not to be confused with the per-object _coprocessors_ (above),
* coprocessorNames is static and stores the set of all coprocessors ever
* loaded by any thread in this JVM. It is strictly additive: coprocessors are
* added to coprocessorNames, by checkAndLoadInstance() but are never removed, since
* the intention is to preserve a history of all loaded coprocessors for
* diagnosis in case of server crash (HBASE-4014).
*/
private static Set coprocessorNames =
Collections.synchronizedSet(new HashSet());
public static Set getLoadedCoprocessors() {
synchronized (coprocessorNames) {
return new HashSet(coprocessorNames);
}
}
/**
* Used to create a parameter to the HServerLoad constructor so that
* HServerLoad can provide information about the coprocessors loaded by this
* regionserver.
* (HBASE-4070: Improve region server metrics to report loaded coprocessors
* to master).
*/
public Set getCoprocessors() {
Set returnValue = new TreeSet<>();
for (E e: coprocEnvironments) {
returnValue.add(e.getInstance().getClass().getSimpleName());
}
return returnValue;
}
/**
* Load system coprocessors once only. Read the class names from configuration.
* Called by constructor.
*/
protected void loadSystemCoprocessors(Configuration conf, String confKey) {
boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
DEFAULT_COPROCESSORS_ENABLED);
if (!coprocessorsEnabled) {
return;
}
Class implClass;
// load default coprocessors from configure file
String[] defaultCPClasses = conf.getStrings(confKey);
if (defaultCPClasses == null || defaultCPClasses.length == 0)
return;
int currentSystemPriority = Coprocessor.PRIORITY_SYSTEM;
for (String className : defaultCPClasses) {
String[] classNameAndPriority = className.split("\\|");
boolean hasPriorityOverride = false;
className = classNameAndPriority[0];
int overridePriority = Coprocessor.PRIORITY_SYSTEM;
if (classNameAndPriority.length > 1){
overridePriority = Integer.parseInt(classNameAndPriority[1]);
hasPriorityOverride = true;
}
className = className.trim();
if (findCoprocessor(className) != null) {
// If already loaded will just continue
LOG.warn("Attempted duplicate loading of " + className + "; skipped");
continue;
}
ClassLoader cl = this.getClass().getClassLoader();
Thread.currentThread().setContextClassLoader(cl);
try {
implClass = cl.loadClass(className);
int coprocPriority = hasPriorityOverride ? overridePriority : currentSystemPriority;
// Add coprocessors as we go to guard against case where a coprocessor is specified twice
// in the configuration
E env = checkAndLoadInstance(implClass, coprocPriority, conf);
if (env != null) {
this.coprocEnvironments.add(env);
LOG.info("System coprocessor {} loaded, priority={}.", className, coprocPriority);
if (!hasPriorityOverride) {
++currentSystemPriority;
}
}
} catch (Throwable t) {
// We always abort if system coprocessors cannot be loaded
abortServer(className, t);
}
}
}
/**
* Load a coprocessor implementation into the host
* @param path path to implementation jar
* @param className the main class name
* @param priority chaining priority
* @param conf configuration for coprocessor
* @throws java.io.IOException Exception
*/
public E load(Path path, String className, int priority,
Configuration conf) throws IOException {
String[] includedClassPrefixes = null;
if (conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY) != null){
String prefixes = conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY);
includedClassPrefixes = prefixes.split(";");
}
return load(path, className, priority, conf, includedClassPrefixes);
}
/**
* Load a coprocessor implementation into the host
* @param path path to implementation jar
* @param className the main class name
* @param priority chaining priority
* @param conf configuration for coprocessor
* @param includedClassPrefixes class name prefixes to include
* @throws java.io.IOException Exception
*/
public E load(Path path, String className, int priority,
Configuration conf, String[] includedClassPrefixes) throws IOException {
Class implClass;
LOG.debug("Loading coprocessor class " + className + " with path " +
path + " and priority " + priority);
boolean skipLoadDuplicateCoprocessor = conf.getBoolean(SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR,
DEFAULT_SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR);
if (skipLoadDuplicateCoprocessor && findCoprocessor(className) != null) {
// If already loaded will just continue
LOG.warn("Attempted duplicate loading of {}; skipped", className);
return null;
}
ClassLoader cl = null;
if (path == null) {
try {
implClass = getClass().getClassLoader().loadClass(className);
} catch (ClassNotFoundException e) {
throw new IOException("No jar path specified for " + className);
}
} else {
cl = CoprocessorClassLoader.getClassLoader(
path, getClass().getClassLoader(), pathPrefix, conf);
try {
implClass = ((CoprocessorClassLoader)cl).loadClass(className, includedClassPrefixes);
} catch (ClassNotFoundException e) {
throw new IOException("Cannot load external coprocessor class " + className, e);
}
}
//load custom code for coprocessor
Thread currentThread = Thread.currentThread();
ClassLoader hostClassLoader = currentThread.getContextClassLoader();
try{
// switch temporarily to the thread classloader for custom CP
currentThread.setContextClassLoader(cl);
E cpInstance = checkAndLoadInstance(implClass, priority, conf);
return cpInstance;
} finally {
// restore the fresh (host) classloader
currentThread.setContextClassLoader(hostClassLoader);
}
}
@VisibleForTesting
public void load(Class implClass, int priority, Configuration conf)
throws IOException {
E env = checkAndLoadInstance(implClass, priority, conf);
coprocEnvironments.add(env);
}
/**
* @param implClass Implementation class
* @param priority priority
* @param conf configuration
* @throws java.io.IOException Exception
*/
public E checkAndLoadInstance(Class implClass, int priority, Configuration conf)
throws IOException {
// create the instance
C impl;
try {
impl = checkAndGetInstance(implClass);
if (impl == null) {
LOG.error("Cannot load coprocessor " + implClass.getSimpleName());
return null;
}
} catch (InstantiationException|IllegalAccessException e) {
throw new IOException(e);
}
// create the environment
E env = createEnvironment(impl, priority, loadSequence.incrementAndGet(), conf);
assert env instanceof BaseEnvironment;
((BaseEnvironment) env).startup();
// HBASE-4014: maintain list of loaded coprocessors for later crash analysis
// if server (master or regionserver) aborts.
coprocessorNames.add(implClass.getName());
return env;
}
/**
* Called when a new Coprocessor class is loaded
*/
public abstract E createEnvironment(C instance, int priority, int sequence, Configuration conf);
/**
* Called when a new Coprocessor class needs to be loaded. Checks if type of the given class
* is what the corresponding host implementation expects. If it is of correct type, returns an
* instance of the coprocessor to be loaded. If not, returns null.
* If an exception occurs when trying to create instance of a coprocessor, it's passed up and
* eventually results into server aborting.
*/
public abstract C checkAndGetInstance(Class implClass)
throws InstantiationException, IllegalAccessException;
public void shutdown(E e) {
assert e instanceof BaseEnvironment;
if (LOG.isDebugEnabled()) {
LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
}
((BaseEnvironment) e).shutdown();
}
/**
* Find coprocessors by full class name or simple name.
*/
public C findCoprocessor(String className) {
for (E env: coprocEnvironments) {
if (env.getInstance().getClass().getName().equals(className) ||
env.getInstance().getClass().getSimpleName().equals(className)) {
return env.getInstance();
}
}
return null;
}
@VisibleForTesting
public T findCoprocessor(Class cls) {
for (E env: coprocEnvironments) {
if (cls.isAssignableFrom(env.getInstance().getClass())) {
return (T) env.getInstance();
}
}
return null;
}
/**
* Find list of coprocessors that extend/implement the given class/interface
* @param cls the class/interface to look for
* @return the list of coprocessors, or null if not found
*/
public List findCoprocessors(Class cls) {
ArrayList ret = new ArrayList<>();
for (E env: coprocEnvironments) {
C cp = env.getInstance();
if(cp != null) {
if (cls.isAssignableFrom(cp.getClass())) {
ret.add((T)cp);
}
}
}
return ret;
}
/**
* Find a coprocessor environment by class name
* @param className the class name
* @return the coprocessor, or null if not found
*/
@VisibleForTesting
public E findCoprocessorEnvironment(String className) {
for (E env: coprocEnvironments) {
if (env.getInstance().getClass().getName().equals(className) ||
env.getInstance().getClass().getSimpleName().equals(className)) {
return env;
}
}
return null;
}
/**
* Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
* jar files.
* @return A set of ClassLoader instances
*/
Set getExternalClassLoaders() {
Set externalClassLoaders = new HashSet<>();
final ClassLoader systemClassLoader = this.getClass().getClassLoader();
for (E env : coprocEnvironments) {
ClassLoader cl = env.getInstance().getClass().getClassLoader();
if (cl != systemClassLoader){
//do not include system classloader
externalClassLoaders.add(cl);
}
}
return externalClassLoaders;
}
/**
* Environment priority comparator.
* Coprocessors are chained in sorted order.
*/
static class EnvironmentPriorityComparator implements Comparator {
@Override
public int compare(final CoprocessorEnvironment env1,
final CoprocessorEnvironment env2) {
if (env1.getPriority() < env2.getPriority()) {
return -1;
} else if (env1.getPriority() > env2.getPriority()) {
return 1;
}
if (env1.getLoadSequence() < env2.getLoadSequence()) {
return -1;
} else if (env1.getLoadSequence() > env2.getLoadSequence()) {
return 1;
}
return 0;
}
}
protected void abortServer(final E environment, final Throwable e) {
abortServer(environment.getInstance().getClass().getName(), e);
}
protected void abortServer(final String coprocessorName, final Throwable e) {
String message = "The coprocessor " + coprocessorName + " threw " + e.toString();
LOG.error(message, e);
if (abortable != null) {
abortable.abort(message, e);
} else {
LOG.warn("No available Abortable, process was not aborted");
}
}
/**
* This is used by coprocessor hooks which are declared to throw IOException
* (or its subtypes). For such hooks, we should handle throwable objects
* depending on the Throwable's type. Those which are instances of
* IOException should be passed on to the client. This is in conformance with
* the HBase idiom regarding IOException: that it represents a circumstance
* that should be passed along to the client for its own handling. For
* example, a coprocessor that implements access controls would throw a
* subclass of IOException, such as AccessDeniedException, in its preGet()
* method to prevent an unauthorized client's performing a Get on a particular
* table.
* @param env Coprocessor Environment
* @param e Throwable object thrown by coprocessor.
* @exception IOException Exception
*/
// Note to devs: Class comments of all observers ({@link MasterObserver}, {@link WALObserver},
// etc) mention this nuance of our exception handling so that coprocessor can throw appropriate
// exceptions depending on situation. If any changes are made to this logic, make sure to
// update all classes' comments.
protected void handleCoprocessorThrowable(final E env, final Throwable e) throws IOException {
if (e instanceof IOException) {
throw (IOException)e;
}
// If we got here, e is not an IOException. A loaded coprocessor has a
// fatal bug, and the server (master or regionserver) should remove the
// faulty coprocessor from its set of active coprocessors. Setting
// 'hbase.coprocessor.abortonerror' to true will cause abortServer(),
// which may be useful in development and testing environments where
// 'failing fast' for error analysis is desired.
if (env.getConfiguration().getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
// server is configured to abort.
abortServer(env, e);
} else {
// If available, pull a table name out of the environment
if(env instanceof RegionCoprocessorEnvironment) {
String tableName = ((RegionCoprocessorEnvironment)env).getRegionInfo().getTable().getNameAsString();
LOG.error("Removing coprocessor '" + env.toString() + "' from table '"+ tableName + "'", e);
} else {
LOG.error("Removing coprocessor '" + env.toString() + "' from " +
"environment",e);
}
coprocEnvironments.remove(env);
try {
shutdown(env);
} catch (Exception x) {
LOG.error("Uncaught exception when shutting down coprocessor '"
+ env.toString() + "'", x);
}
throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
"' threw: '" + e + "' and has been removed from the active " +
"coprocessor set.", e);
}
}
/**
* Used to limit legacy handling to once per Coprocessor class per classloader.
*/
private static final Set> legacyWarning =
new ConcurrentSkipListSet<>(
new Comparator>() {
@Override
public int compare(Class c1, Class c2) {
if (c1.equals(c2)) {
return 0;
}
return c1.getName().compareTo(c2.getName());
}
});
/**
* Implementations defined function to get an observer of type {@code O} from a coprocessor of
* type {@code C}. Concrete implementations of CoprocessorHost define one getter for each
* observer they can handle. For e.g. RegionCoprocessorHost will use 3 getters, one for
* each of RegionObserver, EndpointObserver and BulkLoadObserver.
* These getters are used by {@code ObserverOperation} to get appropriate observer from the
* coprocessor.
*/
@FunctionalInterface
public interface ObserverGetter extends Function> {}
private abstract class ObserverOperation extends ObserverContextImpl {
ObserverGetter observerGetter;
ObserverOperation(ObserverGetter observerGetter) {
this(observerGetter, null);
}
ObserverOperation(ObserverGetter observerGetter, User user) {
this(observerGetter, user, false);
}
ObserverOperation(ObserverGetter observerGetter, boolean bypassable) {
this(observerGetter, null, bypassable);
}
ObserverOperation(ObserverGetter observerGetter, User user, boolean bypassable) {
super(user != null? user: RpcServer.getRequestUser().orElse(null), bypassable);
this.observerGetter = observerGetter;
}
abstract void callObserver() throws IOException;
protected void postEnvCall() {}
}
// Can't derive ObserverOperation from ObserverOperationWithResult (R = Void) because then all
// ObserverCaller implementations will have to have a return statement.
// O = observer, E = environment, C = coprocessor, R=result type
public abstract class ObserverOperationWithoutResult extends ObserverOperation {
protected abstract void call(O observer) throws IOException;
public ObserverOperationWithoutResult(ObserverGetter observerGetter) {
super(observerGetter);
}
public ObserverOperationWithoutResult(ObserverGetter observerGetter, User user) {
super(observerGetter, user);
}
public ObserverOperationWithoutResult(ObserverGetter observerGetter, User user,
boolean bypassable) {
super(observerGetter, user, bypassable);
}
/**
* In case of coprocessors which have many kinds of observers (for eg, {@link RegionCoprocessor}
* has BulkLoadObserver, RegionObserver, etc), some implementations may not need all
* observers, in which case they will return null for that observer's getter.
* We simply ignore such cases.
*/
@Override
void callObserver() throws IOException {
Optional observer = observerGetter.apply(getEnvironment().getInstance());
if (observer.isPresent()) {
call(observer.get());
}
}
}
public abstract class ObserverOperationWithResult extends ObserverOperation {
protected abstract R call(O observer) throws IOException;
private R result;
public ObserverOperationWithResult(ObserverGetter observerGetter, R result) {
this(observerGetter, result, false);
}
public ObserverOperationWithResult(ObserverGetter observerGetter, R result,
boolean bypassable) {
this(observerGetter, result, null, bypassable);
}
public ObserverOperationWithResult(ObserverGetter observerGetter, R result,
User user) {
this(observerGetter, result, user, false);
}
private ObserverOperationWithResult(ObserverGetter observerGetter, R result, User user,
boolean bypassable) {
super(observerGetter, user, bypassable);
this.result = result;
}
protected R getResult() {
return this.result;
}
@Override
void callObserver() throws IOException {
Optional observer = observerGetter.apply(getEnvironment().getInstance());
if (observer.isPresent()) {
result = call(observer.get());
}
}
}
//////////////////////////////////////////////////////////////////////////////////////////
// Functions to execute observer hooks and handle results (if any)
//////////////////////////////////////////////////////////////////////////////////////////
/**
* Do not call with an observerOperation that is null! Have the caller check.
*/
protected R execOperationWithResult(
final ObserverOperationWithResult observerOperation) throws IOException {
boolean bypass = execOperation(observerOperation);
R result = observerOperation.getResult();
return bypass == observerOperation.isBypassable()? result: null;
}
/**
* @return True if we are to bypass (Can only be true
if
* ObserverOperation#isBypassable().
*/
protected boolean execOperation(final ObserverOperation observerOperation)
throws IOException {
boolean bypass = false;
if (observerOperation == null) {
return bypass;
}
List envs = coprocEnvironments.get();
for (E env : envs) {
observerOperation.prepare(env);
Thread currentThread = Thread.currentThread();
ClassLoader cl = currentThread.getContextClassLoader();
try {
currentThread.setContextClassLoader(env.getClassLoader());
observerOperation.callObserver();
} catch (Throwable e) {
handleCoprocessorThrowable(env, e);
} finally {
currentThread.setContextClassLoader(cl);
}
// Internal to shouldBypass, it checks if obeserverOperation#isBypassable().
bypass |= observerOperation.shouldBypass();
observerOperation.postEnvCall();
if (bypass) {
// If CP says bypass, skip out w/o calling any following CPs; they might ruin our response.
// In hbase1, this used to be called 'complete'. In hbase2, we unite bypass and 'complete'.
break;
}
}
return bypass;
}
/**
* Coprocessor classes can be configured in any order, based on that priority is set and
* chained in a sorted order. Should be used preStop*() hooks i.e. when master/regionserver is
* going down. This function first calls coprocessor methods (using ObserverOperation.call())
* and then shutdowns the environment in postEnvCall().
* Need to execute all coprocessor methods first then postEnvCall(), otherwise some coprocessors
* may remain shutdown if any exception occurs during next coprocessor execution which prevent
* master/regionserver stop or cluster shutdown. (Refer:
* HBASE-16663
* @return true if bypaas coprocessor execution, false if not.
* @throws IOException
*/
protected boolean execShutdown(final ObserverOperation observerOperation)
throws IOException {
if (observerOperation == null) return false;
boolean bypass = false;
List envs = coprocEnvironments.get();
// Iterate the coprocessors and execute ObserverOperation's call()
for (E env : envs) {
observerOperation.prepare(env);
Thread currentThread = Thread.currentThread();
ClassLoader cl = currentThread.getContextClassLoader();
try {
currentThread.setContextClassLoader(env.getClassLoader());
observerOperation.callObserver();
} catch (Throwable e) {
handleCoprocessorThrowable(env, e);
} finally {
currentThread.setContextClassLoader(cl);
}
bypass |= observerOperation.shouldBypass();
}
// Iterate the coprocessors and execute ObserverOperation's postEnvCall()
for (E env : envs) {
observerOperation.prepare(env);
observerOperation.postEnvCall();
}
return bypass;
}
}