org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of accumulo-start Show documentation
Show all versions of accumulo-start Show documentation
A library for launching Apache Accumulo services.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.accumulo.start.classloader.vfs;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.ref.WeakReference;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject;
import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.cache.SoftRefFilesCache;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.commons.vfs2.provider.FileReplicator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class builds a hierarchy of Classloaders in the form of:
*
*
* SystemClassLoader that loads JVM classes
* ^
* |
* AccumuloClassLoader loads jars from locations in general.classpaths. Usually the URLs for HADOOP_HOME, ZOOKEEPER_HOME, ACCUMULO_HOME and their associated directories
* ^
* |
* VFSClassLoader that loads jars from locations in general.vfs.classpaths. Can be used to load accumulo jar from HDFS
* ^
* |
* AccumuloReloadingVFSClassLoader That loads jars from locations in general.dynamic.classpaths. Used to load jar dynamically.
*
*
*
*
*/
public class AccumuloVFSClassLoader {
public static class AccumuloVFSClassLoaderShutdownThread implements Runnable {
@Override
public void run() {
try {
AccumuloVFSClassLoader.close();
} catch (Exception e) {
// do nothing, we are shutting down anyway
}
}
}
private static List> vfsInstances = Collections
.synchronizedList(new ArrayList>());
public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
public static final String DEFAULT_DYNAMIC_CLASSPATH_VALUE = "$ACCUMULO_HOME/lib/ext/[^.].*.jar";
public static final String VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY = "general.vfs.classpaths";
public static final String VFS_CONTEXT_CLASSPATH_PROPERTY = "general.vfs.context.classpath.";
public static final String VFS_CACHE_DIR = "general.vfs.cache.dir";
private static ClassLoader parent = null;
private static volatile ReloadingClassLoader loader = null;
private static final Object lock = new Object();
private static ContextManager contextManager;
private static final Logger log = LoggerFactory.getLogger(AccumuloVFSClassLoader.class);
static {
// Register the shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(new AccumuloVFSClassLoaderShutdownThread()));
}
public synchronized static Class loadClass(String classname, Class extension) throws ClassNotFoundException {
try {
return getClassLoader().loadClass(classname).asSubclass(extension);
} catch (IOException e) {
throw new ClassNotFoundException("IO Error loading class " + classname, e);
}
}
public static Class loadClass(String classname) throws ClassNotFoundException {
return loadClass(classname, Object.class).asSubclass(Object.class);
}
static FileObject[] resolve(FileSystemManager vfs, String uris) throws FileSystemException {
return resolve(vfs, uris, new ArrayList());
}
static FileObject[] resolve(FileSystemManager vfs, String uris, ArrayList pathsToMonitor) throws FileSystemException {
if (uris == null)
return new FileObject[0];
ArrayList classpath = new ArrayList();
pathsToMonitor.clear();
for (String path : uris.split(",")) {
path = path.trim();
if (path.equals(""))
continue;
path = AccumuloClassLoader.replaceEnvVars(path, System.getenv());
FileObject fo = vfs.resolveFile(path);
switch (fo.getType()) {
case FILE:
case FOLDER:
classpath.add(fo);
pathsToMonitor.add(fo);
break;
case IMAGINARY:
// assume its a pattern
String pattern = fo.getName().getBaseName();
if (fo.getParent() != null && fo.getParent().getType() == FileType.FOLDER) {
pathsToMonitor.add(fo.getParent());
FileObject[] children = fo.getParent().getChildren();
for (FileObject child : children) {
if (child.getType() == FileType.FILE && child.getName().getBaseName().matches(pattern)) {
classpath.add(child);
}
}
} else {
log.warn("ignoring classpath entry " + fo);
}
break;
default:
log.warn("ignoring classpath entry " + fo);
break;
}
}
return classpath.toArray(new FileObject[classpath.size()]);
}
private static ReloadingClassLoader createDynamicClassloader(final ClassLoader parent) throws FileSystemException, IOException {
String dynamicCPath = AccumuloClassLoader.getAccumuloString(DYNAMIC_CLASSPATH_PROPERTY_NAME, DEFAULT_DYNAMIC_CLASSPATH_VALUE);
String envJars = System.getenv("ACCUMULO_XTRAJARS");
if (null != envJars && !envJars.equals(""))
if (dynamicCPath != null && !dynamicCPath.equals(""))
dynamicCPath = dynamicCPath + "," + envJars;
else
dynamicCPath = envJars;
ReloadingClassLoader wrapper = new ReloadingClassLoader() {
@Override
public ClassLoader getClassLoader() {
return parent;
}
};
if (dynamicCPath == null || dynamicCPath.equals(""))
return wrapper;
// TODO monitor time for lib/ext was 1 sec... should this be configurable? - ACCUMULO-1301
return new AccumuloReloadingVFSClassLoader(dynamicCPath, generateVfs(), wrapper, 1000, true);
}
public static ClassLoader getClassLoader() throws IOException {
ReloadingClassLoader localLoader = loader;
while (null == localLoader) {
synchronized (lock) {
if (null == loader) {
FileSystemManager vfs = generateVfs();
// Set up the 2nd tier class loader
if (null == parent) {
parent = AccumuloClassLoader.getClassLoader();
}
FileObject[] vfsCP = resolve(vfs, AccumuloClassLoader.getAccumuloString(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY, ""));
if (vfsCP.length == 0) {
localLoader = createDynamicClassloader(parent);
loader = localLoader;
return localLoader.getClassLoader();
}
// Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
loader = localLoader;
// An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above.
// The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they
// get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above
// it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted
// and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration)
for (FileObject fo : vfsCP) {
if (fo instanceof HdfsFileObject) {
String uri = fo.getName().getRootURI();
Configuration c = new Configuration(true);
c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
FileSystem fs = FileSystem.get(c);
fs.getConf().setClassLoader(loader.getClassLoader());
}
}
}
}
}
return localLoader.getClassLoader();
}
public static FileSystemManager generateVfs() throws FileSystemException {
DefaultFileSystemManager vfs = new DefaultFileSystemManager();
vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
vfs.addProvider("hdfs", new HdfsFileProvider());
vfs.addExtensionMap("jar", "jar");
vfs.addExtensionMap("zip", "zip");
vfs.addExtensionMap("gz", "gz");
vfs.addExtensionMap("tar", "tar");
vfs.addExtensionMap("tbz2", "tar");
vfs.addExtensionMap("tgz", "tar");
vfs.addExtensionMap("bz2", "bz2");
vfs.addMimeTypeMap("application/x-tar", "tar");
vfs.addMimeTypeMap("application/x-gzip", "gz");
vfs.addMimeTypeMap("application/zip", "zip");
vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
vfs.setFilesCache(new SoftRefFilesCache());
File cacheDir = computeTopCacheDir();
vfs.setReplicator(new UniqueFileReplicator(cacheDir));
vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
vfs.init();
vfsInstances.add(new WeakReference(vfs));
return vfs;
}
private static File computeTopCacheDir() {
String cacheDirPath = AccumuloClassLoader.getAccumuloString(VFS_CACHE_DIR, System.getProperty("java.io.tmpdir"));
String procName = ManagementFactory.getRuntimeMXBean().getName();
return new File(cacheDirPath, "accumulo-vfs-cache-" + procName + "-" + System.getProperty("user.name", "nouser"));
}
public interface Printer {
void print(String s);
}
public static void printClassPath() {
printClassPath(new Printer() {
@Override
public void print(String s) {
System.out.println(s);
}
});
}
public static void printClassPath(Printer out) {
try {
ClassLoader cl = getClassLoader();
ArrayList classloaders = new ArrayList();
while (cl != null) {
classloaders.add(cl);
cl = cl.getParent();
}
Collections.reverse(classloaders);
int level = 0;
for (ClassLoader classLoader : classloaders) {
if (level > 0)
out.print("");
level++;
String classLoaderDescription;
switch (level) {
case 1:
classLoaderDescription = level + ": Java System Classloader (loads Java system resources)";
break;
case 2:
classLoaderDescription = level + ": Java Classloader (loads everything defined by java classpath)";
break;
case 3:
classLoaderDescription = level + ": Accumulo Classloader (loads everything defined by general.classpaths)";
break;
case 4:
classLoaderDescription = level + ": Accumulo Dynamic Classloader (loads everything defined by general.dynamic.classpaths)";
break;
default:
classLoaderDescription = level + ": Mystery Classloader (someone probably added a classloader and didn't update the switch statement in "
+ AccumuloVFSClassLoader.class.getName() + ")";
break;
}
if (classLoader instanceof URLClassLoader) {
// If VFS class loader enabled, but no contexts defined.
out.print("Level " + classLoaderDescription + " URL classpath items are:");
for (URL u : ((URLClassLoader) classLoader).getURLs()) {
out.print("\t" + u.toExternalForm());
}
} else if (classLoader instanceof VFSClassLoader) {
out.print("Level " + classLoaderDescription + " VFS classpaths items are:");
VFSClassLoader vcl = (VFSClassLoader) classLoader;
for (FileObject f : vcl.getFileObjects()) {
out.print("\t" + f.getURL().toExternalForm());
}
} else {
out.print("Unknown classloader configuration " + classLoader.getClass());
}
}
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
public static synchronized ContextManager getContextManager() throws IOException {
if (contextManager == null) {
getClassLoader();
contextManager = new ContextManager(generateVfs(), new ReloadingClassLoader() {
@Override
public ClassLoader getClassLoader() {
try {
return AccumuloVFSClassLoader.getClassLoader();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
}
return contextManager;
}
public static void close() {
for (WeakReference vfsInstance : vfsInstances) {
DefaultFileSystemManager ref = vfsInstance.get();
if (ref != null) {
FileReplicator replicator;
try {
replicator = ref.getReplicator();
if (replicator instanceof UniqueFileReplicator) {
((UniqueFileReplicator) replicator).close();
}
} catch (FileSystemException e) {
log.error("FileSystemException", e);
}
ref.close();
}
}
try {
FileUtils.deleteDirectory(computeTopCacheDir());
} catch (IOException e) {
log.error("IOException", e);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy