org.apache.hadoop.hbase.LocalHBaseCluster Maven / Gradle / Ivy
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Threads;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
/**
* This class creates a single process HBase cluster. One thread is created for
* a master and one per region server.
*
* Call {@link #startup()} to start the cluster running and {@link #shutdown()}
* to close it all down. {@link #join} the cluster is you want to wait on
* shutdown completion.
*
* Runs master on port 60000 by default. Because we can't just kill the
* process -- not till HADOOP-1700 gets fixed and even then.... -- we need to
* be able to find the master with a remote client to run shutdown. To use a
* port other than 60000, set the hbase.master to a value of 'local:PORT':
* that is 'local', not 'localhost', and the port number the master should use
* instead of 60000.
*
*/
public class LocalHBaseCluster {
static final Log LOG = LogFactory.getLog(LocalHBaseCluster.class);
private final List masterThreads =
new CopyOnWriteArrayList();
private final List regionThreads =
new CopyOnWriteArrayList();
private final static int DEFAULT_NO = 1;
/** local mode */
public static final String LOCAL = "local";
/** 'local:' */
public static final String LOCAL_COLON = LOCAL + ":";
private final Configuration conf;
private final Class masterClass;
private final Class regionServerClass;
/**
* Constructor.
* @param conf
* @throws IOException
*/
public LocalHBaseCluster(final Configuration conf)
throws IOException {
this(conf, DEFAULT_NO);
}
/**
* Constructor.
* @param conf Configuration to use. Post construction has the master's
* address.
* @param noRegionServers Count of regionservers to start.
* @throws IOException
*/
public LocalHBaseCluster(final Configuration conf, final int noRegionServers)
throws IOException {
this(conf, 1, noRegionServers, getMasterImplementation(conf),
getRegionServerImplementation(conf));
}
/**
* Constructor.
* @param conf Configuration to use. Post construction has the active master
* address.
* @param noMasters Count of masters to start.
* @param noRegionServers Count of regionservers to start.
* @throws IOException
*/
public LocalHBaseCluster(final Configuration conf, final int noMasters,
final int noRegionServers)
throws IOException {
this(conf, noMasters, noRegionServers, getMasterImplementation(conf),
getRegionServerImplementation(conf));
}
@SuppressWarnings("unchecked")
private static Class getRegionServerImplementation(final Configuration conf) {
return (Class)conf.getClass(HConstants.REGION_SERVER_IMPL,
HRegionServer.class);
}
@SuppressWarnings("unchecked")
private static Class getMasterImplementation(final Configuration conf) {
return (Class)conf.getClass(HConstants.MASTER_IMPL,
HMaster.class);
}
/**
* Constructor.
* @param conf Configuration to use. Post construction has the master's
* address.
* @param noMasters Count of masters to start.
* @param noRegionServers Count of regionservers to start.
* @param masterClass
* @param regionServerClass
* @throws IOException
*/
@SuppressWarnings("unchecked")
public LocalHBaseCluster(final Configuration conf, final int noMasters,
final int noRegionServers, final Class masterClass,
final Class regionServerClass)
throws IOException {
this.conf = conf;
// Always have masters and regionservers come up on port '0' so we don't
// clash over default ports.
conf.set(HConstants.MASTER_PORT, "0");
conf.set(HConstants.REGIONSERVER_PORT, "0");
this.masterClass = (Class)
conf.getClass(HConstants.MASTER_IMPL, masterClass);
// Start the HMasters.
for (int i = 0; i < noMasters; i++) {
addMaster(new Configuration(conf), i);
}
// Start the HRegionServers.
this.regionServerClass =
(Class)conf.getClass(HConstants.REGION_SERVER_IMPL,
regionServerClass);
for (int i = 0; i < noRegionServers; i++) {
addRegionServer(new Configuration(conf), i);
}
}
public JVMClusterUtil.RegionServerThread addRegionServer()
throws IOException {
return addRegionServer(new Configuration(conf), this.regionThreads.size());
}
public JVMClusterUtil.RegionServerThread addRegionServer(
Configuration config, final int index)
throws IOException {
// Create each regionserver with its own Configuration instance so each has
// its HConnection instance rather than share (see HBASE_INSTANCES down in
// the guts of HConnectionManager.
JVMClusterUtil.RegionServerThread rst =
JVMClusterUtil.createRegionServerThread(config,
this.regionServerClass, index);
this.regionThreads.add(rst);
return rst;
}
public JVMClusterUtil.RegionServerThread addRegionServer(
final Configuration config, final int index, User user)
throws IOException, InterruptedException {
return user.runAs(
new PrivilegedExceptionAction() {
public JVMClusterUtil.RegionServerThread run() throws Exception {
return addRegionServer(config, index);
}
});
}
public JVMClusterUtil.MasterThread addMaster() throws IOException {
return addMaster(new Configuration(conf), this.masterThreads.size());
}
public JVMClusterUtil.MasterThread addMaster(Configuration c, final int index)
throws IOException {
// Create each master with its own Configuration instance so each has
// its HConnection instance rather than share (see HBASE_INSTANCES down in
// the guts of HConnectionManager.
JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c,
(Class) conf.getClass(HConstants.MASTER_IMPL, this.masterClass), index);
this.masterThreads.add(mt);
return mt;
}
public JVMClusterUtil.MasterThread addMaster(
final Configuration c, final int index, User user)
throws IOException, InterruptedException {
return user.runAs(
new PrivilegedExceptionAction() {
public JVMClusterUtil.MasterThread run() throws Exception {
return addMaster(c, index);
}
});
}
/**
* @param serverNumber
* @return region server
*/
public HRegionServer getRegionServer(int serverNumber) {
return regionThreads.get(serverNumber).getRegionServer();
}
/**
* @return Read-only list of region server threads.
*/
public List getRegionServers() {
return Collections.unmodifiableList(this.regionThreads);
}
/**
* @return List of running servers (Some servers may have been killed or
* aborted during lifetime of cluster; these servers are not included in this
* list).
*/
public List getLiveRegionServers() {
List liveServers =
new ArrayList();
List list = getRegionServers();
for (JVMClusterUtil.RegionServerThread rst: list) {
if (rst.isAlive()) liveServers.add(rst);
else LOG.info("Not alive " + rst.getName());
}
return liveServers;
}
/**
* Wait for the specified region server to stop
* Removes this thread from list of running threads.
* @param serverNumber
* @return Name of region server that just went down.
*/
public String waitOnRegionServer(int serverNumber) {
JVMClusterUtil.RegionServerThread regionServerThread =
this.regionThreads.remove(serverNumber);
while (regionServerThread.isAlive()) {
try {
LOG.info("Waiting on " +
regionServerThread.getRegionServer().toString());
regionServerThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return regionServerThread.getName();
}
/**
* Wait for the specified region server to stop
* Removes this thread from list of running threads.
* @param rst
* @return Name of region server that just went down.
*/
public String waitOnRegionServer(JVMClusterUtil.RegionServerThread rst) {
while (rst.isAlive()) {
try {
LOG.info("Waiting on " +
rst.getRegionServer().toString());
rst.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
for (int i=0;i getMasters() {
return Collections.unmodifiableList(this.masterThreads);
}
/**
* @return List of running master servers (Some servers may have been killed
* or aborted during lifetime of cluster; these servers are not included in
* this list).
*/
public List getLiveMasters() {
List liveServers =
new ArrayList();
List list = getMasters();
for (JVMClusterUtil.MasterThread mt: list) {
if (mt.isAlive()) {
liveServers.add(mt);
}
}
return liveServers;
}
/**
* Wait for the specified master to stop
* Removes this thread from list of running threads.
* @param serverNumber
* @return Name of master that just went down.
*/
public String waitOnMaster(int serverNumber) {
JVMClusterUtil.MasterThread masterThread =
this.masterThreads.remove(serverNumber);
while (masterThread.isAlive()) {
try {
LOG.info("Waiting on " +
masterThread.getMaster().getServerName().toString());
masterThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return masterThread.getName();
}
/**
* Wait for the specified master to stop
* Removes this thread from list of running threads.
* @param masterThread
* @return Name of master that just went down.
*/
public String waitOnMaster(JVMClusterUtil.MasterThread masterThread) {
while (masterThread.isAlive()) {
try {
LOG.info("Waiting on " +
masterThread.getMaster().getServerName().toString());
masterThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
for (int i=0;i