• Home
• org.hbase
• asynchbase
• 1.8.2
• source code
• HBaseClient.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.hbase.async.HBaseClient Maven / Gradle / Ivy

/*
 * Copyright (C) 2010-2018  The Async HBase Authors.  All rights reserved.
 * This file is part of Async HBase.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *   - Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *   - Redistributions in binary form must reproduce the above copyright notice,
 *     this list of conditions and the following disclaimer in the documentation
 *     and/or other materials provided with the distribution.
 *   - Neither the name of the StumbleUpon nor the names of its contributors
 *     may be used to endorse or promote products derived from this software
 *     without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
package org.hbase.async;

import java.net.Inet6Address;
import com.google.common.cache.LoadingCache;
import com.google.protobuf.InvalidProtocolBufferException;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import com.stumbleupon.async.DeferredGroupException;

import org.apache.zookeeper.AsyncCallback;
import org.apache.zookeeper.KeeperException.Code;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
import org.hbase.async.generated.ZooKeeperPB;
import org.jboss.netty.channel.ChannelEvent;
import org.jboss.netty.channel.ChannelHandler;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.DefaultChannelPipeline;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
import org.jboss.netty.channel.socket.SocketChannel;
import org.jboss.netty.channel.socket.SocketChannelConfig;
import org.jboss.netty.channel.socket.nio.NioChannelConfig;
import org.jboss.netty.channel.socket.nio.NioClientBossPool;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.channel.socket.nio.NioWorkerPool;
import org.jboss.netty.handler.timeout.IdleState;
import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateEvent;
import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.jboss.netty.util.HashedWheelTimer;
import org.jboss.netty.util.ThreadNameDeterminer;
import org.jboss.netty.util.Timeout;
import org.jboss.netty.util.Timer;
import org.jboss.netty.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.UnknownHostException;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

/**
 * A fully asynchronous, thread-safe, modern HBase client.
 * 

 * Unlike the traditional HBase client ({@code HTable}), this client should be
 * instantiated only once.  You can use it with any number of tables at the
 * same time.  The only case where you should have multiple instances is when
 * you want to use multiple different clusters at the same time.
 * 

 * If you play by the rules, this client is (in theory {@code :D}) completely
 * thread-safe.  Read the documentation carefully to know what the requirements
 * are for this guarantee to apply.
 * 

 * This client is fully non-blocking, any blocking operation will return a
 * {@link Deferred} instance to which you can attach a {@link Callback} chain
 * that will execute when the asynchronous operation completes.
 *
 * 
Note regarding {@code HBaseRpc} instances passed to this class
 * Every {@link HBaseRpc} passed to a method of this class should not be
 * changed or re-used until the {@code Deferred} returned by that method
 * calls you back.  Changing or re-using any {@link HBaseRpc} for
 * an RPC in flight will lead to unpredictable results and voids
 * your warranty.
 *
 * 
Data Durability
 * Some methods or RPC types take a {@code durable} argument.  When an edit
 * requests to be durable, the success of the RPC guarantees that the edit is
 * safely and durably stored by HBase and won't be lost.  In case of server
 * failures, the edit won't be lost although it may become momentarily
 * unavailable.  Setting the {@code durable} argument to {@code false} makes
 * the operation complete faster (and puts a lot less strain on HBase), but
 * removes this durability guarantee.  In case of a server failure, the edit
 * may (or may not) be lost forever.  When in doubt, leave it to {@code true}
 * (or use the corresponding method that doesn't accept a {@code durable}
 * argument as it will default to {@code true}).  Setting it to {@code false}
 * is useful in cases where data-loss is acceptable, e.g. during batch imports
 * (where you can re-run the whole import in case of a failure), or when you
 * intend to do statistical analysis on the data (in which case some missing
 * data won't affect the results as long as the data loss caused by machine
 * failures preserves the distribution of your data, which depends on how
 * you're building your row keys and how you're using HBase, so be careful).
 * 

 * Bear in mind that this durability guarantee holds only once the RPC has
 * completed successfully.  Any edit temporarily buffered on the client side
 * or in-flight will be lost if the client itself crashes.  You can control
 * how much buffering is done by the client by using {@link #setFlushInterval}
 * and you can force-flush the buffered edits by calling {@link #flush}.  When
 * you're done using HBase, you must not just give up your
 * reference to your {@code HBaseClient}, you must shut it down gracefully by
 * calling {@link #shutdown}.  If you fail to do this, then all edits still
 * buffered by the client will be lost.
 * 

 * NOTE: This entire section assumes that you use a distributed file
 * system that provides HBase with the required durability semantics.  If
 * you use HDFS, make sure you have a version of HDFS that provides HBase
 * the necessary API and semantics to durability store its data.
 *
 * 
{@code throws} clauses
 * None of the asynchronous methods in this API are expected to throw an
 * exception.  But the {@link Deferred} object they return to you can carry an
 * exception that you should handle (using "errbacks", see the javadoc of
 * {@link Deferred}).  In order to be able to do proper asynchronous error
 * handling, you need to know what types of exceptions you're expected to face
 * in your errbacks.  In order to document that, the methods of this API use
 * javadoc's {@code @throws} to spell out the exception types you should
 * handle in your errback.  Asynchronous exceptions will be indicated as such
 * in the javadoc with "(deferred)".
 * 

 * For instance, if a method {@code foo} pretends to throw an
 * {@link UnknownScannerException} and returns a {@code Deferred},
 * then you should use the method like so:
 * 
 *   HBaseClient client = ...;
 *   {@link Deferred}{@code } d = client.foo();
 *   d.addCallbacks(new {@link Callback}{@code }() {
 *     SomethingElse call(Whatever arg) {
 *       LOG.info("Yay, RPC completed successfully!");
 *       return new SomethingElse(arg.getWhateverResult());
 *     }
 *     String toString() {
 *       return "handle foo response";
 *     }
 *   },
 *   new {@link Callback}{@code }() {
 *     Object call(Exception arg) {
 *       if (arg instanceof {@link UnknownScannerException}) {
 *         LOG.error("Oops, we used the wrong scanner?", arg);
 *         return otherAsyncOperation();  // returns a {@code Deferred}
 *       }
 *       LOG.error("Sigh, the RPC failed and we don't know what to do", arg);
 *       return arg;  // Pass on the error to the next errback (if any);
 *     }
 *     String toString() {
 *       return "foo errback";
 *     }
 *   });
 * 
 * This code calls {@code foo}, and upon successful completion transforms the
 * result from a {@code Whatever} to a {@code SomethingElse} (which will then
 * be given to the next callback in the chain, if any).  When there's a
 * failure, the errback is called instead and it attempts to handle a
 * particular type of exception by retrying the operation differently.
 */
public final class HBaseClient {
  /*
   * TODO(tsuna): Address the following.
   *
   * - Properly handle disconnects.
   *    - Attempt to reconnect a couple of times, see if it was a transient
   *      network blip.
   *    - If the -ROOT- region is unavailable when we start, we should
   *      put a watch in ZK instead of polling it every second.
   * - Stats:
   *     - QPS per RPC type.
   *     - Latency histogram per RPC type (requires open-sourcing the SU Java
   *       stats classes that I wrote in a separate package).
   *     - Cache hit rate in the local META cache.
   *     - RPC errors and retries.
   *     - Typical batch size when flushing edits (is that useful?).
   * - Write unit tests and benchmarks!
   */

  private static final Logger LOG = LoggerFactory.getLogger(HBaseClient.class);

  /**
   * An empty byte array you can use.  This can be useful for instance with
   * {@link Scanner#setStartKey} and {@link Scanner#setStopKey}.
   */
  public static final byte[] EMPTY_ARRAY = new byte[0];

  /** A byte array containing a single zero byte.  */
  private static final byte[] ZERO_ARRAY = new byte[] { 0 };

  protected static final byte[] ROOT = new byte[] { '-', 'R', 'O', 'O', 'T', '-' };
  protected static final byte[] ROOT_REGION = new byte[] { '-', 'R', 'O', 'O', 'T', '-', ',', ',', '0' };
  /** HBase 0.98 and up: -ROOT- is now hbase:root */
  static final byte[] HBASE98_ROOT =
      new byte[] { 'h', 'b', 'a', 's', 'e', ':', 'r', 'o', 'o', 't'};
  static final byte[] HBASE98_ROOT_REGION =
      new byte[] { 'h', 'b', 'a', 's', 'e', ':', 'r', 'o', 'o', 't', ',',',', '0' };
  protected static final byte[] META = new byte[] { '.', 'M', 'E', 'T', 'A', '.' };
  protected static final byte[] INFO = new byte[] { 'i', 'n', 'f', 'o' };
  protected static final byte[] REGIONINFO = new byte[] { 'r', 'e', 'g', 'i', 'o', 'n', 'i', 'n', 'f', 'o' };
  protected static final byte[] SERVER = new byte[] { 's', 'e', 'r', 'v', 'e', 'r' };
  /** HBase 0.95 and up: .META. is now hbase:meta */
  protected static final byte[] HBASE96_META =
    new byte[] { 'h', 'b', 'a', 's', 'e', ':', 'm', 'e', 't', 'a' };
  /** New for HBase 0.95 and up: the name of META is fixed.  */
  protected static final byte[] META_REGION_NAME =
    new byte[] { 'h', 'b', 'a', 's', 'e', ':', 'm', 'e', 't', 'a', ',', ',', '1' };
  /** New for HBase 0.95 and up: the region info for META is fixed.  */
  protected static final RegionInfo META_REGION =
    new RegionInfo(HBASE96_META, META_REGION_NAME, EMPTY_ARRAY);

  /**
   * In HBase 0.95 and up, this magic number is found in a couple places.
   * It's used in the znode that points to the .META. region, to
   * indicate that the contents of the znode is a protocol buffer.
   * It's also used in the value of the KeyValue found in the .META. table
   * that contain a {@link RegionInfo}, to indicate that the value contents
   * is a protocol buffer.
   */
  static final int PBUF_MAGIC = 1346524486;  // 4 bytes: "PBUF"

  /**
   * Timer we use to handle all our timeouts.
   * TODO(tsuna): Get it through the ctor to share it with others.
   */
  private final HashedWheelTimer timer;
  
  /** A separate timer thread used for processing RPC timeout callbacks. We
   * keep it separate as a bad HBase server can cause a timeout storm and we 
   * don't want to block any flushes and operations on other region servers.
   */
  private final HashedWheelTimer rpc_timeout_timer;

  /** Up to how many milliseconds can we buffer an edit on the client side.  */
  private volatile short flush_interval;
  
  /** How many different counters do we want to keep in memory for buffering. */
  private volatile int increment_buffer_size;
  
  /**
   * Low and high watermarks when buffering RPCs due to an NSRE.
   * @see #handleNSRE
   */
  private int nsre_low_watermark;
  private int nsre_high_watermark;
  
  /**
   * Factory through which we will create all its channels / sockets.
   */
  private final ClientSocketChannelFactory channel_factory;

  /** Watcher to keep track of the -ROOT- region in ZooKeeper.  */
  private final ZKClient zkclient;

  /**
   * The client currently connected to the -ROOT- region.
   * If this is {@code null} then we currently don't know where the -ROOT-
   * region is and we're waiting for a notification from ZooKeeper to tell
   * us where it is.
   * Note that with HBase 0.95, {@link #has_root} would be false, and this
   * would instead point to the .META. region.
   */
  private volatile RegionClient rootregion;

  /**
   * Whether or not there is a -ROOT- region.
   * When connecting to HBase 0.95 and up, this would be set to false, so we
   * would go straight to .META. instead (except in the case of Split meta
   * where there are multiple meta regions and a root to route to them).
   */
  volatile boolean has_root = true;

  /**
   * Maps {@code (table, start_key)} pairs to the {@link RegionInfo} that
   * serves this key range for this table.
   * 

   * The keys in this map are region names.
   * @see #createRegionSearchKey
   * Because it's a sorted map, we can efficiently find a region given an
   * arbitrary key.
   * @see #getRegion
   * 

   * This map and the next 2 maps contain the same data, but indexed
   * differently.  There is no consistency guarantee across the maps.
   * They are not updated all at the same time atomically.  This map
   * is always the first to be updated, because that's the map from
   * which all the lookups are done in the fast-path of the requests
   * that need to locate a region.  The second map to be updated is
   * {@link region2client}, because it comes second in the fast-path
   * of every requests that need to locate a region.  The third map
   * is only used to handle RegionServer disconnections gracefully.
   * 

   * Note: before using the {@link RegionInfo} you pull out of this map,
   * you must ensure that {@link RegionInfo#table} doesn't return
   * {@link #EMPTY_ARRAY}.  If it does, it means you got a special entry
   * used to indicate that this region is known to be unavailable right
   * now due to an NSRE.  You must not use this {@link RegionInfo} as
   * if it was a normal entry.
   * @see #handleNSRE
   */
  private final ConcurrentSkipListMap regions_cache =
    new ConcurrentSkipListMap(RegionInfo.REGION_NAME_CMP);

  /**
   * Maps a {@link RegionInfo} to the client currently connected to the
   * RegionServer that serves this region.
   * 

   * The opposite mapping is stored in {@link #client2regions}.
   * There's no consistency guarantee with that other map.
   * See the javadoc for {@link #regions_cache} regarding consistency.
   */
  private final ConcurrentHashMap region2client =
    new ConcurrentHashMap();

  /**
   * Maps a client connected to a RegionServer to the list of regions we know
   * it's serving so far.
   * 

   * The opposite mapping is stored in {@link #region2client}.
   * There's no consistency guarantee with that other map.
   * See the javadoc for {@link #regions_cache} regarding consistency.
   * 

   * Each list in the map is protected by its own monitor lock.
   */
  private final ConcurrentHashMap>
    client2regions = new ConcurrentHashMap>();

  /**
   * Cache that maps a RegionServer address ("ip:port") to the client
   * connected to it.
   * 

   * Access to this map must be synchronized by locking its monitor.
   * Lock ordering: when locking both this map and a RegionClient, the
   * RegionClient must always be locked first to avoid deadlocks.  Logging
   * the contents of this map (or calling toString) requires copying it first.
   * 

   * This isn't a {@link ConcurrentHashMap} because we don't use it frequently
   * (just when connecting to / disconnecting from RegionServers) and when we
   * add something to it, we want to do an atomic get-and-put, but
   * {@code putIfAbsent} isn't a good fit for us since it requires to create
   * an object that may be "wasted" in case another thread wins the insertion
   * race, and we don't want to create unnecessary connections.
   * 

   * Upon disconnection, clients are automatically removed from this map.
   * We don't use a {@code ChannelGroup} because a {@code ChannelGroup} does
   * the clean-up on the {@code channelClosed} event, which is actually the
   * 3rd and last event to be fired when a channel gets disconnected.  The
   * first one to get fired is, {@code channelDisconnected}.  This matters to
   * us because we want to purge disconnected clients from the cache as
   * quickly as possible after the disconnection, to avoid handing out clients
   * that are going to cause unnecessary errors.
   * @see RegionClientPipeline#handleDisconnect
   */
  private final HashMap ip2client =
    new HashMap();

  /**
   * Map of region name to list of pending RPCs for this region.
   * 

   * The array-list isn't expected to be empty, except during rare race
   * conditions.  When the list is non-empty, the first element in the
   * list should be a special "probe" RPC we build to detect when the
   * region NSRE'd is back online.
   * 

   * For more details on how this map is used, please refer to the
   * documentation of {@link #handleNSRE}.
   * 

   * Each list in the map is protected by its own monitor lock.
   */
  private final ConcurrentSkipListMap> got_nsre =
    new ConcurrentSkipListMap>(RegionInfo.REGION_NAME_CMP);

  /**
   * Buffer for atomic increment coalescing.
   * This buffer starts out null, and remains so until the first time we need
   * to buffer an increment.  Once lazily initialized, this buffer will never
   * become null again.
   * 

   * We do this so that we can lazily schedule the flush timer only if we ever
   * have buffered increments.  Applications without buffered increments don't
   * need to pay any memory for the buffer or any CPU time for a useless timer.
   * @see #setupIncrementCoalescing
   */
  private volatile LoadingCache increment_buffer;
  private volatile LoadingCache multi_column_increment_buffer;

  /** The configuration for this client */
  private final Config config;
  
  /** Integers for thread naming */
  final static AtomicInteger BOSS_THREAD_ID = new AtomicInteger();
  final static AtomicInteger WORKER_THREAD_ID = new AtomicInteger();
  final static AtomicInteger TIMER_THREAD_ID = new AtomicInteger();
  
  /** Default RPC timeout in milliseconds from the config */
  private final int rpc_timeout;

  /** Whether or not we have to scan meta instead of making getClosestBeforeRow calls. */
  private volatile boolean scan_meta;
  
  /** Whether or not to split meta is in force. */
  protected boolean split_meta;
  
  private boolean increment_buffer_durable = false;
  
  // ------------------------ //
  // Client usage statistics. //
  // ------------------------ //

  /** Number of connections created by {@link #newClient}.  */
  private final Counter num_connections_created = new Counter();

  /** How many {@code -ROOT-} lookups were made.  */
  private final Counter root_lookups = new Counter();

  /** How many {@code .META.} lookups were made (with a permit).  */
  private final Counter meta_lookups_with_permit = new Counter();

  /** How many {@code .META.} lookups were made (without a permit).  */
  private final Counter meta_lookups_wo_permit = new Counter();

  /** Number of calls to {@link #flush}.  */
  private final Counter num_flushes = new Counter();

  /** Number of NSREs handled by {@link #handleNSRE}.  */
  private final Counter num_nsres = new Counter();

  /** Number of RPCs delayed by {@link #handleNSRE}.  */
  private final Counter num_nsre_rpcs = new Counter();

  /** Number of {@link MultiAction} sent to the network.  */
  final Counter num_multi_rpcs = new Counter();

  /** Number of calls to {@link #get}.  */
  private final Counter num_gets = new Counter();

  /** Number of calls to {@link #openScanner}.  */
  private final Counter num_scanners_opened = new Counter();

  /** Number of calls to {@link #scanNextRows}.  */
  private final Counter num_scans = new Counter();

  /** Number calls to {@link #put}.  */
  private final Counter num_puts = new Counter();

  /** Number calls to {@link #append}.  */
  private final Counter num_appends = new Counter();
  
  /** Number calls to {@link #lockRow}.  */
  private final Counter num_row_locks = new Counter();

  /** Number calls to {@link #delete}.  */
  private final Counter num_deletes = new Counter();

  /** Number of {@link AtomicIncrementRequest} sent.  */
  private final Counter num_atomic_increments = new Counter();
  
  /** Number of region clients closed due to being idle.  */
  private final Counter idle_connections_closed = new Counter();

  /**
   * Constructor.
   * @param quorum_spec The specification of the quorum, e.g.
   * {@code "host1,host2,host3"}.
   */
  public HBaseClient(final String quorum_spec) {
    this(quorum_spec, "/hbase");
  }

  /**
   * Constructor.
   * @param quorum_spec The specification of the quorum, e.g.
   * {@code "host1,host2,host3"}.
   * @param base_path The base path under which is the znode for the
   * -ROOT- region.
   */
  public HBaseClient(final String quorum_spec, final String base_path) {
    this(quorum_spec, base_path, defaultChannelFactory(new Config()));
  }

  /**
   * Constructor for advanced users with special needs.
   * 

   * NOTE: Only advanced users who really know what they're
   * doing should use this constructor.  Passing an inappropriate thread
   * pool, or blocking its threads will prevent this {@code HBaseClient}
   * from working properly or lead to poor performance.
   * @param quorum_spec The specification of the quorum, e.g.
   * {@code "host1,host2,host3"}.
   * @param base_path The base path under which is the znode for the
   * -ROOT- region.
   * @param executor The executor from which to obtain threads for NIO
   * operations.  It is strongly encouraged to use a
   * {@link Executors#newCachedThreadPool} or something equivalent unless
   * you're sure to understand how Netty creates and uses threads.
   * Using a fixed-size thread pool will not work the way you expect.
   * 

   * Note that calling {@link #shutdown} on this client will NOT
   * shut down the executor.
   * @see NioClientSocketChannelFactory
   * @since 1.2
   */
  public HBaseClient(final String quorum_spec, final String base_path,
                     final Executor executor) {
    this(quorum_spec, base_path, new CustomChannelFactory(executor));
  }

  /**
   * Constructor for advanced users with special needs.
   * 

   * Most users don't need to use this constructor.
   * @param quorum_spec The specification of the quorum, e.g.
   * {@code "host1,host2,host3"}.
   * @param base_path The base path under which is the znode for the
   * -ROOT- region.
   * @param channel_factory A custom factory to use to create sockets.
   * 

   * Note that calling {@link #shutdown} on this client will also cause the
   * shutdown and release of the factory and its underlying thread pool.
   * @since 1.2
   */
  public HBaseClient(final String quorum_spec, final String base_path,
                     final ClientSocketChannelFactory channel_factory) {
    this.channel_factory = channel_factory;
    zkclient = new ZKClient(quorum_spec, base_path);
    config = new Config();
    rpc_timeout = config.getInt("hbase.rpc.timeout");
    timer = newTimer(config, "HBaseClient");
    rpc_timeout_timer = newTimer(config, "RPC Timeout Timer");
    flush_interval = config.getShort("hbase.rpcs.buffered_flush_interval");
    increment_buffer_size = config.getInt("hbase.increments.buffer_size");
    nsre_low_watermark = config.getInt("hbase.nsre.low_watermark");
    nsre_high_watermark = config.getInt("hbase.nsre.high_watermark");
    if (config.properties.containsKey("hbase.increments.durable")) {
      increment_buffer_durable = config.getBoolean("hbase.increments.durable");
    }
    if (config.hasProperty("hbase.meta.scan")) {
      scan_meta = config.getBoolean("hbase.meta.scan");
    } else {
      scan_meta = Boolean.parseBoolean(
          System.getProperty("hbase.meta.scan", "false"));
    }
    if (config.hasProperty("hbase.meta.split")) {
      split_meta = config.getBoolean("hbase.meta.split");
    } else {
      split_meta = Boolean.parseBoolean(
          System.getProperty("hbase.meta.split", "false"));
    }
  }
  
  /**
   * Constructor accepting a configuration object with at least the 
   * "hbase.zookeeper.quorum" specified in the format {@code "host1,host2,host3"}.
   * @param config A configuration object
   * @since 1.7
   */
  public HBaseClient(final Config config) {
    this(config, defaultChannelFactory(config));
  }
  
  /**
   * Constructor accepting a configuration object with at least the 
   * "hbase.zookeeper.quorum" specified in the format {@code "host1,host2,host3"}
   * and an executor thread pool.
   * @param config A configuration object
   * @param executor The executor from which to obtain threads for NIO
   * operations.  It is strongly encouraged to use a
   * {@link Executors#newCachedThreadPool} or something equivalent unless
   * you're sure to understand how Netty creates and uses threads.
   * Using a fixed-size thread pool will not work the way you expect.
   * 

   * Note that calling {@link #shutdown} on this client will NOT
   * shut down the executor.
   * @see NioClientSocketChannelFactory
   * @since 1.7
   */
  public HBaseClient(final Config config, final Executor executor) {
    this(config, new CustomChannelFactory(executor));
  }
  
  /**
   * Constructor accepting a configuration object with at least the 
   * "hbase.zookeeper.quorum" specified in the format {@code "host1,host2,host3"}
   * and a custom channel factory for advanced users.
   * 

   * Most users don't need to use this constructor.
   * @param config A configuration object
   * @param channel_factory A custom factory to use to create sockets.
   * 

   * Note that calling {@link #shutdown} on this client will also cause the
   * shutdown and release of the factory and its underlying thread pool.
   * @since 1.7
   */
  public HBaseClient(final Config config, 
      final ClientSocketChannelFactory channel_factory) {
    this.channel_factory = channel_factory;
    zkclient = new ZKClient(config.getString("hbase.zookeeper.quorum"), 
        config.getString("hbase.zookeeper.znode.parent"));
    this.config = config;
    rpc_timeout = config.getInt("hbase.rpc.timeout");
    timer = newTimer(config, "HBaseClient");
    rpc_timeout_timer = newTimer(config, "RPC Timeout Timer");
    flush_interval = config.getShort("hbase.rpcs.buffered_flush_interval");
    increment_buffer_size = config.getInt("hbase.increments.buffer_size");
    nsre_low_watermark = config.getInt("hbase.nsre.low_watermark");
    nsre_high_watermark = config.getInt("hbase.nsre.high_watermark");
    if (config.properties.containsKey("hbase.increments.durable")) {
      increment_buffer_durable = config.getBoolean("hbase.increments.durable");
    }
    if (config.hasProperty("hbase.meta.scan")) {
      scan_meta = config.getBoolean("hbase.meta.scan");
    } else {
      scan_meta = Boolean.parseBoolean(
          System.getProperty("hbase.meta.scan", "false"));
    }
    if (config.hasProperty("hbase.meta.split")) {
      split_meta = config.getBoolean("hbase.meta.split");
    } else {
      split_meta = Boolean.parseBoolean(
          System.getProperty("hbase.meta.split", "false"));
    }
  }
  
  /**
   * Package private timer constructor that provides a useful name for the
   * timer thread.
   * @param config The config object used to pull out the tick interval
   * @param name A name to stash in the timer
   * @return A timer
   */
  static HashedWheelTimer newTimer(final Config config, final String name) {
    class TimerThreadNamer implements ThreadNameDeterminer {
      @Override
      public String determineThreadName(String currentThreadName,
          String proposedThreadName) throws Exception {
        return "AsyncHBase Timer " + name + " #" + TIMER_THREAD_ID.incrementAndGet();
      }
    }
    if (config == null) {
      return new HashedWheelTimer(Executors.defaultThreadFactory(), 
          new TimerThreadNamer(), 100, MILLISECONDS, 512);
    }
    return new HashedWheelTimer(Executors.defaultThreadFactory(), 
        new TimerThreadNamer(), config.getShort("hbase.timer.tick"), 
        MILLISECONDS, config.getInt("hbase.timer.ticks_per_wheel"));
  }
  
  /** Creates a default channel factory in case we haven't been given one. 
   * The factory will use Netty defaults and provide thread naming rules for
   * easier debugging.
   * @param config The config to pull settings from 
   */
  private static NioClientSocketChannelFactory defaultChannelFactory(
      final Config config) {
    class BossThreadNamer implements ThreadNameDeterminer {
      @Override
      public String determineThreadName(String currentThreadName,
          String proposedThreadName) throws Exception {
        return "AsyncHBase I/O Boss #" + BOSS_THREAD_ID.incrementAndGet();
      }
    }
    
    class WorkerThreadNamer implements ThreadNameDeterminer {
      @Override
      public String determineThreadName(String currentThreadName,
          String proposedThreadName) throws Exception {
        return "AsyncHBase I/O Worker #" + WORKER_THREAD_ID.incrementAndGet();
      }
    }
    
    final Executor executor = Executors.newCachedThreadPool();
    final NioClientBossPool boss_pool = 
        new NioClientBossPool(executor, 1, newTimer(config, "Boss Pool"), 
            new BossThreadNamer());
    final int num_workers = config.hasProperty("hbase.workers.size") ? 
        config.getInt("hbase.workers.size") : 
          Runtime.getRuntime().availableProcessors() * 2;
    final NioWorkerPool worker_pool = new NioWorkerPool(executor, 
        num_workers, new WorkerThreadNamer());
    return new NioClientSocketChannelFactory(boss_pool, worker_pool);
  }

  /** A custom channel factory that doesn't shutdown its executor.  */
  private static final class CustomChannelFactory
    extends NioClientSocketChannelFactory {
      CustomChannelFactory(final Executor executor) {
        super(executor, executor);
      }
      @Override
      public void releaseExternalResources() {
        // Do nothing, we don't want to shut down the executor.
      }
  }

  /**
   * Returns a snapshot of usage statistics for this client.
   * @since 1.3
   */
  public ClientStats stats() {
    final LoadingCache cache =
      increment_buffer;
    
    long inflight_rpcs = 0;
    long pending_rpcs = 0;
    long pending_batched_rpcs = 0;
    int dead_region_clients = 0;
    
    final Collection region_clients = client2regions.keySet();
    
    for (final RegionClient rc : region_clients) {
      final RegionClientStats stats = rc.stats();
      inflight_rpcs += stats.inflightRPCs();
      pending_rpcs += stats.pendingRPCs();
      pending_batched_rpcs += stats.pendingBatchedRPCs();
      if (stats.isDead()) {
        dead_region_clients++;
      }
    }
    
    return new ClientStats(
      num_connections_created.get(),
      root_lookups.get(),
      meta_lookups_with_permit.get(),
      meta_lookups_wo_permit.get(),
      num_flushes.get(),
      num_nsres.get(),
      num_nsre_rpcs.get(),
      num_multi_rpcs.get(),
      num_gets.get(),
      num_scanners_opened.get(),
      num_scans.get(),
      num_puts.get(),
      num_appends.get(),
      num_row_locks.get(),
      num_deletes.get(),
      num_atomic_increments.get(),
      cache != null ? cache.stats() : BufferedIncrement.ZERO_STATS,
      inflight_rpcs,
      pending_rpcs,
      pending_batched_rpcs,
      dead_region_clients,
      region_clients.size(),
      idle_connections_closed.get()
    );
  }

  /**
   * Returns a list of region client stats objects for debugging.
   * @return A list of region client statistics
   * @since 1.7
   */
  public List regionStats() {
    final Collection region_clients = client2regions.keySet();
    final List stats = 
        new ArrayList(region_clients.size());
    for (final RegionClient rc : region_clients) {
      stats.add(rc.stats());
    }
    return stats;
  }
  
  /**
   * Flushes to HBase any buffered client-side write operation.
   * 

   * @return A {@link Deferred}, whose callback chain will be invoked when
   * everything that was buffered at the time of the call has been flushed.
   * 

   * Note that this doesn't guarantee that ALL outstanding RPCs have
   * completed.  This doesn't introduce any sort of global sync point.  All
   * it does really is it sends any buffered RPCs to HBase.
   */
  public Deferred