All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hadoop.hbase.util.RegionMover Maven / Gradle / Ivy

There is a newer version: 3.0.0-beta-1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.util;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Predicate;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.RackManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;

/**
 * Tool for loading/unloading regions to/from given regionserver This tool can be run from Command
 * line directly as a utility. Supports Ack/No Ack mode for loading/unloading operations.Ack mode
 * acknowledges if regions are online after movement while noAck mode is best effort mode that
 * improves performance but will still move on if region is stuck/not moved. Motivation behind noAck
 * mode being RS shutdown where even if a Region is stuck, upon shutdown master will move it
 * anyways. This can also be used by constructiong an Object using the builder and then calling
 * {@link #load()} or {@link #unload()} methods for the desired operations.
 */
@InterfaceAudience.Public
public class RegionMover extends AbstractHBaseTool implements Closeable {
  public static final String MOVE_RETRIES_MAX_KEY = "hbase.move.retries.max";
  public static final String MOVE_WAIT_MAX_KEY = "hbase.move.wait.max";
  public static final String SERVERSTART_WAIT_MAX_KEY = "hbase.serverstart.wait.max";
  public static final int DEFAULT_MOVE_RETRIES_MAX = 5;
  public static final int DEFAULT_MOVE_WAIT_MAX = 60;
  public static final int DEFAULT_SERVERSTART_WAIT_MAX = 180;

  private static final Logger LOG = LoggerFactory.getLogger(RegionMover.class);

  private RegionMoverBuilder rmbuilder;
  private boolean ack = true;
  private int maxthreads = 1;
  private int timeout;
  private String loadUnload;
  private String hostname;
  private String filename;
  private String excludeFile;
  private String designatedFile;
  private int port;
  private Connection conn;
  private Admin admin;
  private RackManager rackManager;

  private RegionMover(RegionMoverBuilder builder) throws IOException {
    this.hostname = builder.hostname;
    this.filename = builder.filename;
    this.excludeFile = builder.excludeFile;
    this.designatedFile = builder.designatedFile;
    this.maxthreads = builder.maxthreads;
    this.ack = builder.ack;
    this.port = builder.port;
    this.timeout = builder.timeout;
    setConf(builder.conf);
    this.conn = ConnectionFactory.createConnection(conf);
    this.admin = conn.getAdmin();
    // Only while running unit tests, builder.rackManager will not be null for the convenience of
    // providing custom rackManager. Otherwise for regular workflow/user triggered action,
    // builder.rackManager is supposed to be null. Hence, setter of builder.rackManager is
    // provided as @InterfaceAudience.Private and it is commented that this is just
    // to be used by unit test.
    rackManager = builder.rackManager == null ? new RackManager(conf) : builder.rackManager;
  }

  private RegionMover() {
  }

  @Override
  public void close() {
    IOUtils.closeQuietly(this.admin, e -> LOG.warn("failed to close admin", e));
    IOUtils.closeQuietly(this.conn, e -> LOG.warn("failed to close conn", e));
  }

  /**
   * Builder for Region mover. Use the {@link #build()} method to create RegionMover object. Has
   * {@link #filename(String)}, {@link #excludeFile(String)}, {@link #maxthreads(int)},
   * {@link #ack(boolean)}, {@link #timeout(int)}, {@link #designatedFile(String)} methods to set
   * the corresponding options.
   */
  public static class RegionMoverBuilder {
    private boolean ack = true;
    private int maxthreads = 1;
    private int timeout = Integer.MAX_VALUE;
    private String hostname;
    private String filename;
    private String excludeFile = null;
    private String designatedFile = null;
    private String defaultDir = System.getProperty("java.io.tmpdir");
    @InterfaceAudience.Private
    final int port;
    private final Configuration conf;
    private RackManager rackManager;

    public RegionMoverBuilder(String hostname) {
      this(hostname, createConf());
    }

    /**
     * Creates a new configuration and sets region mover specific overrides
     */
    private static Configuration createConf() {
      Configuration conf = HBaseConfiguration.create();
      conf.setInt("hbase.client.prefetch.limit", 1);
      conf.setInt("hbase.client.pause", 500);
      conf.setInt("hbase.client.retries.number", 100);
      return conf;
    }

    /**
     * @param hostname Hostname to unload regions from or load regions to. Can be either hostname or
     *                 hostname:port.
     * @param conf     Configuration object
     */
    public RegionMoverBuilder(String hostname, Configuration conf) {
      String[] splitHostname = hostname.toLowerCase().split(":");
      this.hostname = splitHostname[0];
      if (splitHostname.length == 2) {
        this.port = Integer.parseInt(splitHostname[1]);
      } else {
        this.port = conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT);
      }
      this.filename = defaultDir + File.separator + System.getProperty("user.name") + this.hostname
        + ":" + Integer.toString(this.port);
      this.conf = conf;
    }

    /**
     * Path of file where regions will be written to during unloading/read from during loading
     * @return RegionMoverBuilder object
     */
    public RegionMoverBuilder filename(String filename) {
      this.filename = filename;
      return this;
    }

    /**
     * Set the max number of threads that will be used to move regions
     */
    public RegionMoverBuilder maxthreads(int threads) {
      this.maxthreads = threads;
      return this;
    }

    /**
     * Path of file containing hostnames to be excluded during region movement. Exclude file should
     * have 'host:port' per line. Port is mandatory here as we can have many RS running on a single
     * host.
     */
    public RegionMoverBuilder excludeFile(String excludefile) {
      this.excludeFile = excludefile;
      return this;
    }

    /**
     * Set the designated file. Designated file contains hostnames where region moves. Designated
     * file should have 'host:port' per line. Port is mandatory here as we can have many RS running
     * on a single host.
     * @param designatedFile The designated file
     * @return RegionMoverBuilder object
     */
    public RegionMoverBuilder designatedFile(String designatedFile) {
      this.designatedFile = designatedFile;
      return this;
    }

    /**
     * Set ack/noAck mode.
     * 

* In ack mode regions are acknowledged before and after moving and the move is retried * hbase.move.retries.max times, if unsuccessful we quit with exit code 1.No Ack mode is a best * effort mode,each region movement is tried once.This can be used during graceful shutdown as * even if we have a stuck region,upon shutdown it'll be reassigned anyway. *

* @return RegionMoverBuilder object */ public RegionMoverBuilder ack(boolean ack) { this.ack = ack; return this; } /** * Set the timeout for Load/Unload operation in seconds.This is a global timeout,threadpool for * movers also have a separate time which is hbase.move.wait.max * number of regions to * load/unload * @param timeout in seconds * @return RegionMoverBuilder object */ public RegionMoverBuilder timeout(int timeout) { this.timeout = timeout; return this; } /** * Set specific rackManager implementation. This setter method is for testing purpose only. * @param rackManager rackManager impl * @return RegionMoverBuilder object */ @InterfaceAudience.Private public RegionMoverBuilder rackManager(RackManager rackManager) { this.rackManager = rackManager; return this; } /** * This method builds the appropriate RegionMover object which can then be used to load/unload * using load and unload methods * @return RegionMover object */ public RegionMover build() throws IOException { return new RegionMover(this); } } /** * Loads the specified {@link #hostname} with regions listed in the {@link #filename} RegionMover * Object has to be created using {@link #RegionMover(RegionMoverBuilder)} * @return true if loading succeeded, false otherwise */ public boolean load() throws ExecutionException, InterruptedException, TimeoutException { ExecutorService loadPool = Executors.newFixedThreadPool(1); Future loadTask = loadPool.submit(getMetaRegionMovePlan()); boolean isMetaMoved = waitTaskToFinish(loadPool, loadTask, "loading"); if (!isMetaMoved) { return false; } loadPool = Executors.newFixedThreadPool(1); loadTask = loadPool.submit(getNonMetaRegionsMovePlan()); return waitTaskToFinish(loadPool, loadTask, "loading"); } private Callable getMetaRegionMovePlan() { return getRegionsMovePlan(true); } private Callable getNonMetaRegionsMovePlan() { return getRegionsMovePlan(false); } private Callable getRegionsMovePlan(boolean moveMetaRegion) { return () -> { try { List regionsToMove = readRegionsFromFile(filename); if (regionsToMove.isEmpty()) { LOG.info("No regions to load.Exiting"); return true; } Optional metaRegion = getMetaRegionInfoIfToBeMoved(regionsToMove); if (moveMetaRegion) { if (metaRegion.isPresent()) { loadRegions(Collections.singletonList(metaRegion.get())); } } else { metaRegion.ifPresent(regionsToMove::remove); loadRegions(regionsToMove); } } catch (Exception e) { LOG.error("Error while loading regions to " + hostname, e); return false; } return true; }; } private Optional getMetaRegionInfoIfToBeMoved(List regionsToMove) { return regionsToMove.stream().filter(RegionInfo::isMetaRegion).findFirst(); } private void loadRegions(List regionsToMove) throws Exception { ServerName server = getTargetServer(); List movedRegions = Collections.synchronizedList(new ArrayList<>()); LOG.info("Moving " + regionsToMove.size() + " regions to " + server + " using " + this.maxthreads + " threads.Ack mode:" + this.ack); final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); List> taskList = new ArrayList<>(); int counter = 0; while (counter < regionsToMove.size()) { RegionInfo region = regionsToMove.get(counter); ServerName currentServer = MoveWithAck.getServerNameForRegion(region, admin, conn); if (currentServer == null) { LOG .warn("Could not get server for Region:" + region.getRegionNameAsString() + " moving on"); counter++; continue; } else if (server.equals(currentServer)) { LOG.info( "Region " + region.getRegionNameAsString() + " is already on target server=" + server); counter++; continue; } if (ack) { Future task = moveRegionsPool .submit(new MoveWithAck(conn, region, currentServer, server, movedRegions)); taskList.add(task); } else { Future task = moveRegionsPool .submit(new MoveWithoutAck(admin, region, currentServer, server, movedRegions)); taskList.add(task); } counter++; } moveRegionsPool.shutdown(); long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration().getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); waitMoveTasksToFinish(moveRegionsPool, taskList, timeoutInSeconds); } /** * Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}.In * noAck mode we do not make sure that region is successfully online on the target region * server,hence it is best effort.We do not unload regions to hostnames given in * {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions * to hostnames provided in {@link #designatedFile} * @return true if unloading succeeded, false otherwise */ public boolean unload() throws InterruptedException, ExecutionException, TimeoutException { return unloadRegions(false); } /** * Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}.In * noAck mode we do not make sure that region is successfully online on the target region * server,hence it is best effort.We do not unload regions to hostnames given in * {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions * to hostnames provided in {@link #designatedFile}. While unloading regions, destination * RegionServers are selected from different rack i.e regions should not move to any RegionServers * that belong to same rack as source RegionServer. * @return true if unloading succeeded, false otherwise */ public boolean unloadFromRack() throws InterruptedException, ExecutionException, TimeoutException { return unloadRegions(true); } private boolean unloadRegions(boolean unloadFromRack) throws InterruptedException, ExecutionException, TimeoutException { deleteFile(this.filename); ExecutorService unloadPool = Executors.newFixedThreadPool(1); Future unloadTask = unloadPool.submit(() -> { List movedRegions = Collections.synchronizedList(new ArrayList<>()); try { // Get Online RegionServers List regionServers = new ArrayList<>(); regionServers.addAll(admin.getRegionServers()); // Remove the host Region server from target Region Servers list ServerName server = stripServer(regionServers, hostname, port); if (server == null) { LOG.info("Could not find server '{}:{}' in the set of region servers. giving up.", hostname, port); LOG.debug("List of region servers: {}", regionServers); return false; } // Remove RS not present in the designated file includeExcludeRegionServers(designatedFile, regionServers, true); // Remove RS present in the exclude file includeExcludeRegionServers(excludeFile, regionServers, false); if (unloadFromRack) { // remove regionServers that belong to same rack (as source host) since the goal is to // unload regions from source regionServer to destination regionServers // that belong to different rack only. String sourceRack = rackManager.getRack(server); List racks = rackManager.getRack(regionServers); Iterator iterator = regionServers.iterator(); int i = 0; while (iterator.hasNext()) { iterator.next(); if (racks.size() > i && racks.get(i) != null && racks.get(i).equals(sourceRack)) { iterator.remove(); } i++; } } // Remove decommissioned RS Set decommissionedRS = new HashSet<>(admin.listDecommissionedRegionServers()); if (CollectionUtils.isNotEmpty(decommissionedRS)) { regionServers.removeIf(decommissionedRS::contains); LOG.debug("Excluded RegionServers from unloading regions to because they " + "are marked as decommissioned. Servers: {}", decommissionedRS); } stripMaster(regionServers); if (regionServers.isEmpty()) { LOG.warn("No Regions were moved - no servers available"); return false; } unloadRegions(server, regionServers, movedRegions); } catch (Exception e) { LOG.error("Error while unloading regions ", e); return false; } finally { if (movedRegions != null) { writeFile(filename, movedRegions); } } return true; }); return waitTaskToFinish(unloadPool, unloadTask, "unloading"); } private void unloadRegions(ServerName server, List regionServers, List movedRegions) throws Exception { while (true) { List regionsToMove = admin.getRegions(server); regionsToMove.removeAll(movedRegions); if (regionsToMove.isEmpty()) { LOG.info("No Regions to move....Quitting now"); break; } LOG.info("Moving {} regions from {} to {} servers using {} threads .Ack Mode: {}", regionsToMove.size(), this.hostname, regionServers.size(), this.maxthreads, ack); Optional metaRegion = getMetaRegionInfoIfToBeMoved(regionsToMove); if (metaRegion.isPresent()) { RegionInfo meta = metaRegion.get(); submitRegionMovesWhileUnloading(server, regionServers, movedRegions, Collections.singletonList(meta)); regionsToMove.remove(meta); } submitRegionMovesWhileUnloading(server, regionServers, movedRegions, regionsToMove); } } private void submitRegionMovesWhileUnloading(ServerName server, List regionServers, List movedRegions, List regionsToMove) throws Exception { final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); List> taskList = new ArrayList<>(); int serverIndex = 0; for (RegionInfo regionToMove : regionsToMove) { if (ack) { Future task = moveRegionsPool.submit(new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), movedRegions)); taskList.add(task); } else { Future task = moveRegionsPool.submit(new MoveWithoutAck(admin, regionToMove, server, regionServers.get(serverIndex), movedRegions)); taskList.add(task); } serverIndex = (serverIndex + 1) % regionServers.size(); } moveRegionsPool.shutdown(); long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration().getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); waitMoveTasksToFinish(moveRegionsPool, taskList, timeoutInSeconds); } private boolean waitTaskToFinish(ExecutorService pool, Future task, String operation) throws TimeoutException, InterruptedException, ExecutionException { pool.shutdown(); try { if (!pool.awaitTermination((long) this.timeout, TimeUnit.SECONDS)) { LOG.warn("Timed out before finishing the " + operation + " operation. Timeout: " + this.timeout + "sec"); pool.shutdownNow(); } } catch (InterruptedException e) { pool.shutdownNow(); Thread.currentThread().interrupt(); } try { return task.get(5, TimeUnit.SECONDS); } catch (InterruptedException e) { LOG.warn("Interrupted while " + operation + " Regions on " + this.hostname, e); throw e; } catch (ExecutionException e) { LOG.error("Error while " + operation + " regions on RegionServer " + this.hostname, e); throw e; } } private void waitMoveTasksToFinish(ExecutorService moveRegionsPool, List> taskList, long timeoutInSeconds) throws Exception { try { if (!moveRegionsPool.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS)) { moveRegionsPool.shutdownNow(); } } catch (InterruptedException e) { moveRegionsPool.shutdownNow(); Thread.currentThread().interrupt(); } for (Future future : taskList) { try { // if even after shutdownNow threads are stuck we wait for 5 secs max if (!future.get(5, TimeUnit.SECONDS)) { LOG.error("Was Not able to move region....Exiting Now"); throw new Exception("Could not move region Exception"); } } catch (InterruptedException e) { LOG.error("Interrupted while waiting for Thread to Complete " + e.getMessage(), e); throw e; } catch (ExecutionException e) { boolean ignoreFailure = ignoreRegionMoveFailure(e); if (ignoreFailure) { LOG.debug("Ignore region move failure, it might have been split/merged.", e); } else { LOG.error("Got Exception From Thread While moving region {}", e.getMessage(), e); throw e; } } catch (CancellationException e) { LOG.error("Thread for moving region cancelled. Timeout for cancellation:" + timeoutInSeconds + "secs", e); throw e; } } } private boolean ignoreRegionMoveFailure(ExecutionException e) { boolean ignoreFailure = false; if (e.getCause() instanceof UnknownRegionException) { // region does not exist anymore ignoreFailure = true; } else if ( e.getCause() instanceof DoNotRetryRegionException && e.getCause().getMessage() != null && e.getCause().getMessage() .contains(AssignmentManager.UNEXPECTED_STATE_REGION + "state=SPLIT,") ) { // region is recently split ignoreFailure = true; } return ignoreFailure; } private ServerName getTargetServer() throws Exception { ServerName server = null; int maxWaitInSeconds = admin.getConfiguration().getInt(SERVERSTART_WAIT_MAX_KEY, DEFAULT_SERVERSTART_WAIT_MAX); long maxWait = EnvironmentEdgeManager.currentTime() + maxWaitInSeconds * 1000; while (EnvironmentEdgeManager.currentTime() < maxWait) { try { List regionServers = new ArrayList<>(); regionServers.addAll(admin.getRegionServers()); // Remove the host Region server from target Region Servers list server = stripServer(regionServers, hostname, port); if (server != null) { break; } else { LOG.warn("Server " + hostname + ":" + port + " is not up yet, waiting"); } } catch (IOException e) { LOG.warn("Could not get list of region servers", e); } Thread.sleep(500); } if (server == null) { LOG.error("Server " + hostname + ":" + port + " is not up. Giving up."); throw new Exception("Server " + hostname + ":" + port + " to load regions not online"); } return server; } private List readRegionsFromFile(String filename) throws IOException { List regions = new ArrayList<>(); File f = new File(filename); if (!f.exists()) { return regions; } try ( DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(f)))) { int numRegions = dis.readInt(); int index = 0; while (index < numRegions) { regions.add(RegionInfo.parseFromOrNull(Bytes.readByteArray(dis))); index++; } } catch (IOException e) { LOG.error("Error while reading regions from file:" + filename, e); throw e; } return regions; } /** * Write the number of regions moved in the first line followed by regions moved in subsequent * lines */ private void writeFile(String filename, List movedRegions) throws IOException { try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(filename)))) { dos.writeInt(movedRegions.size()); for (RegionInfo region : movedRegions) { Bytes.writeByteArray(dos, RegionInfo.toByteArray(region)); } } catch (IOException e) { LOG.error("ERROR: Was Not able to write regions moved to output file but moved " + movedRegions.size() + " regions", e); throw e; } } private void deleteFile(String filename) { File f = new File(filename); if (f.exists()) { f.delete(); } } /** * @param filename The file should have 'host:port' per line * @return List of servers from the file in format 'hostname:port'. */ private List readServersFromFile(String filename) throws IOException { List servers = new ArrayList<>(); if (filename != null) { try { Files.readAllLines(Paths.get(filename)).stream().map(String::trim) .filter(((Predicate) String::isEmpty).negate()).map(String::toLowerCase) .forEach(servers::add); } catch (IOException e) { LOG.error("Exception while reading servers from file,", e); throw e; } } return servers; } /** * Designates or excludes the servername whose hostname and port portion matches the list given in * the file. Example:
* If you want to designated RSs, suppose designatedFile has RS1, regionServers has RS1, RS2 and * RS3. When we call includeExcludeRegionServers(designatedFile, regionServers, true), RS2 and RS3 * are removed from regionServers list so that regions can move to only RS1. If you want to * exclude RSs, suppose excludeFile has RS1, regionServers has RS1, RS2 and RS3. When we call * includeExcludeRegionServers(excludeFile, servers, false), RS1 is removed from regionServers * list so that regions can move to only RS2 and RS3. */ private void includeExcludeRegionServers(String fileName, List regionServers, boolean isInclude) throws IOException { if (fileName != null) { List servers = readServersFromFile(fileName); if (servers.isEmpty()) { LOG.warn("No servers provided in the file: {}." + fileName); return; } Iterator i = regionServers.iterator(); while (i.hasNext()) { String rs = i.next().getServerName(); String rsPort = rs.split(ServerName.SERVERNAME_SEPARATOR)[0].toLowerCase() + ":" + rs.split(ServerName.SERVERNAME_SEPARATOR)[1]; if (isInclude != servers.contains(rsPort)) { i.remove(); } } } } /** * Exclude master from list of RSs to move regions to */ private void stripMaster(List regionServers) throws IOException { ServerName master = admin.getClusterMetrics(EnumSet.of(Option.MASTER)).getMasterName(); stripServer(regionServers, master.getHostname(), master.getPort()); } /** * Remove the servername whose hostname and port portion matches from the passed array of servers. * Returns as side-effect the servername removed. * @return server removed from list of Region Servers */ private ServerName stripServer(List regionServers, String hostname, int port) { for (Iterator iter = regionServers.iterator(); iter.hasNext();) { ServerName server = iter.next(); if ( server.getAddress().getHostname().equalsIgnoreCase(hostname) && server.getAddress().getPort() == port ) { iter.remove(); return server; } } return null; } @Override protected void addOptions() { this.addRequiredOptWithArg("r", "regionserverhost", "region server |"); this.addRequiredOptWithArg("o", "operation", "Expected: load/unload/unload_from_rack"); this.addOptWithArg("m", "maxthreads", "Define the maximum number of threads to use to unload and reload the regions"); this.addOptWithArg("x", "excludefile", "File with per line to exclude as unload targets; default excludes only " + "target host; useful for rack decommisioning."); this.addOptWithArg("d", "designatedfile", "File with per line as unload targets;" + "default is all online hosts"); this.addOptWithArg("f", "filename", "File to save regions list into unloading, or read from loading; " + "default /tmp/"); this.addOptNoArg("n", "noack", "Turn on No-Ack mode(default: false) which won't check if region is online on target " + "RegionServer, hence best effort. This is more performant in unloading and loading " + "but might lead to region being unavailable for some time till master reassigns it " + "in case the move failed"); this.addOptWithArg("t", "timeout", "timeout in seconds after which the tool will exit " + "irrespective of whether it finished or not;default Integer.MAX_VALUE"); } @Override protected void processOptions(CommandLine cmd) { String hostname = cmd.getOptionValue("r"); rmbuilder = new RegionMoverBuilder(hostname); if (cmd.hasOption('m')) { rmbuilder.maxthreads(Integer.parseInt(cmd.getOptionValue('m'))); } if (cmd.hasOption('n')) { rmbuilder.ack(false); } if (cmd.hasOption('f')) { rmbuilder.filename(cmd.getOptionValue('f')); } if (cmd.hasOption('x')) { rmbuilder.excludeFile(cmd.getOptionValue('x')); } if (cmd.hasOption('d')) { rmbuilder.designatedFile(cmd.getOptionValue('d')); } if (cmd.hasOption('t')) { rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t'))); } this.loadUnload = cmd.getOptionValue("o").toLowerCase(Locale.ROOT); } @Override protected int doWork() throws Exception { boolean success; try (RegionMover rm = rmbuilder.build()) { if (loadUnload.equalsIgnoreCase("load")) { success = rm.load(); } else if (loadUnload.equalsIgnoreCase("unload")) { success = rm.unload(); } else if (loadUnload.equalsIgnoreCase("unload_from_rack")) { success = rm.unloadFromRack(); } else { printUsage(); success = false; } } return (success ? 0 : 1); } public static void main(String[] args) { try (RegionMover mover = new RegionMover()) { mover.doStaticMain(args); } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy