org.apache.hadoop.hbase.mapreduce.SimpleTotalOrderPartitioner Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of hbase-mapreduce Show documentation

This module contains implementations of InputFormat, OutputFormat, Mapper, Reducer, etc which are needed for running MR jobs on tables, WALs, HFiles and other HBase specific constructs. It also contains a bunch of tools: RowCounter, ImportTsv, Import, Export, CompactionTool, ExportSnapshot, WALPlayer, etc

There is a newer version: 3.0.0-beta-1

Show newest version

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.mapreduce;

import java.util.Base64;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A partitioner that takes start and end keys and uses bigdecimal to figure which reduce a key
 * belongs to. Pass the start and end keys in the Configuration using
 * hbase.simpletotalorder.start and hbase.simpletotalorder.end. The end
 * key needs to be exclusive; i.e. one larger than the biggest key in your key space. You may be
 * surprised at how this class partitions the space; it may not align with preconceptions; e.g. a
 * start key of zero and an end key of 100 divided in ten will not make regions whose range is 0-10,
 * 10-20, and so on. Make your own partitioner if you need the region spacing to come out a
 * particular way.
 * @param 
 * @see #START
 * @see #END
 */
@InterfaceAudience.Public
public class SimpleTotalOrderPartitioner extends Partitioner
  implements Configurable {
  private final static Logger LOG = LoggerFactory.getLogger(SimpleTotalOrderPartitioner.class);

  /**
   * @deprecated since 0.90.0
   * @see HBASE-1923
   */
  @Deprecated
  public static final String START = "hbase.simpletotalorder.start";

  /**
   * @deprecated since 0.90.0
   * @see HBASE-1923
   */
  @Deprecated
  public static final String END = "hbase.simpletotalorder.end";

  static final String START_BASE64 = "hbase.simpletotalorder.start.base64";
  static final String END_BASE64 = "hbase.simpletotalorder.end.base64";

  private Configuration c;
  private byte[] startkey;
  private byte[] endkey;
  private byte[][] splits;
  private int lastReduces = -1;

  public static void setStartKey(Configuration conf, byte[] startKey) {
    conf.set(START_BASE64, Bytes.toString(Base64.getEncoder().encode(startKey)));
  }

  public static void setEndKey(Configuration conf, byte[] endKey) {
    conf.set(END_BASE64, Bytes.toString(Base64.getEncoder().encode(endKey)));
  }

  @SuppressWarnings("deprecation")
  static byte[] getStartKey(Configuration conf) {
    return getKeyFromConf(conf, START_BASE64, START);
  }

  @SuppressWarnings("deprecation")
  static byte[] getEndKey(Configuration conf) {
    return getKeyFromConf(conf, END_BASE64, END);
  }

  private static byte[] getKeyFromConf(Configuration conf, String base64Key, String deprecatedKey) {
    String encoded = conf.get(base64Key);
    if (encoded != null) {
      return Base64.getDecoder().decode(encoded);
    }
    String oldStyleVal = conf.get(deprecatedKey);
    if (oldStyleVal == null) {
      return null;
    }
    LOG.warn("Using deprecated configuration " + deprecatedKey
      + " - please use static accessor methods instead.");
    return Bytes.toBytesBinary(oldStyleVal);
  }

  @Override
  public int getPartition(final ImmutableBytesWritable key, final VALUE value, final int reduces) {
    if (reduces == 1) return 0;
    if (this.lastReduces != reduces) {
      this.splits = Bytes.split(this.startkey, this.endkey, reduces - 1);
      for (int i = 0; i < splits.length; i++) {
        LOG.info(Bytes.toStringBinary(splits[i]));
      }
      this.lastReduces = reduces;
    }
    int pos = Bytes.binarySearch(this.splits, key.get(), key.getOffset(), key.getLength());
    // Below code is from hfile index search.
    if (pos < 0) {
      pos++;
      pos *= -1;
      if (pos == 0) {
        // falls before the beginning of the file.
        throw new RuntimeException("Key outside start/stop range: " + key.toString());
      }
      pos--;
    }
    return pos;
  }

  @Override
  public Configuration getConf() {
    return this.c;
  }

  @Override
  public void setConf(Configuration conf) {
    this.c = conf;
    this.startkey = getStartKey(conf);
    this.endkey = getEndKey(conf);
    if (startkey == null || endkey == null) {
      throw new RuntimeException(this.getClass() + " not configured");
    }
    LOG.info(
      "startkey=" + Bytes.toStringBinary(startkey) + ", endkey=" + Bytes.toStringBinary(endkey));
    // Reset last reduces count on change of Start / End key
    this.lastReduces = -1;
  }
}