org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.exec.persistence;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.text.NumberFormat;
import java.util.HashMap;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;


/**
 * Simple wrapper for persistent Hashmap implementing only the put/get/remove/clear interface. The
 * main memory hash table acts as a cache and all put/get will operate on it first. If the size of
 * the main memory hash table exceeds a certain threshold, new elements will go into the persistent
 * hash table.
 */

public class HashMapWrapper implements Serializable {

  private static final long serialVersionUID = 1L;
  protected Log LOG = LogFactory.getLog(this.getClass().getName());

  // default threshold for using main memory based HashMap

  private static final int THRESHOLD = 1000000;
  private static final float LOADFACTOR = 0.75f;
  private static final float MEMORYUSAGE = 1;

  private float maxMemoryUsage;
  private HashMap mHash; // main memory HashMap
  protected transient LogHelper console;

  private File dumpFile;
  public static MemoryMXBean memoryMXBean;
  private long maxMemory;
  private long currentMemory;
  private NumberFormat num;

  /**
   * Constructor.
   *
   * @param threshold
   *          User specified threshold to store new values into persistent storage.
   */
  public HashMapWrapper(int threshold, float loadFactor, float memoryUsage) {
    maxMemoryUsage = memoryUsage;
    mHash = new HashMap(threshold, loadFactor);
    memoryMXBean = ManagementFactory.getMemoryMXBean();
    maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
    LOG.info("maximum memory: " + maxMemory);
    num = NumberFormat.getInstance();
    num.setMinimumFractionDigits(2);
  }

  public HashMapWrapper(int threshold) {
    this(threshold, LOADFACTOR, MEMORYUSAGE);
  }

  public HashMapWrapper() {
    this(THRESHOLD, LOADFACTOR, MEMORYUSAGE);
  }

  public V get(K key) {
    return mHash.get(key);
  }

  public boolean put(K key, V value) throws HiveException {
    // isAbort();
    mHash.put(key, value);
    return false;
  }


  public void remove(K key) {
    mHash.remove(key);
  }

  /**
   * Flush the main memory hash table into the persistent cache file
   *
   * @return persistent cache file
   */
  public long flushMemoryCacheToPersistent(File file) throws IOException {
    ObjectOutputStream outputStream = null;
    outputStream = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file), 4096));
    outputStream.writeObject(mHash);
    outputStream.flush();
    outputStream.close();

    return file.length();
  }

  public void initilizePersistentHash(String fileName) throws IOException, ClassNotFoundException {
    ObjectInputStream inputStream = null;
    inputStream = new ObjectInputStream(new BufferedInputStream(new FileInputStream(fileName), 4096));
    HashMap hashtable = (HashMap) inputStream.readObject();
    this.setMHash(hashtable);

    inputStream.close();
  }

  public int size() {
    return mHash.size();
  }

  public Set keySet() {
    return mHash.keySet();
  }


  /**
   * Close the persistent hash table and clean it up.
   *
   * @throws HiveException
   */
  public void close() throws HiveException {
    mHash.clear();
  }

  public void clear() throws HiveException {
    mHash.clear();
  }

  public int getKeySize() {
    return mHash.size();
  }

  public boolean isAbort(long numRows,LogHelper console) {
    int size = mHash.size();
    long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
    double rate = (double) usedMemory / (double) maxMemory;
    console.printInfo(Utilities.now() + "\tProcessing rows:\t" + numRows + "\tHashtable size:\t"
        + size + "\tMemory usage:\t" + usedMemory + "\trate:\t" + num.format(rate));
    if (rate > (double) maxMemoryUsage) {
      return true;
    }
    return false;
  }

  public void setLOG(Log log) {
    LOG = log;
  }

  public HashMap getMHash() {
    return mHash;
  }

  public void setMHash(HashMap hash) {
    mHash = hash;
  }

  public LogHelper getConsole() {
    return console;
  }

  public void setConsole(LogHelper console) {
    this.console = console;
  }

  public File getDumpFile() {
    return dumpFile;
  }

  public void setDumpFile(File dumpFile) {
    this.dumpFile = dumpFile;
  }

  public static MemoryMXBean getMemoryMXBean() {
    return memoryMXBean;
  }

  public static void setMemoryMXBean(MemoryMXBean memoryMXBean) {
    HashMapWrapper.memoryMXBean = memoryMXBean;
  }

  public long getMaxMemory() {
    return maxMemory;
  }

  public void setMaxMemory(long maxMemory) {
    this.maxMemory = maxMemory;
  }

  public long getCurrentMemory() {
    return currentMemory;
  }

  public void setCurrentMemory(long currentMemory) {
    this.currentMemory = currentMemory;
  }

  public NumberFormat getNum() {
    return num;
  }

  public void setNum(NumberFormat num) {
    this.num = num;
  }

  public static int getTHRESHOLD() {
    return THRESHOLD;
  }

}