org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableConf Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.exec.persistence;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * This conf class is a wrapper of a list of HybridHashTableContainers and some common info shared
 * among them, which is used in n-way join (multiple small tables are involved).
 */
public class HybridHashTableConf {
  private List loadedContainerList; // A list of alrady loaded containers
  private int numberOfPartitions = 0; // Number of partitions each table should have
  private int nextSpillPartition = -1;       // The partition to be spilled next

  public HybridHashTableConf() {
    loadedContainerList = new ArrayList();
  }

  public int getNumberOfPartitions() {
    return numberOfPartitions;
  }

  public void setNumberOfPartitions(int numberOfPartitions) {
    this.numberOfPartitions = numberOfPartitions;
    this.nextSpillPartition = numberOfPartitions - 1;
  }

  public int getNextSpillPartition() {
    return this.nextSpillPartition;
  }

  public void setNextSpillPartition(int nextSpillPartition) {
    this.nextSpillPartition = nextSpillPartition;
  }


  public List getLoadedContainerList() {
    return loadedContainerList;
  }

  /**
   * Spill one in-memory partition from tail for all previously loaded HybridHashTableContainers.
   * Also mark that partition number as spill-on-creation for future created containers.
   * @return amount of memory freed; 0 if only one last partition is in memory for each container
   */
  public long spill() throws IOException {
    if (nextSpillPartition == 0) {
      return 0;
    }
    long memFreed = 0;
    for (HybridHashTableContainer container : loadedContainerList) {
      memFreed += container.spillPartition(nextSpillPartition);
      container.setSpill(true);
    }
    nextSpillPartition--;
    return memFreed;
  }

  /**
   * Check if a partition should be spilled directly on creation
   * @param partitionId the partition to create
   * @return true if it should be spilled directly, false otherwise
   */
  public boolean doSpillOnCreation(int partitionId) {
    return nextSpillPartition != -1 && partitionId > nextSpillPartition;
  }
}