org.apache.hadoop.mapred.BasicReducePartition Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-core Show documentation
Hadoop is the distributed computing framework of Apache; hadoop-core contains the filesystem, job tracker and map/reduce modules
The newest version!
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.mapred;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapred.Task.TaskReporter;
import org.apache.hadoop.util.IndexedSortable;
import org.apache.hadoop.util.QuickSort;

public abstract class BasicReducePartition
    implements MemoryBlockHolder {

  class OffsetSortable implements IndexedSortable {

    private int[] offsets;
    private int[] keyLenArray;
    private int[] valLenArray;
    private byte[] kvbuffer;

    public OffsetSortable(int[] offsets, int[] keyLenArray, int[] valLenArray,
        byte[] kvbuffer) {
      this.offsets = offsets;
      this.keyLenArray = keyLenArray;
      this.valLenArray = valLenArray;
      this.kvbuffer = kvbuffer;
    }

    public OffsetSortable(MemoryBlock memBlock, byte[] kvbuffer) {
      this.offsets = memBlock.getOffsets();
      this.keyLenArray = memBlock.getKeyLenArray();
      this.valLenArray = memBlock.getValueLenArray();
      this.kvbuffer = kvbuffer;
    }

    @Override
    public int compare(int i, int j) {
      return LexicographicalComparerHolder.compareBytes(kvbuffer, offsets[i],
          keyLenArray[i], offsets[j], keyLenArray[j]);
    }

    @Override
    public void swap(int i, int j) {
      swapElement(offsets, i, j);
      swapElement(keyLenArray, i, j);
      swapElement(valLenArray, i, j);
    }

    private void swapElement(int[] array, int i, int j) {
      int tmp = array[i];
      array[i] = array[j];
      array[j] = tmp;
    }
  }

  protected MemoryBlockAllocator memoryBlockAllocator;
  protected byte[] kvbuffer;
  protected final TaskReporter reporter;
  protected BlockMapOutputCollector collector;

  protected int partition;

  protected int collectedBytesSize;
  protected int collectedRecordsNum;
  
  protected MemoryBlock currentBlock;
  protected List memoryBlocks;


  public BasicReducePartition(int reduceNum,
      MemoryBlockAllocator memoryBlockAllocator, byte[] kvBuffer,
      BlockMapOutputCollector collector, TaskReporter reporter)
      throws IOException {
    this.partition = reduceNum;
    this.collectedBytesSize = 0;
    this.collectedRecordsNum = 0;
    this.memoryBlockAllocator = memoryBlockAllocator;
    this.kvbuffer = kvBuffer;
    this.collector = collector;
    this.reporter = reporter;
    this.memoryBlockAllocator.registerMemoryBlockHolder(this);
    initMemoryBlocks();
  }
  
  protected void initMemoryBlocks() {
    memoryBlocks = new ArrayList();
  }

  protected void sortMemBlock(MemoryBlock memBlock) {
    if (memBlock.currentPtr <= 0) {
      return;
    }
    // quick sort the offsets
    OffsetSortable sortableObj = new OffsetSortable(memBlock, kvbuffer);
    QuickSort quickSort = new QuickSort();
    quickSort.sort(sortableObj, 0, memBlock.currentPtr);
  }

  protected void sortIndividualMemoryBlock(List memBlks) {
    if (memBlks == null) {
      return;
    }
    for (MemoryBlock memBlk : memBlks) {
      if (memBlk != null) {
        sortMemBlock(memBlk);
      }
    }
  }

  public int getCollectedRecordsNum() {
    return collectedRecordsNum;
  }

  public int getCollectedBytesSize() {
    return collectedBytesSize;
  }

  abstract void groupOrSort();

  public abstract KeyValueSpillIterator getKeyValueSpillIterator();

  public abstract IndexRecord spill(JobConf job, FSDataOutputStream out,
      Class keyClass, Class valClass, CompressionCodec codec,
      Counter spillCounter) throws IOException;
  
  public abstract int collect(K key, V value) throws IOException;

  @Override
  public MemoryBlock getCurrentOpenMemoryBlock() {
    return currentBlock;
  }

  @Override
  public List getClosedMemoryBlocks() {
    return memoryBlocks;
  }
}