org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hive-serde
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hive.serde2.lazybinary;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyObject;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.RecordInfo;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

/**
 * LazyBinaryArray is serialized as follows: start A b b b b b b end bytes[] ->
 * |--------|---|---|---|---| ... |---|---|
 * 
 * Section A is the null-bytes. Suppose the list has N elements, then there are
 * (N+7)/8 bytes used as null-bytes. Each bit corresponds to an element and it
 * indicates whether that element is null (0) or not null (1).
 * 
 * After A, all b(s) represent the elements of the list. Each of them is again a
 * LazyBinaryObject.
 * 
 */

public class LazyBinaryArray extends
    LazyBinaryNonPrimitive {

  /**
   * Whether the data is already parsed or not.
   */
  boolean parsed = false;
  /**
   * The length of the array. Only valid when the data is parsed.
   */
  int arraySize = 0;

  /**
   * The start positions and lengths of array elements. Only valid when the data
   * is parsed.
   */
  int[] elementStart;
  int[] elementLength;

  /**
   * Whether an element is initialized or not.
   */
  boolean[] elementInited;

  /**
   * Whether an element is null or not. Because length is 0 does not means the
   * field is null. In particular, a 0-length string is not null.
   */
  boolean[] elementIsNull;

  /**
   * The elements of the array. Note that we call arrayElements[i].init(bytes,
   * begin, length) only when that element is accessed.
   */
  LazyBinaryObject[] arrayElements;

  /**
   * Construct a LazyBinaryArray object with the ObjectInspector.
   * 
   * @param oi
   *          the oi representing the type of this LazyBinaryArray
   */
  protected LazyBinaryArray(LazyBinaryListObjectInspector oi) {
    super(oi);
  }

  /**
   * Set the row data for this LazyBinaryArray.
   * 
   * @see LazyObject#init(ByteArrayRef, int, int)
   */
  @Override
  public void init(ByteArrayRef bytes, int start, int length) {
    super.init(bytes, start, length);
    parsed = false;
  }

  /**
   * Enlarge the size of arrays storing information for the elements inside the
   * array.
   */
  private void adjustArraySize(int newSize) {
    if (elementStart == null || elementStart.length < newSize) {
      elementStart = new int[newSize];
      elementLength = new int[newSize];
      elementInited = new boolean[newSize];
      elementIsNull = new boolean[newSize];
      arrayElements = new LazyBinaryObject[newSize];
    }
  }

  VInt vInt = new LazyBinaryUtils.VInt();
  RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();

  /**
   * Parse the bytes and fill elementStart, elementLength, elementInited and
   * elementIsNull.
   */
  private void parse() {

    byte[] bytes = this.bytes.getData();

    // get the vlong that represents the map size
    LazyBinaryUtils.readVInt(bytes, start, vInt);
    arraySize = vInt.value;
    if (0 == arraySize) {
      parsed = true;
      return;
    }

    // adjust arrays
    adjustArraySize(arraySize);
    // find out the null-bytes
    int arrayByteStart = start + vInt.length;
    int nullByteCur = arrayByteStart;
    int nullByteEnd = arrayByteStart + (arraySize + 7) / 8;
    // the begin the real elements
    int lastElementByteEnd = nullByteEnd;
    // the list element object inspector
    ObjectInspector listEleObjectInspector = ((ListObjectInspector) oi)
        .getListElementObjectInspector();
    // parsing elements one by one
    for (int i = 0; i < arraySize; i++) {
      elementIsNull[i] = true;
      if ((bytes[nullByteCur] & (1 << (i % 8))) != 0) {
        elementIsNull[i] = false;
        LazyBinaryUtils.checkObjectByteInfo(listEleObjectInspector, bytes,
            lastElementByteEnd, recordInfo, vInt);
        elementStart[i] = lastElementByteEnd + recordInfo.elementOffset;
        elementLength[i] = recordInfo.elementSize;
        lastElementByteEnd = elementStart[i] + elementLength[i];
      }
      // move onto the next null byte
      if (7 == (i % 8)) {
        nullByteCur++;
      }
    }

    Arrays.fill(elementInited, 0, arraySize, false);
    parsed = true;
  }

  /**
   * Returns the actual primitive object at the index position inside the array
   * represented by this LazyBinaryObject.
   */
  public Object getListElementObject(int index) {
    if (!parsed) {
      parse();
    }
    if (index < 0 || index >= arraySize) {
      return null;
    }
    return uncheckedGetElement(index);
  }

  /**
   * Get the element without checking out-of-bound index.
   * 
   * @param index
   *          index to the array element
   */
  private Object uncheckedGetElement(int index) {

    if (elementIsNull[index]) {
      return null;
    } else {
      if (!elementInited[index]) {
        elementInited[index] = true;
        if (arrayElements[index] == null) {
          arrayElements[index] = LazyBinaryFactory.createLazyBinaryObject((oi)
              .getListElementObjectInspector());
        }
        arrayElements[index].init(bytes, elementStart[index],
            elementLength[index]);
      }
    }
    return arrayElements[index].getObject();
  }

  /**
   * Returns the array size.
   */
  public int getListLength() {
    if (!parsed) {
      parse();
    }
    return arraySize;
  }

  /**
   * cachedList is reused every time getList is called. Different
   * LazyBinaryArray instances cannot share the same cachedList.
   */
  ArrayList