org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUnion Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.serde2.lazybinary;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryUnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.*;
/**
* LazyBinaryUnion is serialized as follows: start TAG FIELD end bytes[] ->
* |-----|---------|--- ... ---|-----|---------|
*
* Section TAG is one byte, corresponding to tag of set union field
* FIELD is a LazyBinaryObject corresponding to set union field value.
*
*/
public class LazyBinaryUnion extends
LazyBinaryNonPrimitive implements SerDeStatsStruct {
private static Logger LOG = LoggerFactory.getLogger(LazyBinaryUnion.class.getName());
/**
* Whether the data is already parsed or not.
*/
boolean parsed;
/**
* Size of serialized data
*/
long serializedSize;
/**
* The field of the union which contains the value.
*/
LazyBinaryObject field;
boolean fieldInited;
/**
* The start positions and lengths of union fields. Only valid when the data
* is parsed.
*/
int fieldStart;
int fieldLength;
byte tag;
final LazyBinaryUtils.VInt vInt = new LazyBinaryUtils.VInt();
/**
* Construct a LazyBinaryUnion object with an ObjectInspector.
*/
protected LazyBinaryUnion(LazyBinaryUnionObjectInspector oi) {
super(oi);
}
@Override
public void init(ByteArrayRef bytes, int start, int length) {
super.init(bytes, start, length);
parsed = false;
serializedSize = length;
fieldInited = false;
field = null;
cachedObject = null;
}
LazyBinaryUtils.RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
boolean missingFieldWarned = false;
boolean extraFieldWarned = false;
/**
* Parse the byte[] and fill fieldStart, fieldLength, fieldInited and
* fieldIsNull.
*/
private void parse() {
LazyBinaryUnionObjectInspector uoi = (LazyBinaryUnionObjectInspector) oi;
/**
* Please note that tag is followed by field
*/
int unionByteEnd = start + length;
byte[] byteArr = this.bytes.getData();
//Tag of union field is the first byte to be parsed
final int tagEnd = start + 1;
tag = byteArr[start];
field = LazyBinaryFactory.createLazyBinaryObject(uoi.getObjectInspectors().get(tag));
//Check the union field's length and offset
LazyBinaryUtils.checkObjectByteInfo(uoi.getObjectInspectors().get(tag), byteArr, tagEnd, recordInfo, vInt);
fieldStart = tagEnd + recordInfo.elementOffset;
// Add 1 for tag
fieldLength = recordInfo.elementSize;
// Extra bytes at the end?
if (!extraFieldWarned && (fieldStart + fieldLength) < unionByteEnd) {
extraFieldWarned = true;
LOG.warn("Extra bytes detected at the end of the row! Ignoring similar "
+ "problems.");
}
// Missing fields?
if (!missingFieldWarned && (fieldStart + fieldLength) > unionByteEnd) {
missingFieldWarned = true;
LOG.info("Missing fields! Expected 1 fields but "
+ "only got " + field + "! Ignoring similar problems.");
}
parsed = true;
}
/**
* Get the set field out of the union.
*
* If the field is a primitive field, return the actual object. Otherwise
* return the LazyObject. This is because PrimitiveObjectInspector does not
* have control over the object used by the user - the user simply directly
* use the Object instead of going through Object
* PrimitiveObjectInspector.get(Object).
* @return The field as a LazyObject
*/
public Object getField() {
if (!parsed) {
parse();
}
if(cachedObject == null) {
return uncheckedGetField();
}
return cachedObject;
}
/**
* Get the field out of the row without checking parsed. This is called by
* both getField and getFieldsAsList.
*
* @param fieldID
* The id of the field starting from 0.
* @return The value of the field
*/
private Object uncheckedGetField() {
// Test the length first so in most cases we avoid doing a byte[]
// comparison.
if (!fieldInited) {
fieldInited = true;
field.init(bytes, fieldStart, fieldLength);
}
cachedObject = field.getObject();
return field.getObject();
}
Object cachedObject;
@Override
public Object getObject() {
return this;
}
public long getRawDataSerializedSize() {
return serializedSize;
}
/**
* Get the set field's tag
*
*
* @return The tag of the field set in the union
*/
public byte getTag() {
if (!parsed) {
parse();
}
return tag;
}
}