org.apache.hadoop.hbase.io.encoding.PrefixKeyDeltaEncoder Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hbase-common Show documentation
Show all versions of hbase-common Show documentation
Common functionality for HBase
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.io.encoding;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Compress key by storing size of common prefix with previous KeyValue and storing raw size of
* rest. Format: 1-5 bytes: compressed key length minus prefix (7-bit encoding) 1-5 bytes:
* compressed value length (7-bit encoding) 1-3 bytes: compressed length of common key prefix ...
* bytes: rest of key (including timestamp) ... bytes: value In a worst case compressed KeyValue
* will be three bytes longer than original.
*/
@InterfaceAudience.Private
public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
@Override
public int internalEncode(Cell cell, HFileBlockDefaultEncodingContext encodingContext,
DataOutputStream out) throws IOException {
int klength = KeyValueUtil.keyLength(cell);
int vlength = cell.getValueLength();
EncodingState state = encodingContext.getEncodingState();
if (state.prevCell == null) {
// copy the key, there is no common prefix with none
ByteBufferUtils.putCompressedInt(out, klength);
ByteBufferUtils.putCompressedInt(out, vlength);
ByteBufferUtils.putCompressedInt(out, 0);
PrivateCellUtil.writeFlatKey(cell, (DataOutput) out);
} else {
// find a common prefix and skip it
int common = PrivateCellUtil.findCommonPrefixInFlatKey(cell, state.prevCell, true, true);
ByteBufferUtils.putCompressedInt(out, klength - common);
ByteBufferUtils.putCompressedInt(out, vlength);
ByteBufferUtils.putCompressedInt(out, common);
writeKeyExcludingCommon(cell, common, out);
}
// Write the value part
PrivateCellUtil.writeValue(out, cell, vlength);
int size = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
size += afterEncodingKeyValue(cell, out, encodingContext);
state.prevCell = cell;
return size;
}
private void writeKeyExcludingCommon(Cell cell, int commonPrefix, DataOutputStream out)
throws IOException {
short rLen = cell.getRowLength();
if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
// Previous and current rows are different. Need to write the differing part followed by
// cf,q,ts and type
PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
byte fLen = cell.getFamilyLength();
out.writeByte(fLen);
PrivateCellUtil.writeFamily(out, cell, fLen);
PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
out.writeLong(cell.getTimestamp());
out.writeByte(cell.getTypeByte());
} else {
// The full row key part is common. CF part will be common for sure as we deal with Cells in
// same family. Just need write the differing part in q, ts and type
commonPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
- (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
int qLen = cell.getQualifierLength();
int commonQualPrefix = Math.min(commonPrefix, qLen);
int qualPartLenToWrite = qLen - commonQualPrefix;
if (qualPartLenToWrite > 0) {
PrivateCellUtil.writeQualifierSkippingBytes(out, cell, qLen, commonQualPrefix);
}
commonPrefix -= commonQualPrefix;
// Common part in TS also?
if (commonPrefix > 0) {
int commonTimestampPrefix = Math.min(commonPrefix, KeyValue.TIMESTAMP_SIZE);
if (commonTimestampPrefix < KeyValue.TIMESTAMP_SIZE) {
byte[] curTsBuf = Bytes.toBytes(cell.getTimestamp());
out.write(curTsBuf, commonTimestampPrefix,
KeyValue.TIMESTAMP_SIZE - commonTimestampPrefix);
}
commonPrefix -= commonTimestampPrefix;
if (commonPrefix == 0) {
out.writeByte(cell.getTypeByte());
}
} else {
out.writeLong(cell.getTimestamp());
out.writeByte(cell.getTypeByte());
}
}
}
@Override
protected ByteBuffer internalDecodeKeyValues(DataInputStream source, int allocateHeaderLength,
int skipLastBytes, HFileBlockDefaultDecodingContext decodingCtx) throws IOException {
int decompressedSize = source.readInt();
ByteBuffer buffer = ByteBuffer.allocate(decompressedSize + allocateHeaderLength);
buffer.position(allocateHeaderLength);
int prevKeyOffset = 0;
while (source.available() > skipLastBytes) {
prevKeyOffset = decodeKeyValue(source, buffer, prevKeyOffset);
afterDecodingKeyValue(source, buffer, decodingCtx);
}
if (source.available() != skipLastBytes) {
throw new IllegalStateException("Read too many bytes.");
}
buffer.limit(buffer.position());
return buffer;
}
private int decodeKeyValue(DataInputStream source, ByteBuffer buffer, int prevKeyOffset)
throws IOException, EncoderBufferTooSmallException {
int keyLength = ByteBufferUtils.readCompressedInt(source);
int valueLength = ByteBufferUtils.readCompressedInt(source);
int commonLength = ByteBufferUtils.readCompressedInt(source);
int keyOffset;
keyLength += commonLength;
ensureSpace(buffer, keyLength + valueLength + KeyValue.ROW_OFFSET);
buffer.putInt(keyLength);
buffer.putInt(valueLength);
// copy the prefix
if (commonLength > 0) {
keyOffset = buffer.position();
ByteBufferUtils.copyFromBufferToBuffer(buffer, buffer, prevKeyOffset, commonLength);
} else {
keyOffset = buffer.position();
}
// copy rest of the key and value
int len = keyLength - commonLength + valueLength;
ByteBufferUtils.copyFromStreamToBuffer(buffer, source, len);
return keyOffset;
}
@Override
public Cell getFirstKeyCellInBlock(ByteBuff block) {
block.mark();
block.position(Bytes.SIZEOF_INT);
int keyLength = ByteBuff.readCompressedInt(block);
// TODO : See if we can avoid these reads as the read values are not getting used
ByteBuff.readCompressedInt(block);
int commonLength = ByteBuff.readCompressedInt(block);
if (commonLength != 0) {
throw new AssertionError(
"Nonzero common length in the first key in " + "block: " + commonLength);
}
ByteBuffer key = block.asSubByteBuffer(keyLength).duplicate();
block.reset();
return createFirstKeyCell(key, keyLength);
}
@Override
public String toString() {
return PrefixKeyDeltaEncoder.class.getSimpleName();
}
@Override
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
return new SeekerStateBufferedEncodedSeeker(decodingCtx);
}
private static class SeekerStateBufferedEncodedSeeker extends BufferedEncodedSeeker {
private SeekerStateBufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
super(decodingCtx);
}
@Override
protected void decodeNext() {
current.keyLength = ByteBuff.readCompressedInt(currentBuffer);
current.valueLength = ByteBuff.readCompressedInt(currentBuffer);
current.lastCommonPrefix = ByteBuff.readCompressedInt(currentBuffer);
current.keyLength += current.lastCommonPrefix;
current.ensureSpaceForKey();
currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
current.keyLength - current.lastCommonPrefix);
current.valueOffset = currentBuffer.position();
currentBuffer.skip(current.valueLength);
if (includesTags()) {
decodeTags();
}
if (includesMvcc()) {
current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
} else {
current.memstoreTS = 0;
}
current.nextKvOffset = currentBuffer.position();
}
@Override
protected void decodeFirst() {
currentBuffer.skip(Bytes.SIZEOF_INT);
decodeNext();
}
}
}