com.kamikaze.pfordelta.LCPForDelta Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of JavaFastPFOR Show documentation
Show all versions of JavaFastPFOR Show documentation
It is a library to compress and uncompress arrays of integers
very fast. The assumption is that most (but not all) values in
your array use less than 32 bits.
package com.kamikaze.pfordelta;
/**
* This is a version of the kamikaze PForDelta library that
* was slightly cleaned up by D. Lemire. It is included in the
* JavaFastPFOR library for comparison purposes. As the original
*/
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.IntBuffer;
import java.util.Arrays;
/**
* Implementation of the optimized PForDelta algorithm for sorted integer
* arrays. The basic ideas are based on
*
* 1. Original algorithm from
* http://homepages.cwi.nl/~heman/downloads/msthesis.pdf
*
* 2. Optimization and variation from
* http://www2008.org/papers/pdf/p387-zhangA.pdf
*
* 3. Further optimization http://www2009.org/proceedings/pdf/p401.pdf
*
* As a part of the PForDelta implementation, Simple16 is used to compress
* exceptions. The original Simple16 algorithm can also be found in the above
* literature.
*
* This implementation overcomes the problem that Simple16 cannot deal with
* greater than 2^28 numbers.
*
* This implementation is almost same as PForDelta in the same package, except
* that it is tuned especially for Lucene-4.0 Codec to achieve the best
* performance in Lucene-4.0.
*
* @author hao yan, [email protected]
*/
public class LCPForDelta {
// NOTE: we expect the blockSize is always < (1<<(31-POSSIBLE_B_BITS)).
// For example, in the current default settings,
// the blockSize < (1<<(31-5)), that is, < 2^27
// All possible values of b in the PForDelta algorithm
private static final int[] POSSIBLE_B = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 16, 20, 28 };
// POSSIBLE_B.length < (1<>> 5)];
int[] expPosBuffer = new int[blockSize];
int[] expHighBitsBuffer = new int[blockSize];
// compress the b-bit slots
for (int i = 0; i < blockSize; ++i) {
int value = inputBlock[i];
if (value < expUpperBound) {
writeBits(tmpCompBuffer, value, outputOffset,
bits);
} else // exp
{
// store the lower bits-bits of the exception
writeBits(tmpCompBuffer, value & MASK[bits],
outputOffset, bits);
// write the position of exception
expPosBuffer[expNum] = i;
// write the higher 32-bits bits of the
// exception
expHighBitsBuffer[expNum] = (value >>> bits)
& MASK[32 - bits];
expNum++;
}
outputOffset += bits;
}
tmpCompBuffer[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31 - POSSIBLE_B_BITS))
| (expNum & MASK[31 - POSSIBLE_B_BITS]);
// compress exceptions
if (expNum > 0) {
int compressedBitSize;
compressedBitSize = compressBlockByS16(tmpCompBuffer,
outputOffset, expPosBuffer, expNum, blockSize,
inputBlock);
outputOffset += compressedBitSize;
compressedBitSize = compressBlockByS16(tmpCompBuffer,
outputOffset, expHighBitsBuffer, expNum,
blockSize, inputBlock);
outputOffset += compressedBitSize;
}
// discard the redundant parts in the tmpCompressedBlock
int compressedSizeInInts = (outputOffset + 31) >>> 5;
compBuffer = tmpCompBuffer;
return compressedSizeInInts;
}
protected int compressOneBlockCore2(int[] inputBlock, int blockSize,
int bits) throws IllegalArgumentException {
int outputOffset = HEADER_SIZE;
int expUpperBound = 1 << bits;
int expNum = 0;
int maxCompBitSize = HEADER_SIZE + blockSize
* (MAX_BITS + MAX_BITS + MAX_BITS) + 32;
int[] tmpCompBuffer = new int[(maxCompBitSize >>> 5)];
int[] expPosBuffer = new int[blockSize];
int[] expHighBitsBuffer = new int[blockSize];
// compress the b-bit slots
for (int i = 0; i < blockSize; ++i) {
int value = inputBlock[i];
if (value < expUpperBound) {
writeBits(tmpCompBuffer, value, outputOffset,
bits);
} else // exp
{
// store the lower bits-bits of the exception
writeBits(tmpCompBuffer, value & MASK[bits],
outputOffset, bits);
// write the position of exception
expPosBuffer[expNum] = i;
// write the higher 32-bits bits of the
// exception
expHighBitsBuffer[expNum] = (value >>> bits)
& MASK[32 - bits];
expNum++;
}
outputOffset += bits;
}
tmpCompBuffer[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31 - POSSIBLE_B_BITS))
| (expNum & MASK[31 - POSSIBLE_B_BITS]);
// compress exceptions
if (expNum > 0) {
int compressedBitSize;
int[] expBuffer = new int[expNum * 2];
System.arraycopy(expPosBuffer, 0, expBuffer, 0, expNum);
System.arraycopy(expHighBitsBuffer, 0, expBuffer,
expNum, expNum);
compressedBitSize = compressBlockByS16(tmpCompBuffer,
outputOffset, expBuffer, expNum * 2, blockSize,
inputBlock);
outputOffset += compressedBitSize;
}
// discard the redundant parts in the tmpCompressedBlock
int compressedSizeInInts = (outputOffset + 31) >>> 5;
compBuffer = tmpCompBuffer;
return compressedSizeInInts;
}
/**
* Decompress one block using PForDelta
*
* @param decompBlock
* the block that was decompressed
* @param inBlock
* the block to be decompressed
* @param blockSize
* the number of elements in the decompressed block
*/
public static void decompressOneBlock(int[] decompBlock, int[] inBlock,
int blockSize) {
int expNum = inBlock[0] & MASK[31 - POSSIBLE_B_BITS];
int bits = (inBlock[0] >>> (31 - POSSIBLE_B_BITS)) & (0x1f);
int[] expPosBuffer = new int[blockSize];
int[] expHighBitsBuffer = new int[blockSize];
// decompress the b-bit slots
int offset = HEADER_SIZE;
int compressedBits = 0;
if (bits == 0) {
Arrays.fill(decompBlock, 0);
} else {
compressedBits = decompressBBitSlots(decompBlock,
inBlock, blockSize, bits);
// compressedBits =
// decompressBBitSlotsWithHardCodes(decompBlock,
// inBlock, blockSize, bits);
}
offset += compressedBits;
// decompress exceptions
if (expNum > 0) {
compressedBits = decompressBlockByS16(expPosBuffer,
inBlock, offset, expNum);
offset += compressedBits;
compressedBits = decompressBlockByS16(
expHighBitsBuffer, inBlock, offset, expNum);
offset += compressedBits;
for (int i = 0; i < expNum; i++) {
int curExpPos = expPosBuffer[i];
int curHighBits = expHighBitsBuffer[i];
decompBlock[curExpPos] = (decompBlock[curExpPos] & MASK[bits])
| ((curHighBits & MASK[32 - bits]) << bits);
}
}
}
protected static void decompressOneBlockWithSize(int[] decompBlock,
int[] inBlock, int blockSize, int[] expPosBuffer,
int[] expHighBitsBuffer, int inBlockLen) {
int expNum = inBlock[0] & MASK[31 - POSSIBLE_B_BITS];
int bits = (inBlock[0] >>> (31 - POSSIBLE_B_BITS)) & (0x1f);
// decompress the b-bit slots
int offset = HEADER_SIZE;
int compressedBits = 0;
if (bits == 0) {
Arrays.fill(decompBlock, 0, inBlockLen, 0);
} else {
// compressedBits =
// decompressBBitSlotsWithHardCodes(decompBlock,
// inBlock, blockSize, bits);
compressedBits = decompressBBitSlots(decompBlock,
inBlock, blockSize, bits);
}
offset += compressedBits;
// decompress exceptions
if (expNum > 0) {
compressedBits = decompressBlockByS16(expPosBuffer,
inBlock, offset, expNum);
offset += compressedBits;
compressedBits = decompressBlockByS16(
expHighBitsBuffer, inBlock, offset, expNum);
offset += compressedBits;
for (int i = 0; i < expNum; i++) {
int curExpPos = expPosBuffer[i];
int curHighBits = expHighBitsBuffer[i];
decompBlock[curExpPos] = (decompBlock[curExpPos] & MASK[bits])
| ((curHighBits & MASK[32 - bits]) << bits);
}
}
}
protected static void decompressOneBlockWithSizeWithIntBuffer(
final int[] decompBlock, final IntBuffer inBlock,
final int blockSize, final int[] expPosBuffer,
final int[] expHighBitsBuffer, final int inBlockLen) {
final int flag = inBlock.get();
final int expNum = flag & MASK[31 - POSSIBLE_B_BITS];
final int bits = (flag >>> (31 - POSSIBLE_B_BITS)) & (0x1f);
if (bits == 0) {
Arrays.fill(decompBlock, 0, inBlockLen, 0);
} else {
PForDeltaUnpack128WIthIntBuffer.unpack(decompBlock,
inBlock, bits);
}
if (expNum > 0) {
// decompress expPos
int num, outOffset = 0, numLeft;
for (numLeft = expNum; numLeft > 0; numLeft -= num) {
num = Simple16WithHardCodes
.s16DecompressWithIntBufferWithHardCodes(
expPosBuffer, outOffset,
inBlock.get(), numLeft);
outOffset += num;
}
// decompress expHighBits and decompBlock at the same
// time
for (outOffset = 0, numLeft = expNum; numLeft > 0; numLeft -= num) {
num = Simple16WithHardCodes
.s16DecompressWithIntBufferIntegrated2(
decompBlock, outOffset,
inBlock.get(), numLeft,
expPosBuffer, bits);
outOffset += num;
}
}
}
/**
* Estimate the compressed size in ints of a block
*
* @param inputBlock
* the block to be compressed
* @param bits
* the value of the parameter b
* @param blockSize
* the block size
* @return the compressed size in ints
*/
public static int estimateCompressedSize(int[] inputBlock,
int blockSize, int bits) {
int maxNoExp = (1 << bits) - 1;
// Size of the header and the bits-bit slots
int outputOffset = HEADER_SIZE + bits * blockSize;
int expNum = 0;
for (int i = 0; i < blockSize; ++i) {
if (inputBlock[i] > maxNoExp) {
expNum++;
}
}
outputOffset += (expNum << 5);
return outputOffset;
}
/**
* Check if the block contains big numbers that is greater than ((1<<
* bits)-1)
*
* @param inputBlock
* the block to be compressed
* @param bits
* the numbers of bits to decide whether a number is a
* big number
* @param blockSize
* the block size
* @return true if there is any big numbers in the block
*/
public static boolean checkBigNumbers(int[] inputBlock, int blockSize,
int bits) {
int maxNoExp = (1 << bits) - 1;
for (int i = 0; i < blockSize; ++i) {
if (inputBlock[i] > maxNoExp)
return true;
}
return false;
}
/**
* Decompress b-bit slots
*
* @param outDecompSlots
* decompressed block which is the output
* @param inCompBlock
* the compressed block which is the input
* @param blockSize
* the block size
* @param bits
* the value of the parameter b
* @return the compressed size in bits of the data that has been
* decompressed
*/
public static int decompressBBitSlots(int[] outDecompSlots,
int[] inCompBlock, int blockSize, int bits) {
int compressedBitSize = 0;
int offset = HEADER_SIZE;
for (int i = 0; i < blockSize; i++) {
outDecompSlots[i] = readBits(inCompBlock, offset, bits);
offset += bits;
}
compressedBitSize = bits * blockSize;
return compressedBitSize;
}
protected static int decompressBBitSlotsWithHardCodes(
final int[] outDecompSlots, final int[] inCompBlock,
final int blockSize, final int bits) {
int compressedBitSize = 0;
PForDeltaUnpack128.unpack(outDecompSlots, inCompBlock, bits);
compressedBitSize = bits * blockSize;
return compressedBitSize;
}
protected static int decompressBBitSlotsWithHardCodesWithIntBuffer(
final int[] outDecompSlots, final IntBuffer inCompBlock,
final int blockSize, final int bits) {
PForDeltaUnpack128WIthIntBuffer.unpack(outDecompSlots,
inCompBlock, bits);
return bits * blockSize;
}
/**
* Compress a block of blockSize integers using Simple16 algorithm
*
* @param outCompBlock
* the compressed block which is the output
* @param outStartOffsetInBits
* the start offset in bits of the compressed block
* @param inBlock
* the block to be compressed
* @param blockSize
* the block size
* @return the compressed size in bits
*/
private static int compressBlockByS16(int[] outCompBlock,
int outStartOffsetInBits, int[] inBlock, int blockSize,
int oriBlockSize, int[] oriInputBlock) {
int outOffset = (outStartOffsetInBits + 31) >>> 5;
int num, inOffset = 0, numLeft;
for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
num = Simple16WithHardCodes.s16Compress(outCompBlock,
outOffset, inBlock, inOffset, numLeft,
blockSize, oriBlockSize, oriInputBlock);
outOffset++;
inOffset += num;
}
int compressedBitSize = (outOffset << 5) - outStartOffsetInBits;
return compressedBitSize;
}
/**
* Decompress a block of blockSize integers using Simple16 algorithm
*
* @param outDecompBlock
* the decompressed block which is the output
* @param inCompBlock
* the compressed block which is the input
* @param blockSize
* the block size
* @param inStartOffsetInBits
* the start offset in bits of the compressed block
* @return the compressed size in bits of the data that has been
* decompressed
*/
public static int decompressBlockByS16(int[] outDecompBlock,
int[] inCompBlock, int inStartOffsetInBits, int blockSize) {
int inOffset = (inStartOffsetInBits + 31) >>> 5;
int num, outOffset = 0, numLeft;
for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
num = Simple16.s16Decompress(outDecompBlock, outOffset,
inCompBlock, inOffset, numLeft);
outOffset += num;
inOffset++;
}
int compressedBitSize = (inOffset << 5) - inStartOffsetInBits;
return compressedBitSize;
}
protected static void decompressBlockByS16WithIntBuffer(
final int[] outDecompBlock, final IntBuffer inCompBlock,
final int blockSize) {
int num, outOffset = 0, numLeft;
for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
num = Simple16WithHardCodes.s16DecompressWithIntBuffer(
outDecompBlock, outOffset, inCompBlock.get(),
numLeft);
outOffset += num;
}
}
protected static void decompressBlockByS16WithIntBufferIntegrated(
final int[] outDecompBlock, final IntBuffer inCompBlock,
final int blockSize, int[] expPosBuffer, int oribits) {
int num, outOffset = 0, numLeft;
for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
num = Simple16WithHardCodes
.s16DecompressWithIntBufferIntegrated(
outDecompBlock, outOffset,
inCompBlock.get(), numLeft,
expPosBuffer, oribits);
outOffset += num;
}
}
/**
* Write a certain number of bits of an integer into an integer array
* starting from the given start offset
*
* @param out
* the output array
* @param val
* the integer to be written
* @param outOffset
* the start offset in bits in the output array
* @param bits
* the number of bits to be written (bits greater or equal to 0)
*/
public static final void writeBits(int[] out, int val, int outOffset,
int bits) {
if (bits == 0)
return;
final int index = outOffset >>> 5;
final int skip = outOffset & 0x1f;
val &= (0xffffffff >>> (32 - bits));
out[index] |= (val << skip);
if (32 - skip < bits) {
out[index + 1] |= (val >>> (32 - skip));
}
}
/**
* Read a certain number of bits of an integer into an integer array
* starting from the given start offset
*
* @param in
* the input array
* @param inOffset
* the start offset in bits in the input array
* @param bits
* the number of bits to be read, unlike writeBits(),
* readBits() does not deal with bits==0 and thus bits
* must be greater than 0. When bits ==0, the calling functions will
* just skip the entire bits-bit slots without decoding
* them
* @return the bits bits of the input
*/
public static final int readBits(int[] in, final int inOffset,
final int bits) {
final int index = inOffset >>> 5;
final int skip = inOffset & 0x1f;
int val = in[index] >>> skip;
if (32 - skip < bits) {
val |= (in[index + 1] << (32 - skip));
}
return val & (0xffffffff >>> (32 - bits));
}
protected static final int readBitsWithBuffer(int[] in,
final int inOffset, final int bits) {
final int index = inOffset >>> 5;
final int skip = inOffset & 0x1f;
int val = in[index] >>> skip;
if (32 - skip < bits) {
val |= (in[index + 1] << (32 - skip));
}
return val & (0xffffffff >>> (32 - bits));
}
}