org.roaringbitmap.longlong.Roaring64Bitmap Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of RoaringBitmap Show documentation
Show all versions of RoaringBitmap Show documentation
Roaring bitmaps are compressed bitmaps (also called bitsets) which tend to outperform
conventional compressed bitmaps such as WAH or Concise.
package org.roaringbitmap.longlong;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Objects;
import org.roaringbitmap.*;
import org.roaringbitmap.art.ContainerIterator;
import org.roaringbitmap.art.KeyIterator;
import org.roaringbitmap.art.LeafNode;
import org.roaringbitmap.art.LeafNodeIterator;
/**
* Roaring64Bitmap is a compressed 64 bit bitmap. It can contain all the numbers of long
* rang[Long.MAX_VALUE, Long.MIN_VALUE]. Since java has no unsigned long,we treat the negative value
* as a successor of the positive value. So the ascending ordering of all the long value is:
* 0,1,2...Long.MAX_VALUE,Long.MIN_VALUE,Long.MIN_VALUE+1.......-1. See Long.toUnsignedString()
*/
public class Roaring64Bitmap implements Externalizable, LongBitmapDataProvider {
private HighLowContainer highLowContainer;
public Roaring64Bitmap() {
highLowContainer = new HighLowContainer();
}
public void addInt(int x) {
addLong(Util.toUnsignedLong(x));
}
/**
* Add the value to the container (set the value to "true"), whether it already appears or not.
*
* Java lacks native unsigned longs but the x argument is considered to be unsigned. Within
* bitmaps, numbers are ordered according to{@link Long#toUnsignedString}. We order the numbers
* like 0, 1, ..., 9223372036854775807, -9223372036854775808, -9223372036854775807,..., -1.
*
* @param x long value
*/
@Override
public void addLong(long x) {
byte[] high = LongUtils.highPart(x);
char low = LongUtils.lowPart(x);
ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(high);
if (containerWithIndex != null) {
Container container = containerWithIndex.getContainer();
Container freshOne = container.add(low);
highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
} else {
ArrayContainer arrayContainer = new ArrayContainer();
arrayContainer.add(low);
highLowContainer.put(high, arrayContainer);
}
}
/**
* Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
*
* @return the cardinality
*/
@Override
public long getLongCardinality() {
if (highLowContainer.isEmpty()) {
return 0L;
}
Iterator containerIterator = highLowContainer.containerIterator();
long cardinality = 0L;
while (containerIterator.hasNext()) {
Container container = containerIterator.next();
cardinality += container.getCardinality();
}
return cardinality;
}
/**
* @return the cardinality as an int
* @throws UnsupportedOperationException if the cardinality does not fit in an int
*/
public int getIntCardinality() throws UnsupportedOperationException {
long cardinality = getLongCardinality();
if (cardinality > Integer.MAX_VALUE) {
// TODO: we should handle cardinality fitting in an unsigned int
throw new UnsupportedOperationException(
"Can not call .getIntCardinality as the cardinality is bigger than Integer.MAX_VALUE");
}
return (int) cardinality;
}
/**
* Return the jth value stored in this bitmap.
*
* @param j index of the value
* @return the value
* @throws IllegalArgumentException if j is out of the bounds of the bitmap cardinality
*/
@Override
public long select(final long j) throws IllegalArgumentException {
long left = j;
LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(false);
while (leafNodeIterator.hasNext()) {
LeafNode leafNode = leafNodeIterator.next();
long containerIdx = leafNode.getContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
int card = container.getCardinality();
if (left >= card) {
left = left - card;
} else {
byte[] high = leafNode.getKeyBytes();
int leftAsUnsignedInt = (int) left;
char low = container.select(leftAsUnsignedInt);
return LongUtils.toLong(high, low);
}
}
return throwSelectInvalidIndex(j);
}
private long throwSelectInvalidIndex(long j) {
throw new IllegalArgumentException(
"select " + j + " when the cardinality is " + this.getLongCardinality());
}
/**
* Get the first (smallest) integer in this RoaringBitmap,
* that is, returns the minimum of the set.
* @return the first (smallest) integer
* @throws NoSuchElementException if empty
*/
@Override
public long first() {
return highLowContainer.first();
}
/**
* Get the last (largest) integer in this RoaringBitmap,
* that is, returns the maximum of the set.
* @return the last (largest) integer
* @throws NoSuchElementException if empty
*/
@Override
public long last() {
return highLowContainer.last();
}
/**
* For better performance, consider the Use the {@link #forEach forEach} method.
*
* @return a custom iterator over set bits, the bits are traversed in ascending sorted order
*/
public Iterator iterator() {
final LongIterator it = getLongIterator();
return new Iterator() {
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public Long next() {
return it.next();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public void forEach(final LongConsumer lc) {
KeyIterator keyIterator = highLowContainer.highKeyIterator();
while (keyIterator.hasNext()) {
byte[] high = keyIterator.next();
long containerIdx = keyIterator.currentContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
PeekableCharIterator charIterator = container.getCharIterator();
while (charIterator.hasNext()) {
char low = charIterator.next();
long v = LongUtils.toLong(high, low);
lc.accept(v);
}
}
}
/**
* Consume presence information for all values in the range [start, start + length).
*
* @param start Lower bound of values to consume.
* @param length Maximum number of values to consume.
* @param rrc Code to be executed for each present or absent value.
*/
public void forAllInRange(long start, int length, final RelativeRangeConsumer rrc) {
final LeafNodeIterator leafIterator =
highLowContainer.highKeyLeafNodeIteratorFrom(start, false);
if (!leafIterator.hasNext()) {
rrc.acceptAllAbsent(0, length);
return; // nothing else to do
}
final long end = start + length;
final long endHigh = LongUtils.rightShiftHighPart(end);
long filledUntil = start;
LeafNode node = leafIterator.next();
long high = node.getKey();
while (high <= endHigh) {
// fill missing values until start of container
long containerStart = LongUtils.toLong(high, (char) 0);
if (filledUntil < containerStart) {
rrc.acceptAllAbsent((int) (filledUntil - start), (int) (containerStart - start));
filledUntil = containerStart;
}
// Inspect Container
long containerIdx = node.getContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
long containerEnd = LongUtils.toLong(high, Character.MAX_VALUE) + 1;
int containerRangeStartOffset = (int) (filledUntil - start);
boolean startInContainer = containerStart < start;
boolean endInContainer = end < containerEnd;
if (startInContainer && endInContainer) {
// Only part of the container is in range
char containerRangeStart = LongUtils.lowPart(start);
char containerRangeEnd = LongUtils.lowPart(end);
container.forAllInRange(
LongUtils.lowPart(start),
LongUtils.lowPart(end),
rrc);
filledUntil += containerRangeEnd - containerRangeStart;
} else if (startInContainer) {// && !endInContainer
// range begins within the container
char containerRangeStart = LongUtils.lowPart(start);
container.forAllFrom(containerRangeStart, rrc);
filledUntil += BitmapContainer.MAX_CAPACITY - containerRangeStart;
} else if (endInContainer) {// && !startInContainer
// range end within the container
char containerRangeEnd = LongUtils.lowPart(end);
container.forAllUntil(containerRangeStartOffset, containerRangeEnd, rrc);
filledUntil += containerRangeEnd;
} else {
container.forAll(containerRangeStartOffset, rrc);
filledUntil += BitmapContainer.MAX_CAPACITY;
}
if (leafIterator.hasNext()) {
node = leafIterator.next();
high = node.getKey();
} else {
break;
}
}
// next container (if any) is beyond the end, but there may be missing values in between
if (filledUntil < end) {
rrc.acceptAllAbsent((int) (filledUntil - start), length);
}
}
/**
* Consume each value present in the range [start, start + length).
*
* @param start Lower bound of values to consume.
* @param length Maximum number of values to consume.
* @param lc Code to be executed for each present value.
*/
public void forEachInRange(long start, int length, final LongConsumer lc) {
forAllInRange(start, length, new LongConsumerRelativeRangeAdapter(start, lc));
}
@Override
public long rankLong(long id) {
long result = 0;
long high = LongUtils.rightShiftHighPart(id);
byte[] highBytes = LongUtils.highPart(id);
char low = LongUtils.lowPart(id);
ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(highBytes);
KeyIterator keyIterator = highLowContainer.highKeyIterator();
if (containerWithIndex == null) {
while (keyIterator.hasNext()) {
long highKey = keyIterator.nextKey();
if (highKey > high) {
break;
} else {
long containerIdx = keyIterator.currentContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
result += container.getCardinality();
}
}
} else {
while (keyIterator.hasNext()) {
long key = keyIterator.nextKey();
long containerIdx = keyIterator.currentContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
if (key == high) {
result += container.rank(low);
break;
} else {
result += container.getCardinality();
}
}
}
return result;
}
/**
* In-place bitwise OR (union) operation. The current bitmap is modified.
*
* @param x2 other bitmap
*/
public void or(final Roaring64Bitmap x2) {
if(this == x2) { return; }
KeyIterator highIte2 = x2.highLowContainer.highKeyIterator();
while (highIte2.hasNext()) {
byte[] high = highIte2.next();
long containerIdx = highIte2.currentContainerIdx();
Container container2 = x2.highLowContainer.getContainer(containerIdx);
ContainerWithIndex containerWithIdx = this.highLowContainer.searchContainer(high);
if (containerWithIdx == null) {
Container container2clone = container2.clone();
this.highLowContainer.put(high, container2clone);
} else {
Container freshContainer = containerWithIdx.getContainer().ior(container2);
this.highLowContainer.replaceContainer(containerWithIdx.getContainerIdx(), freshContainer);
}
}
}
/**
* In-place bitwise XOR (symmetric difference) operation. The current bitmap is modified.
*
* @param x2 other bitmap
*/
public void xor(final Roaring64Bitmap x2) {
if(x2 == this) {
clear();
return;
}
KeyIterator keyIterator = x2.highLowContainer.highKeyIterator();
while (keyIterator.hasNext()) {
byte[] high = keyIterator.next();
long containerIdx = keyIterator.currentContainerIdx();
Container container = x2.highLowContainer.getContainer(containerIdx);
ContainerWithIndex containerWithIndex = this.highLowContainer.searchContainer(high);
if (containerWithIndex == null) {
Container containerClone2 = container.clone();
this.highLowContainer.put(high, containerClone2);
} else {
Container freshOne = containerWithIndex.getContainer().ixor(container);
this.highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
}
}
}
/**
* In-place bitwise AND (intersection) operation. The current bitmap is modified.
*
* @param x2 other bitmap
*/
public void and(final Roaring64Bitmap x2) {
if(x2 == this) { return; }
KeyIterator thisIterator = highLowContainer.highKeyIterator();
while (thisIterator.hasNext()) {
byte[] highKey = thisIterator.next();
long containerIdx = thisIterator.currentContainerIdx();
ContainerWithIndex containerWithIdx = x2.highLowContainer.searchContainer(highKey);
if (containerWithIdx == null) {
thisIterator.remove();
} else {
Container container1 = highLowContainer.getContainer(containerIdx);
Container freshContainer = container1.iand(containerWithIdx.getContainer());
if (!freshContainer.isEmpty()) {
highLowContainer.replaceContainer(containerIdx, freshContainer);
} else {
thisIterator.remove();
}
}
}
}
/**
* In-place bitwise ANDNOT (difference) operation. The current bitmap is modified.
*
* @param x2 other bitmap
*/
public void andNot(final Roaring64Bitmap x2) {
if(x2 == this) {
clear();
return;
}
KeyIterator thisKeyIterator = highLowContainer.highKeyIterator();
while (thisKeyIterator.hasNext()) {
byte[] high = thisKeyIterator.next();
long containerIdx = thisKeyIterator.currentContainerIdx();
ContainerWithIndex containerWithIdx2 = x2.highLowContainer.searchContainer(high);
if (containerWithIdx2 != null) {
Container thisContainer = highLowContainer.getContainer(containerIdx);
Container freshContainer = thisContainer.iandNot(containerWithIdx2.getContainer());
highLowContainer.replaceContainer(containerIdx, freshContainer);
if (!freshContainer.isEmpty()) {
highLowContainer.replaceContainer(containerIdx, freshContainer);
} else {
thisKeyIterator.remove();
}
}
}
}
/**
* Complements the bits in the given range, from rangeStart (inclusive) rangeEnd (exclusive). The
* given bitmap is unchanged.
*
* @param rangeStart inclusive beginning of range, in [0, 0xffffffffffffffff]
* @param rangeEnd exclusive ending of range, in [0, 0xffffffffffffffff + 1]
*/
public void flip(final long rangeStart, final long rangeEnd) {
if(rangeEnd >= 0 && rangeStart >= rangeEnd){
// both numbers in positive range, and start is beyond end, nothing to do.
return;
} else if(rangeStart < 0 && rangeStart >= rangeEnd){
// both numbers in negative range, and start is beyond end, nothing to do.
return;
} else if(rangeStart < 0 && rangeEnd > 0) {
// start is neg which is "higher" and end is above zero thus, nothing to do.
return;
}
byte[] hbStart = LongUtils.highPart(rangeStart);
char lbStart = LongUtils.lowPart(rangeStart);
char lbLast = LongUtils.lowPart(rangeEnd - 1L);
long shStart = LongUtils.rightShiftHighPart(rangeStart);
long shEnd = LongUtils.rightShiftHighPart(rangeEnd - 1L);
// TODO:this can be accelerated considerably
for (long hb = shStart; hb <= shEnd; ++hb) {
// first container may contain partial range
final int containerStart = (hb == shStart) ? lbStart : 0;
// last container may contain partial range
final int containerLast = (hb == shEnd) ? lbLast : LongUtils.maxLowBitAsInteger();
ContainerWithIndex cwi = highLowContainer.searchContainer(
LongUtils.highPartInPlace(LongUtils.leftShiftHighPart(hb), hbStart));
if (cwi != null) {
final long i = cwi.getContainerIdx();
final Container c = cwi.getContainer().inot(containerStart, containerLast + 1);
if (!c.isEmpty()) {
highLowContainer.replaceContainer(i, c);
} else {
highLowContainer.remove(hbStart);
}
} else {
Container newContainer = Container.rangeOfOnes(containerStart, containerLast + 1);
highLowContainer.put(hbStart, newContainer);
}
}
}
/**
* {@link Roaring64NavigableMap} are serializable. However, contrary to RoaringBitmap, the
* serialization format is not well-defined: for now, it is strongly coupled with Java standard
* serialization. Just like the serialization may be incompatible between various Java versions,
* {@link Roaring64NavigableMap} are subject to incompatibilities. Moreover, even on a given Java
* versions, the serialization format may change from one RoaringBitmap version to another
*/
@Override
public void writeExternal(ObjectOutput out) throws IOException {
serialize(out);
}
@Override
public void readExternal(ObjectInput in) throws IOException {
deserialize(in);
}
/**
* A string describing the bitmap.
*
* @return the string
*/
@Override
public String toString() {
final StringBuilder answer = new StringBuilder("{}".length() + "-1234567890123456789,".length()
* 256);
final LongIterator i = this.getLongIterator();
answer.append('{');
if (i.hasNext()) {
answer.append(i.next());
}
while (i.hasNext()) {
answer.append(',');
// to avoid using too much memory, we limit the size
if (answer.length() > 0x80000) {
answer.append('.').append('.').append('.');
break;
}
answer.append(i.next());
}
answer.append("}");
return answer.toString();
}
/**
* For better performance, consider the Use the {@link #forEach forEach} method.
*
* @return a custom iterator over set bits, the bits are traversed in ascending sorted order
*/
@Override
public PeekableLongIterator getLongIterator() {
LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(false);
return new ForwardPeekableIterator(leafNodeIterator);
}
// for testing only
LeafNodeIterator getLeafNodeIterator() {
return highLowContainer.highKeyLeafNodeIterator(false);
}
/**
* Produce an iterator over the values in this bitmap starting from `minval`.
*
* @param minval the lower bound of the iterator returned
* @return a custom iterator over set bits, the bits are traversed in ascending sorted order
*/
public PeekableLongIterator getLongIteratorFrom(long minval) {
LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIteratorFrom(minval, false);
ForwardPeekableIterator fpi = new ForwardPeekableIterator(leafNodeIterator);
fpi.advanceIfNeeded(minval); // make sure the lower end is advanced as well
return fpi;
}
@Override
public boolean contains(long x) {
byte[] high = LongUtils.highPart(x);
ContainerWithIndex containerWithIdx = highLowContainer.searchContainer(high);
if (containerWithIdx == null) {
return false;
}
char low = LongUtils.lowPart(x);
return containerWithIdx.getContainer().contains(low);
}
@Override
public int getSizeInBytes() {
return (int) getLongSizeInBytes();
}
/**
* Estimate of the memory usage of this data structure. This can be expected to be within 1% of
* the true memory usage in common usage scenarios.
* If exact measures are needed, we recommend using dedicated libraries
* such as ehcache-sizeofengine.
*
* In adversarial cases, this estimate may be 10x the actual memory usage. For example, if
* you insert a single random value in a bitmap, then over a 100 bytes may be used by the JVM
* whereas this function may return an estimate of 32 bytes.
*
* The same will be true in the "sparse" scenario where you have a small set of
* random-looking integers spanning a wide range of values.
*
* These are considered adversarial cases because, as a general rule,
* if your data looks like a set
* of random integers, Roaring bitmaps are probably not the right data structure.
*
* Note that you can serialize your Roaring Bitmaps to disk and then construct
* ImmutableRoaringBitmap instances from a ByteBuffer. In such cases, the Java heap
* usage will be significantly less than
* what is reported.
*
* If your main goal is to compress arrays of integers, there are other libraries
* that are maybe more appropriate
* such as JavaFastPFOR.
*
* Note, however, that in general, random integers (as produced by random number
* generators or hash functions) are not compressible.
* Trying to compress random data is an adversarial use case.
*
* @see JavaFastPFOR
*
*
* @return estimated memory usage.
*/
@Override
public long getLongSizeInBytes() {
// 'serializedSizeInBytes' is a better than nothing estimation of the memory footprint
// It would generally be an optimistic estimator (by underestimating the size in memory)
return serializedSizeInBytes();
}
@Override
public boolean isEmpty() {
return getLongCardinality() == 0L;
}
@Override
public ImmutableLongBitmapDataProvider limit(long x) {
throw new UnsupportedOperationException("TODO");
}
/**
* Use a run-length encoding where it is estimated as more space efficient
*
* @return whether a change was applied
*/
public boolean runOptimize() {
boolean hasChanged = false;
ContainerIterator containerIterator = highLowContainer.containerIterator();
while (containerIterator.hasNext()) {
Container container = containerIterator.next();
Container freshContainer = container.runOptimize();
if (freshContainer instanceof RunContainer) {
hasChanged = true;
containerIterator.replace(freshContainer);
}
}
return hasChanged;
}
/**
* Serialize this bitmap.
*
* Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
* another, and from one RoaringBitmap version to another.
*
* Consider calling {@link #runOptimize} before serialization to improve compression.
*
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
@Override
public void serialize(DataOutput out) throws IOException {
highLowContainer.serialize(out);
}
/**
* Serialize this bitmap, please make sure the size of the serialized bytes is
* smaller enough that ByteBuffer can hold it.
* @param byteBuffer the ByteBuffer
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(ByteBuffer byteBuffer) throws IOException {
highLowContainer.serialize(byteBuffer);
}
/**
* Deserialize (retrieve) this bitmap.
*
* Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
* another, and from one RoaringBitmap version to another.
*
* The current bitmap is overwritten.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(DataInput in) throws IOException {
this.clear();
highLowContainer.deserialize(in);
}
/**
* Deserialize (retrieve) this bitmap.
*
* Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
* another, and from one RoaringBitmap version to another.
*
* The current bitmap is overwritten.
*
* @param in the ByteBuffer stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(ByteBuffer in) throws IOException {
this.clear();
highLowContainer.deserialize(in);
}
@Override
public long serializedSizeInBytes() {
long nbBytes = highLowContainer.serializedSizeInBytes();
return nbBytes;
}
/**
* reset to an empty bitmap; result occupies as much space a newly created bitmap.
*/
public void clear() {
this.highLowContainer.clear();
}
/**
* Return the set values as an array, if the cardinality is smaller than 2147483648. The long
* values are in sorted order.
*
* @return array representing the set values.
*/
@Override
public long[] toArray() {
long cardinality = this.getLongCardinality();
if (cardinality > Integer.MAX_VALUE) {
throw new IllegalStateException("The cardinality does not fit in an array");
}
final long[] array = new long[(int) cardinality];
int pos = 0;
LongIterator it = getLongIterator();
while (it.hasNext()) {
array[pos++] = it.next();
}
return array;
}
/**
* Generate a bitmap with the specified values set to true. The provided longs values don't have
* to be in sorted order, but it may be preferable to sort them from a performance point of view.
*
* @param dat set values
* @return a new bitmap
*/
public static Roaring64Bitmap bitmapOf(final long... dat) {
final Roaring64Bitmap ans = new Roaring64Bitmap();
ans.add(dat);
return ans;
}
/**
* Set all the specified values to true. This can be expected to be slightly faster than calling
* "add" repeatedly. The provided integers values don't have to be in sorted order, but it may be
* preferable to sort them from a performance point of view.
*
* @param dat set values
*/
public void add(long... dat) {
for (long oneLong : dat) {
addLong(oneLong);
}
}
/**
* Add to the current bitmap all longs in [rangeStart,rangeEnd).
*
* @param rangeStart inclusive beginning of range
* @param rangeEnd exclusive ending of range
* @deprecated as this may be confused with adding individual longs
*/
@Deprecated
public void add(final long rangeStart, final long rangeEnd) {
addRange(rangeStart, rangeEnd);
}
/**
* Add to the current bitmap all longs in [rangeStart,rangeEnd).
*
* @param rangeStart inclusive beginning of range
* @param rangeEnd exclusive ending of range
*/
public void addRange(final long rangeStart, final long rangeEnd) {
if (rangeEnd == 0 || Long.compareUnsigned(rangeStart, rangeEnd) >= 0) {
throw new IllegalArgumentException("Invalid range [" + rangeStart + "," + rangeEnd + ")");
}
long startHigh = LongUtils.rightShiftHighPart(rangeStart);
int startLow = LongUtils.lowPart(rangeStart);
long endHigh = LongUtils.rightShiftHighPart(rangeEnd - 1);
int endLow = LongUtils.lowPart(rangeEnd - 1);
long rangeStartVal = rangeStart;
long startHighKey = LongUtils.rightShiftHighPart(rangeStart);
byte[] startHighKeyBytes = LongUtils.highPart(rangeStart);
while (startHighKey <= endHigh) {
final int containerStart = startHighKey == startHigh ? startLow : 0;
// last container may contain partial range
final int containerLast = startHighKey == endHigh ? endLow : Util.maxLowBitAsInteger();
ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(startHighKeyBytes);
if (containerWithIndex != null) {
long containerIdx = containerWithIndex.getContainerIdx();
Container freshContainer = highLowContainer.getContainer(containerIdx)
.iadd(containerStart, containerLast + 1);
highLowContainer.replaceContainer(containerIdx, freshContainer);
} else {
Container freshContainer = Container.rangeOfOnes(containerStart, containerLast + 1);
highLowContainer.put(startHighKeyBytes, freshContainer);
}
if (LongUtils.isMaxHigh(startHighKey)) {
break;
}
//increase the high
rangeStartVal = rangeStartVal + (containerLast - containerStart) + 1;
startHighKey = LongUtils.rightShiftHighPart(rangeStartVal);
startHighKeyBytes = LongUtils.highPart(rangeStartVal);
}
}
@Override
public PeekableLongIterator getReverseLongIterator() {
LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(true);
return new ReversePeekableIterator(leafNodeIterator);
}
/**
* Produce an iterator over the values in this bitmap starting from `maxval`.
*
* @param maxval the upper bound of the iterator returned
* @return a custom iterator over set bits, the bits are traversed in descending sorted order
*/
public PeekableLongIterator getReverseLongIteratorFrom(long maxval) {
LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIteratorFrom(maxval, true);
ReversePeekableIterator rpi = new ReversePeekableIterator(leafNodeIterator);
rpi.advanceIfNeeded(maxval); // make sure the lower end is advanced as well
return rpi;
}
@Override
public void removeLong(long x) {
byte[] high = LongUtils.highPart(x);
ContainerWithIndex containerWithIdx = highLowContainer.searchContainer(high);
if (containerWithIdx != null) {
char low = LongUtils.lowPart(x);
Container container = containerWithIdx.getContainer();
Container freshContainer = container.remove(low);
if (freshContainer.isEmpty()) {
// Attempt to remove empty container to save memory
highLowContainer.remove(high);
} else {
highLowContainer.replaceContainer(containerWithIdx.getContainerIdx(), freshContainer);
}
}
}
/**
* remove the allocated unused memory space
*/
@Override
public void trim() {
if (highLowContainer.isEmpty()) {
return;
}
KeyIterator keyIterator = highLowContainer.highKeyIterator();
while (keyIterator.hasNext()) {
long containerIdx = keyIterator.currentContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
if (container.isEmpty()) {
keyIterator.remove();
} else {
//TODO
container.trim();
}
}
}
@Override
public int hashCode() {
return highLowContainer.hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
Roaring64Bitmap other = (Roaring64Bitmap) obj;
return Objects.equals(highLowContainer, other.highLowContainer);
}
/**
* Add the value if it is not already present, otherwise remove it.
*
* @param x long value
*/
public void flip(final long x) {
byte[] high = LongUtils.highPart(x);
ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(high);
if (containerWithIndex == null) {
addLong(x);
} else {
char low = LongUtils.lowPart(x);
Container freshOne = containerWithIndex.getContainer().flip(low);
highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
}
}
//mainly used for benchmark
@Override
public Roaring64Bitmap clone() {
long sizeInBytesL = this.serializedSizeInBytes();
if (sizeInBytesL >= Integer.MAX_VALUE) {
throw new UnsupportedOperationException();
}
int sizeInBytesInt = (int) sizeInBytesL;
ByteBuffer byteBuffer = ByteBuffer.allocate(sizeInBytesInt).order(ByteOrder.LITTLE_ENDIAN);
try {
this.serialize(byteBuffer);
byteBuffer.flip();
Roaring64Bitmap freshOne = new Roaring64Bitmap();
freshOne.deserialize(byteBuffer);
return freshOne;
} catch (Exception e) {
throw new RuntimeException("fail to clone thorough the ser/deser", e);
}
}
private abstract class PeekableIterator implements PeekableLongIterator {
private final LeafNodeIterator keyIte;
private byte[] high;
private PeekableCharIterator charIterator;
PeekableIterator(final LeafNodeIterator keyIte) {
this.keyIte = keyIte;
}
abstract PeekableCharIterator getIterator(Container container);
abstract boolean compare(long next, long val);
@Override
public boolean hasNext() {
if (charIterator != null && charIterator.hasNext()) {
return true;
}
while (keyIte.hasNext()) {
LeafNode leafNode = keyIte.next();
high = leafNode.getKeyBytes();
long containerIdx = leafNode.getContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
charIterator = getIterator(container);
if(charIterator.hasNext()){
return true;
}
}
return false;
}
@Override
public long next() {
if (hasNext()) {
char low = charIterator.next();
return LongUtils.toLong(high, low);
} else {
throw new IllegalStateException("empty");
}
}
@Override
public void advanceIfNeeded(long minval) {
if (!hasNext()) {
return;
}
if(compare(this.peekNext(), minval)) {
return;
}
//empty bitset
if(this.high == null) {
return;
}
long minHigh = LongUtils.rightShiftHighPart(minval);
long high = LongUtils.toLong(this.high);
if (minHigh != high) {
// advance outer
if (keyIte.hasNext()) {
LeafNode leafNode = keyIte.next();
this.high = leafNode.getKeyBytes();
if (compare(leafNode.getKey(), minHigh)) {
long containerIdx = leafNode.getContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
charIterator = getIterator(container);
if(!charIterator.hasNext()){
return;
}
} else {
keyIte.seek(minval);
if (keyIte.hasNext()) {
leafNode = keyIte.next();
this.high = leafNode.getKeyBytes();
long containerIdx = leafNode.getContainerIdx();
Container container = highLowContainer.getContainer(containerIdx);
charIterator = getIterator(container);
if(!charIterator.hasNext()){
return;
}
} else {
// make sure we don't accidentally continue at the previous iterator position
// after stepping to the end.
charIterator = null;
return;
}
}
}
}
byte[] minHighBytes = LongUtils.highPart(minval);
if (Arrays.equals(this.high, minHighBytes)) {
// advance inner
char low = LongUtils.lowPart(minval);
charIterator.advanceIfNeeded(low);
}
}
@Override
public long peekNext() {
if (hasNext()) {
char low = charIterator.peekNext();
return LongUtils.toLong(high, low);
} else {
throw new IllegalStateException("empty");
}
}
@Override
public PeekableLongIterator clone() {
throw new UnsupportedOperationException("TODO");
}
}
private class ForwardPeekableIterator extends PeekableIterator {
public ForwardPeekableIterator(final LeafNodeIterator keyIte) {
super(keyIte);
}
@Override
PeekableCharIterator getIterator(Container container) {
return container.getCharIterator();
}
@Override
boolean compare(long next, long val) {
return Long.compareUnsigned(next, val) >= 0;
}
}
private class ReversePeekableIterator extends PeekableIterator {
public ReversePeekableIterator(final LeafNodeIterator keyIte) {
super(keyIte);
}
@Override
PeekableCharIterator getIterator(Container container) {
return container.getReverseCharIterator();
}
@Override
boolean compare(long next, long val) {
return Long.compareUnsigned(next, val) <= 0;
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy