All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.roaringbitmap.longlong.Roaring64Bitmap Maven / Gradle / Ivy

Go to download

Roaring bitmaps are compressed bitmaps (also called bitsets) which tend to outperform conventional compressed bitmaps such as WAH or Concise.

There is a newer version: 1.3.0
Show newest version
package org.roaringbitmap.longlong;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Objects;

import org.roaringbitmap.*;
import org.roaringbitmap.art.ContainerIterator;
import org.roaringbitmap.art.KeyIterator;
import org.roaringbitmap.art.LeafNode;
import org.roaringbitmap.art.LeafNodeIterator;

/**
 * Roaring64Bitmap is a compressed 64 bit bitmap. It can contain all the numbers of long
 * rang[Long.MAX_VALUE, Long.MIN_VALUE]. Since java has no unsigned long,we treat the negative value
 * as a successor of the positive value. So the ascending ordering of all the long value is:
 * 0,1,2...Long.MAX_VALUE,Long.MIN_VALUE,Long.MIN_VALUE+1.......-1. See Long.toUnsignedString()
 */
public class Roaring64Bitmap implements Externalizable, LongBitmapDataProvider {

  private HighLowContainer highLowContainer;

  public Roaring64Bitmap() {
    highLowContainer = new HighLowContainer();
  }

  public void addInt(int x) {
    addLong(Util.toUnsignedLong(x));
  }

  /**
   * Add the value to the container (set the value to "true"), whether it already appears or not.
   *
   * Java lacks native unsigned longs but the x argument is considered to be unsigned. Within
   * bitmaps, numbers are ordered according to{@link Long#toUnsignedString}. We order the numbers
   * like 0, 1, ..., 9223372036854775807, -9223372036854775808, -9223372036854775807,..., -1.
   *
   * @param x long value
   */
  @Override
  public void addLong(long x) {
    byte[] high = LongUtils.highPart(x);
    char low = LongUtils.lowPart(x);
    ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(high);
    if (containerWithIndex != null) {
      Container container = containerWithIndex.getContainer();
      Container freshOne = container.add(low);
      highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
    } else {
      ArrayContainer arrayContainer = new ArrayContainer();
      arrayContainer.add(low);
      highLowContainer.put(high, arrayContainer);
    }
  }

  /**
   * Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
   *
   * @return the cardinality
   */
  @Override
  public long getLongCardinality() {
    if (highLowContainer.isEmpty()) {
      return 0L;
    }
    Iterator containerIterator = highLowContainer.containerIterator();
    long cardinality = 0L;
    while (containerIterator.hasNext()) {
      Container container = containerIterator.next();
      cardinality += container.getCardinality();
    }
    return cardinality;
  }

  /**
   * @return the cardinality as an int
   * @throws UnsupportedOperationException if the cardinality does not fit in an int
   */
  public int getIntCardinality() throws UnsupportedOperationException {
    long cardinality = getLongCardinality();
    if (cardinality > Integer.MAX_VALUE) {
      // TODO: we should handle cardinality fitting in an unsigned int
      throw new UnsupportedOperationException(
          "Can not call .getIntCardinality as the cardinality is bigger than Integer.MAX_VALUE");
    }
    return (int) cardinality;
  }

  /**
   * Return the jth value stored in this bitmap.
   *
   * @param j index of the value
   * @return the value
   * @throws IllegalArgumentException if j is out of the bounds of the bitmap cardinality
   */
  @Override
  public long select(final long j) throws IllegalArgumentException {
    long left = j;
    LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(false);
    while (leafNodeIterator.hasNext()) {
      LeafNode leafNode = leafNodeIterator.next();
      long containerIdx = leafNode.getContainerIdx();
      Container container = highLowContainer.getContainer(containerIdx);
      int card = container.getCardinality();
      if (left >= card) {
        left = left - card;
      } else {
        byte[] high = leafNode.getKeyBytes();
        int leftAsUnsignedInt = (int) left;
        char low = container.select(leftAsUnsignedInt);
        return LongUtils.toLong(high, low);
      }
    }
    return throwSelectInvalidIndex(j);
  }

  private long throwSelectInvalidIndex(long j) {
    throw new IllegalArgumentException(
        "select " + j + " when the cardinality is " + this.getLongCardinality());
  }

  /**
   * Get the first (smallest) integer in this RoaringBitmap,
   * that is, returns the minimum of the set.
   * @return the first (smallest) integer
   * @throws NoSuchElementException if empty
   */
  @Override
  public long first() {
    return highLowContainer.first();
  }

  /**
   * Get the last (largest) integer in this RoaringBitmap,
   * that is, returns the maximum of the set.
   * @return the last (largest) integer
   * @throws NoSuchElementException if empty
   */
  @Override
  public long last() {
    return highLowContainer.last();
  }

  /**
   * For better performance, consider the Use the {@link #forEach forEach} method.
   *
   * @return a custom iterator over set bits, the bits are traversed in ascending sorted order
   */
  public Iterator iterator() {
    final LongIterator it = getLongIterator();

    return new Iterator() {

      @Override
      public boolean hasNext() {
        return it.hasNext();
      }

      @Override
      public Long next() {
        return it.next();
      }

      @Override
      public void remove() {
        throw new UnsupportedOperationException();
      }
    };
  }

  @Override
  public void forEach(final LongConsumer lc) {
    KeyIterator keyIterator = highLowContainer.highKeyIterator();
    while (keyIterator.hasNext()) {
      byte[] high = keyIterator.next();
      long containerIdx = keyIterator.currentContainerIdx();
      Container container = highLowContainer.getContainer(containerIdx);
      PeekableCharIterator charIterator = container.getCharIterator();
      while (charIterator.hasNext()) {
        char low = charIterator.next();
        long v = LongUtils.toLong(high, low);
        lc.accept(v);
      }
    }
  }

  /**
   * Consume presence information for all values in the range [start, start + length).
   *
   * @param start Lower bound of values to consume.
   * @param length Maximum number of values to consume.
   * @param rrc Code to be executed for each present or absent value.
   */
  public void forAllInRange(long start, int length, final RelativeRangeConsumer rrc) {
    final LeafNodeIterator leafIterator =
        highLowContainer.highKeyLeafNodeIteratorFrom(start, false);
    if (!leafIterator.hasNext()) {
      rrc.acceptAllAbsent(0, length);
      return; // nothing else to do
    }
    final long end = start + length;
    final long endHigh = LongUtils.rightShiftHighPart(end);
    long filledUntil = start;

    LeafNode node = leafIterator.next();
    long high = node.getKey();
    while (high <= endHigh) {
      // fill missing values until start of container
      long containerStart = LongUtils.toLong(high, (char) 0);
      if (filledUntil < containerStart) {
        rrc.acceptAllAbsent((int) (filledUntil - start), (int) (containerStart - start));
        filledUntil = containerStart;
      }
      // Inspect Container
      long containerIdx = node.getContainerIdx();
      Container container = highLowContainer.getContainer(containerIdx);
      long containerEnd = LongUtils.toLong(high, Character.MAX_VALUE) + 1;
      int containerRangeStartOffset = (int) (filledUntil - start);

      boolean startInContainer = containerStart < start;
      boolean endInContainer = end < containerEnd;

      if (startInContainer && endInContainer) {
        // Only part of the container is in range
        char containerRangeStart = LongUtils.lowPart(start);
        char containerRangeEnd = LongUtils.lowPart(end);
        container.forAllInRange(
            LongUtils.lowPart(start),
            LongUtils.lowPart(end),
            rrc);
        filledUntil += containerRangeEnd - containerRangeStart;
      } else if (startInContainer) {//  && !endInContainer
        // range begins within the container
        char containerRangeStart = LongUtils.lowPart(start);
        container.forAllFrom(containerRangeStart, rrc);
        filledUntil += BitmapContainer.MAX_CAPACITY - containerRangeStart;
      } else if (endInContainer) {// && !startInContainer
        // range end within the container
        char containerRangeEnd = LongUtils.lowPart(end);
        container.forAllUntil(containerRangeStartOffset, containerRangeEnd, rrc);
        filledUntil += containerRangeEnd;
      } else {
        container.forAll(containerRangeStartOffset, rrc);
        filledUntil += BitmapContainer.MAX_CAPACITY;
      }
      if (leafIterator.hasNext()) {
        node = leafIterator.next();
        high = node.getKey();
      } else {
        break;
      }
    }
    // next container (if any) is beyond the end, but there may be missing values in between
    if (filledUntil < end) {
      rrc.acceptAllAbsent((int) (filledUntil - start), length);
    }
  }

  /**
   * Consume each value present in the range [start, start + length).
   *
   * @param start Lower bound of values to consume.
   * @param length Maximum number of values to consume.
   * @param lc Code to be executed for each present value.
   */
  public void forEachInRange(long start, int length, final LongConsumer lc) {
    forAllInRange(start, length, new LongConsumerRelativeRangeAdapter(start, lc));
  }

  @Override
  public long rankLong(long id) {
    long result = 0;
    long high = LongUtils.rightShiftHighPart(id);
    byte[] highBytes = LongUtils.highPart(id);
    char low = LongUtils.lowPart(id);
    ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(highBytes);
    KeyIterator keyIterator = highLowContainer.highKeyIterator();
    if (containerWithIndex == null) {
      while (keyIterator.hasNext()) {
        long highKey = keyIterator.nextKey();
        if (highKey > high) {
          break;
        } else {
          long containerIdx = keyIterator.currentContainerIdx();
          Container container = highLowContainer.getContainer(containerIdx);
          result += container.getCardinality();
        }
      }
    } else {
      while (keyIterator.hasNext()) {
        long key = keyIterator.nextKey();
        long containerIdx = keyIterator.currentContainerIdx();
        Container container = highLowContainer.getContainer(containerIdx);
        if (key == high) {
          result += container.rank(low);
          break;
        } else {
          result += container.getCardinality();
        }
      }
    }
    return result;
  }

  /**
   * In-place bitwise OR (union) operation. The current bitmap is modified.
   *
   * @param x2 other bitmap
   */
  public void or(final Roaring64Bitmap x2) {
    if(this == x2) { return; }
    KeyIterator highIte2 = x2.highLowContainer.highKeyIterator();
    while (highIte2.hasNext()) {
      byte[] high = highIte2.next();
      long containerIdx = highIte2.currentContainerIdx();
      Container container2 = x2.highLowContainer.getContainer(containerIdx);
      ContainerWithIndex containerWithIdx = this.highLowContainer.searchContainer(high);
      if (containerWithIdx == null) {
        Container container2clone = container2.clone();
        this.highLowContainer.put(high, container2clone);
      } else {
        Container freshContainer = containerWithIdx.getContainer().ior(container2);
        this.highLowContainer.replaceContainer(containerWithIdx.getContainerIdx(), freshContainer);
      }
    }
  }

  /**
   * In-place bitwise XOR (symmetric difference) operation. The current bitmap is modified.
   *
   * @param x2 other bitmap
   */
  public void xor(final Roaring64Bitmap x2) {
    if(x2 == this) {
      clear();
      return;
    }
    KeyIterator keyIterator = x2.highLowContainer.highKeyIterator();
    while (keyIterator.hasNext()) {
      byte[] high = keyIterator.next();
      long containerIdx = keyIterator.currentContainerIdx();
      Container container = x2.highLowContainer.getContainer(containerIdx);
      ContainerWithIndex containerWithIndex = this.highLowContainer.searchContainer(high);
      if (containerWithIndex == null) {
        Container containerClone2 = container.clone();
        this.highLowContainer.put(high, containerClone2);
      } else {
        Container freshOne = containerWithIndex.getContainer().ixor(container);
        this.highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
      }
    }
  }

  /**
   * In-place bitwise AND (intersection) operation. The current bitmap is modified.
   *
   * @param x2 other bitmap
   */
  public void and(final Roaring64Bitmap x2) {
    if(x2 == this) { return; }
    KeyIterator thisIterator = highLowContainer.highKeyIterator();
    while (thisIterator.hasNext()) {
      byte[] highKey = thisIterator.next();
      long containerIdx = thisIterator.currentContainerIdx();
      ContainerWithIndex containerWithIdx = x2.highLowContainer.searchContainer(highKey);
      if (containerWithIdx == null) {
        thisIterator.remove();
      } else {
        Container container1 = highLowContainer.getContainer(containerIdx);
        Container freshContainer = container1.iand(containerWithIdx.getContainer());
        if (!freshContainer.isEmpty()) {
          highLowContainer.replaceContainer(containerIdx, freshContainer);
        } else {
          thisIterator.remove();
        }
      }
    }
  }


  /**
   * In-place bitwise ANDNOT (difference) operation. The current bitmap is modified.
   *
   * @param x2 other bitmap
   */
  public void andNot(final Roaring64Bitmap x2) {
    if(x2 == this) {
      clear();
      return;
    }
    KeyIterator thisKeyIterator = highLowContainer.highKeyIterator();
    while (thisKeyIterator.hasNext()) {
      byte[] high = thisKeyIterator.next();
      long containerIdx = thisKeyIterator.currentContainerIdx();
      ContainerWithIndex containerWithIdx2 = x2.highLowContainer.searchContainer(high);
      if (containerWithIdx2 != null) {
        Container thisContainer = highLowContainer.getContainer(containerIdx);
        Container freshContainer = thisContainer.iandNot(containerWithIdx2.getContainer());
        highLowContainer.replaceContainer(containerIdx, freshContainer);
        if (!freshContainer.isEmpty()) {
          highLowContainer.replaceContainer(containerIdx, freshContainer);
        } else {
          thisKeyIterator.remove();
        }
      }
    }
  }

  /**
   * Complements the bits in the given range, from rangeStart (inclusive) rangeEnd (exclusive). The
   * given bitmap is unchanged.
   *
   * @param rangeStart inclusive beginning of range, in [0, 0xffffffffffffffff]
   * @param rangeEnd exclusive ending of range, in [0, 0xffffffffffffffff + 1]
   */
  public void flip(final long rangeStart, final long rangeEnd) {

    if(rangeEnd >= 0 && rangeStart >= rangeEnd){
      // both numbers in positive range, and start is beyond end, nothing to do.
      return;
    } else if(rangeStart < 0 && rangeStart >= rangeEnd){
      // both numbers in negative range, and start is beyond end, nothing to do.
      return;
    } else if(rangeStart < 0 && rangeEnd > 0) {
      // start is neg which is "higher" and end is above zero thus, nothing to do.
      return;
    }

    byte[] hbStart = LongUtils.highPart(rangeStart);
    char lbStart = LongUtils.lowPart(rangeStart);
    char lbLast = LongUtils.lowPart(rangeEnd - 1L);

    long shStart = LongUtils.rightShiftHighPart(rangeStart);
    long shEnd = LongUtils.rightShiftHighPart(rangeEnd - 1L);

    // TODO:this can be accelerated considerably
    for (long hb = shStart; hb <= shEnd; ++hb) {
      // first container may contain partial range
      final int containerStart = (hb == shStart) ? lbStart : 0;
      // last container may contain partial range
      final int containerLast = (hb == shEnd) ? lbLast : LongUtils.maxLowBitAsInteger();

      ContainerWithIndex cwi = highLowContainer.searchContainer(
              LongUtils.highPartInPlace(LongUtils.leftShiftHighPart(hb), hbStart));

      if (cwi != null) {
        final long i = cwi.getContainerIdx();
        final Container c = cwi.getContainer().inot(containerStart, containerLast + 1);
        if (!c.isEmpty()) {
          highLowContainer.replaceContainer(i, c);
        } else {
          highLowContainer.remove(hbStart);
        }
      } else {
        Container newContainer = Container.rangeOfOnes(containerStart, containerLast + 1);
        highLowContainer.put(hbStart, newContainer);
      }
    }
  }

  /**
   * {@link Roaring64NavigableMap} are serializable. However, contrary to RoaringBitmap, the
   * serialization format is not well-defined: for now, it is strongly coupled with Java standard
   * serialization. Just like the serialization may be incompatible between various Java versions,
   * {@link Roaring64NavigableMap} are subject to incompatibilities. Moreover, even on a given Java
   * versions, the serialization format may change from one RoaringBitmap version to another
   */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    serialize(out);
  }

  @Override
  public void readExternal(ObjectInput in) throws IOException {
    deserialize(in);
  }

  /**
   * A string describing the bitmap.
   *
   * @return the string
   */
  @Override
  public String toString() {
    final StringBuilder answer = new StringBuilder("{}".length() + "-1234567890123456789,".length()
        * 256);
    final LongIterator i = this.getLongIterator();
    answer.append('{');
    if (i.hasNext()) {
      answer.append(i.next());
    }
    while (i.hasNext()) {
      answer.append(',');
      // to avoid using too much memory, we limit the size
      if (answer.length() > 0x80000) {
        answer.append('.').append('.').append('.');
        break;
      }
      answer.append(i.next());
    }
    answer.append("}");
    return answer.toString();
  }


  /**
   * For better performance, consider the Use the {@link #forEach forEach} method.
   *
   * @return a custom iterator over set bits, the bits are traversed in ascending sorted order
   */
  @Override
  public PeekableLongIterator getLongIterator() {
    LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(false);
    return new ForwardPeekableIterator(leafNodeIterator);
  }

  // for testing only
  LeafNodeIterator getLeafNodeIterator() {
    return highLowContainer.highKeyLeafNodeIterator(false);
  }

  /**
   * Produce an iterator over the values in this bitmap starting from `minval`.
   *
   * @param minval the lower bound of the iterator returned
   * @return a custom iterator over set bits, the bits are traversed in ascending sorted order
   */
  public PeekableLongIterator getLongIteratorFrom(long minval) {
    LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIteratorFrom(minval, false);
    ForwardPeekableIterator fpi = new ForwardPeekableIterator(leafNodeIterator);
    fpi.advanceIfNeeded(minval); // make sure the lower end is advanced as well
    return fpi;
  }

  @Override
  public boolean contains(long x) {
    byte[] high = LongUtils.highPart(x);
    ContainerWithIndex containerWithIdx = highLowContainer.searchContainer(high);
    if (containerWithIdx == null) {
      return false;
    }
    char low = LongUtils.lowPart(x);
    return containerWithIdx.getContainer().contains(low);
  }

  @Override
  public int getSizeInBytes() {
    return (int) getLongSizeInBytes();
  }


  /**
   * Estimate of the memory usage of this data structure. This can be expected to be within 1% of
   * the true memory usage in common usage scenarios.
   * If exact measures are needed, we recommend using dedicated libraries
   * such as ehcache-sizeofengine.
   *
   * In adversarial cases, this estimate may be 10x the actual memory usage. For example, if
   * you insert a single random value in a bitmap, then over a 100 bytes may be used by the JVM
   * whereas this function may return an estimate of 32 bytes.
   *
   * The same will be true in the "sparse" scenario where you have a small set of
   * random-looking integers spanning a wide range of values.
   *
   * These are considered adversarial cases because, as a general rule,
   * if your data looks like a set
   * of random integers, Roaring bitmaps are probably not the right data structure.
   *
   * Note that you can serialize your Roaring Bitmaps to disk and then construct
   * ImmutableRoaringBitmap instances from a ByteBuffer. In such cases, the Java heap
   * usage will be significantly less than
   * what is reported.
   *
   * If your main goal is to compress arrays of integers, there are other libraries
   * that are maybe more appropriate
   * such as JavaFastPFOR.
   *
   * Note, however, that in general, random integers (as produced by random number
   * generators or hash functions) are not compressible.
   * Trying to compress random data is an adversarial use case.
   *
   * @see JavaFastPFOR
   *
   *
   * @return estimated memory usage.
   */
  @Override
  public long getLongSizeInBytes() {
    // 'serializedSizeInBytes' is a better than nothing estimation of the memory footprint
    // It would generally be an optimistic estimator (by underestimating the size in memory)
    return serializedSizeInBytes();
  }

  @Override
  public boolean isEmpty() {
    return getLongCardinality() == 0L;
  }

  @Override
  public ImmutableLongBitmapDataProvider limit(long x) {
    throw new UnsupportedOperationException("TODO");
  }

  /**
   * Use a run-length encoding where it is estimated as more space efficient
   *
   * @return whether a change was applied
   */
  public boolean runOptimize() {
    boolean hasChanged = false;
    ContainerIterator containerIterator = highLowContainer.containerIterator();
    while (containerIterator.hasNext()) {
      Container container = containerIterator.next();
      Container freshContainer = container.runOptimize();
      if (freshContainer instanceof RunContainer) {
        hasChanged = true;
        containerIterator.replace(freshContainer);
      }
    }
    return hasChanged;
  }


  /**
   * Serialize this bitmap.
   *
   * Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
   * another, and from one RoaringBitmap version to another.
   *
   * Consider calling {@link #runOptimize} before serialization to improve compression.
   *
   * The current bitmap is not modified.
   *
   * @param out the DataOutput stream
   * @throws IOException Signals that an I/O exception has occurred.
   */
  @Override
  public void serialize(DataOutput out) throws IOException {
    highLowContainer.serialize(out);
  }

  /**
   * Serialize this bitmap, please make sure the size of the serialized bytes is
   * smaller enough that ByteBuffer can hold it.
   * @param byteBuffer the ByteBuffer
   * @throws IOException Signals that an I/O exception has occurred.
   */
  public void serialize(ByteBuffer byteBuffer) throws IOException {
    highLowContainer.serialize(byteBuffer);
  }

  /**
   * Deserialize (retrieve) this bitmap.
   *
   * Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
   * another, and from one RoaringBitmap version to another.
   *
   * The current bitmap is overwritten.
   *
   * @param in the DataInput stream
   * @throws IOException Signals that an I/O exception has occurred.
   */
  public void deserialize(DataInput in) throws IOException {
    this.clear();
    highLowContainer.deserialize(in);
  }

  /**
   * Deserialize (retrieve) this bitmap.
   *
   * Unlike RoaringBitmap, there is no specification for now: it may change from one java version to
   * another, and from one RoaringBitmap version to another.
   *
   * The current bitmap is overwritten.
   *
   * @param in the ByteBuffer stream
   * @throws IOException Signals that an I/O exception has occurred.
   */
  public void deserialize(ByteBuffer in) throws IOException {
    this.clear();
    highLowContainer.deserialize(in);
  }

  @Override
  public long serializedSizeInBytes() {
    long nbBytes = highLowContainer.serializedSizeInBytes();
    return nbBytes;
  }

  /**
   * reset to an empty bitmap; result occupies as much space a newly created bitmap.
   */
  public void clear() {
    this.highLowContainer.clear();
  }

  /**
   * Return the set values as an array, if the cardinality is smaller than 2147483648. The long
   * values are in sorted order.
   *
   * @return array representing the set values.
   */
  @Override
  public long[] toArray() {
    long cardinality = this.getLongCardinality();
    if (cardinality > Integer.MAX_VALUE) {
      throw new IllegalStateException("The cardinality does not fit in an array");
    }

    final long[] array = new long[(int) cardinality];

    int pos = 0;
    LongIterator it = getLongIterator();

    while (it.hasNext()) {
      array[pos++] = it.next();
    }
    return array;
  }

  /**
   * Generate a bitmap with the specified values set to true. The provided longs values don't have
   * to be in sorted order, but it may be preferable to sort them from a performance point of view.
   *
   * @param dat set values
   * @return a new bitmap
   */
  public static Roaring64Bitmap bitmapOf(final long... dat) {
    final Roaring64Bitmap ans = new Roaring64Bitmap();
    ans.add(dat);
    return ans;
  }

  /**
   * Set all the specified values to true. This can be expected to be slightly faster than calling
   * "add" repeatedly. The provided integers values don't have to be in sorted order, but it may be
   * preferable to sort them from a performance point of view.
   *
   * @param dat set values
   */
  public void add(long... dat) {
    for (long oneLong : dat) {
      addLong(oneLong);
    }
  }

  /**
   * Add to the current bitmap all longs in [rangeStart,rangeEnd).
   *
   * @param rangeStart inclusive beginning of range
   * @param rangeEnd exclusive ending of range
   * @deprecated as this may be confused with adding individual longs
   */
  @Deprecated
  public void add(final long rangeStart, final long rangeEnd) {
    addRange(rangeStart, rangeEnd);
  }

  /**
   * Add to the current bitmap all longs in [rangeStart,rangeEnd).
   *
   * @param rangeStart inclusive beginning of range
   * @param rangeEnd exclusive ending of range
   */
  public void addRange(final long rangeStart, final long rangeEnd) {
    if (rangeEnd == 0 || Long.compareUnsigned(rangeStart, rangeEnd) >= 0) {
      throw new IllegalArgumentException("Invalid range [" + rangeStart + "," + rangeEnd + ")");
    }

    long startHigh = LongUtils.rightShiftHighPart(rangeStart);
    int startLow = LongUtils.lowPart(rangeStart);
    long endHigh = LongUtils.rightShiftHighPart(rangeEnd - 1);
    int endLow = LongUtils.lowPart(rangeEnd - 1);

    long rangeStartVal = rangeStart;
    long startHighKey = LongUtils.rightShiftHighPart(rangeStart);
    byte[] startHighKeyBytes = LongUtils.highPart(rangeStart);
    while (startHighKey <= endHigh) {
      final int containerStart = startHighKey == startHigh ? startLow : 0;
      // last container may contain partial range
      final int containerLast = startHighKey == endHigh ? endLow : Util.maxLowBitAsInteger();
      ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(startHighKeyBytes);
      if (containerWithIndex != null) {
        long containerIdx = containerWithIndex.getContainerIdx();
        Container freshContainer = highLowContainer.getContainer(containerIdx)
            .iadd(containerStart, containerLast + 1);
        highLowContainer.replaceContainer(containerIdx, freshContainer);
      } else {
        Container freshContainer = Container.rangeOfOnes(containerStart, containerLast + 1);
        highLowContainer.put(startHighKeyBytes, freshContainer);
      }

      if (LongUtils.isMaxHigh(startHighKey)) {
        break;
      }
      //increase the high
      rangeStartVal = rangeStartVal + (containerLast - containerStart) + 1;
      startHighKey = LongUtils.rightShiftHighPart(rangeStartVal);
      startHighKeyBytes = LongUtils.highPart(rangeStartVal);
    }
  }

  @Override
  public PeekableLongIterator getReverseLongIterator() {
    LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIterator(true);
    return new ReversePeekableIterator(leafNodeIterator);
  }

  /**
   * Produce an iterator over the values in this bitmap starting from `maxval`.
   *
   * @param maxval the upper bound of the iterator returned
   * @return a custom iterator over set bits, the bits are traversed in descending sorted order
   */
  public PeekableLongIterator getReverseLongIteratorFrom(long maxval) {
    LeafNodeIterator leafNodeIterator = highLowContainer.highKeyLeafNodeIteratorFrom(maxval, true);
    ReversePeekableIterator rpi = new ReversePeekableIterator(leafNodeIterator);
    rpi.advanceIfNeeded(maxval); // make sure the lower end is advanced as well
    return rpi;
  }

  @Override
  public void removeLong(long x) {
    byte[] high = LongUtils.highPart(x);
    ContainerWithIndex containerWithIdx = highLowContainer.searchContainer(high);
    if (containerWithIdx != null) {
      char low = LongUtils.lowPart(x);
      Container container = containerWithIdx.getContainer();
      Container freshContainer = container.remove(low);
      if (freshContainer.isEmpty()) {
        // Attempt to remove empty container to save memory
        highLowContainer.remove(high);
      } else {
        highLowContainer.replaceContainer(containerWithIdx.getContainerIdx(), freshContainer);
      }
    }
  }

  /**
   * remove the allocated unused memory space
   */
  @Override
  public void trim() {
    if (highLowContainer.isEmpty()) {
      return;
    }
    KeyIterator keyIterator = highLowContainer.highKeyIterator();
    while (keyIterator.hasNext()) {
      long containerIdx = keyIterator.currentContainerIdx();
      Container container = highLowContainer.getContainer(containerIdx);
      if (container.isEmpty()) {
        keyIterator.remove();
      } else {
        //TODO
        container.trim();
      }
    }
  }

  @Override
  public int hashCode() {
    return highLowContainer.hashCode();
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj) {
      return true;
    }
    if (obj == null) {
      return false;
    }
    if (getClass() != obj.getClass()) {
      return false;
    }
    Roaring64Bitmap other = (Roaring64Bitmap) obj;
    return Objects.equals(highLowContainer, other.highLowContainer);
  }


  /**
   * Add the value if it is not already present, otherwise remove it.
   *
   * @param x long value
   */
  public void flip(final long x) {
    byte[] high = LongUtils.highPart(x);
    ContainerWithIndex containerWithIndex = highLowContainer.searchContainer(high);
    if (containerWithIndex == null) {
      addLong(x);
    } else {
      char low = LongUtils.lowPart(x);
      Container freshOne = containerWithIndex.getContainer().flip(low);
      highLowContainer.replaceContainer(containerWithIndex.getContainerIdx(), freshOne);
    }
  }

  //mainly used for benchmark
  @Override
  public Roaring64Bitmap clone() {
    long sizeInBytesL = this.serializedSizeInBytes();
    if (sizeInBytesL >= Integer.MAX_VALUE) {
      throw new UnsupportedOperationException();
    }
    int sizeInBytesInt = (int) sizeInBytesL;
    ByteBuffer byteBuffer = ByteBuffer.allocate(sizeInBytesInt).order(ByteOrder.LITTLE_ENDIAN);
    try {
      this.serialize(byteBuffer);
      byteBuffer.flip();
      Roaring64Bitmap freshOne = new Roaring64Bitmap();
      freshOne.deserialize(byteBuffer);
      return freshOne;
    } catch (Exception e) {
      throw new RuntimeException("fail to clone thorough the ser/deser", e);
    }
  }


  private abstract class PeekableIterator implements PeekableLongIterator {
    private final LeafNodeIterator keyIte;
    private byte[] high;
    private PeekableCharIterator charIterator;

    PeekableIterator(final LeafNodeIterator keyIte) {
      this.keyIte = keyIte;
    }
    
    abstract PeekableCharIterator getIterator(Container container);
    abstract boolean compare(long next, long val);

    @Override
    public boolean hasNext() {
      if (charIterator != null && charIterator.hasNext()) {
        return true;
      }
      while (keyIte.hasNext()) {
        LeafNode leafNode = keyIte.next();
        high = leafNode.getKeyBytes();
        long containerIdx = leafNode.getContainerIdx();
        Container container = highLowContainer.getContainer(containerIdx);
        charIterator = getIterator(container);
        if(charIterator.hasNext()){
          return true;
        }
      }
      return false;
    }

    @Override
    public long next() {
      if (hasNext()) {
        char low = charIterator.next();
        return LongUtils.toLong(high, low);
      } else {
        throw new IllegalStateException("empty");
      }
    }

    @Override
    public void advanceIfNeeded(long minval) {
      if (!hasNext()) {
        return;
      }
      if(compare(this.peekNext(), minval)) {
        return;
      }
      //empty bitset
      if(this.high == null) {
        return;
      }

      long minHigh = LongUtils.rightShiftHighPart(minval);
      long high = LongUtils.toLong(this.high);
      if (minHigh != high) {
        // advance outer
        if (keyIte.hasNext()) {
          LeafNode leafNode = keyIte.next();
          this.high = leafNode.getKeyBytes();
          if (compare(leafNode.getKey(), minHigh)) {
            long containerIdx = leafNode.getContainerIdx();
            Container container = highLowContainer.getContainer(containerIdx);
            charIterator = getIterator(container);
            if(!charIterator.hasNext()){
              return;
            }
          } else {
            keyIte.seek(minval);
            if (keyIte.hasNext()) {
              leafNode = keyIte.next();
              this.high = leafNode.getKeyBytes();
              long containerIdx = leafNode.getContainerIdx();
              Container container = highLowContainer.getContainer(containerIdx);
              charIterator = getIterator(container);
              if(!charIterator.hasNext()){
                return;
              }
            } else {
              // make sure we don't accidentally continue at the previous iterator position
              // after stepping to the end.
              charIterator = null;
              return;
            }
          }
        }
      }

      byte[] minHighBytes = LongUtils.highPart(minval);
      if (Arrays.equals(this.high, minHighBytes)) {
        // advance inner
        char low = LongUtils.lowPart(minval);
        charIterator.advanceIfNeeded(low);
      }
    }

    @Override
    public long peekNext() {
      if (hasNext()) {
        char low = charIterator.peekNext();
        return LongUtils.toLong(high, low);
      } else {
        throw new IllegalStateException("empty");
      }
    }

    @Override
    public PeekableLongIterator clone() {
      throw new UnsupportedOperationException("TODO");
    }
  }


  private class ForwardPeekableIterator extends PeekableIterator {

    public ForwardPeekableIterator(final LeafNodeIterator keyIte) {
      super(keyIte);
    }
    
    @Override
    PeekableCharIterator getIterator(Container container) {
      return container.getCharIterator();
    }
    
    @Override
    boolean compare(long next, long val) {
      return Long.compareUnsigned(next, val) >= 0;
    }
  }

  private class ReversePeekableIterator extends PeekableIterator {
    public ReversePeekableIterator(final LeafNodeIterator keyIte) {
      super(keyIte);
    }
    
    @Override
    PeekableCharIterator getIterator(Container container) {
      return container.getReverseCharIterator();
    }
    
    @Override
    boolean compare(long next, long val) {
      return Long.compareUnsigned(next, val) <= 0;
    }
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy