All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.arcadedb.index.lsm.LSMTreeIndexCursor Maven / Gradle / Ivy

/*
 * Copyright © 2021-present Arcade Data Ltd ([email protected])
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd ([email protected])
 * SPDX-License-Identifier: Apache-2.0
 */
package com.arcadedb.index.lsm;

import com.arcadedb.database.Binary;
import com.arcadedb.database.Identifiable;
import com.arcadedb.database.RID;
import com.arcadedb.database.TransactionContext;
import com.arcadedb.database.TransactionIndexContext;
import com.arcadedb.engine.BasePage;
import com.arcadedb.engine.PageId;
import com.arcadedb.index.IndexCursor;
import com.arcadedb.index.IndexCursorEntry;
import com.arcadedb.index.TempIndexCursor;
import com.arcadedb.serializer.BinaryComparator;
import com.arcadedb.serializer.BinarySerializer;

import java.io.*;
import java.util.*;

/**
 * Index cursor doesn't remove the deleted entries.
 */
public class LSMTreeIndexCursor implements IndexCursor {
  private final LSMTreeIndexMutable                    index;
  private final boolean                                ascendingOrder;
  private final Object[]                               fromKeys;
  private final Object[]                               toKeys;
  private final Object[]                               serializedToKeys;
  private final boolean                                toKeysInclusive;
  private final LSMTreeIndexUnderlyingAbstractCursor[] pageCursors;
  private       Object[]                               currentKeys;
  private       RID[]                                  currentValues;
  private       int                                    currentValueIndex = 0;
  private final int                                    totalCursors;
  private final byte[]                                 binaryKeyTypes;
  private final Object[][]                             cursorKeys;
  private final BinaryComparator                       comparator;
  private       int                                    validIterators;
  private       TempIndexCursor                        txCursor;

  public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendingOrder) throws IOException {
    this(index, ascendingOrder, null, true, null, true);
  }

  public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendingOrder, final Object[] fromKeys, final boolean beginKeysInclusive,
      final Object[] toKeys, final boolean endKeysInclusive) throws IOException {
    this.index = index;
    this.ascendingOrder = ascendingOrder;
    this.binaryKeyTypes = index.getBinaryKeyTypes();

    index.checkForNulls(fromKeys);
    index.checkForNulls(toKeys);

    final Object[] serializedFromKeys = index.convertKeys(fromKeys, binaryKeyTypes);

    this.fromKeys = fromKeys;

    this.toKeys = toKeys != null && toKeys.length == 0 ? null : toKeys;
    this.serializedToKeys = index.convertKeys(this.toKeys, binaryKeyTypes);
    this.toKeysInclusive = endKeysInclusive;

    final BinarySerializer serializer = index.getDatabase().getSerializer();
    this.comparator = serializer.getComparator();

    final LSMTreeIndexCompacted compacted = index.getSubIndex();

    final List compactedSeriesIterators;

    if (compacted != null)
      // INCLUDE COMPACTED
      compactedSeriesIterators = compacted.newIterators(ascendingOrder, serializedFromKeys, serializedToKeys);
    else
      compactedSeriesIterators = Collections.emptyList();

    final int totalPages = index.getTotalPages();

    totalCursors = totalPages + compactedSeriesIterators.size();

    // CREATE AN ARRAY OF CURSOR. SINCE WITH LSM THE LATEST PAGE IS THE MOST UPDATED, IN THE ARRAY ARE SET FIRST THE MUTABLE ONES BECAUSE THEY ARE MORE UPDATED.
    // FROM THE LAST TO THE FIRST. THEN THE COMPACTED, FROM THE LAST TO THE FIRST
    pageCursors = new LSMTreeIndexUnderlyingAbstractCursor[totalCursors];
    cursorKeys = new Object[totalCursors][binaryKeyTypes.length];

    validIterators = 0;

    for (int i = 0; i < compactedSeriesIterators.size(); ++i) {
      LSMTreeIndexUnderlyingCompactedSeriesCursor pageCursor = compactedSeriesIterators.get(i);
      if (pageCursor != null) {
        if (pageCursor.hasNext()) {
          pageCursor.next();
          cursorKeys[totalPages + i] = pageCursor.getKeys();
        } else
          pageCursor = null;
        pageCursors[totalPages + i] = pageCursor;
      }
    }

    int pageCounter = 0;
    for (int pageId = totalPages - 1; pageId > -1; --pageId) {
      final int cursorIdx = pageCounter;

      if (serializedFromKeys != null) {
        // SEEK FOR THE FROM RANGE
        final BasePage currentPage = index.getDatabase().getTransaction().getPage(new PageId(index.getFileId(), pageId), index.getPageSize());
        final Binary currentPageBuffer = new Binary(currentPage.slice());
        final int count = index.getCount(currentPage);

        if (count > 0) {
          final LSMTreeIndexMutable.LookupResult lookupResult = index.lookupInPage(currentPage.getPageId().getPageNumber(), count, currentPageBuffer,
              serializedFromKeys, ascendingOrder ? 2 : 3);

          if (!lookupResult.outside) {
            pageCursors[cursorIdx] = index.newPageIterator(pageId, lookupResult.keyIndex, ascendingOrder);
            cursorKeys[cursorIdx] = pageCursors[cursorIdx].getKeys();

            if (ascendingOrder) {
              if (LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[cursorIdx], fromKeys) < 0) {
                pageCursors[cursorIdx] = null;
                cursorKeys[cursorIdx] = null;
              }
            } else {
              if (LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[cursorIdx], fromKeys) > 0) {
                pageCursors[cursorIdx] = null;
                cursorKeys[cursorIdx] = null;
              }
            }
          }
        }

      } else {
        if (ascendingOrder) {
          pageCursors[cursorIdx] = index.newPageIterator(pageId, -1, true);
        } else {
          final BasePage currentPage = index.getDatabase().getTransaction().getPage(new PageId(index.getFileId(), pageId), index.getPageSize());
          pageCursors[cursorIdx] = index.newPageIterator(pageId, index.getCount(currentPage), false);
        }

        if (pageCursors[cursorIdx].hasNext()) {
          pageCursors[cursorIdx].next();
          cursorKeys[cursorIdx] = pageCursors[cursorIdx].getKeys();
        } else
          pageCursors[cursorIdx] = null;
      }
      ++pageCounter;
    }

    final Set removedRIDs = new HashSet<>();
    final Set validRIDs = new HashSet<>();

    // CHECK THE VALIDITY OF CURSORS
    for (int i = 0; i < pageCursors.length; ++i) {
      final LSMTreeIndexUnderlyingAbstractCursor pageCursor = pageCursors[i];

      if (pageCursor != null) {
        if (fromKeys != null && !beginKeysInclusive) {
          if (LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[i], fromKeys) == 0) {
            // SKIP THIS
            if (pageCursor.hasNext()) {
              pageCursor.next();
              cursorKeys[i] = pageCursor.getKeys();
            } else
              // INVALID
              pageCursors[i] = null;
          }
        }

        if (this.serializedToKeys != null) {
          //final Object[] cursorKey = index.convertKeys(index.checkForNulls(pageCursor.getKeys()), keyTypes);
          final int compare = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, pageCursor.getKeys(), this.toKeys);

          if ((ascendingOrder && ((endKeysInclusive && compare <= 0) || (!endKeysInclusive && compare < 0))) || //
              (!ascendingOrder && ((endKeysInclusive && compare >= 0) || (!endKeysInclusive && compare > 0))))
            ;
          else
            // INVALID
            pageCursors[i] = null;
        }

        if (pageCursors[i] != null) {
          final RID[] rids = pageCursors[i].getValue();
          if (rids != null) {
            for (int j = rids.length - 1; j > -1; --j) {
              final RID r = rids[j];
              if (r.getBucketId() < 0) {
                final RID originalRID = index.getOriginalRID(r);
                if (!validRIDs.remove(originalRID))
                  removedRIDs.add(originalRID);
                continue;
              } else if (removedRIDs.contains(r))
                // HAS BEEN DELETED
                continue;

              validRIDs.add(r);
              validIterators++;
            }
          }
        }
      }
    }

    getClosestEntryInTx(fromKeys, beginKeysInclusive);
  }

  @Override
  public String dumpStats() {
    final StringBuilder buffer = new StringBuilder(1024);

    buffer.append(String.format("%nDUMP OF %s UNDERLYING-CURSORS on index %s", pageCursors.length, index.getName()));
    for (int i = 0; i < pageCursors.length; ++i) {
      final LSMTreeIndexUnderlyingAbstractCursor cursor = pageCursors[i];

      if (cursor == null)
        buffer.append(String.format("%n- Cursor[%d] = null", i));
      else {
        buffer.append(String.format("%n- Cursor[%d] %s=%s index=%s compacted=%s totalKeys=%d ascending=%s keyTypes=%s currentPageId=%s currentPosInPage=%d", i,
            Arrays.toString(cursorKeys[i]), Arrays.toString(cursor.getValue()), cursor.index, cursor instanceof LSMTreeIndexUnderlyingCompactedSeriesCursor,
            cursor.totalKeys, cursor.ascendingOrder, Arrays.toString(cursor.keyTypes), cursor.getCurrentPageId(), cursor.getCurrentPositionInPage()));
      }
    }

    return buffer.toString();
  }

  @Override
  public BinaryComparator getComparator() {
    return comparator;
  }

  @Override
  public byte[] getBinaryKeyTypes() {
    return binaryKeyTypes;
  }

  @Override
  public long estimateSize() {
    return -1L;
  }

  @Override
  public boolean hasNext() {
    return validIterators > 0 || (currentValues != null && currentValueIndex < currentValues.length) || txCursor != null;
  }

  @Override
  public RID next() {
    do {
      if (currentValues != null && currentValueIndex < currentValues.length) {
        final RID value = currentValues[currentValueIndex++];
        if (!index.isDeletedEntry(value))
          return value;

        continue;
      }

      currentValueIndex = 0;

      Object[] minorKey = null;
      final List minorKeyIndexes = new ArrayList<>();

      // FIND THE MINOR KEY
      for (int p = 0; p < totalCursors; ++p) {
        if (pageCursors[p] != null) {
          if (minorKey == null) {
            minorKey = cursorKeys[p];
            minorKeyIndexes.add(p);
          } else {
            if (cursorKeys[p] != null) {
              final int compare = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[p], minorKey);
              if (compare == 0) {
                minorKeyIndexes.add(p);
              } else if ((ascendingOrder && compare < 0) || (!ascendingOrder && compare > 0)) {
                minorKey = cursorKeys[p];
                minorKeyIndexes.clear();
                minorKeyIndexes.add(p);
              }
            }
          }
        }
      }

      if (txCursor != null && txCursor.hasNext()) {
        currentValues = new RID[] { (RID) txCursor.next() };

        final Object[] txKeys = txCursor.getKeys();
        if (minorKey != null) {
          final int compare = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, txKeys, minorKey);
          if (compare == 0) {
          } else if ((ascendingOrder && compare < 0) || (!ascendingOrder && compare > 0)) {
            minorKey = txKeys;
            minorKeyIndexes.clear();
          }
        } else
          minorKey = txKeys;
      }

      if (minorKey == null) {
        validIterators = 0;
        return null;//throw new NoSuchElementException();
      }

      currentKeys = minorKey;

      // FILTER DELETED ITEMS
      final Set removedRIDs = new HashSet<>();
      final Set validRIDs = new HashSet<>();

      boolean removedEntry = false;
      for (int i = 0; i < minorKeyIndexes.size(); ++i) {
        final int minorKeyIndex = minorKeyIndexes.get(i);

        final LSMTreeIndexUnderlyingAbstractCursor currentCursor = pageCursors[minorKeyIndex];
        currentKeys = currentCursor.getKeys();

        final RID[] tempCurrentValues = currentCursor.getValue();

        if (i == 0 || currentValues == null)
          currentValues = tempCurrentValues;
        else if (tempCurrentValues.length > 0) {
          // MERGE VALUES
          final RID[] newArray = Arrays.copyOf(currentValues, currentValues.length + tempCurrentValues.length);
          System.arraycopy(tempCurrentValues, 0, newArray, currentValues.length, newArray.length - currentValues.length);
          currentValues = newArray;
        }

        // START FROM THE LAST ENTRY
        for (int k = currentValues.length - 1; k > -1; --k) {
          final RID rid = currentValues[k];

          if (index.REMOVED_ENTRY_RID.equals(rid)) {
            removedEntry = true;
            break;
          }

          if (rid.getBucketId() < 0) {
            // RID DELETED, SKIP THE RID
            final RID originalRID = index.getOriginalRID(rid);
            if (!validRIDs.contains(originalRID))
              removedRIDs.add(originalRID);
            continue;
          }

          if (removedRIDs.contains(rid))
            // HAS BEEN DELETED
            continue;

          validRIDs.add(rid);
        }

        // PREPARE THE NEXT ENTRY
        if (currentCursor.hasNext()) {
          currentCursor.next();
          cursorKeys[minorKeyIndex] = currentCursor.getKeys();

          if (serializedToKeys != null) {
            final int compare = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[minorKeyIndex], toKeys);

            if ((ascendingOrder && ((toKeysInclusive && compare > 0) || (!toKeysInclusive && compare >= 0))) || (!ascendingOrder && (
                (toKeysInclusive && compare < 0) || (!toKeysInclusive && compare <= 0)))) {
              currentCursor.close();
              pageCursors[minorKeyIndex] = null;
              cursorKeys[minorKeyIndex] = null;
              --validIterators;
            }
          }
        } else {
          currentCursor.close();
          pageCursors[minorKeyIndex] = null;
          cursorKeys[minorKeyIndex] = null;
          --validIterators;
        }

        if (removedEntry) {
          currentValues = null;
          break;
        }

        if (validRIDs.isEmpty())
          currentValues = null;
        else
          validRIDs.toArray(currentValues);
      }

      if (txCursor == null || !txCursor.hasNext())
        getClosestEntryInTx(currentKeys != null ? currentKeys : fromKeys, false);

    } while ((currentValues == null || currentValues.length == 0 || (currentValueIndex < currentValues.length && index.isDeletedEntry(
        currentValues[currentValueIndex]))) && hasNext());

    return currentValues == null || currentValueIndex >= currentValues.length ? null : currentValues[currentValueIndex++];
  }

  private void getClosestEntryInTx(final Object[] keys, final boolean inclusive) {
    txCursor = null;
    if (index.getDatabase().getTransaction().getStatus() == TransactionContext.STATUS.BEGUN) {
      Set txChanges = null;

      final TreeMap> indexChanges = index.getDatabase()
          .getTransaction().getIndexChanges().getIndexKeys(index.getName());
      if (indexChanges != null) {
        final Map.Entry> entry;
        if (ascendingOrder) {
          if (keys == null)
            entry = indexChanges.firstEntry();
          else if (inclusive)
            entry = indexChanges.ceilingEntry(new TransactionIndexContext.ComparableKey(keys));
          else
            entry = indexChanges.higherEntry(new TransactionIndexContext.ComparableKey(keys));
        } else {
          if (keys == null)
            entry = indexChanges.lastEntry();
          else if (inclusive)
            entry = indexChanges.floorEntry(new TransactionIndexContext.ComparableKey(keys));
          else
            entry = indexChanges.lowerEntry(new TransactionIndexContext.ComparableKey(keys));
        }

        final Map values = entry != null ? entry.getValue() : null;
        if (values != null) {
          for (final TransactionIndexContext.IndexKey value : values.values()) {
            if (value != null) {
              if (!value.addOperation)
                // REMOVED
                break;

              final Object[] tmpKeys = entry.getKey().values;

              if (toKeys != null) {
                final int cmp = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, tmpKeys, toKeys);

                if (cmp > 0)
                  continue;
                else if (!toKeysInclusive && cmp == 0)
                  continue;
              }

              if (txChanges == null)
                txChanges = new HashSet<>();

              txChanges.add(new IndexCursorEntry(tmpKeys, value.rid, 1));
            }
          }
        }
      }

      if (txChanges != null) {
        // MERGE SETS
        txCursor = new TempIndexCursor(txChanges);
      }
    }

  }

  @Override
  public Object[] getKeys() {
    return currentKeys;
  }

  @Override
  public Identifiable getRecord() {
    if (currentValues != null && currentValueIndex < currentValues.length) {
      final RID value = currentValues[currentValueIndex];
      if (!index.isDeletedEntry(value))
        return value;
    }
    return null;
  }

  @Override
  public int getScore() {
    return 1;
  }

  @Override
  public void close() {
    for (final LSMTreeIndexUnderlyingAbstractCursor it : pageCursors)
      if (it != null)
        it.close();
    Arrays.fill(pageCursors, null);
  }

  @Override
  public Iterator iterator() {
    return this;
  }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy