All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.deephaven.engine.table.impl.ReverseOperation Maven / Gradle / Ivy

There is a newer version: 0.37.1
Show newest version
/**
 * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
 */
package io.deephaven.engine.table.impl;

import io.deephaven.base.verify.Assert;
import io.deephaven.engine.rowset.*;
import io.deephaven.engine.rowset.RowSetFactory;
import io.deephaven.engine.rowset.impl.rsp.RspArray;
import io.deephaven.engine.table.ColumnSource;
import io.deephaven.engine.table.ModifiedColumnSet;
import io.deephaven.engine.table.TableUpdate;
import io.deephaven.engine.table.TableUpdateListener;
import io.deephaven.engine.table.impl.sources.ReversedColumnSource;
import io.deephaven.util.annotations.VisibleForTesting;
import org.jetbrains.annotations.NotNull;

import java.util.LinkedHashMap;
import java.util.Map;

public class ReverseOperation implements QueryTable.MemoizableOperation {

    private final QueryTable parent;
    private QueryTable resultTable;
    private ModifiedColumnSet.Transformer mcsTransformer;

    /**
     * Minimum pivot is RSP container size. This guarantees that we only generate shifts that are a multiple of
     * container size, which is important if we're using an RSP-backed OrderedLongSet to implement our RowSet.
     */
    @VisibleForTesting
    static final long MINIMUM_PIVOT = RspArray.BLOCK_SIZE;
    /**
     * Maximum pivot is the maximum possible row key.
     */
    private static final long MAXIMUM_PIVOT = Long.MAX_VALUE;
    /**
     * Since we are using highest one bit of maximum parent row key, this should be a power of two greater than one.
     */
    private static final int PIVOT_GROWTH_FACTOR = 4;

    private long pivotPoint;
    private long prevPivotPoint;
    private long lastPivotPointChange;

    public ReverseOperation(QueryTable parent) {
        this.parent = parent;
    }

    @Override
    public String getDescription() {
        return "reverse()";
    }

    @Override
    public String getLogPrefix() {
        return "reverse";
    }

    @Override
    public MemoizedOperationKey getMemoizedOperationKey() {
        return MemoizedOperationKey.reverse();
    }

    @Override
    public OperationSnapshotControl newSnapshotControl(QueryTable queryTable) {
        return new OperationSnapshotControl(queryTable) {
            @Override
            public synchronized boolean snapshotCompletedConsistently(
                    final long afterClockValue,
                    final boolean usedPreviousValues) {
                final boolean success = super.snapshotCompletedConsistently(afterClockValue, usedPreviousValues);
                if (success) {
                    QueryTable.startTrackingPrev(resultTable.getColumnSources());
                }
                return success;
            }
        };
    }

    @Override
    public Result initialize(boolean usePrev, long beforeClock) {
        final RowSet rowSetToReverse = usePrev ? parent.getRowSet().prev() : parent.getRowSet();
        prevPivotPoint = pivotPoint = computePivot(rowSetToReverse.lastRowKey());
        lastPivotPointChange = usePrev ? beforeClock - 1 : beforeClock;

        final Map> resultColumnSources = new LinkedHashMap<>();
        for (Map.Entry> entry : parent.getColumnSourceMap().entrySet()) {
            resultColumnSources.put(entry.getKey(), new ReversedColumnSource<>(entry.getValue(), this));
        }

        final TrackingWritableRowSet resultRowSet = transform(rowSetToReverse).toTracking();
        Assert.eq(resultRowSet.size(), "resultRowSet.size()", rowSetToReverse.size(), "rowSetToReverse.size()");

        resultTable = new QueryTable(parent.getDefinition(), resultRowSet, resultColumnSources);
        mcsTransformer = parent.newModifiedColumnSetIdentityTransformer(resultTable);
        parent.copyAttributes(resultTable, BaseTable.CopyAttributeOperation.Reverse);

        if (!parent.isRefreshing()) {
            return new Result<>(resultTable);
        }

        final TableUpdateListener listener =
                new BaseTable.ListenerImpl(getDescription(), parent, resultTable) {
                    @Override
                    public void onUpdate(final TableUpdate upstream) {
                        ReverseOperation.this.onUpdate(upstream);
                    }
                };

        return new Result<>(resultTable, listener);
    }

    private void onUpdate(final TableUpdate upstream) {
        final WritableRowSet resultRowSet = resultTable.getRowSet().writableCast();
        final TrackingRowSet parentRowSet = parent.getRowSet();
        if (resultRowSet.size() != parentRowSet.sizePrev()) {
            QueryTable.log.error()
                    .append("Result Size Mismatch: Pre-Update Result RowSet: ")
                    .append(resultRowSet).append(" size=").append(resultRowSet.size())
                    .append(", Parent RowSet: ")
                    .append(parentRowSet).append(" size=").append(parentRowSet.size())
                    .append(", Parent Previous RowSet: ")
                    .append(parentRowSet.prev()).append(" size=").append(parentRowSet.sizePrev())
                    .append(", Added: ").append(upstream.added()).append(" size=").append(upstream.added().size())
                    .append(", Removed: ").append(upstream.removed()).append(" size=").append(upstream.removed().size())
                    .endl();
            Assert.eq(resultRowSet.size(), "resultRowSet.size()", parentRowSet.sizePrev(), "parentRowSet.sizePrev()");
        }

        if (parentRowSet.size() != (resultRowSet.size() + upstream.added().size() - upstream.removed().size())) {
            QueryTable.log.error()
                    .append("Parent Size Mismatch: Pre-Update Result RowSet: ")
                    .append(resultRowSet).append(" size=").append(resultRowSet.size())
                    .append(", Parent RowSet: ")
                    .append(parentRowSet).append(" size=").append(parentRowSet.size())
                    .append(", Added: ").append(upstream.added()).append(" size=").append(upstream.added().size())
                    .append(", Removed: ").append(upstream.removed()).append(" size=").append(upstream.removed().size())
                    .endl();
            Assert.eq(parentRowSet.size(), "parentRowSet.size()",
                    resultRowSet.size() + upstream.added().size() - upstream.removed().size(),
                    "resultRowSet.size() + upstream.added().size() - upstream.removed().size()");
        }

        final TableUpdateImpl downstream = new TableUpdateImpl();

        // removed is in pre-shift keyspace
        downstream.removed = transform(upstream.removed());
        resultRowSet.remove(downstream.removed());

        // transform shifted and apply to our RowSet
        final long newShift =
                (parentRowSet.lastRowKey() > pivotPoint) ? computePivot(parentRowSet.lastRowKey()) - pivotPoint : 0;
        if (upstream.shifted().nonempty() || newShift > 0) {
            // Only compute downstream shifts if there are retained rows to shift
            if (resultRowSet.isEmpty()) {
                downstream.shifted = RowSetShiftData.EMPTY;
            } else {
                long watermarkKey = 0;
                final RowSetShiftData.Builder oShiftedBuilder = new RowSetShiftData.Builder();

                // Bounds seem weird because we might need to shift all keys outside of shifts too.
                for (int idx = upstream.shifted().size(); idx >= 0; --idx) {
                    final long nextShiftEnd;
                    final long nextShiftStart;
                    final long nextShiftDelta;
                    if (idx == 0) {
                        nextShiftStart = nextShiftEnd = pivotPoint + 1;
                        nextShiftDelta = 0;
                    } else {
                        // Note: begin/end flip responsibilities in the transformation
                        nextShiftDelta = -upstream.shifted().getShiftDelta(idx - 1);
                        final long minStart = Math.max(-nextShiftDelta - newShift, 0);
                        nextShiftStart = Math.max(minStart, transform(upstream.shifted().getEndRange(idx - 1)));
                        nextShiftEnd = transform(upstream.shifted().getBeginRange(idx - 1));
                        if (nextShiftEnd < nextShiftStart) {
                            continue;
                        }
                    }

                    // insert range prior to here; note shift ends are inclusive so we need the -1 for endRange
                    long innerEnd = nextShiftStart - 1 + (nextShiftDelta < 0 ? nextShiftDelta : 0);
                    oShiftedBuilder.shiftRange(watermarkKey, innerEnd, newShift);

                    if (idx == 0) {
                        continue;
                    }

                    // insert this range
                    oShiftedBuilder.shiftRange(nextShiftStart, nextShiftEnd, newShift + nextShiftDelta);
                    watermarkKey = nextShiftEnd + 1 + (nextShiftDelta > 0 ? nextShiftDelta : 0);
                }

                downstream.shifted = oShiftedBuilder.build();
                downstream.shifted().apply(resultRowSet);
            }

            // Update pivot logic.
            lastPivotPointChange = parent.getUpdateGraph().clock().currentStep();
            prevPivotPoint = pivotPoint;
            pivotPoint += newShift;
        } else {
            downstream.shifted = RowSetShiftData.EMPTY;
        }

        // added/modified are in post-shift keyspace
        downstream.added = transform(upstream.added());
        resultRowSet.insert(downstream.added());
        downstream.modified = transform(upstream.modified());

        Assert.eq(downstream.added().size(), "update.added.size()", upstream.added().size(), "upstream.added.size()");
        Assert.eq(downstream.removed().size(), "update.removed.size()", upstream.removed().size(),
                "upstream.removed.size()");
        Assert.eq(downstream.modified().size(), "update.modified.size()", upstream.modified().size(),
                "upstream.modified.size()");

        downstream.modifiedColumnSet = resultTable.getModifiedColumnSetForUpdates();
        downstream.modifiedColumnSet.clear();
        if (downstream.modified().isNonempty()) {
            mcsTransformer.transform(upstream.modifiedColumnSet(), downstream.modifiedColumnSet);
        }

        if (resultRowSet.size() != parentRowSet.size()) {
            QueryTable.log.error()
                    .append("Size Mismatch: Result RowSet: ").append(resultRowSet)
                    .append("Parent RowSet: ").append(parentRowSet)
                    .append("Upstream Update: ").append(upstream)
                    .append("Downstream Update: ").append(downstream)
                    .endl();
            Assert.eq(resultRowSet.size(), "resultRowSet.size()", parentRowSet.size(), "parentRowSet.size()");
        }

        resultTable.notifyListeners(downstream);
    }

    private static long computePivot(final long parentLastRowKey) {
        final long highestOneBit = Long.highestOneBit(parentLastRowKey);
        if (highestOneBit > (MAXIMUM_PIVOT / PIVOT_GROWTH_FACTOR)) {
            return MAXIMUM_PIVOT;
        } else {
            // Make it big enough that we should be able to accommodate what we are adding now, plus a bit more
            return Math.max(highestOneBit * PIVOT_GROWTH_FACTOR - 1, MINIMUM_PIVOT);
        }
    }

    private long getPrevPivotPoint() {
        if ((prevPivotPoint != pivotPoint)) {
            if (parent.getUpdateGraph().clock().currentStep() != lastPivotPointChange) {
                prevPivotPoint = pivotPoint;
            }
        }
        return prevPivotPoint;
    }

    /**
     * Transform an outer (reversed) RowSet to the inner (un-reversed) RowSet, or vice versa.
     *
     * @param outerRowSet The outer (reversed) RowSet
     * @return The corresponding inner RowSet
     */
    public WritableRowSet transform(@NotNull final RowSet outerRowSet) {
        return transform(outerRowSet, false);
    }

    /**
     * Transform an outer (reversed) RowSet to the inner (un-reversed) RowSet as of the previous cycle, or vice versa.
     *
     * @param outerRowSet The outer (reversed) RowSet
     * @return The corresponding inner RowSet
     */
    public WritableRowSet transformPrev(@NotNull final RowSet outerRowSet) {
        return transform(outerRowSet, true);
    }

    private WritableRowSet transform(@NotNull final RowSet outerRowSet, final boolean usePrev) {
        final long pivot = usePrev ? getPrevPivotPoint() : pivotPoint;
        final RowSetBuilderRandom reversedBuilder = RowSetFactory.builderRandom();

        for (final RowSet.RangeIterator rangeIterator = outerRowSet.rangeIterator(); rangeIterator.hasNext();) {
            rangeIterator.next();
            final long startValue = rangeIterator.currentRangeStart();
            final long endValue = rangeIterator.currentRangeEnd();
            final long transformedStart = (startValue < 0) ? startValue : pivot - startValue;
            final long transformedEnd = (endValue < 0) ? endValue : pivot - endValue;
            Assert.geqZero(transformedStart, "transformedStart");
            Assert.geqZero(transformedEnd, "transformedEnd");
            Assert.leq(transformedEnd, "transformedEnd", transformedStart, "transformedStart");
            reversedBuilder.addRange(transformedEnd, transformedStart);
        }

        return reversedBuilder.build();
    }

    /**
     * Transform an outer (reversed) row key to the inner (un-reversed) row key, or vice versa.
     *
     * @param outerRowKey The outer (reversed) row key
     * @return The corresponding inner row key
     */
    public long transform(final long outerRowKey) {
        return (outerRowKey < 0) ? outerRowKey : pivotPoint - outerRowKey;
    }

    /**
     * Transform an outer (reversed) row key to the inner (un-reversed) row key as of the previous cycle, or vice versa.
     *
     * @param outerRowKey The outer (reversed) row key
     * @return The corresponding inner row key
     */
    public long transformPrev(final long outerRowKey) {
        return (outerRowKey < 0) ? outerRowKey : getPrevPivotPoint() - outerRowKey;
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy