All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.deephaven.engine.table.impl.SortOperation Maven / Gradle / Ivy

There is a newer version: 0.37.1
Show newest version
/**
 * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
 */
package io.deephaven.engine.table.impl;

import io.deephaven.base.verify.Assert;
import io.deephaven.base.verify.Require;
import io.deephaven.engine.rowset.*;
import io.deephaven.engine.rowset.RowSetFactory;
import io.deephaven.engine.table.*;
import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
import io.deephaven.engine.table.iterators.ChunkedLongColumnIterator;
import io.deephaven.engine.table.iterators.LongColumnIterator;
import io.deephaven.util.datastructures.hash.HashMapK4V4;
import io.deephaven.util.datastructures.hash.HashMapLockFreeK4V4;
import io.deephaven.engine.table.impl.sources.RedirectedColumnSource;
import io.deephaven.engine.table.impl.sources.SwitchColumnSource;
import io.deephaven.chunk.LongChunk;
import io.deephaven.chunk.WritableLongChunk;
import io.deephaven.engine.table.impl.sources.chunkcolumnsource.LongChunkColumnSource;
import io.deephaven.engine.table.impl.util.*;
import io.deephaven.util.SafeCloseableList;

import org.apache.commons.lang3.mutable.MutableObject;
import org.jetbrains.annotations.NotNull;

import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.LongUnaryOperator;

import static io.deephaven.engine.table.Table.SORT_REVERSE_LOOKUP_ATTRIBUTE;

public class SortOperation implements QueryTable.MemoizableOperation {

    private final QueryTable parent;
    private QueryTable resultTable;
    private RowRedirection sortMapping;

    private final SortPair[] sortPairs;
    private final SortingOrder[] sortOrder;
    private final String[] sortColumnNames;
    private final ColumnSource>[] sortColumns;

    public SortOperation(QueryTable parent, SortPair[] sortPairs) {
        this.parent = parent;
        this.sortPairs = sortPairs;
        this.sortOrder = Arrays.stream(sortPairs).map(SortPair::getOrder).toArray(SortingOrder[]::new);
        this.sortColumnNames = Arrays.stream(sortPairs).map(SortPair::getColumn).toArray(String[]::new);

        // noinspection unchecked
        sortColumns = new ColumnSource[sortColumnNames.length];

        for (int ii = 0; ii < sortColumnNames.length; ++ii) {
            // noinspection unchecked
            sortColumns[ii] = (ColumnSource>) ReinterpretUtils
                    .maybeConvertToPrimitive(parent.getColumnSource(sortColumnNames[ii]));

            Require.requirement(
                    Comparable.class.isAssignableFrom(sortColumns[ii].getType())
                            || sortColumns[ii].getType().isPrimitive(),
                    "Comparable.class.isAssignableFrom(sortColumns[ii].getType()) || sortColumns[ii].getType().isPrimitive()",
                    sortColumnNames[ii], "sortColumnNames[ii]", sortColumns[ii].getType(), "sortColumns[ii].getType()");
        }

        parent.assertSortable(sortColumnNames);
    }

    @Override
    public String getDescription() {
        return "sort(" + Arrays.toString(sortPairs) + ")";
    }

    @Override
    public String getLogPrefix() {
        return "sort";
    }

    @Override
    public MemoizedOperationKey getMemoizedOperationKey() {
        return MemoizedOperationKey.sort(sortPairs);
    }

    @Override
    public OperationSnapshotControl newSnapshotControl(QueryTable queryTable) {
        return new OperationSnapshotControl(queryTable) {
            @Override
            public synchronized boolean snapshotCompletedConsistently(
                    final long afterClockValue,
                    final boolean usedPreviousValues) {
                final boolean success = super.snapshotCompletedConsistently(afterClockValue, usedPreviousValues);
                if (success) {
                    QueryTable.startTrackingPrev(resultTable.getColumnSources());
                    if (sortMapping.isWritable()) {
                        sortMapping.writableCast().startTrackingPrevValues();
                    }
                }
                return success;
            }
        };
    }

    private static boolean alreadySorted(final QueryTable parent, @NotNull final SortHelpers.SortMapping sortedKeys) {
        if (sortedKeys.size() == 0) {
            return true;
        }
        final RowSet.Iterator it = parent.getRowSet().iterator();
        return sortedKeys.forEachLong(currentKey -> currentKey == it.nextLong());
    }

    @NotNull
    private QueryTable historicalSort(SortHelpers.SortMapping sortedKeys) {
        if (alreadySorted(parent, sortedKeys)) {
            return withSorted(parent);
        }

        final WritableRowRedirection sortMapping = sortedKeys.makeHistoricalRowRedirection();
        final TrackingRowSet resultRowSet = RowSetFactory.flat(sortedKeys.size()).toTracking();

        final Map> resultMap = new LinkedHashMap<>();
        for (Map.Entry> stringColumnSourceEntry : parent.getColumnSourceMap().entrySet()) {
            resultMap.put(stringColumnSourceEntry.getKey(),
                    RedirectedColumnSource.maybeRedirect(sortMapping, stringColumnSourceEntry.getValue()));
        }

        resultTable = new QueryTable(resultRowSet, resultMap);
        parent.copyAttributes(resultTable, BaseTable.CopyAttributeOperation.Sort);
        resultTable.setFlat();
        setSorted(resultTable);
        return resultTable;
    }

    @NotNull
    private Result blinkTableSort(@NotNull final SortHelpers.SortMapping initialSortedKeys) {
        final LongChunkColumnSource initialInnerRedirectionSource = new LongChunkColumnSource();
        if (initialSortedKeys.size() > 0) {
            initialInnerRedirectionSource
                    .addChunk(WritableLongChunk.writableChunkWrap(initialSortedKeys.getArrayMapping()));
        }
        final MutableObject recycledInnerRedirectionSource = new MutableObject<>();
        final SwitchColumnSource redirectionSource = new SwitchColumnSource<>(initialInnerRedirectionSource,
                (final ColumnSource previousInnerRedirectionSource) -> {
                    final LongChunkColumnSource recycled = (LongChunkColumnSource) previousInnerRedirectionSource;
                    recycled.clear();
                    recycledInnerRedirectionSource.setValue(recycled);
                });

        sortMapping = new LongColumnSourceRowRedirection<>(redirectionSource);
        final TrackingWritableRowSet resultRowSet =
                RowSetFactory.flat(initialSortedKeys.size()).toTracking();

        final Map> resultMap = new LinkedHashMap<>();
        for (Map.Entry> stringColumnSourceEntry : parent.getColumnSourceMap().entrySet()) {
            resultMap.put(stringColumnSourceEntry.getKey(),
                    RedirectedColumnSource.maybeRedirect(sortMapping, stringColumnSourceEntry.getValue()));
        }

        resultTable = new QueryTable(resultRowSet, resultMap);
        parent.copyAttributes(resultTable, BaseTable.CopyAttributeOperation.Sort);
        resultTable.setFlat();
        setSorted(resultTable);

        final TableUpdateListener resultListener =
                new BaseTable.ListenerImpl("Stream sort listener", parent, resultTable) {
                    @Override
                    public void onUpdate(@NotNull final TableUpdate upstream) {
                        Assert.assertion(upstream.modified().isEmpty() && upstream.shifted().empty(),
                                "upstream.modified.empty() && upstream.shifted.empty()");
                        Assert.eq(resultRowSet.size(), "resultRowSet.size()", upstream.removed().size(),
                                "upstream.removed.size()");
                        if (upstream.empty()) {
                            return;
                        }

                        final SortHelpers.SortMapping updateSortedKeys =
                                SortHelpers.getSortedKeys(sortOrder, sortColumns, upstream.added(), false, false);
                        final LongChunkColumnSource recycled = recycledInnerRedirectionSource.getValue();
                        recycledInnerRedirectionSource.setValue(null);
                        final LongChunkColumnSource updateInnerRedirectSource =
                                recycled == null ? new LongChunkColumnSource() : recycled;
                        if (updateSortedKeys.size() > 0) {
                            updateInnerRedirectSource
                                    .addChunk(WritableLongChunk.writableChunkWrap(updateSortedKeys.getArrayMapping()));
                        }
                        redirectionSource.setNewCurrent(updateInnerRedirectSource);

                        final RowSet added = RowSetFactory.flat(upstream.added().size());
                        final RowSet removed = RowSetFactory.flat(upstream.removed().size());
                        if (added.size() > removed.size()) {
                            resultRowSet.insertRange(removed.size(), added.size() - 1);
                        } else if (removed.size() > added.size()) {
                            resultRowSet.removeRange(added.size(), removed.size() - 1);
                        }
                        resultTable.notifyListeners(new TableUpdateImpl(added, removed, RowSetFactory.empty(),
                                RowSetShiftData.EMPTY, ModifiedColumnSet.EMPTY));
                    }
                };

        return new Result<>(resultTable, resultListener);
    }

    private void setSorted(QueryTable table) {
        // no matter what we are always sorted by the first column
        SortedColumnsAttribute.setOrderForColumn(table, sortColumnNames[0], sortOrder[0]);
    }

    private QueryTable withSorted(QueryTable table) {
        return (QueryTable) SortedColumnsAttribute.withOrderForColumn(table, sortColumnNames[0], sortOrder[0]);
    }

    @Override
    public Result initialize(boolean usePrev, long beforeClock) {
        if (!parent.isRefreshing()) {
            final SortHelpers.SortMapping sortedKeys =
                    SortHelpers.getSortedKeys(sortOrder, sortColumns, parent.getRowSet(), false);
            return new Result<>(historicalSort(sortedKeys));
        }
        if (parent.isBlink()) {
            try (final RowSet prevIndex = usePrev ? parent.getRowSet().copyPrev() : null) {
                final RowSet indexToUse = usePrev ? prevIndex : parent.getRowSet();
                final SortHelpers.SortMapping sortedKeys =
                        SortHelpers.getSortedKeys(sortOrder, sortColumns, indexToUse, usePrev);
                return blinkTableSort(sortedKeys);
            }
        }

        try (final SafeCloseableList closer = new SafeCloseableList()) {
            // reset the sort data structures that we share between invocations
            final Map> resultMap = new LinkedHashMap<>();

            final RowSet rowSetToSort = usePrev ? closer.add(parent.getRowSet().copyPrev()) : parent.getRowSet();

            if (rowSetToSort.size() >= Integer.MAX_VALUE) {
                throw new UnsupportedOperationException("Can not perform ticking sort for table larger than "
                        + Integer.MAX_VALUE + " rows, table is" + rowSetToSort.size());
            }

            final long[] sortedKeys =
                    SortHelpers.getSortedKeys(sortOrder, sortColumns, rowSetToSort, usePrev).getArrayMapping();

            final HashMapK4V4 reverseLookup = new HashMapLockFreeK4V4(sortedKeys.length, .75f, -3);
            sortMapping = SortHelpers.createSortRowRedirection();

            // Center the keys around middleKeyToUse
            final long offset = SortListener.REBALANCE_MIDPOINT - sortedKeys.length / 2;
            final TrackingRowSet resultRowSet = (sortedKeys.length == 0
                    ? RowSetFactory.empty()
                    : RowSetFactory.fromRange(offset, offset + sortedKeys.length - 1)).toTracking();

            for (int i = 0; i < sortedKeys.length; i++) {
                reverseLookup.put(sortedKeys[i], i + offset);
            }

            // fillFromChunk may convert the provided RowSequence to a KeyRanges (or RowKeys) chunk that is owned by
            // the RowSequence and is not closed until the RowSequence is closed.
            ChunkSink.FillFromContext fillFromContext =
                    closer.add(sortMapping.writableCast().makeFillFromContext(sortedKeys.length));
            sortMapping.writableCast().fillFromChunk(fillFromContext, LongChunk.chunkWrap(sortedKeys),
                    closer.add(resultRowSet.copy()));

            for (Map.Entry> stringColumnSourceEntry : parent.getColumnSourceMap().entrySet()) {
                resultMap.put(stringColumnSourceEntry.getKey(),
                        RedirectedColumnSource.maybeRedirect(sortMapping, stringColumnSourceEntry.getValue()));
            }

            // noinspection unchecked
            final ColumnSource>[] sortedColumnsToSortBy =
                    Arrays.stream(sortColumnNames).map(resultMap::get).toArray(ColumnSource[]::new);
            // we also reinterpret our sortedColumnsToSortBy, which are guaranteed to be redirected sources of the inner
            // source
            for (int ii = 0; ii < sortedColumnsToSortBy.length; ++ii) {
                // noinspection unchecked
                sortedColumnsToSortBy[ii] = (ColumnSource>) ReinterpretUtils
                        .maybeConvertToPrimitive(sortedColumnsToSortBy[ii]);
            }

            resultTable = new QueryTable(resultRowSet, resultMap);
            parent.copyAttributes(resultTable, BaseTable.CopyAttributeOperation.Sort);
            setReverseLookup(resultTable, (final long innerRowKey) -> {
                final long outerRowKey = reverseLookup.get(innerRowKey);
                return outerRowKey == reverseLookup.getNoEntryValue() ? RowSequence.NULL_ROW_KEY : outerRowKey;
            });

            final SortListener listener = new SortListener(parent, resultTable, reverseLookup, sortColumns, sortOrder,
                    sortMapping.writableCast(), sortedColumnsToSortBy,
                    parent.newModifiedColumnSetIdentityTransformer(resultTable),
                    parent.newModifiedColumnSet(sortColumnNames));

            setSorted(resultTable);

            return new Result<>(resultTable, listener);
        }
    }

    /**
     * Get the row redirection for a sort result.
     *
     * @param sortResult The sort result table; must be the direct result of a sort.
     * @return The row redirection if at least one column required redirection, otherwise {@code null}
     */
    public static RowRedirection getRowRedirection(@NotNull final Table sortResult) {
        for (final ColumnSource columnSource : sortResult.getColumnSources()) {
            if (columnSource instanceof RedirectedColumnSource) {
                return ((RedirectedColumnSource) columnSource).getRowRedirection();
            }
        }
        return null;
    }

    /**
     * Get a reverse lookup for a sort result, providing a mapping from input row key (that is, in the parent table's
     * row key space) to output row key (that is, in the sorted table's row space). Unknown input row keys are mapped to
     * the {@link RowSequence#NULL_ROW_KEY null row key}. This is effectively the reverse of the mapping provided by the
     * sort's {@link RowRedirection}.
     * 

* Unsupported if the sort result's parent was a {@link BaseTable#isBlink() blink table}. *

* For refreshing tables, using the reverse lookup concurrently requires careful consideration. The mappings are * always against "current" data. It is only safe to use before the parent table notifies on a given cycle, or after * the sorted table notifies (and during idle phases). *

* For static tables, do note that the reverse lookup will be produced on-demand within this method. * * @param parent The sort input table; must have been sorted in order to produce {@code sortResult} * @param sortResult The sort result table; must be the direct result of a sort on {@code parent} * @return The reverse lookup */ public static LongUnaryOperator getReverseLookup(@NotNull final Table parent, @NotNull final Table sortResult) { if (BlinkTableTools.isBlink(parent)) { throw new UnsupportedOperationException("Blink tables do not support sort reverse lookup"); } final Object value = sortResult.getAttribute(SORT_REVERSE_LOOKUP_ATTRIBUTE); if (sortResult.isRefreshing()) { Assert.neqNull(value, "sort result reverse lookup"); } if (value != null) { Assert.instanceOf(value, "sort result reverse lookup", LongUnaryOperator.class); return (LongUnaryOperator) value; } final RowRedirection sortRedirection = getRowRedirection(sortResult); if (sortRedirection == null || sortRedirection == getRowRedirection(parent)) { // Static table was already sorted return LongUnaryOperator.identity(); } final HashMapK4V4 reverseLookup = new HashMapLockFreeK4V4(sortResult.intSize(), .75f, RowSequence.NULL_ROW_KEY); try (final LongColumnIterator innerRowKeys = new ChunkedLongColumnIterator(sortRedirection, sortResult.getRowSet()); final RowSet.Iterator outerRowKeys = sortResult.getRowSet().iterator()) { while (outerRowKeys.hasNext()) { reverseLookup.put(innerRowKeys.nextLong(), outerRowKeys.nextLong()); } } return reverseLookup::get; } private static void setReverseLookup( @NotNull final QueryTable sortResult, @NotNull final LongUnaryOperator reverseLookup) { sortResult.setAttribute(SORT_REVERSE_LOOKUP_ATTRIBUTE, reverseLookup); } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy