io.trino.util.MergeSortedPages Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of trino-main Show documentation
Trino - Engine
There is a newer version: 465
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.trino.util;

import io.trino.memory.context.AggregatedMemoryContext;
import io.trino.memory.context.LocalMemoryContext;
import io.trino.operator.DriverYieldSignal;
import io.trino.operator.PageWithPositionComparator;
import io.trino.operator.WorkProcessor;
import io.trino.operator.WorkProcessor.ProcessState;
import io.trino.operator.WorkProcessor.TransformationState;
import io.trino.spi.Page;
import io.trino.spi.PageBuilder;
import io.trino.spi.block.Block;
import io.trino.spi.type.Type;

import java.util.Comparator;
import java.util.List;
import java.util.function.BiPredicate;
import java.util.stream.IntStream;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.trino.operator.WorkProcessor.mergeSorted;
import static java.util.Objects.requireNonNull;

public final class MergeSortedPages
{
    private MergeSortedPages() {}

    public static WorkProcessor mergeSortedPages(
            List> pageProducers,
            PageWithPositionComparator comparator,
            List outputTypes,
            AggregatedMemoryContext aggregatedMemoryContext,
            DriverYieldSignal yieldSignal)
    {
        return mergeSortedPages(
                pageProducers,
                comparator,
                IntStream.range(0, outputTypes.size()).boxed().collect(toImmutableList()),
                outputTypes,
                (pageBuilder, pageWithPosition) -> pageBuilder.isFull(),
                false,
                aggregatedMemoryContext,
                yieldSignal);
    }

    public static WorkProcessor mergeSortedPages(
            List> pageProducers,
            PageWithPositionComparator comparator,
            List outputChannels,
            List outputTypes,
            BiPredicate pageBreakPredicate,
            boolean updateMemoryAfterEveryPosition,
            AggregatedMemoryContext aggregatedMemoryContext,
            DriverYieldSignal yieldSignal)
    {
        requireNonNull(pageProducers, "pageProducers is null");
        requireNonNull(comparator, "comparator is null");
        requireNonNull(outputChannels, "outputChannels is null");
        requireNonNull(outputTypes, "outputTypes is null");
        requireNonNull(pageBreakPredicate, "pageBreakPredicate is null");
        requireNonNull(aggregatedMemoryContext, "aggregatedMemoryContext is null");
        requireNonNull(yieldSignal, "yieldSignal is null");

        List> pageWithPositionProducers = pageProducers.stream()
                .map(pageProducer -> pageWithPositions(pageProducer, aggregatedMemoryContext))
                .collect(toImmutableList());

        Comparator pageWithPositionComparator = (firstPageWithPosition, secondPageWithPosition) -> comparator.compareTo(
                firstPageWithPosition.getPage(), firstPageWithPosition.getPosition(),
                secondPageWithPosition.getPage(), secondPageWithPosition.getPosition());

        return buildPage(
                mergeSorted(pageWithPositionProducers, pageWithPositionComparator),
                outputChannels,
                outputTypes,
                pageBreakPredicate,
                updateMemoryAfterEveryPosition,
                aggregatedMemoryContext,
                yieldSignal);
    }

    private static WorkProcessor buildPage(
            WorkProcessor pageWithPositions,
            List outputChannels,
            List outputTypes,
            BiPredicate pageBreakPredicate,
            boolean updateMemoryAfterEveryPosition,
            AggregatedMemoryContext aggregatedMemoryContext,
            DriverYieldSignal yieldSignal)
    {
        LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
        PageBuilder pageBuilder = new PageBuilder(outputTypes);
        return pageWithPositions
                .yielding(yieldSignal::isSet)
                .transform(pageWithPosition -> {
                    boolean finished = pageWithPosition == null;
                    if (finished && pageBuilder.isEmpty()) {
                        memoryContext.close();
                        return TransformationState.finished();
                    }

                    if (finished || pageBreakPredicate.test(pageBuilder, pageWithPosition)) {
                        if (!updateMemoryAfterEveryPosition) {
                            // update memory usage just before producing page to cap from top
                            memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
                        }

                        Page page = pageBuilder.build();
                        pageBuilder.reset();
                        if (!finished) {
                            pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes);
                        }

                        if (updateMemoryAfterEveryPosition) {
                            memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
                        }

                        return TransformationState.ofResult(page, !finished);
                    }

                    pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes);

                    if (updateMemoryAfterEveryPosition) {
                        memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
                    }

                    return TransformationState.needsMoreData();
                });
    }

    private static WorkProcessor pageWithPositions(WorkProcessor pages, AggregatedMemoryContext aggregatedMemoryContext)
    {
        return pages.flatMap(page -> {
            LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
            memoryContext.setBytes(page.getRetainedSizeInBytes());

            return WorkProcessor.create(new WorkProcessor.Process()
            {
                int position;

                @Override
                public ProcessState process()
                {
                    if (position >= page.getPositionCount()) {
                        memoryContext.close();
                        return ProcessState.finished();
                    }

                    return ProcessState.ofResult(new PageWithPosition(page, position++));
                }
            });
        });
    }

    public static class PageWithPosition
    {
        private final Page page;
        private final int position;

        private PageWithPosition(Page page, int position)
        {
            this.page = requireNonNull(page, "page is null");
            this.position = position;
        }

        public Page getPage()
        {
            return page;
        }

        public int getPosition()
        {
            return position;
        }

        public void appendTo(PageBuilder pageBuilder, List outputChannels, List outputTypes)
        {
            pageBuilder.declarePosition();
            for (int i = 0; i < outputChannels.size(); i++) {
                Type type = outputTypes.get(i);
                Block block = page.getBlock(outputChannels.get(i));
                type.appendTo(block, position, pageBuilder.getBlockBuilder(i));
            }
        }
    }
}