Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.operator;
import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ListenableFuture;
import io.trino.memory.context.LocalMemoryContext;
import io.trino.spi.Page;
import io.trino.spi.connector.SortOrder;
import io.trino.spi.type.Type;
import io.trino.spiller.Spiller;
import io.trino.spiller.SpillerFactory;
import io.trino.sql.gen.OrderingCompiler;
import io.trino.sql.planner.plan.PlanNodeId;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Verify.verify;
import static com.google.common.base.Verify.verifyNotNull;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.Iterators.transform;
import static com.google.common.util.concurrent.Futures.immediateVoidFuture;
import static io.airlift.concurrent.MoreFutures.checkSuccess;
import static io.airlift.concurrent.MoreFutures.getFutureValue;
import static io.trino.util.MergeSortedPages.mergeSortedPages;
import static java.util.Objects.requireNonNull;
public class OrderByOperator
implements Operator
{
public static class OrderByOperatorFactory
implements OperatorFactory
{
private final int operatorId;
private final PlanNodeId planNodeId;
private final List sourceTypes;
private final List outputChannels;
private final int expectedPositions;
private final List sortChannels;
private final List sortOrder;
private final PagesIndex.Factory pagesIndexFactory;
private final boolean spillEnabled;
private final Optional spillerFactory;
private final OrderingCompiler orderingCompiler;
private boolean closed;
public OrderByOperatorFactory(
int operatorId,
PlanNodeId planNodeId,
List sourceTypes,
List outputChannels,
int expectedPositions,
List sortChannels,
List sortOrder,
PagesIndex.Factory pagesIndexFactory,
boolean spillEnabled,
Optional spillerFactory,
OrderingCompiler orderingCompiler)
{
this.operatorId = operatorId;
this.planNodeId = requireNonNull(planNodeId, "planNodeId is null");
this.sourceTypes = ImmutableList.copyOf(requireNonNull(sourceTypes, "sourceTypes is null"));
this.outputChannels = requireNonNull(outputChannels, "outputChannels is null");
this.expectedPositions = expectedPositions;
this.sortChannels = ImmutableList.copyOf(requireNonNull(sortChannels, "sortChannels is null"));
this.sortOrder = ImmutableList.copyOf(requireNonNull(sortOrder, "sortOrder is null"));
this.pagesIndexFactory = requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.spillEnabled = spillEnabled;
this.spillerFactory = requireNonNull(spillerFactory, "spillerFactory is null");
this.orderingCompiler = requireNonNull(orderingCompiler, "orderingCompiler is null");
checkArgument(!spillEnabled || spillerFactory.isPresent(), "Spiller Factory is not present when spill is enabled");
}
@Override
public Operator createOperator(DriverContext driverContext)
{
checkState(!closed, "Factory is already closed");
OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, OrderByOperator.class.getSimpleName());
return new OrderByOperator(
operatorContext,
sourceTypes,
outputChannels,
expectedPositions,
sortChannels,
sortOrder,
pagesIndexFactory,
spillEnabled,
spillerFactory,
orderingCompiler);
}
@Override
public void noMoreOperators()
{
closed = true;
}
@Override
public OperatorFactory duplicate()
{
return new OrderByOperatorFactory(
operatorId,
planNodeId,
sourceTypes,
outputChannels,
expectedPositions,
sortChannels,
sortOrder,
pagesIndexFactory,
spillEnabled,
spillerFactory,
orderingCompiler);
}
}
private enum State
{
NEEDS_INPUT,
HAS_OUTPUT,
FINISHED
}
private final OperatorContext operatorContext;
private final List sortChannels;
private final List sortOrder;
private final int[] outputChannels;
private final LocalMemoryContext revocableMemoryContext;
private final LocalMemoryContext localUserMemoryContext;
private final PagesIndex pageIndex;
private final List sourceTypes;
private final boolean spillEnabled;
private final Optional spillerFactory;
private final OrderingCompiler orderingCompiler;
private Optional spiller = Optional.empty();
private ListenableFuture spillInProgress = immediateVoidFuture();
private Optional finishMemoryRevoke = Optional.empty();
private Iterator> sortedPages;
private State state = State.NEEDS_INPUT;
public OrderByOperator(
OperatorContext operatorContext,
List sourceTypes,
List outputChannels,
int expectedPositions,
List sortChannels,
List sortOrder,
PagesIndex.Factory pagesIndexFactory,
boolean spillEnabled,
Optional spillerFactory,
OrderingCompiler orderingCompiler)
{
requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");
this.outputChannels = Ints.toArray(requireNonNull(outputChannels, "outputChannels is null"));
this.sortChannels = ImmutableList.copyOf(requireNonNull(sortChannels, "sortChannels is null"));
this.sortOrder = ImmutableList.copyOf(requireNonNull(sortOrder, "sortOrder is null"));
this.sourceTypes = ImmutableList.copyOf(requireNonNull(sourceTypes, "sourceTypes is null"));
this.localUserMemoryContext = operatorContext.localUserMemoryContext();
this.revocableMemoryContext = operatorContext.localRevocableMemoryContext();
this.pageIndex = pagesIndexFactory.newPagesIndex(sourceTypes, expectedPositions);
this.spillEnabled = spillEnabled;
this.spillerFactory = requireNonNull(spillerFactory, "spillerFactory is null");
this.orderingCompiler = requireNonNull(orderingCompiler, "orderingCompiler is null");
checkArgument(!spillEnabled || spillerFactory.isPresent(), "Spiller Factory is not present when spill is enabled");
}
@Override
public OperatorContext getOperatorContext()
{
return operatorContext;
}
@Override
public void finish()
{
if (!spillInProgress.isDone()) {
return;
}
checkSuccess(spillInProgress, "spilling failed");
if (finishMemoryRevoke.isPresent()) {
return;
}
if (state == State.NEEDS_INPUT) {
state = State.HAS_OUTPUT;
// Convert revocable memory to user memory as sortedPages holds on to memory so we no longer can revoke.
if (revocableMemoryContext.getBytes() > 0) {
long currentRevocableBytes = revocableMemoryContext.getBytes();
revocableMemoryContext.setBytes(0);
if (!localUserMemoryContext.trySetBytes(localUserMemoryContext.getBytes() + currentRevocableBytes)) {
// TODO: this might fail (even though we have just released memory), but we don't
// have a proper way to atomically convert memory reservations
revocableMemoryContext.setBytes(currentRevocableBytes);
// spill since revocable memory could not be converted to user memory immediately
// TODO: this should be asynchronous
getFutureValue(spillToDisk());
finishMemoryRevoke.orElseThrow().run();
finishMemoryRevoke = Optional.empty();
}
}
pageIndex.sort(sortChannels, sortOrder);
Iterator sortedPagesIndex = pageIndex.getSortedPages();
List> spilledPages = getSpilledPages();
if (spilledPages.isEmpty()) {
sortedPages = transform(sortedPagesIndex, Optional::of);
}
else {
sortedPages = mergeSpilledAndMemoryPages(spilledPages, sortedPagesIndex).yieldingIterator();
}
}
}
@Override
public boolean isFinished()
{
return state == State.FINISHED;
}
@Override
public boolean needsInput()
{
return state == State.NEEDS_INPUT;
}
@Override
public void addInput(Page page)
{
checkState(state == State.NEEDS_INPUT, "Operator is already finishing");
requireNonNull(page, "page is null");
checkSuccess(spillInProgress, "spilling failed");
// TODO: remove when retained memory accounting for pages does not
// count shared data structures multiple times
page.compact();
pageIndex.addPage(page);
updateMemoryUsage();
}
@Override
public Page getOutput()
{
checkSuccess(spillInProgress, "spilling failed");
if (state != State.HAS_OUTPUT) {
return null;
}
verifyNotNull(sortedPages, "sortedPages is null");
if (!sortedPages.hasNext()) {
state = State.FINISHED;
return null;
}
Optional next = sortedPages.next();
if (next.isEmpty()) {
return null;
}
Page nextPage = next.get();
return nextPage.getColumns(outputChannels);
}
@Override
public ListenableFuture startMemoryRevoke()
{
verify(state == State.NEEDS_INPUT || revocableMemoryContext.getBytes() == 0, "Cannot spill in state: %s", state);
return spillToDisk();
}
private ListenableFuture spillToDisk()
{
checkSuccess(spillInProgress, "spilling failed");
if (revocableMemoryContext.getBytes() == 0) {
verify(pageIndex.getPositionCount() == 0 || state == State.HAS_OUTPUT);
finishMemoryRevoke = Optional.of(() -> {});
return immediateVoidFuture();
}
// TODO try pageIndex.compact(); before spilling, as in HashBuilderOperator.startMemoryRevoke()
if (spiller.isEmpty()) {
spiller = Optional.of(spillerFactory.get().create(
sourceTypes,
operatorContext.getSpillContext(),
operatorContext.newAggregateUserMemoryContext()));
}
pageIndex.sort(sortChannels, sortOrder);
spillInProgress = spiller.get().spill(pageIndex.getSortedPages());
finishMemoryRevoke = Optional.of(() -> {
pageIndex.clear();
updateMemoryUsage();
});
return spillInProgress;
}
@Override
public void finishMemoryRevoke()
{
finishMemoryRevoke.orElseThrow().run();
finishMemoryRevoke = Optional.empty();
}
private List> getSpilledPages()
{
if (spiller.isEmpty()) {
return ImmutableList.of();
}
return spiller.get().getSpills().stream()
.map(WorkProcessor::fromIterator)
.collect(toImmutableList());
}
private WorkProcessor mergeSpilledAndMemoryPages(List> spilledPages, Iterator sortedPagesIndex)
{
List> sortedStreams = ImmutableList.>builder()
.addAll(spilledPages)
.add(WorkProcessor.fromIterator(sortedPagesIndex))
.build();
return mergeSortedPages(
sortedStreams,
orderingCompiler.compilePageWithPositionComparator(sourceTypes, sortChannels, sortOrder),
sourceTypes,
operatorContext.aggregateUserMemoryContext(),
operatorContext.getDriverContext().getYieldSignal());
}
private void updateMemoryUsage()
{
if (spillEnabled && state == State.NEEDS_INPUT) {
if (pageIndex.getPositionCount() == 0) {
localUserMemoryContext.setBytes(pageIndex.getEstimatedSize().toBytes());
revocableMemoryContext.setBytes(0L);
}
else {
localUserMemoryContext.setBytes(0);
revocableMemoryContext.setBytes(pageIndex.getEstimatedSize().toBytes());
}
}
else {
revocableMemoryContext.setBytes(0);
if (!localUserMemoryContext.trySetBytes(pageIndex.getEstimatedSize().toBytes())) {
pageIndex.compact();
localUserMemoryContext.setBytes(pageIndex.getEstimatedSize().toBytes());
}
}
}
@Override
public void close()
{
pageIndex.clear();
sortedPages = null;
spiller.ifPresent(Spiller::close);
}
}
