Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.operator.join.unspilled;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.errorprone.annotations.ThreadSafe;
import io.trino.memory.context.CoarseGrainLocalMemoryContext;
import io.trino.operator.DriverContext;
import io.trino.operator.HashArraySizeSupplier;
import io.trino.operator.Operator;
import io.trino.operator.OperatorContext;
import io.trino.operator.OperatorFactory;
import io.trino.operator.PagesIndex;
import io.trino.operator.join.JoinBridgeManager;
import io.trino.operator.join.LookupSourceSupplier;
import io.trino.spi.Page;
import io.trino.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory;
import io.trino.sql.planner.plan.PlanNodeId;
import jakarta.annotation.Nullable;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Verify.verify;
import static io.trino.memory.context.CoarseGrainLocalMemoryContext.DEFAULT_GRANULARITY;
import static java.util.Objects.requireNonNull;
/**
* Like {@link io.trino.operator.join.HashBuilderOperator} but simplified,
* without spill support.
*/
@ThreadSafe
public class HashBuilderOperator
implements Operator
{
public static class HashBuilderOperatorFactory
implements OperatorFactory
{
private final int operatorId;
private final PlanNodeId planNodeId;
private final JoinBridgeManager lookupSourceFactoryManager;
private final List outputChannels;
private final List hashChannels;
private final OptionalInt preComputedHashChannel;
private final Optional filterFunctionFactory;
private final Optional sortChannel;
private final List searchFunctionFactories;
private final PagesIndex.Factory pagesIndexFactory;
private final int expectedPositions;
private final HashArraySizeSupplier hashArraySizeSupplier;
private int partitionIndex;
private boolean closed;
public HashBuilderOperatorFactory(
int operatorId,
PlanNodeId planNodeId,
JoinBridgeManager lookupSourceFactoryManager,
List outputChannels,
List hashChannels,
OptionalInt preComputedHashChannel,
Optional filterFunctionFactory,
Optional sortChannel,
List searchFunctionFactories,
int expectedPositions,
PagesIndex.Factory pagesIndexFactory,
HashArraySizeSupplier hashArraySizeSupplier)
{
this.operatorId = operatorId;
this.planNodeId = requireNonNull(planNodeId, "planNodeId is null");
requireNonNull(sortChannel, "sortChannel cannot be null");
requireNonNull(searchFunctionFactories, "searchFunctionFactories is null");
checkArgument(sortChannel.isPresent() != searchFunctionFactories.isEmpty(), "both or none sortChannel and searchFunctionFactories must be set");
this.lookupSourceFactoryManager = requireNonNull(lookupSourceFactoryManager, "lookupSourceFactoryManager is null");
this.outputChannels = ImmutableList.copyOf(requireNonNull(outputChannels, "outputChannels is null"));
this.hashChannels = ImmutableList.copyOf(requireNonNull(hashChannels, "hashChannels is null"));
this.preComputedHashChannel = requireNonNull(preComputedHashChannel, "preComputedHashChannel is null");
this.filterFunctionFactory = requireNonNull(filterFunctionFactory, "filterFunctionFactory is null");
this.sortChannel = sortChannel;
this.searchFunctionFactories = ImmutableList.copyOf(searchFunctionFactories);
this.pagesIndexFactory = requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.hashArraySizeSupplier = requireNonNull(hashArraySizeSupplier, "hashArraySizeSupplier is null");
this.expectedPositions = expectedPositions;
}
@Override
public HashBuilderOperator createOperator(DriverContext driverContext)
{
checkState(!closed, "Factory is already closed");
OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName());
PartitionedLookupSourceFactory lookupSourceFactory = this.lookupSourceFactoryManager.getJoinBridge();
verify(partitionIndex < lookupSourceFactory.partitions());
partitionIndex++;
return new HashBuilderOperator(
operatorContext,
lookupSourceFactory,
partitionIndex - 1,
outputChannels,
hashChannels,
preComputedHashChannel,
filterFunctionFactory,
sortChannel,
searchFunctionFactories,
expectedPositions,
pagesIndexFactory,
hashArraySizeSupplier);
}
@Override
public void noMoreOperators()
{
closed = true;
}
@Override
public OperatorFactory duplicate()
{
throw new UnsupportedOperationException("Parallel hash build cannot be duplicated");
}
}
@VisibleForTesting
public enum State
{
/**
* Operator accepts input
*/
CONSUMING_INPUT,
/**
* LookupSource has been built and passed on without any spill occurring
*/
LOOKUP_SOURCE_BUILT,
/**
* No longer needed
*/
CLOSED
}
private final OperatorContext operatorContext;
private final CoarseGrainLocalMemoryContext localUserMemoryContext;
private final PartitionedLookupSourceFactory lookupSourceFactory;
private final ListenableFuture lookupSourceFactoryDestroyed;
private final int partitionIndex;
private final List outputChannels;
private final List hashChannels;
private final OptionalInt preComputedHashChannel;
private final Optional filterFunctionFactory;
private final Optional sortChannel;
private final List searchFunctionFactories;
private final HashArraySizeSupplier hashArraySizeSupplier;
private State state = State.CONSUMING_INPUT;
@Nullable
private PagesIndex index;
private Optional> lookupSourceNotNeeded = Optional.empty();
@Nullable
private LookupSourceSupplier lookupSourceSupplier;
public HashBuilderOperator(
OperatorContext operatorContext,
PartitionedLookupSourceFactory lookupSourceFactory,
int partitionIndex,
List outputChannels,
List hashChannels,
OptionalInt preComputedHashChannel,
Optional filterFunctionFactory,
Optional sortChannel,
List searchFunctionFactories,
int expectedPositions,
PagesIndex.Factory pagesIndexFactory,
HashArraySizeSupplier hashArraySizeSupplier)
{
this(operatorContext, lookupSourceFactory, partitionIndex, outputChannels, hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, expectedPositions, pagesIndexFactory, hashArraySizeSupplier, DEFAULT_GRANULARITY);
}
@VisibleForTesting
HashBuilderOperator(
OperatorContext operatorContext,
PartitionedLookupSourceFactory lookupSourceFactory,
int partitionIndex,
List outputChannels,
List hashChannels,
OptionalInt preComputedHashChannel,
Optional filterFunctionFactory,
Optional sortChannel,
List searchFunctionFactories,
int expectedPositions,
PagesIndex.Factory pagesIndexFactory,
HashArraySizeSupplier hashArraySizeSupplier,
long memorySyncThreshold)
{
requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.operatorContext = operatorContext;
this.partitionIndex = partitionIndex;
this.filterFunctionFactory = filterFunctionFactory;
this.sortChannel = sortChannel;
this.searchFunctionFactories = searchFunctionFactories;
this.localUserMemoryContext = new CoarseGrainLocalMemoryContext(operatorContext.localUserMemoryContext(), memorySyncThreshold);
this.index = pagesIndexFactory.newPagesIndex(lookupSourceFactory.getTypes(), expectedPositions);
this.lookupSourceFactory = lookupSourceFactory;
lookupSourceFactoryDestroyed = lookupSourceFactory.isDestroyed();
this.outputChannels = outputChannels;
this.hashChannels = hashChannels;
this.preComputedHashChannel = preComputedHashChannel;
this.hashArraySizeSupplier = requireNonNull(hashArraySizeSupplier, "hashArraySizeSupplier is null");
}
@Override
public OperatorContext getOperatorContext()
{
return operatorContext;
}
@Override
public ListenableFuture isBlocked()
{
return switch (state) {
case CONSUMING_INPUT -> NOT_BLOCKED;
case LOOKUP_SOURCE_BUILT -> lookupSourceNotNeeded.orElseThrow(() -> new IllegalStateException("Lookup source built, but disposal future not set"));
case CLOSED -> NOT_BLOCKED;
};
}
@Override
public boolean needsInput()
{
boolean stateNeedsInput = (state == State.CONSUMING_INPUT);
return stateNeedsInput && !lookupSourceFactoryDestroyed.isDone();
}
@Override
public void addInput(Page page)
{
requireNonNull(page, "page is null");
if (lookupSourceFactoryDestroyed.isDone()) {
close();
return;
}
checkState(state == State.CONSUMING_INPUT);
updateIndex(page);
}
private void updateIndex(Page page)
{
checkState(index != null, "index is null");
index.addPage(page);
if (!localUserMemoryContext.trySetBytes(index.getEstimatedSize().toBytes())) {
index.compact();
localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes());
}
operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount());
}
@Override
public Page getOutput()
{
return null;
}
@Override
public void finish()
{
if (lookupSourceFactoryDestroyed.isDone()) {
close();
return;
}
switch (state) {
case CONSUMING_INPUT:
finishInput();
return;
case LOOKUP_SOURCE_BUILT:
disposeLookupSourceIfRequested();
return;
case CLOSED:
// no-op
return;
}
throw new IllegalStateException("Unhandled state: " + state);
}
private void finishInput()
{
checkState(state == State.CONSUMING_INPUT);
if (lookupSourceFactoryDestroyed.isDone()) {
close();
return;
}
checkState(index != null, "index is null");
ListenableFuture reserved = localUserMemoryContext.setBytes(index.getEstimatedMemoryRequiredToCreateLookupSource(
hashArraySizeSupplier,
sortChannel,
hashChannels));
if (!reserved.isDone()) {
// Yield when not enough memory is available to proceed, finish is expected to be called again when some memory is freed
return;
}
LookupSourceSupplier partition = buildLookupSource();
localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes());
lookupSourceNotNeeded = Optional.of(lookupSourceFactory.lendPartitionLookupSource(partitionIndex, partition));
index = null;
state = State.LOOKUP_SOURCE_BUILT;
}
private void disposeLookupSourceIfRequested()
{
checkState(state == State.LOOKUP_SOURCE_BUILT);
verify(lookupSourceNotNeeded.isPresent());
if (!lookupSourceNotNeeded.get().isDone()) {
return;
}
close();
}
private LookupSourceSupplier buildLookupSource()
{
checkState(index != null, "index is null");
LookupSourceSupplier partition = index.createLookupSourceSupplier(operatorContext.getSession(), hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, Optional.of(outputChannels), hashArraySizeSupplier);
checkState(lookupSourceSupplier == null, "lookupSourceSupplier is already set");
this.lookupSourceSupplier = partition;
return partition;
}
@Override
public boolean isFinished()
{
if (lookupSourceFactoryDestroyed.isDone()) {
// Finish early when the probe side is empty
close();
return true;
}
return state == State.CLOSED;
}
@Override
public void close()
{
if (state == State.CLOSED) {
return;
}
// close() can be called in any state, due for example to query failure, and must clean resource up unconditionally
lookupSourceSupplier = null;
index = null;
localUserMemoryContext.setBytes(0);
state = State.CLOSED;
}
@VisibleForTesting
State getState()
{
return state;
}
}
