org.apache.flink.runtime.operators.DriverStrategy Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flink-runtime_2.11 Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.operators;

import static org.apache.flink.runtime.operators.DamBehavior.FULL_DAM;
import static org.apache.flink.runtime.operators.DamBehavior.MATERIALIZING;
import static org.apache.flink.runtime.operators.DamBehavior.PIPELINED;

import org.apache.flink.runtime.operators.chaining.ChainedDriver;
import org.apache.flink.runtime.operators.chaining.ChainedAllReduceDriver;
import org.apache.flink.runtime.operators.chaining.ChainedFlatMapDriver;
import org.apache.flink.runtime.operators.chaining.ChainedMapDriver;
import org.apache.flink.runtime.operators.chaining.SynchronousChainedCombineDriver;
import org.apache.flink.runtime.operators.chaining.ChainedReduceCombineDriver;

/**
 * Enumeration of all available operator strategies. 
 */
public enum DriverStrategy {
	// no local strategy, as for sources and sinks
	NONE(null, null, PIPELINED, 0),
	// a unary no-op operator
	UNARY_NO_OP(NoOpDriver.class, NoOpChainedDriver.class, PIPELINED, PIPELINED, 0),
	// a binary no-op operator. non implementation available
	BINARY_NO_OP(null, null, PIPELINED, PIPELINED, 0),

	// the proper mapper
	MAP(MapDriver.class, ChainedMapDriver.class, PIPELINED, 0),

	// the proper map partition
	MAP_PARTITION(MapPartitionDriver.class, null, PIPELINED, 0),

	// the flat mapper
	FLAT_MAP(FlatMapDriver.class, ChainedFlatMapDriver.class, PIPELINED, 0),

	// group everything together into one group and apply the Reduce function
	ALL_REDUCE(AllReduceDriver.class, ChainedAllReduceDriver.class, PIPELINED, 0),
	// group everything together into one group and apply the GroupReduce function
	ALL_GROUP_REDUCE(AllGroupReduceDriver.class, null, PIPELINED, 0),
	// group everything together into one group and apply the GroupReduce's combine function
	ALL_GROUP_REDUCE_COMBINE(AllGroupReduceDriver.class, null, PIPELINED, 0),

	// grouping the inputs and apply the Reduce Function
	SORTED_REDUCE(ReduceDriver.class, null, PIPELINED, 1),
	// sorted partial reduce is a combiner for the Reduce. same function, but potentially not fully sorted
	SORTED_PARTIAL_REDUCE(ReduceCombineDriver.class, ChainedReduceCombineDriver.class, MATERIALIZING, 1),

	// hashed partial reduce is a combiner for the Reduce
	HASHED_PARTIAL_REDUCE(ReduceCombineDriver.class, ChainedReduceCombineDriver.class, MATERIALIZING, 1),
	
	// grouping the inputs and apply the GroupReduce function
	SORTED_GROUP_REDUCE(GroupReduceDriver.class, null, PIPELINED, 1),
	// partially grouping inputs (best effort resulting possibly in duplicates --> combiner)
	SORTED_GROUP_COMBINE(GroupReduceCombineDriver.class, SynchronousChainedCombineDriver.class, MATERIALIZING, 2),

	// group combine on all inputs within a partition (without grouping)
	ALL_GROUP_COMBINE(AllGroupCombineDriver.class, null, PIPELINED, 0),

	// both inputs are merged, but materialized to the side for block-nested-loop-join among values with equal key
	INNER_MERGE(JoinDriver.class, null, MATERIALIZING, MATERIALIZING, 2),
	LEFT_OUTER_MERGE(LeftOuterJoinDriver.class, null, MATERIALIZING, MATERIALIZING, 2),
	RIGHT_OUTER_MERGE(RightOuterJoinDriver.class, null, MATERIALIZING, MATERIALIZING, 2),
	FULL_OUTER_MERGE(FullOuterJoinDriver.class, null, MATERIALIZING, MATERIALIZING, 2),

	// co-grouping inputs
	CO_GROUP(CoGroupDriver.class, null, PIPELINED, PIPELINED, 2),
	// python-cogroup
	CO_GROUP_RAW(CoGroupRawDriver.class, null, PIPELINED, PIPELINED, 0),
	
	
	// the first input is build side, the second side is probe side of a hybrid hash table
	HYBRIDHASH_BUILD_FIRST(JoinDriver.class, null, FULL_DAM, MATERIALIZING, 2),
	// the second input is build side, the first side is probe side of a hybrid hash table
	HYBRIDHASH_BUILD_SECOND(JoinDriver.class, null, MATERIALIZING, FULL_DAM, 2),
	// a cached variant of HYBRIDHASH_BUILD_FIRST, that can only be used inside of iterations
	HYBRIDHASH_BUILD_FIRST_CACHED(BuildFirstCachedJoinDriver.class, null, FULL_DAM, MATERIALIZING, 2),
	//  cached variant of HYBRIDHASH_BUILD_SECOND, that can only be used inside of iterations
	HYBRIDHASH_BUILD_SECOND_CACHED(BuildSecondCachedJoinDriver.class, null, MATERIALIZING, FULL_DAM, 2),
	
	// right outer join, the first input is build side, the second input is probe side of a hybrid hash table.
	RIGHT_HYBRIDHASH_BUILD_FIRST(RightOuterJoinDriver.class, null, FULL_DAM, MATERIALIZING, 2),
	// right outer join, the first input is probe side, the second input is build side of a hybrid hash table.
	RIGHT_HYBRIDHASH_BUILD_SECOND(RightOuterJoinDriver.class, null, FULL_DAM, MATERIALIZING, 2),
	// left outer join, the first input is build side, the second input is probe side of a hybrid hash table.
	LEFT_HYBRIDHASH_BUILD_FIRST(LeftOuterJoinDriver.class, null, MATERIALIZING, FULL_DAM, 2),
	// left outer join, the first input is probe side, the second input is build side of a hybrid hash table.
	LEFT_HYBRIDHASH_BUILD_SECOND(LeftOuterJoinDriver.class, null, MATERIALIZING, FULL_DAM, 2),
	// full outer join, the first input is build side, the second input is the probe side of a hybrid hash table.
	FULL_OUTER_HYBRIDHASH_BUILD_FIRST(FullOuterJoinDriver.class, null, FULL_DAM, MATERIALIZING, 2),
	// full outer join, the first input is probe side, the second input is the build side of a hybrid hash table.
	FULL_OUTER_HYBRIDHASH_BUILD_SECOND(FullOuterJoinDriver.class, null, MATERIALIZING, FULL_DAM, 2),

	// the second input is inner loop, the first input is outer loop and block-wise processed
	NESTEDLOOP_BLOCKED_OUTER_FIRST(CrossDriver.class, null, MATERIALIZING, FULL_DAM, 0),
	// the first input is inner loop, the second input is outer loop and block-wise processed
	NESTEDLOOP_BLOCKED_OUTER_SECOND(CrossDriver.class, null, FULL_DAM, MATERIALIZING, 0),
	// the second input is inner loop, the first input is outer loop and stream-processed
	NESTEDLOOP_STREAMED_OUTER_FIRST(CrossDriver.class, null, PIPELINED, FULL_DAM, 0),
	// the first input is inner loop, the second input is outer loop and stream-processed
	NESTEDLOOP_STREAMED_OUTER_SECOND(CrossDriver.class, null, FULL_DAM, PIPELINED, 0),
	
	// union utility op. unions happen implicitly on the network layer (in the readers) when bundling streams
	UNION(null, null, PIPELINED, PIPELINED, 0),
	// explicit binary union between a streamed and a cached input
	UNION_WITH_CACHED(UnionWithTempOperator.class, null, FULL_DAM, PIPELINED, 0),
	
	// some enumeration constants to mark sources and sinks
	SOURCE(null, null, PIPELINED, 0),
	SINK(null, null, PIPELINED, 0);
	
	// --------------------------------------------------------------------------------------------
	
	private final Class> driverClass;
	
	private final Class> pushChainDriver;
	
	private final DamBehavior dam1;
	private final DamBehavior dam2;
	
	private final int numInputs;
	
	private final int numRequiredComparators;
	

	@SuppressWarnings("unchecked")
	private DriverStrategy(
			@SuppressWarnings("rawtypes") Class driverClass,
			@SuppressWarnings("rawtypes") Class pushChainDriverClass, 
			DamBehavior dam, int numComparator)
	{
		this.driverClass = (Class>) driverClass;
		this.pushChainDriver = (Class>) pushChainDriverClass;
		this.numInputs = 1;
		this.dam1 = dam;
		this.dam2 = null;
		this.numRequiredComparators = numComparator;
	}
	
	@SuppressWarnings("unchecked")
	private DriverStrategy(
			@SuppressWarnings("rawtypes") Class driverClass,
			@SuppressWarnings("rawtypes") Class pushChainDriverClass, 
			DamBehavior firstDam, DamBehavior secondDam, int numComparator)
	{
		this.driverClass = (Class>) driverClass;
		this.pushChainDriver = (Class>) pushChainDriverClass;
		this.numInputs = 2;
		this.dam1 = firstDam;
		this.dam2 = secondDam;
		this.numRequiredComparators = numComparator;
	}
	
	// --------------------------------------------------------------------------------------------
	
	public Class> getDriverClass() {
		return this.driverClass;
	}
	
	public Class> getPushChainDriverClass() {
		return this.pushChainDriver;
	}
	
	public int getNumInputs() {
		return this.numInputs;
	}
	
	public DamBehavior firstDam() {
		return this.dam1;
	}
	
	public DamBehavior secondDam() {
		if (this.numInputs == 2) {
			return this.dam2;
		} else {
			throw new IllegalArgumentException("The given strategy does not work on two inputs.");
		}
	}
	
	public DamBehavior damOnInput(int num) {
		if (num < this.numInputs) {
			if (num == 0) {
				return this.dam1;
			} else if (num == 1) {
				return this.dam2;
			}
		}
		throw new IllegalArgumentException();
	}
	
	public boolean isMaterializing() {
		return this.dam1.isMaterializing() || (this.dam2 != null && this.dam2.isMaterializing());
	}
	
	public int getNumRequiredComparators() {
		return this.numRequiredComparators;
	}
}