org.apache.flink.runtime.jobgraph.topology.DefaultLogicalTopology Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of flink-runtime_2.11 Show documentation
There is a newer version: 1.13.6
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.jobgraph.topology;

import org.apache.flink.runtime.executiongraph.failover.flip1.PipelinedRegionComputeUtil;
import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.IterableUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;

import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * Default implementation of {@link LogicalTopology}.
 * It is an adapter of {@link JobGraph}.
 */
public class DefaultLogicalTopology implements LogicalTopology {

	private final boolean containsCoLocationConstraints;

	private final List verticesSorted;

	private final Map idToVertexMap;

	private final Map idToResultMap;

	public DefaultLogicalTopology(final JobGraph jobGraph) {
		checkNotNull(jobGraph);

		this.containsCoLocationConstraints = IterableUtils.toStream(jobGraph.getVertices())
			.map(JobVertex::getCoLocationGroup)
			.anyMatch(Objects::nonNull);

		this.verticesSorted = new ArrayList<>(jobGraph.getNumberOfVertices());
		this.idToVertexMap = new HashMap<>();
		this.idToResultMap = new HashMap<>();

		buildVerticesAndResults(jobGraph);
	}

	private void buildVerticesAndResults(final JobGraph jobGraph) {
		final Function vertexRetriever = this::getVertex;
		final Function resultRetriever = this::getResult;

		for (JobVertex jobVertex : jobGraph.getVerticesSortedTopologicallyFromSources()) {
			final DefaultLogicalVertex logicalVertex = new DefaultLogicalVertex(jobVertex, resultRetriever);
			this.verticesSorted.add(logicalVertex);
			this.idToVertexMap.put(logicalVertex.getId(), logicalVertex);

			for (IntermediateDataSet intermediateDataSet : jobVertex.getProducedDataSets()) {
				final DefaultLogicalResult logicalResult = new DefaultLogicalResult(intermediateDataSet, vertexRetriever);
				idToResultMap.put(logicalResult.getId(), logicalResult);
			}
		}
	}

	@Override
	public Iterable getVertices() {
		return verticesSorted;
	}

	@Override
	public boolean containsCoLocationConstraints() {
		return containsCoLocationConstraints;
	}

	private DefaultLogicalVertex getVertex(final JobVertexID vertexId) {
		return Optional.ofNullable(idToVertexMap.get(vertexId))
			.orElseThrow(() -> new IllegalArgumentException("can not find vertex: " + vertexId));
	}

	private DefaultLogicalResult getResult(final IntermediateDataSetID resultId) {
		return Optional.ofNullable(idToResultMap.get(resultId))
			.orElseThrow(() -> new IllegalArgumentException("can not find result: " + resultId));
	}

	public Set getLogicalPipelinedRegions() {
		final Set> regionsRaw = PipelinedRegionComputeUtil.computePipelinedRegions(this);

		final Set regions = new HashSet<>();
		for (Set regionVertices : regionsRaw) {
			regions.add(new LogicalPipelinedRegion(regionVertices));
		}
		return regions;
	}
}