org.apache.flink.runtime.scheduler.adaptive.CreatingExecutionGraph Maven / Gradle / Ivy

Go to download
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.scheduler.adaptive;

import org.apache.flink.api.common.JobStatus;
import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionVertex;
import org.apache.flink.runtime.jobgraph.JobType;
import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
import org.apache.flink.runtime.scheduler.DefaultOperatorCoordinatorHandler;
import org.apache.flink.runtime.scheduler.ExecutionGraphHandler;
import org.apache.flink.runtime.scheduler.GlobalFailureHandler;
import org.apache.flink.runtime.scheduler.OperatorCoordinatorHandler;
import org.apache.flink.runtime.scheduler.adaptive.allocator.VertexParallelism;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.concurrent.FutureUtils;

import org.slf4j.Logger;

import javax.annotation.Nullable;

import java.time.Duration;
import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledFuture;
import java.util.stream.StreamSupport;

/**
 * State which waits for the creation of the {@link ExecutionGraph}. If the creation fails, then the
 * state transitions to {@link Finished}. If the creation succeeds, then the system tries to assign
 * the required slots. If the set of available slots has changed so that the created {@link
 * ExecutionGraph} cannot be executed, the state transitions back into {@link WaitingForResources}.
 * If there are enough slots for the {@link ExecutionGraph} to run, the state transitions to {@link
 * Executing}.
 */
public class CreatingExecutionGraph implements State {

    private final Context context;
    private final Logger logger;
    private final OperatorCoordinatorHandlerFactory operatorCoordinatorHandlerFactory;

    public CreatingExecutionGraph(
            Context context,
            CompletableFuture
                    executionGraphWithParallelismFuture,
            Logger logger,
            OperatorCoordinatorHandlerFactory operatorCoordinatorFactory) {
        this.context = context;
        this.logger = logger;
        this.operatorCoordinatorHandlerFactory = operatorCoordinatorFactory;

        FutureUtils.assertNoException(
                executionGraphWithParallelismFuture.handle(
                        (executionGraphWithVertexParallelism, throwable) -> {
                            context.runIfState(
                                    this,
                                    () ->
                                            handleExecutionGraphCreation(
                                                    executionGraphWithVertexParallelism, throwable),
                                    Duration.ZERO);
                            return null;
                        }));
    }

    private void handleExecutionGraphCreation(
            @Nullable ExecutionGraphWithVertexParallelism executionGraphWithVertexParallelism,
            @Nullable Throwable throwable) {
        if (throwable != null) {
            logger.info(
                    "Failed to go from {} to {} because the ExecutionGraph creation failed.",
                    CreatingExecutionGraph.class.getSimpleName(),
                    Executing.class.getSimpleName(),
                    throwable);
            context.goToFinished(context.getArchivedExecutionGraph(JobStatus.FAILED, throwable));
        } else {
            for (ExecutionVertex vertex :
                    executionGraphWithVertexParallelism.executionGraph.getAllExecutionVertices()) {
                vertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
            }

            final AssignmentResult result =
                    context.tryToAssignSlots(executionGraphWithVertexParallelism);

            if (result.isSuccess()) {
                logger.debug(
                        "Successfully reserved and assigned the required slots for the ExecutionGraph.");
                final ExecutionGraph executionGraph = result.getExecutionGraph();
                final ExecutionGraphHandler executionGraphHandler =
                        new ExecutionGraphHandler(
                                executionGraph,
                                getLogger(),
                                context.getIOExecutor(),
                                context.getMainThreadExecutor());
                // Operator coordinator outlives the current state, so we need to use context as a
                // global failure handler.
                final OperatorCoordinatorHandler operatorCoordinatorHandler =
                        operatorCoordinatorHandlerFactory.create(executionGraph, context);
                operatorCoordinatorHandler.initializeOperatorCoordinators(
                        context.getMainThreadExecutor());
                operatorCoordinatorHandler.startAllOperatorCoordinators();
                String updatedPlan =
                        JsonPlanGenerator.generatePlan(
                                executionGraph.getJobID(),
                                executionGraph.getJobName(),
                                JobType.STREAMING, // Adaptive scheduler works only with STREAMING
                                // jobs
                                () ->
                                        StreamSupport.stream(
                                                        executionGraph
                                                                .getAllExecutionVertices()
                                                                .spliterator(),
                                                        false)
                                                .map(v -> v.getJobVertex().getJobVertex())
                                                .iterator(),
                                executionGraphWithVertexParallelism.getVertexParallelism());
                executionGraph.setJsonPlan(updatedPlan);
                context.goToExecuting(
                        result.getExecutionGraph(),
                        executionGraphHandler,
                        operatorCoordinatorHandler,
                        Collections.emptyList());
            } else {
                logger.debug(
                        "Failed to reserve and assign the required slots. Waiting for new resources.");
                context.goToWaitingForResources();
            }
        }
    }

    @Override
    public void cancel() {
        context.goToFinished(context.getArchivedExecutionGraph(JobStatus.CANCELED, null));
    }

    @Override
    public void suspend(Throwable cause) {
        context.goToFinished(context.getArchivedExecutionGraph(JobStatus.SUSPENDED, cause));
    }

    @Override
    public JobStatus getJobStatus() {
        return JobStatus.CREATED;
    }

    @Override
    public ArchivedExecutionGraph getJob() {
        return context.getArchivedExecutionGraph(getJobStatus(), null);
    }

    @Override
    public void handleGlobalFailure(Throwable cause) {
        context.goToFinished(context.getArchivedExecutionGraph(JobStatus.FAILED, cause));
    }

    @Override
    public Logger getLogger() {
        return logger;
    }

    /** Context for the {@link CreatingExecutionGraph} state. */
    interface Context
            extends GlobalFailureHandler,
                    StateTransitions.ToExecuting,
                    StateTransitions.ToFinished,
                    StateTransitions.ToWaitingForResources {

        /**
         * Creates the {@link ArchivedExecutionGraph} for the given job status and cause. Cause can
         * be null if there is no failure.
         *
         * @param jobStatus jobStatus to initialize the {@link ArchivedExecutionGraph} with
         * @param cause cause describing a failure cause; {@code null} if there is none
         * @return the created {@link ArchivedExecutionGraph}
         */
        ArchivedExecutionGraph getArchivedExecutionGraph(
                JobStatus jobStatus, @Nullable Throwable cause);

        /**
         * Runs the given action after a delay if the state at this time equals the expected state.
         *
         * @param expectedState expectedState describes the required state at the time of running
         *     the action
         * @param action action to run if the expected state equals the actual state
         * @param delay delay after which to run the action
         * @return a ScheduledFuture representing pending completion of the task
         */
        ScheduledFuture runIfState(State expectedState, Runnable action, Duration delay);

        /**
         * Try to assign slots to the created {@link ExecutionGraph}. If it is possible, then this
         * method returns a successful {@link AssignmentResult} which contains the assigned {@link
         * ExecutionGraph}. If not, then the assignment result is a failure.
         *
         * @param executionGraphWithVertexParallelism executionGraphWithVertexParallelism to assign
         *     slots to resources
         * @return {@link AssignmentResult} representing the result of the assignment
         */
        AssignmentResult tryToAssignSlots(
                ExecutionGraphWithVertexParallelism executionGraphWithVertexParallelism);

        /**
         * Gets the I/O executor.
         *
         * @return the I/O executor
         */
        Executor getIOExecutor();

        /**
         * Gets the main thread executor.
         *
         * @return the main thread executor
         */
        ComponentMainThreadExecutor getMainThreadExecutor();
    }

    @FunctionalInterface
    interface OperatorCoordinatorHandlerFactory {

        /**
         * Creates a new {@link OperatorCoordinatorHandler}. This interface is primarily intended
         * for easier testing.
         *
         * @param executionGraph Current execution graph, that contains operator coordinators that
         *     we want to start.
         * @param globalFailureHandler Global failure handler.
         * @return An {@link OperatorCoordinatorHandler} instance.
         */
        OperatorCoordinatorHandler create(
                ExecutionGraph executionGraph, GlobalFailureHandler globalFailureHandler);
    }

    /**
     * Class representing the assignment result of the slots to the {@link ExecutionGraph}. The
     * assignment is either successful or not possible. If it is successful, the assignment also
     * contains the assigned {@link ExecutionGraph}.
     */
    static final class AssignmentResult {

        private static final AssignmentResult NOT_POSSIBLE = new AssignmentResult(null);

        @Nullable private final ExecutionGraph executionGraph;

        private AssignmentResult(@Nullable ExecutionGraph executionGraph) {
            this.executionGraph = executionGraph;
        }

        boolean isSuccess() {
            return executionGraph != null;
        }

        ExecutionGraph getExecutionGraph() {
            Preconditions.checkState(
                    isSuccess(), "Can only return the ExecutionGraph if it is a success.");
            return executionGraph;
        }

        static AssignmentResult success(ExecutionGraph executionGraph) {
            return new AssignmentResult(
                    Preconditions.checkNotNull(
                            executionGraph,
                            "AssignmentResult.success expects a non-null ExecutionGraph."));
        }

        static AssignmentResult notPossible() {
            return NOT_POSSIBLE;
        }
    }

    /** Factory for the {@link CreatingExecutionGraph} state. */
    static class Factory implements StateFactory {

        private final Context context;

        private final CompletableFuture
                executionGraphWithParallelismFuture;

        private final Logger log;

        Factory(
                Context context,
                CompletableFuture
                        executionGraphWithParallelismFuture,
                Logger log) {
            this.context = context;
            this.executionGraphWithParallelismFuture = executionGraphWithParallelismFuture;
            this.log = log;
        }

        @Override
        public Class getStateClass() {
            return CreatingExecutionGraph.class;
        }

        @Override
        public CreatingExecutionGraph getState() {
            return new CreatingExecutionGraph(
                    context,
                    executionGraphWithParallelismFuture,
                    log,
                    DefaultOperatorCoordinatorHandler::new);
        }
    }

    static class ExecutionGraphWithVertexParallelism {
        private final ExecutionGraph executionGraph;

        private final VertexParallelism vertexParallelism;

        private ExecutionGraphWithVertexParallelism(
                ExecutionGraph executionGraph, VertexParallelism vertexParallelism) {
            this.executionGraph = executionGraph;
            this.vertexParallelism = vertexParallelism;
        }

        public static ExecutionGraphWithVertexParallelism create(
                ExecutionGraph executionGraph, VertexParallelism vertexParallelism) {
            return new ExecutionGraphWithVertexParallelism(executionGraph, vertexParallelism);
        }

        public ExecutionGraph getExecutionGraph() {
            return executionGraph;
        }

        public VertexParallelism getVertexParallelism() {
            return vertexParallelism;
        }
    }
}