All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.client.program.PerJobMiniClusterFactory Maven / Gradle / Ivy

The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.client.program;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.configuration.RestOptions;
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.minicluster.MiniCluster;
import org.apache.flink.runtime.minicluster.MiniClusterConfiguration;
import org.apache.flink.runtime.minicluster.MiniClusterJobClient;
import org.apache.flink.runtime.minicluster.RpcServiceSharing;
import org.apache.flink.util.MathUtils;
import org.apache.flink.util.function.FunctionUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;

/**
 * Starts a {@link MiniCluster} for every submitted job. This class guarantees to tear down the
 * MiniCluster in case of normal or exceptional job completion.
 */
public final class PerJobMiniClusterFactory {

    private static final Logger LOG = LoggerFactory.getLogger(PerJobMiniClusterFactory.class);

    private final Configuration configuration;
    private final Function
            miniClusterFactory;

    public static PerJobMiniClusterFactory create() {
        return new PerJobMiniClusterFactory(new Configuration(), MiniCluster::new);
    }

    public static PerJobMiniClusterFactory createWithFactory(
            Configuration configuration,
            Function miniClusterFactory) {
        return new PerJobMiniClusterFactory(configuration, miniClusterFactory);
    }

    private PerJobMiniClusterFactory(
            Configuration configuration,
            Function miniClusterFactory) {
        this.configuration = configuration;
        this.miniClusterFactory = miniClusterFactory;
    }

    /** Starts a {@link MiniCluster} and submits a job. */
    public CompletableFuture submitJob(
            JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
        MiniClusterConfiguration miniClusterConfig =
                getMiniClusterConfig(jobGraph.getMaximumParallelism());
        MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
        miniCluster.start();

        return miniCluster
                .submitJob(jobGraph)
                .thenApplyAsync(
                        FunctionUtils.uncheckedFunction(
                                submissionResult -> {
                                    org.apache.flink.client.ClientUtils
                                            .waitUntilJobInitializationFinished(
                                                    () ->
                                                            miniCluster
                                                                    .getJobStatus(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    () ->
                                                            miniCluster
                                                                    .requestJobResult(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    userCodeClassloader);
                                    return submissionResult;
                                }))
                .thenApply(
                        result ->
                                new MiniClusterJobClient(
                                        result.getJobID(),
                                        miniCluster,
                                        userCodeClassloader,
                                        MiniClusterJobClient.JobFinalizationBehavior
                                                .SHUTDOWN_CLUSTER))
                .whenComplete(
                        (ignored, throwable) -> {
                            if (throwable != null) {
                                // We failed to create the JobClient and must shutdown to ensure
                                // cleanup.
                                shutDownCluster(miniCluster);
                            }
                        })
                .thenApply(Function.identity());
    }

    private MiniClusterConfiguration getMiniClusterConfig(int maximumParallelism) {
        Configuration configuration = new Configuration(this.configuration);

        if (!configuration.contains(RestOptions.BIND_PORT)) {
            configuration.set(RestOptions.BIND_PORT, "0");
        }

        int numTaskManagers = configuration.get(TaskManagerOptions.MINI_CLUSTER_NUM_TASK_MANAGERS);

        Map overwriteParallelisms =
                configuration.get(PipelineOptions.PARALLELISM_OVERRIDES);
        if (overwriteParallelisms != null) {
            for (String overrideParallelism : overwriteParallelisms.values()) {
                maximumParallelism =
                        Math.max(maximumParallelism, Integer.parseInt(overrideParallelism));
            }
        }

        int finalMaximumParallelism = maximumParallelism;
        int numSlotsPerTaskManager =
                configuration
                        .getOptional(TaskManagerOptions.NUM_TASK_SLOTS)
                        .orElseGet(
                                () ->
                                        finalMaximumParallelism > 0
                                                ? MathUtils.divideRoundUp(
                                                        finalMaximumParallelism, numTaskManagers)
                                                : TaskManagerOptions.NUM_TASK_SLOTS.defaultValue());

        return new MiniClusterConfiguration.Builder()
                .setConfiguration(configuration)
                .setNumTaskManagers(numTaskManagers)
                .setRpcServiceSharing(RpcServiceSharing.SHARED)
                .setNumSlotsPerTaskManager(numSlotsPerTaskManager)
                .build();
    }

    private static void shutDownCluster(MiniCluster miniCluster) {
        miniCluster
                .closeAsync()
                .whenComplete(
                        (ignored, throwable) -> {
                            if (throwable != null) {
                                LOG.warn("Shutdown of MiniCluster failed.", throwable);
                            }
                        });
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy