All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.runtime.jobmaster.JobMaster Maven / Gradle / Ivy

There is a newer version: 1.13.6
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.jobmaster;

import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.CheckpointMetrics;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.SubtaskState;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.Execution;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.IntermediateResult;
import org.apache.flink.runtime.executiongraph.JobStatusListener;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.heartbeat.HeartbeatListener;
import org.apache.flink.runtime.heartbeat.HeartbeatManager;
import org.apache.flink.runtime.heartbeat.HeartbeatServices;
import org.apache.flink.runtime.heartbeat.HeartbeatTarget;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.highavailability.LeaderIdMismatchException;
import org.apache.flink.runtime.instance.Slot;
import org.apache.flink.runtime.instance.SlotPool;
import org.apache.flink.runtime.instance.SlotPoolGateway;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException;
import org.apache.flink.runtime.jobmanager.slots.AllocatedSlot;
import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.query.KvStateID;
import org.apache.flink.runtime.query.KvStateLocation;
import org.apache.flink.runtime.query.KvStateLocationRegistry;
import org.apache.flink.runtime.query.KvStateServerAddress;
import org.apache.flink.runtime.query.UnknownKvStateLocation;
import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.registration.RetryingRegistration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.StartStoppable;
import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;

import org.slf4j.Logger;

import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;

import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * JobMaster implementation. The job master is responsible for the execution of a single
 * {@link JobGraph}.
 * 

* It offers the following methods as part of its rpc interface to interact with the JobMaster * remotely: *

    *
  • {@link #updateTaskExecutionState} updates the task execution state for * given task
  • *
*/ public class JobMaster extends RpcEndpoint { /** Default names for Flink's distributed components */ public static final String JOB_MANAGER_NAME = "jobmanager"; public static final String ARCHIVE_NAME = "archive"; private static final AtomicReferenceFieldUpdater LEADER_ID_UPDATER = AtomicReferenceFieldUpdater.newUpdater(JobMaster.class, UUID.class, "leaderSessionID"); // ------------------------------------------------------------------------ private final ResourceID resourceId; /** Logical representation of the job */ private final JobGraph jobGraph; /** Configuration of the JobManager */ private final Configuration configuration; private final Time rpcTimeout; /** Service to contend for and retrieve the leadership of JM and RM */ private final HighAvailabilityServices highAvailabilityServices; /** Blob cache manager used across jobs */ private final BlobLibraryCacheManager libraryCacheManager; /** The metrics for the JobManager itself */ private final MetricGroup jobManagerMetricGroup; /** The metrics for the job */ private final MetricGroup jobMetricGroup; /** The heartbeat manager with task managers */ private final HeartbeatManager taskManagerHeartbeatManager; /** The heartbeat manager with resource manager */ private final HeartbeatManager resourceManagerHeartbeatManager; /** The execution context which is used to execute futures */ private final Executor executor; private final OnCompletionActions jobCompletionActions; private final FatalErrorHandler errorHandler; private final ClassLoader userCodeLoader; /** The execution graph of this job */ private final ExecutionGraph executionGraph; private final SlotPool slotPool; private final SlotPoolGateway slotPoolGateway; private volatile UUID leaderSessionID; // --------- ResourceManager -------- /** Leader retriever service used to locate ResourceManager's address */ private LeaderRetrievalService resourceManagerLeaderRetriever; /** Connection with ResourceManager, null if not located address yet or we close it initiative */ private ResourceManagerConnection resourceManagerConnection; // --------- TaskManagers -------- private final Map> registeredTaskManagers; // ------------------------------------------------------------------------ public JobMaster( RpcService rpcService, ResourceID resourceId, JobGraph jobGraph, Configuration configuration, HighAvailabilityServices highAvailabilityService, HeartbeatServices heartbeatServices, ScheduledExecutorService executor, BlobLibraryCacheManager libraryCacheManager, RestartStrategyFactory restartStrategyFactory, Time rpcAskTimeout, @Nullable JobManagerMetricGroup jobManagerMetricGroup, OnCompletionActions jobCompletionActions, FatalErrorHandler errorHandler, ClassLoader userCodeLoader) throws Exception { super(rpcService, AkkaRpcServiceUtils.createRandomName(JobMaster.JOB_MANAGER_NAME)); this.resourceId = checkNotNull(resourceId); this.jobGraph = checkNotNull(jobGraph); this.configuration = checkNotNull(configuration); this.rpcTimeout = rpcAskTimeout; this.highAvailabilityServices = checkNotNull(highAvailabilityService); this.libraryCacheManager = checkNotNull(libraryCacheManager); this.executor = checkNotNull(executor); this.jobCompletionActions = checkNotNull(jobCompletionActions); this.errorHandler = checkNotNull(errorHandler); this.userCodeLoader = checkNotNull(userCodeLoader); this.taskManagerHeartbeatManager = heartbeatServices.createHeartbeatManagerSender( resourceId, new TaskManagerHeartbeatListener(), rpcService.getScheduledExecutor(), log); this.resourceManagerHeartbeatManager = heartbeatServices.createHeartbeatManager( resourceId, new ResourceManagerHeartbeatListener(), rpcService.getScheduledExecutor(), log); final String jobName = jobGraph.getName(); final JobID jid = jobGraph.getJobID(); if (jobManagerMetricGroup != null) { this.jobManagerMetricGroup = jobManagerMetricGroup; this.jobMetricGroup = jobManagerMetricGroup.addJob(jobGraph); } else { this.jobManagerMetricGroup = new UnregisteredMetricsGroup(); this.jobMetricGroup = new UnregisteredMetricsGroup(); } log.info("Initializing job {} ({}).", jobName, jid); final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration = jobGraph.getSerializedExecutionConfig() .deserializeValue(userCodeLoader) .getRestartStrategy(); final RestartStrategy restartStrategy = (restartStrategyConfiguration != null) ? RestartStrategyFactory.createRestartStrategy(restartStrategyConfiguration) : restartStrategyFactory.createRestartStrategy(); log.info("Using restart strategy {} for {} ({}).", restartStrategy, jobName, jid); CheckpointRecoveryFactory checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory(); resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever(); this.slotPool = new SlotPool(rpcService, jobGraph.getJobID()); this.slotPoolGateway = slotPool.getSelf(); this.executionGraph = ExecutionGraphBuilder.buildGraph( null, jobGraph, configuration, executor, executor, slotPool.getSlotProvider(), userCodeLoader, checkpointRecoveryFactory, rpcAskTimeout, restartStrategy, jobMetricGroup, -1, log); // register self as job status change listener executionGraph.registerJobStatusListener(new JobManagerJobStatusListener()); this.registeredTaskManagers = new HashMap<>(4); } //---------------------------------------------------------------------------------------------- // Lifecycle management //---------------------------------------------------------------------------------------------- @Override public void start() { throw new UnsupportedOperationException("Should never call start() without leader ID"); } /** * Start the rpc service and begin to run the job. * * @param leaderSessionID The necessary leader id for running the job. */ public void start(final UUID leaderSessionID) throws Exception { if (LEADER_ID_UPDATER.compareAndSet(this, null, leaderSessionID)) { // make sure we receive RPC and async calls super.start(); log.info("JobManager started as leader {} for job {}", leaderSessionID, jobGraph.getJobID()); getSelf().startJobExecution(); } else { log.warn("Job already started with leader ID {}, ignoring this start request.", leaderSessionID); } } /** * Suspend the job and shutdown all other services including rpc. */ @Override public void shutDown() throws Exception { taskManagerHeartbeatManager.stop(); resourceManagerHeartbeatManager.stop(); // make sure there is a graceful exit getSelf().suspendExecution(new Exception("JobManager is shutting down.")); super.shutDown(); } //---------------------------------------------------------------------------------------------- // RPC methods //---------------------------------------------------------------------------------------------- //-- job starting and stopping ----------------------------------------------------------------- @RpcMethod public void startJobExecution() { // double check that the leader status did not change if (leaderSessionID == null) { log.info("Aborting job startup - JobManager lost leader status"); return; } log.info("Starting execution of job {} ({})", jobGraph.getName(), jobGraph.getJobID()); try { // start the slot pool make sure the slot pool now accepts messages for this leader log.debug("Staring SlotPool component"); slotPool.start(leaderSessionID, getAddress()); } catch (Exception e) { log.error("Faild to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), e); handleFatalError(new Exception("Could not start job execution: Failed to start the slot pool.", e)); } try { // job is ready to go, try to establish connection with resource manager // - activate leader retrieval for the resource manager // - on notification of the leader, the connection will be established and // the slot pool will start requesting slots resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Throwable t) { log.error("Failed to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t); handleFatalError(new Exception( "Could not start job execution: Failed to start leader service for Resource Manager", t)); return; } // start scheduling job in another thread executor.execute(new Runnable() { @Override public void run() { try { executionGraph.scheduleForExecution(); } catch (Throwable t) { executionGraph.failGlobal(t); } } }); } /** * Suspending job, all the running tasks will be cancelled, and communication with other components * will be disposed. * *

Mostly job is suspended because of the leadership has been revoked, one can be restart this job by * calling the {@link #start(UUID)} method once we take the leadership back again. * * @param cause The reason of why this job been suspended. */ @RpcMethod public void suspendExecution(final Throwable cause) { if (leaderSessionID == null) { log.debug("Job has already been suspended or shutdown."); return; } // not leader any more - should not accept any leader messages any more leaderSessionID = null; try { resourceManagerLeaderRetriever.stop(); } catch (Throwable t) { log.warn("Failed to stop resource manager leader retriever when suspending.", t); } // tell the execution graph (JobManager is still processing messages here) executionGraph.suspend(cause); // receive no more messages until started again, should be called before we clear self leader id ((StartStoppable) getSelf()).stop(); // the slot pool stops receiving messages and clears its pooled slots slotPoolGateway.suspend(); // disconnect from resource manager: closeResourceManagerConnection(new Exception("Execution was suspended.", cause)); } //---------------------------------------------------------------------------------------------- /** * Updates the task execution state for a given task. * * @param taskExecutionState New task execution state for a given task * @return Acknowledge the task execution state update */ @RpcMethod public Acknowledge updateTaskExecutionState( final UUID leaderSessionID, final TaskExecutionState taskExecutionState) throws Exception { checkNotNull(taskExecutionState, "taskExecutionState"); validateLeaderSessionId(leaderSessionID); if (executionGraph.updateState(taskExecutionState)) { return Acknowledge.get(); } else { throw new ExecutionGraphException("The execution attempt " + taskExecutionState.getID() + " was not found."); } } @RpcMethod public SerializedInputSplit requestNextInputSplit( final UUID leaderSessionID, final JobVertexID vertexID, final ExecutionAttemptID executionAttempt) throws Exception { validateLeaderSessionId(leaderSessionID); final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt); if (execution == null) { // can happen when JobManager had already unregistered this execution upon on task failure, // but TaskManager get some delay to aware of that situation if (log.isDebugEnabled()) { log.debug("Can not find Execution for attempt {}.", executionAttempt); } // but we should TaskManager be aware of this throw new Exception("Can not find Execution for attempt " + executionAttempt); } final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID); if (vertex == null) { log.error("Cannot find execution vertex for vertex ID {}.", vertexID); throw new Exception("Cannot find execution vertex for vertex ID " + vertexID); } final InputSplitAssigner splitAssigner = vertex.getSplitAssigner(); if (splitAssigner == null) { log.error("No InputSplitAssigner for vertex ID {}.", vertexID); throw new Exception("No InputSplitAssigner for vertex ID " + vertexID); } final Slot slot = execution.getAssignedResource(); final int taskId = execution.getVertex().getParallelSubtaskIndex(); final String host = slot != null ? slot.getTaskManagerLocation().getHostname() : null; final InputSplit nextInputSplit = splitAssigner.getNextInputSplit(host, taskId); if (log.isDebugEnabled()) { log.debug("Send next input split {}.", nextInputSplit); } try { final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit); return new SerializedInputSplit(serializedInputSplit); } catch (Exception ex) { log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex); IOException reason = new IOException("Could not serialize the next input split of class " + nextInputSplit.getClass() + ".", ex); vertex.fail(reason); throw reason; } } @RpcMethod public ExecutionState requestPartitionState( final UUID leaderSessionID, final IntermediateDataSetID intermediateResultId, final ResultPartitionID resultPartitionId) throws Exception { validateLeaderSessionId(leaderSessionID); final Execution execution = executionGraph.getRegisteredExecutions().get(resultPartitionId.getProducerId()); if (execution != null) { return execution.getState(); } else { final IntermediateResult intermediateResult = executionGraph.getAllIntermediateResults().get(intermediateResultId); if (intermediateResult != null) { // Try to find the producing execution Execution producerExecution = intermediateResult .getPartitionById(resultPartitionId.getPartitionId()) .getProducer() .getCurrentExecutionAttempt(); if (producerExecution.getAttemptId() == resultPartitionId.getProducerId()) { return producerExecution.getState(); } else { throw new PartitionProducerDisposedException(resultPartitionId); } } else { throw new IllegalArgumentException("Intermediate data set with ID " + intermediateResultId + " not found."); } } } @RpcMethod public Acknowledge scheduleOrUpdateConsumers( final UUID leaderSessionID, final ResultPartitionID partitionID) throws Exception { validateLeaderSessionId(leaderSessionID); executionGraph.scheduleOrUpdateConsumers(partitionID); return Acknowledge.get(); } @RpcMethod public void disconnectTaskManager(final ResourceID resourceID, final Exception cause) { taskManagerHeartbeatManager.unmonitorTarget(resourceID); slotPoolGateway.releaseTaskManager(resourceID); Tuple2 taskManagerConnection = registeredTaskManagers.remove(resourceID); if (taskManagerConnection != null) { taskManagerConnection.f1.disconnectJobManager(jobGraph.getJobID(), cause); } } // TODO: This method needs a leader session ID @RpcMethod public void acknowledgeCheckpoint( final JobID jobID, final ExecutionAttemptID executionAttemptID, final long checkpointId, final CheckpointMetrics checkpointMetrics, final SubtaskState checkpointState) throws CheckpointException { final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint(jobID, executionAttemptID, checkpointId, checkpointMetrics, checkpointState); if (checkpointCoordinator != null) { getRpcService().execute(new Runnable() { @Override public void run() { try { checkpointCoordinator.receiveAcknowledgeMessage(ackMessage); } catch (Throwable t) { log.warn("Error while processing checkpoint acknowledgement message"); } } }); } else { log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator", jobGraph.getJobID()); } } // TODO: This method needs a leader session ID @RpcMethod public void declineCheckpoint( final JobID jobID, final ExecutionAttemptID executionAttemptID, final long checkpointID, final Throwable reason) { final DeclineCheckpoint decline = new DeclineCheckpoint( jobID, executionAttemptID, checkpointID, reason); final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); if (checkpointCoordinator != null) { getRpcService().execute(new Runnable() { @Override public void run() { try { checkpointCoordinator.receiveDeclineMessage(decline); } catch (Exception e) { log.error("Error in CheckpointCoordinator while processing {}", decline, e); } } }); } else { log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator", jobGraph.getJobID()); } } @RpcMethod public KvStateLocation lookupKvStateLocation(final String registrationName) throws Exception { if (log.isDebugEnabled()) { log.debug("Lookup key-value state for job {} with registration " + "name {}.", jobGraph.getJobID(), registrationName); } final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry(); final KvStateLocation location = registry.getKvStateLocation(registrationName); if (location != null) { return location; } else { throw new UnknownKvStateLocation(registrationName); } } @RpcMethod public void notifyKvStateRegistered( final JobVertexID jobVertexId, final KeyGroupRange keyGroupRange, final String registrationName, final KvStateID kvStateId, final KvStateServerAddress kvStateServerAddress) { if (log.isDebugEnabled()) { log.debug("Key value state registered for job {} under name {}.", jobGraph.getJobID(), registrationName); } try { executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered( jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress); } catch (Exception e) { log.error("Failed to notify KvStateRegistry about registration {}.", registrationName); } } @RpcMethod public void notifyKvStateUnregistered( JobVertexID jobVertexId, KeyGroupRange keyGroupRange, String registrationName) { if (log.isDebugEnabled()) { log.debug("Key value state unregistered for job {} under name {}.", jobGraph.getJobID(), registrationName); } try { executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered( jobVertexId, keyGroupRange, registrationName); } catch (Exception e) { log.error("Failed to notify KvStateRegistry about registration {}.", registrationName); } } @RpcMethod public ClassloadingProps requestClassloadingProps() throws Exception { return new ClassloadingProps(libraryCacheManager.getBlobServerPort(), executionGraph.getRequiredJarFiles(), executionGraph.getRequiredClasspaths()); } @RpcMethod public Future> offerSlots( final ResourceID taskManagerId, final Iterable slots, final UUID leaderId) throws Exception { validateLeaderSessionId(leaderId); Tuple2 taskManager = registeredTaskManagers.get(taskManagerId); if (taskManager == null) { throw new Exception("Unknown TaskManager " + taskManagerId); } final JobID jid = jobGraph.getJobID(); final TaskManagerLocation taskManagerLocation = taskManager.f0; final TaskExecutorGateway taskExecutorGateway = taskManager.f1; final ArrayList> slotsAndOffers = new ArrayList<>(); final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, leaderId); for (SlotOffer slotOffer : slots) { final AllocatedSlot slot = new AllocatedSlot( slotOffer.getAllocationId(), jid, taskManagerLocation, slotOffer.getSlotIndex(), slotOffer.getResourceProfile(), rpcTaskManagerGateway); slotsAndOffers.add(new Tuple2<>(slot, slotOffer)); } return slotPoolGateway.offerSlots(slotsAndOffers); } @RpcMethod public void failSlot(final ResourceID taskManagerId, final AllocationID allocationId, final UUID leaderId, final Exception cause) throws Exception { validateLeaderSessionId(leaderSessionID); if (!registeredTaskManagers.containsKey(taskManagerId)) { throw new Exception("Unknown TaskManager " + taskManagerId); } slotPoolGateway.failAllocation(allocationId, cause); } @RpcMethod public Future registerTaskManager( final String taskManagerRpcAddress, final TaskManagerLocation taskManagerLocation, final UUID leaderId) throws Exception { if (!JobMaster.this.leaderSessionID.equals(leaderId)) { log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " + "leader session ID {} did not equal the received leader session ID {}.", taskManagerLocation.getResourceID(), taskManagerRpcAddress, JobMaster.this.leaderSessionID, leaderId); throw new Exception("Leader id not match, expected: " + JobMaster.this.leaderSessionID + ", actual: " + leaderId); } final ResourceID taskManagerId = taskManagerLocation.getResourceID(); if (registeredTaskManagers.containsKey(taskManagerId)) { final RegistrationResponse response = new JMTMRegistrationSuccess( resourceId, libraryCacheManager.getBlobServerPort()); return FlinkCompletableFuture.completed(response); } else { return getRpcService().execute(new Callable() { @Override public TaskExecutorGateway call() throws Exception { return getRpcService().connect(taskManagerRpcAddress, TaskExecutorGateway.class) .get(rpcTimeout.getSize(), rpcTimeout.getUnit()); } }).handleAsync(new BiFunction() { @Override public RegistrationResponse apply(final TaskExecutorGateway taskExecutorGateway, Throwable throwable) { if (throwable != null) { return new RegistrationResponse.Decline(throwable.getMessage()); } if (!JobMaster.this.leaderSessionID.equals(leaderId)) { log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " + "leader session ID {} did not equal the received leader session ID {}.", taskManagerId, taskManagerRpcAddress, JobMaster.this.leaderSessionID, leaderId); return new RegistrationResponse.Decline("Invalid leader session id"); } slotPoolGateway.registerTaskManager(taskManagerId); registeredTaskManagers.put(taskManagerId, Tuple2.of(taskManagerLocation, taskExecutorGateway)); // monitor the task manager as heartbeat target taskManagerHeartbeatManager.monitorTarget(taskManagerId, new HeartbeatTarget() { @Override public void receiveHeartbeat(ResourceID resourceID, Void payload) { // the task manager will not request heartbeat, so this method will never be called currently } @Override public void requestHeartbeat(ResourceID resourceID, Void payload) { taskExecutorGateway.heartbeatFromJobManager(resourceID); } }); return new JMTMRegistrationSuccess(resourceId, libraryCacheManager.getBlobServerPort()); } }, getMainThreadExecutor()); } } @RpcMethod public void disconnectResourceManager( final UUID jobManagerLeaderId, final UUID resourceManagerLeaderId, final Exception cause) throws Exception { validateLeaderSessionId(jobManagerLeaderId); if (resourceManagerConnection != null && resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderId)) { closeResourceManagerConnection(cause); } } @RpcMethod public void heartbeatFromTaskManager(final ResourceID resourceID) { taskManagerHeartbeatManager.receiveHeartbeat(resourceID, null); } @RpcMethod public void heartbeatFromResourceManager(final ResourceID resourceID) { resourceManagerHeartbeatManager.requestHeartbeat(resourceID, null); } //---------------------------------------------------------------------------------------------- // Internal methods //---------------------------------------------------------------------------------------------- private void handleFatalError(final Throwable cause) { runAsync(new Runnable() { @Override public void run() { log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause); try { shutDown(); } catch (Exception e) { cause.addSuppressed(e); } errorHandler.onFatalError(cause); } }); } private void jobStatusChanged(final JobStatus newJobStatus, long timestamp, final Throwable error) { validateRunsInMainThread(); final JobID jobID = executionGraph.getJobID(); final String jobName = executionGraph.getJobName(); log.info("Status of job {} ({}) changed to {}.", jobID, jobName, newJobStatus, error); if (newJobStatus.isGloballyTerminalState()) { switch (newJobStatus) { case FINISHED: try { // TODO get correct job duration // job done, let's get the accumulators Map accumulatorResults = executionGraph.getAccumulators(); JobExecutionResult result = new JobExecutionResult(jobID, 0L, accumulatorResults); jobCompletionActions.jobFinished(result); } catch (Exception e) { log.error("Cannot fetch final accumulators for job {} ({})", jobName, jobID, e); final JobExecutionException exception = new JobExecutionException(jobID, "Failed to retrieve accumulator results. " + "The job is registered as 'FINISHED (successful), but this notification describes " + "a failure, since the resulting accumulators could not be fetched.", e); jobCompletionActions.jobFailed(exception); } break; case CANCELED: { final JobExecutionException exception = new JobExecutionException( jobID, "Job was cancelled.", new Exception("The job was cancelled")); jobCompletionActions.jobFailed(exception); break; } case FAILED: { final Throwable unpackedError = SerializedThrowable.get(error, userCodeLoader); final JobExecutionException exception = new JobExecutionException( jobID, "Job execution failed.", unpackedError); jobCompletionActions.jobFailed(exception); break; } default: // this can happen only if the enum is buggy throw new IllegalStateException(newJobStatus.toString()); } } } private void notifyOfNewResourceManagerLeader(final String resourceManagerAddress, final UUID resourceManagerLeaderId) { if (resourceManagerConnection != null) { if (resourceManagerAddress != null) { if (resourceManagerAddress.equals(resourceManagerConnection.getTargetAddress()) && resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId())) { // both address and leader id are not changed, we can keep the old connection return; } closeResourceManagerConnection(new Exception( "ResourceManager leader changed to new address " + resourceManagerAddress)); log.info("ResourceManager leader changed from {} to {}. Registering at new leader.", resourceManagerConnection.getTargetAddress(), resourceManagerAddress); } else { log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.", resourceManagerConnection.getTargetAddress()); } } if (resourceManagerAddress != null) { log.info("Attempting to register at ResourceManager {}", resourceManagerAddress); resourceManagerConnection = new ResourceManagerConnection( log, jobGraph.getJobID(), resourceId, getAddress(), leaderSessionID, resourceManagerAddress, resourceManagerLeaderId, executor); resourceManagerConnection.start(); } } private void establishResourceManagerConnection(final JobMasterRegistrationSuccess success) { final UUID resourceManagerLeaderId = success.getResourceManagerLeaderId(); // verify the response with current connection if (resourceManagerConnection != null && resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderId)) { log.info("JobManager successfully registered at ResourceManager, leader id: {}.", resourceManagerLeaderId); final ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway(); slotPoolGateway.connectToResourceManager(resourceManagerLeaderId, resourceManagerGateway); resourceManagerHeartbeatManager.monitorTarget(success.getResourceManagerResourceId(), new HeartbeatTarget() { @Override public void receiveHeartbeat(ResourceID resourceID, Void payload) { resourceManagerGateway.heartbeatFromJobManager(resourceID); } @Override public void requestHeartbeat(ResourceID resourceID, Void payload) { // request heartbeat will never be called on the job manager side } }); } } private void closeResourceManagerConnection(Exception cause) { if (resourceManagerConnection != null) { log.info("Close ResourceManager connection {}.", resourceManagerConnection.getResourceManagerResourceID(), cause); resourceManagerHeartbeatManager.unmonitorTarget(resourceManagerConnection.getResourceManagerResourceID()); ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway(); resourceManagerGateway.disconnectJobManager(resourceManagerConnection.getJobID(), cause); resourceManagerConnection.close(); resourceManagerConnection = null; } slotPoolGateway.disconnectResourceManager(); } private void validateLeaderSessionId(UUID leaderSessionID) throws LeaderIdMismatchException { if (this.leaderSessionID == null || !this.leaderSessionID.equals(leaderSessionID)) { throw new LeaderIdMismatchException(this.leaderSessionID, leaderSessionID); } } //---------------------------------------------------------------------------------------------- // Utility classes //---------------------------------------------------------------------------------------------- private class ResourceManagerLeaderListener implements LeaderRetrievalListener { @Override public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) { runAsync(new Runnable() { @Override public void run() { notifyOfNewResourceManagerLeader(leaderAddress, leaderSessionID); } }); } @Override public void handleError(final Exception exception) { handleFatalError(new Exception("Fatal error in the ResourceManager leader service", exception)); } } //---------------------------------------------------------------------------------------------- private class ResourceManagerConnection extends RegisteredRpcConnection { private final JobID jobID; private final ResourceID jobManagerResourceID; private final String jobManagerRpcAddress; private final UUID jobManagerLeaderID; private ResourceID resourceManagerResourceID; ResourceManagerConnection( final Logger log, final JobID jobID, final ResourceID jobManagerResourceID, final String jobManagerRpcAddress, final UUID jobManagerLeaderID, final String resourceManagerAddress, final UUID resourceManagerLeaderID, final Executor executor) { super(log, resourceManagerAddress, resourceManagerLeaderID, executor); this.jobID = checkNotNull(jobID); this.jobManagerResourceID = checkNotNull(jobManagerResourceID); this.jobManagerRpcAddress = checkNotNull(jobManagerRpcAddress); this.jobManagerLeaderID = checkNotNull(jobManagerLeaderID); } @Override protected RetryingRegistration generateRegistration() { return new RetryingRegistration( log, getRpcService(), "ResourceManager", ResourceManagerGateway.class, getTargetAddress(), getTargetLeaderId()) { @Override protected Future invokeRegistration( ResourceManagerGateway gateway, UUID leaderId, long timeoutMillis) throws Exception { Time timeout = Time.milliseconds(timeoutMillis); return gateway.registerJobManager( leaderId, jobManagerLeaderID, jobManagerResourceID, jobManagerRpcAddress, jobID, timeout); } }; } @Override protected void onRegistrationSuccess(final JobMasterRegistrationSuccess success) { runAsync(new Runnable() { @Override public void run() { resourceManagerResourceID = success.getResourceManagerResourceId(); establishResourceManagerConnection(success); } }); } @Override protected void onRegistrationFailure(final Throwable failure) { handleFatalError(failure); } public ResourceID getResourceManagerResourceID() { return resourceManagerResourceID; } public JobID getJobID() { return jobID; } } //---------------------------------------------------------------------------------------------- private class JobManagerJobStatusListener implements JobStatusListener { @Override public void jobStatusChanges( final JobID jobId, final JobStatus newJobStatus, final long timestamp, final Throwable error) { // run in rpc thread to avoid concurrency runAsync(new Runnable() { @Override public void run() { jobStatusChanged(newJobStatus, timestamp, error); } }); } } private class TaskManagerHeartbeatListener implements HeartbeatListener { @Override public void notifyHeartbeatTimeout(ResourceID resourceID) { log.info("Heartbeat of TaskManager with id {} timed out.", resourceID); getSelf().disconnectTaskManager( resourceID, new TimeoutException("Heartbeat of TaskManager with id " + resourceID + " timed out.")); } @Override public void reportPayload(ResourceID resourceID, Void payload) { // nothing to do since there is no payload } @Override public Future retrievePayload() { return FlinkCompletableFuture.completed(null); } } private class ResourceManagerHeartbeatListener implements HeartbeatListener { @Override public void notifyHeartbeatTimeout(final ResourceID resourceId) { runAsync(new Runnable() { @Override public void run() { log.info("The heartbeat of ResourceManager with id {} timed out.", resourceId); closeResourceManagerConnection( new TimeoutException( "The heartbeat of ResourceManager with id " + resourceId + " timed out.")); } }); } @Override public void reportPayload(ResourceID resourceID, Void payload) { // nothing to do since the payload is of type Void } @Override public Future retrievePayload() { return FlinkCompletableFuture.completed(null); } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy