All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph Maven / Gradle / Ivy

There is a newer version: 1.13.6
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.executiongraph;

import org.apache.flink.api.common.ArchivedExecutionConfig;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;

import javax.annotation.Nullable;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;

/** An archived execution graph represents a serializable form of an {@link ExecutionGraph}. */
public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializable {

    private static final long serialVersionUID = 7231383912742578428L;
    // --------------------------------------------------------------------------------------------

    /** The ID of the job this graph has been built for. */
    private final JobID jobID;

    /** The name of the original job graph. */
    private final String jobName;

    /** All job vertices that are part of this graph. */
    private final Map tasks;

    /** All vertices, in the order in which they were created. * */
    private final List verticesInCreationOrder;

    /**
     * Timestamps (in milliseconds as returned by {@code System.currentTimeMillis()} when the
     * execution graph transitioned into a certain state. The index into this array is the ordinal
     * of the enum value, i.e. the timestamp when the graph went into state "RUNNING" is at {@code
     * stateTimestamps[RUNNING.ordinal()]}.
     */
    private final long[] stateTimestamps;

    // ------ Configuration of the Execution -------

    // ------ Execution status and progress. These values are volatile, and accessed under the lock
    // -------

    /** Current status of the job execution. */
    private final JobStatus state;

    /**
     * The exception that caused the job to fail. This is set to the first root exception that was
     * not recoverable and triggered job failure
     */
    @Nullable private final ErrorInfo failureCause;

    // ------ Fields that are only relevant for archived execution graphs ------------
    private final String jsonPlan;
    private final StringifiedAccumulatorResult[] archivedUserAccumulators;
    private final ArchivedExecutionConfig archivedExecutionConfig;
    private final boolean isStoppable;
    private final Map>> serializedUserAccumulators;

    @Nullable private final CheckpointCoordinatorConfiguration jobCheckpointingConfiguration;

    @Nullable private final CheckpointStatsSnapshot checkpointStatsSnapshot;

    @Nullable private final String stateBackendName;

    @Nullable private final String checkpointStorageName;

    public ArchivedExecutionGraph(
            JobID jobID,
            String jobName,
            Map tasks,
            List verticesInCreationOrder,
            long[] stateTimestamps,
            JobStatus state,
            @Nullable ErrorInfo failureCause,
            String jsonPlan,
            StringifiedAccumulatorResult[] archivedUserAccumulators,
            Map>> serializedUserAccumulators,
            ArchivedExecutionConfig executionConfig,
            boolean isStoppable,
            @Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
            @Nullable CheckpointStatsSnapshot checkpointStatsSnapshot,
            @Nullable String stateBackendName,
            @Nullable String checkpointStorageName) {

        this.jobID = Preconditions.checkNotNull(jobID);
        this.jobName = Preconditions.checkNotNull(jobName);
        this.tasks = Preconditions.checkNotNull(tasks);
        this.verticesInCreationOrder = Preconditions.checkNotNull(verticesInCreationOrder);
        this.stateTimestamps = Preconditions.checkNotNull(stateTimestamps);
        this.state = Preconditions.checkNotNull(state);
        this.failureCause = failureCause;
        this.jsonPlan = Preconditions.checkNotNull(jsonPlan);
        this.archivedUserAccumulators = Preconditions.checkNotNull(archivedUserAccumulators);
        this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators);
        this.archivedExecutionConfig = Preconditions.checkNotNull(executionConfig);
        this.isStoppable = isStoppable;
        this.jobCheckpointingConfiguration = jobCheckpointingConfiguration;
        this.checkpointStatsSnapshot = checkpointStatsSnapshot;
        this.stateBackendName = stateBackendName;
        this.checkpointStorageName = checkpointStorageName;
    }

    // --------------------------------------------------------------------------------------------

    @Override
    public String getJsonPlan() {
        return jsonPlan;
    }

    @Override
    public JobID getJobID() {
        return jobID;
    }

    @Override
    public String getJobName() {
        return jobName;
    }

    @Override
    public JobStatus getState() {
        return state;
    }

    @Nullable
    @Override
    public ErrorInfo getFailureInfo() {
        return failureCause;
    }

    @Override
    public ArchivedExecutionJobVertex getJobVertex(JobVertexID id) {
        return this.tasks.get(id);
    }

    @Override
    public Map getAllVertices() {
        return Collections.unmodifiableMap(this.tasks);
    }

    @Override
    public Iterable getVerticesTopologically() {
        // we return a specific iterator that does not fail with concurrent modifications
        // the list is append only, so it is safe for that
        final int numElements = this.verticesInCreationOrder.size();

        return new Iterable() {
            @Override
            public Iterator iterator() {
                return new Iterator() {
                    private int pos = 0;

                    @Override
                    public boolean hasNext() {
                        return pos < numElements;
                    }

                    @Override
                    public ArchivedExecutionJobVertex next() {
                        if (hasNext()) {
                            return verticesInCreationOrder.get(pos++);
                        } else {
                            throw new NoSuchElementException();
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException();
                    }
                };
            }
        };
    }

    @Override
    public Iterable getAllExecutionVertices() {
        return new Iterable() {
            @Override
            public Iterator iterator() {
                return new AllVerticesIterator(getVerticesTopologically().iterator());
            }
        };
    }

    @Override
    public long getStatusTimestamp(JobStatus status) {
        return this.stateTimestamps[status.ordinal()];
    }

    @Override
    public CheckpointCoordinatorConfiguration getCheckpointCoordinatorConfiguration() {
        return jobCheckpointingConfiguration;
    }

    @Override
    public CheckpointStatsSnapshot getCheckpointStatsSnapshot() {
        return checkpointStatsSnapshot;
    }

    @Override
    public ArchivedExecutionConfig getArchivedExecutionConfig() {
        return archivedExecutionConfig;
    }

    @Override
    public boolean isStoppable() {
        return isStoppable;
    }

    @Override
    public StringifiedAccumulatorResult[] getAccumulatorResultsStringified() {
        return archivedUserAccumulators;
    }

    @Override
    public Map>> getAccumulatorsSerialized() {
        return serializedUserAccumulators;
    }

    @Override
    public Optional getStateBackendName() {
        return Optional.ofNullable(stateBackendName);
    }

    @Override
    public Optional getCheckpointStorageName() {
        return Optional.ofNullable(checkpointStorageName);
    }

    class AllVerticesIterator implements Iterator {

        private final Iterator jobVertices;

        private ArchivedExecutionVertex[] currVertices;

        private int currPos;

        public AllVerticesIterator(Iterator jobVertices) {
            this.jobVertices = jobVertices;
        }

        @Override
        public boolean hasNext() {
            while (true) {
                if (currVertices != null) {
                    if (currPos < currVertices.length) {
                        return true;
                    } else {
                        currVertices = null;
                    }
                } else if (jobVertices.hasNext()) {
                    currVertices = jobVertices.next().getTaskVertices();
                    currPos = 0;
                } else {
                    return false;
                }
            }
        }

        @Override
        public ArchivedExecutionVertex next() {
            if (hasNext()) {
                return currVertices[currPos++];
            } else {
                throw new NoSuchElementException();
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }

    /**
     * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
     *
     * @param executionGraph to create the ArchivedExecutionGraph from
     * @return ArchivedExecutionGraph created from the given ExecutionGraph
     */
    public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) {
        return createFrom(executionGraph, null);
    }

    /**
     * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
     *
     * @param executionGraph to create the ArchivedExecutionGraph from
     * @param statusOverride optionally overrides the JobStatus of the ExecutionGraph with a
     *     non-globally-terminal state and clears timestamps of globally-terminal states
     * @return ArchivedExecutionGraph created from the given ExecutionGraph
     */
    public static ArchivedExecutionGraph createFrom(
            ExecutionGraph executionGraph, @Nullable JobStatus statusOverride) {
        Preconditions.checkArgument(
                statusOverride == null || !statusOverride.isGloballyTerminalState(),
                "Status override is only allowed for non-globally-terminal states.");

        final int numberVertices = executionGraph.getTotalNumberOfVertices();

        Map archivedTasks = new HashMap<>(numberVertices);
        List archivedVerticesInCreationOrder =
                new ArrayList<>(numberVertices);

        for (ExecutionJobVertex task : executionGraph.getVerticesTopologically()) {
            ArchivedExecutionJobVertex archivedTask = task.archive();
            archivedVerticesInCreationOrder.add(archivedTask);
            archivedTasks.put(task.getJobVertexId(), archivedTask);
        }

        final Map>> serializedUserAccumulators =
                executionGraph.getAccumulatorsSerialized();

        final long[] timestamps = new long[JobStatus.values().length];

        // if the state is overridden with a non-globally-terminal state then we need to erase
        // traces of globally-terminal states for consistency
        final boolean clearGloballyTerminalStateTimestamps = statusOverride != null;

        for (JobStatus jobStatus : JobStatus.values()) {
            final int ordinal = jobStatus.ordinal();
            if (!(clearGloballyTerminalStateTimestamps && jobStatus.isGloballyTerminalState())) {
                timestamps[ordinal] = executionGraph.getStatusTimestamp(jobStatus);
            }
        }

        return new ArchivedExecutionGraph(
                executionGraph.getJobID(),
                executionGraph.getJobName(),
                archivedTasks,
                archivedVerticesInCreationOrder,
                timestamps,
                statusOverride == null ? executionGraph.getState() : statusOverride,
                executionGraph.getFailureInfo(),
                executionGraph.getJsonPlan(),
                executionGraph.getAccumulatorResultsStringified(),
                serializedUserAccumulators,
                executionGraph.getArchivedExecutionConfig(),
                executionGraph.isStoppable(),
                executionGraph.getCheckpointCoordinatorConfiguration(),
                executionGraph.getCheckpointStatsSnapshot(),
                executionGraph.getStateBackendName().orElse(null),
                executionGraph.getCheckpointStorageName().orElse(null));
    }

    /**
     * Create a sparse ArchivedExecutionGraph for a job while it is still initializing. Most fields
     * will be empty, only job status and error-related fields are set.
     */
    public static ArchivedExecutionGraph createFromInitializingJob(
            JobID jobId,
            String jobName,
            JobStatus jobStatus,
            @Nullable Throwable throwable,
            @Nullable JobCheckpointingSettings checkpointingSettings,
            long initializationTimestamp) {
        Map archivedTasks = Collections.emptyMap();
        List archivedVerticesInCreationOrder = Collections.emptyList();
        final Map>> serializedUserAccumulators =
                Collections.emptyMap();
        StringifiedAccumulatorResult[] archivedUserAccumulators =
                new StringifiedAccumulatorResult[] {};

        final long[] timestamps = new long[JobStatus.values().length];
        timestamps[JobStatus.INITIALIZING.ordinal()] = initializationTimestamp;

        String jsonPlan = "{}";

        ErrorInfo failureInfo = null;
        if (throwable != null) {
            Preconditions.checkState(
                    jobStatus == JobStatus.FAILED || jobStatus == JobStatus.SUSPENDED);
            long failureTime = System.currentTimeMillis();
            failureInfo = new ErrorInfo(throwable, failureTime);
            timestamps[jobStatus.ordinal()] = failureTime;
        }

        return new ArchivedExecutionGraph(
                jobId,
                jobName,
                archivedTasks,
                archivedVerticesInCreationOrder,
                timestamps,
                jobStatus,
                failureInfo,
                jsonPlan,
                archivedUserAccumulators,
                serializedUserAccumulators,
                new ExecutionConfig().archive(),
                false,
                checkpointingSettings == null
                        ? null
                        : checkpointingSettings.getCheckpointCoordinatorConfiguration(),
                checkpointingSettings == null ? null : CheckpointStatsSnapshot.empty(),
                checkpointingSettings == null ? null : "Unknown",
                checkpointingSettings == null ? null : "Unknown");
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy