org.apache.flink.runtime.history.FsJobArchivist Maven / Gradle / Ivy

Go to download
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.history;

import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
import org.apache.flink.runtime.webmonitor.history.ArchivedJson;
import org.apache.flink.util.IOUtils;

import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonEncoding;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;

/** Utility class for writing an archive file to a {@link FileSystem} and reading it back. */
public class FsJobArchivist {

    private static final Logger LOG = LoggerFactory.getLogger(FsJobArchivist.class);
    private static final JsonFactory jacksonFactory = new JsonFactory();
    private static final ObjectMapper mapper = new ObjectMapper();

    private static final String ARCHIVE = "archive";
    private static final String PATH = "path";
    private static final String JSON = "json";

    private FsJobArchivist() {}

    /**
     * Writes the given {@link AccessExecutionGraph} to the {@link FileSystem} pointed to by {@link
     * JobManagerOptions#ARCHIVE_DIR}.
     *
     * @param rootPath directory to which the archive should be written to
     * @param jobId job id
     * @param jsonToArchive collection of json-path pairs to that should be archived
     * @return path to where the archive was written, or null if no archive was created
     * @throws IOException
     */
    public static Path archiveJob(
            Path rootPath, JobID jobId, Collection jsonToArchive) throws IOException {
        try {
            FileSystem fs = rootPath.getFileSystem();
            Path path = new Path(rootPath, jobId.toString());
            OutputStream out = fs.create(path, FileSystem.WriteMode.NO_OVERWRITE);

            try (JsonGenerator gen = jacksonFactory.createGenerator(out, JsonEncoding.UTF8)) {
                gen.writeStartObject();
                gen.writeArrayFieldStart(ARCHIVE);
                for (ArchivedJson archive : jsonToArchive) {
                    gen.writeStartObject();
                    gen.writeStringField(PATH, archive.getPath());
                    gen.writeStringField(JSON, archive.getJson());
                    gen.writeEndObject();
                }
                gen.writeEndArray();
                gen.writeEndObject();
            } catch (Exception e) {
                fs.delete(path, false);
                throw e;
            }
            LOG.info("Job {} has been archived at {}.", jobId, path);
            return path;
        } catch (IOException e) {
            LOG.error("Failed to archive job.", e);
            throw e;
        }
    }

    /**
     * Reads the given archive file and returns a {@link Collection} of contained {@link
     * ArchivedJson}.
     *
     * @param file archive to extract
     * @return collection of archived jsons
     * @throws IOException if the file can't be opened, read or doesn't contain valid json
     */
    public static Collection getArchivedJsons(Path file) throws IOException {
        try (FSDataInputStream input = file.getFileSystem().open(file);
                ByteArrayOutputStream output = new ByteArrayOutputStream()) {
            IOUtils.copyBytes(input, output);

            try {
                JsonNode archive = mapper.readTree(output.toByteArray());

                Collection archives = new ArrayList<>();
                for (JsonNode archivePart : archive.get(ARCHIVE)) {
                    String path = archivePart.get(PATH).asText();
                    String json = archivePart.get(JSON).asText();
                    archives.add(new ArchivedJson(path, json));
                }
                return archives;
            } catch (NullPointerException npe) {
                // occurs if the archive is empty or any of the expected fields are not present
                throw new IOException(
                        "Job archive (" + file.getPath() + ") did not conform to expected format.");
            }
        }
    }
}