org.apache.flink.runtime.highavailability.filesystem.FileSystemHaServices Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.highavailability.filesystem;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.blob.BlobStore;
import org.apache.flink.runtime.blob.BlobStoreService;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneRunningJobsRegistry;
import org.apache.flink.runtime.jobmanager.FileSystemSubmittedJobGraphStore;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderelection.StandaloneLeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
import org.apache.flink.util.ExceptionUtils;
import javax.annotation.concurrent.GuardedBy;
import java.io.IOException;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
/**
* An implementation of the {@link HighAvailabilityServices} using file system.
*
* This implementation has no dependencies on any external services. It returns a fix
* pre-configured ResourceManager and JobManager, and stores checkpoints and metadata simply on a
* file system and therefore relies on the high availability of the file system.
*/
public class FileSystemHaServices implements HighAvailabilityServices {
/** The fix address of the ResourceManager */
private final String resourceManagerAddress;
/** The fix address of the Dispatcher */
private final String dispatcherAddress;
/** The fix address of the JobManager */
private final String jobManagerAddress;
private final String webMonitorAddress;
private final String jobGraphPath;
protected final Object lock = new Object();
private final RunningJobsRegistry runningJobsRegistry;
/** Store for arbitrary blobs */
private final BlobStoreService blobStoreService;
private boolean shutdown;
/**
* Creates a new services class for the fix pre-defined leaders.
*
* @param resourceManagerAddress The fix address of the ResourceManager
* @param webMonitorAddress
*/
public FileSystemHaServices(
String resourceManagerAddress,
String dispatcherAddress,
String jobManagerAddress,
String webMonitorAddress,
String jobGraphPath,
BlobStoreService blobStoreService) {
this.resourceManagerAddress = checkNotNull(resourceManagerAddress, "resourceManagerAddress");
this.dispatcherAddress = checkNotNull(dispatcherAddress, "dispatcherAddress");
this.jobManagerAddress = checkNotNull(jobManagerAddress, "jobManagerAddress");
this.webMonitorAddress = checkNotNull(webMonitorAddress, webMonitorAddress);
this.jobGraphPath = checkNotNull(jobGraphPath, "jobGraphPath");
this.blobStoreService = checkNotNull(blobStoreService, "blobStoreService");
this.runningJobsRegistry = new StandaloneRunningJobsRegistry();
shutdown = false;
}
// ------------------------------------------------------------------------
// Services
// ------------------------------------------------------------------------
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
}
}
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
}
}
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderElectionService();
}
}
@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderElectionService();
}
}
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
}
}
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
}
}
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderElectionService();
}
}
@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
}
}
@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneLeaderElectionService();
}
}
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
synchronized (lock) {
checkNotShutdown();
return new StandaloneCheckpointRecoveryFactory();
}
}
@Override
public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
synchronized (lock) {
checkNotShutdown();
return new FileSystemSubmittedJobGraphStore(jobGraphPath);
}
}
@Override
public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
synchronized (lock) {
checkNotShutdown();
return runningJobsRegistry;
}
}
@Override
public BlobStore createBlobStore() throws IOException {
synchronized (lock) {
checkNotShutdown();
return blobStoreService;
}
}
// ------------------------------------------------------------------------
// Shutdown and Cleanup
// ------------------------------------------------------------------------
@Override
public void close() throws Exception {
Throwable exception = null;
try {
blobStoreService.close();
} catch (Throwable t) {
exception = t;
}
internalClose();
if (exception != null) {
ExceptionUtils.rethrowException(exception, "Could not properly close the FileSystemHaServices.");
}
}
@Override
public void closeAndCleanupAllData() throws Exception {
Throwable exception = null;
try {
blobStoreService.closeAndCleanupAllData();
} catch (Throwable t) {
exception = t;
}
internalClose();
if (exception != null) {
ExceptionUtils.rethrowException(exception, "Could not properly close and clean up all data of FileSystemHaServices.");
}
}
/**
* Closes components which don't distinguish between close and closeAndCleanupAllData
*/
private void internalClose() {
synchronized (lock) {
if (!shutdown) {
shutdown = true;
}
}
}
// ----------------------------------------------------------------------
// Helper methods
// ----------------------------------------------------------------------
@GuardedBy("lock")
private void checkNotShutdown() {
checkState(!shutdown, "high availability services are shut down");
}
}