All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.runtime.state.StateBackendLoader Maven / Gradle / Ivy

There is a newer version: 1.13.6
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state;

import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackendFactory;
import org.apache.flink.util.DynamicCodeLoadingException;

import org.slf4j.Logger;

import javax.annotation.Nullable;

import java.io.IOException;
import java.util.UUID;

import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * This class contains utility methods to load state backends from configurations.
 */
public class StateBackendLoader {

	// ------------------------------------------------------------------------
	//  Configuration shortcut names
	// ------------------------------------------------------------------------

	/** The shortcut configuration name for the MemoryState backend that checkpoints to the JobManager */
	public static final String MEMORY_STATE_BACKEND_NAME = "jobmanager";

	/** The shortcut configuration name for the FileSystem State backend */
	public static final String FS_STATE_BACKEND_NAME = "filesystem";

	/** The shortcut configuration name for the RocksDB State Backend */
	public static final String ROCKSDB_STATE_BACKEND_NAME = "rocksdb";

	// ------------------------------------------------------------------------
	//  Loading the state backend from a configuration 
	// ------------------------------------------------------------------------

	/**
	 * Loads the state backend from the configuration, from the parameter 'state.backend', as defined
	 * in {@link CheckpointingOptions#STATE_BACKEND}.
	 *
	 * 

The state backends can be specified either via their shortcut name, or via the class name * of a {@link StateBackendFactory}. If a StateBackendFactory class name is specified, the factory * is instantiated (via its zero-argument constructor) and its * {@link StateBackendFactory#createFromConfig(Configuration)} method is called. * *

Recognized shortcut names are '{@value StateBackendLoader#MEMORY_STATE_BACKEND_NAME}', * '{@value StateBackendLoader#FS_STATE_BACKEND_NAME}', and * '{@value StateBackendLoader#ROCKSDB_STATE_BACKEND_NAME}'. * * @param config The configuration to load the state backend from * @param classLoader The class loader that should be used to load the state backend * @param logger Optionally, a logger to log actions to (may be null) * * @return The instantiated state backend. * * @throws DynamicCodeLoadingException * Thrown if a state backend factory is configured and the factory class was not * found or the factory could not be instantiated * @throws IllegalConfigurationException * May be thrown by the StateBackendFactory when creating / configuring the state * backend in the factory * @throws IOException * May be thrown by the StateBackendFactory when instantiating the state backend */ public static StateBackend loadStateBackendFromConfig( Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException { checkNotNull(config, "config"); checkNotNull(classLoader, "classLoader"); final String backendName = config.getString(CheckpointingOptions.STATE_BACKEND); if (backendName == null) { return null; } // by default the factory class is the backend name String factoryClassName = backendName; switch (backendName.toLowerCase()) { case MEMORY_STATE_BACKEND_NAME: MemoryStateBackend memBackend = new MemoryStateBackendFactory().createFromConfig(config); if (logger != null) { Path memExternalized = memBackend.getCheckpointPath(); String extern = memExternalized == null ? "" : " (externalized to " + memExternalized + ')'; logger.info("State backend is set to heap memory (checkpoint to JobManager) {}", extern); } return memBackend; case FS_STATE_BACKEND_NAME: FsStateBackend fsBackend = new FsStateBackendFactory().createFromConfig(config); if (logger != null) { logger.info("State backend is set to heap memory (checkpoints to filesystem \"{}\")", fsBackend.getCheckpointPath()); } return fsBackend; case ROCKSDB_STATE_BACKEND_NAME: factoryClassName = "org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory"; // fall through to the 'default' case that uses reflection to load the backend // that way we can keep RocksDB in a separate module default: if (logger != null) { logger.info("Loading state backend via factory {}", factoryClassName); } StateBackendFactory factory; try { @SuppressWarnings("rawtypes") Class clazz = Class.forName(factoryClassName, false, classLoader) .asSubclass(StateBackendFactory.class); factory = clazz.newInstance(); } catch (ClassNotFoundException e) { throw new DynamicCodeLoadingException( "Cannot find configured state backend factory class: " + backendName, e); } catch (ClassCastException | InstantiationException | IllegalAccessException e) { throw new DynamicCodeLoadingException("The class configured under '" + CheckpointingOptions.STATE_BACKEND.key() + "' is not a valid state backend factory (" + backendName + ')', e); } return factory.createFromConfig(config); } } /** * Checks if an application-defined state backend is given, and if not, loads the state * backend from the configuration, from the parameter 'state.backend', as defined * in {@link CheckpointingOptions#STATE_BACKEND}. If no state backend is configured, this instantiates the * default state backend (the {@link MemoryStateBackend}). * *

If an application-defined state backend is found, and the state backend is a * {@link ConfigurableStateBackend}, this methods calls {@link ConfigurableStateBackend#configure(Configuration)} * on the state backend. * *

Refer to {@link #loadStateBackendFromConfig(Configuration, ClassLoader, Logger)} for details on * how the state backend is loaded from the configuration. * * @param config The configuration to load the state backend from * @param classLoader The class loader that should be used to load the state backend * @param logger Optionally, a logger to log actions to (may be null) * * @return The instantiated state backend. * * @throws DynamicCodeLoadingException * Thrown if a state backend factory is configured and the factory class was not * found or the factory could not be instantiated * @throws IllegalConfigurationException * May be thrown by the StateBackendFactory when creating / configuring the state * backend in the factory * @throws IOException * May be thrown by the StateBackendFactory when instantiating the state backend */ public static StateBackend fromApplicationOrConfigOrDefault( @Nullable StateBackend fromApplication, Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException { checkNotNull(config, "config"); checkNotNull(classLoader, "classLoader"); final StateBackend backend; // (1) the application defined state backend has precedence if (fromApplication != null) { if (logger != null) { logger.info("Using application-defined state backend: {}", fromApplication); } // see if this is supposed to pick up additional configuration parameters if (fromApplication instanceof ConfigurableStateBackend) { // needs to pick up configuration if (logger != null) { logger.info("Configuring application-defined state backend with job/cluster config"); } backend = ((ConfigurableStateBackend) fromApplication).configure(config); } else { // keep as is! backend = fromApplication; } } else { // (2) check if the config defines a state backend final StateBackend fromConfig = loadStateBackendFromConfig(config, classLoader, logger); if (fromConfig != null) { backend = fromConfig; } else { // (3) use the default backend = new MemoryStateBackendFactory().createFromConfig(config); if (logger != null) { logger.info("No state backend has been configured, using default (Memory / JobManager) {}", backend); } } } // to keep supporting the old behavior where default (JobManager) Backend + HA mode = checkpoints in HA store // we add the HA persistence dir as the checkpoint directory if none other is set if (backend instanceof MemoryStateBackend) { final MemoryStateBackend memBackend = (MemoryStateBackend) backend; if (memBackend.getCheckpointPath() == null && HighAvailabilityMode.isHighAvailabilityModeActivated(config)) { final String haStoragePath = config.getString(HighAvailabilityOptions.HA_STORAGE_PATH); if (haStoragePath != null) { try { Path checkpointDirPath = new Path(haStoragePath, UUID.randomUUID().toString()); if (checkpointDirPath.toUri().getScheme() == null) { checkpointDirPath = checkpointDirPath.makeQualified(checkpointDirPath.getFileSystem()); } Configuration tempConfig = new Configuration(config); tempConfig.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirPath.toString()); return memBackend.configure(tempConfig); } catch (Exception ignored) {} } } } return backend; } // ------------------------------------------------------------------------ /** This class is not meant to be instantiated */ private StateBackendLoader() {} }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy