All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.runtime.state.AbstractStateBackend Maven / Gradle / Ivy

There is a newer version: 1.13.6
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state;

import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.CoreOptions;
import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.filesystem.FsStateBackendFactory;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.util.DynamicCodeLoadingException;

import org.slf4j.Logger;

import javax.annotation.Nullable;
import java.io.IOException;

import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * An abstract base implementation of the {@link StateBackend} interface.
 * 
 * 

*/ @PublicEvolving public abstract class AbstractStateBackend implements StateBackend, java.io.Serializable { private static final long serialVersionUID = 4620415814639230247L; // ------------------------------------------------------------------------ // Configuration shortcut names // ------------------------------------------------------------------------ /** The shortcut configuration name for the MemoryState backend that checkpoints to the JobManager */ public static final String MEMORY_STATE_BACKEND_NAME = "jobmanager"; /** The shortcut configuration name for the FileSystem State backend */ public static final String FS_STATE_BACKEND_NAME = "filesystem"; /** The shortcut configuration name for the RocksDB State Backend */ public static final String ROCKSDB_STATE_BACKEND_NAME = "rocksdb"; // ------------------------------------------------------------------------ // State Backend - Persisting Byte Storage // ------------------------------------------------------------------------ @Override public abstract CheckpointStreamFactory createStreamFactory( JobID jobId, String operatorIdentifier) throws IOException; @Override public abstract CheckpointStreamFactory createSavepointStreamFactory( JobID jobId, String operatorIdentifier, @Nullable String targetLocation) throws IOException; // ------------------------------------------------------------------------ // State Backend - State-Holding Backends // ------------------------------------------------------------------------ @Override public abstract AbstractKeyedStateBackend createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry) throws IOException; @Override public abstract OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier) throws Exception; // ------------------------------------------------------------------------ // Loading the state backend from a configuration // ------------------------------------------------------------------------ /** * Loads the state backend from the configuration, from the parameter 'state.backend', as defined * in {@link CoreOptions#STATE_BACKEND}. * *

The state backends can be specified either via their shortcut name, or via the class name * of a {@link StateBackendFactory}. If a StateBackendFactory class name is specified, the factory * is instantiated (via its zero-argument constructor) and its * {@link StateBackendFactory#createFromConfig(Configuration)} method is called. * *

Recognized shortcut names are '{@value AbstractStateBackend#MEMORY_STATE_BACKEND_NAME}', * '{@value AbstractStateBackend#FS_STATE_BACKEND_NAME}', and * '{@value AbstractStateBackend#ROCKSDB_STATE_BACKEND_NAME}'. * * @param config The configuration to load the state backend from * @param classLoader The class loader that should be used to load the state backend * @param logger Optionally, a logger to log actions to (may be null) * * @return The instantiated state backend. * * @throws DynamicCodeLoadingException * Thrown if a state backend factory is configured and the factory class was not * found or the factory could not be instantiated * @throws IllegalConfigurationException * May be thrown by the StateBackendFactory when creating / configuring the state * backend in the factory * @throws IOException * May be thrown by the StateBackendFactory when instantiating the state backend */ public static StateBackend loadStateBackendFromConfig( Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException { checkNotNull(config, "config"); checkNotNull(classLoader, "classLoader"); final String backendName = config.getString(CoreOptions.STATE_BACKEND); if (backendName == null) { return null; } // by default the factory class is the backend name String factoryClassName = backendName; switch (backendName.toLowerCase()) { case MEMORY_STATE_BACKEND_NAME: if (logger != null) { logger.info("State backend is set to heap memory (checkpoint to JobManager)"); } return new MemoryStateBackend(); case FS_STATE_BACKEND_NAME: FsStateBackend fsBackend = new FsStateBackendFactory().createFromConfig(config); if (logger != null) { logger.info("State backend is set to heap memory (checkpoints to filesystem \"{}\")", fsBackend.getBasePath()); } return fsBackend; case ROCKSDB_STATE_BACKEND_NAME: factoryClassName = "org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory"; // fall through to the 'default' case that uses reflection to load the backend // that way we can keep RocksDB in a separate module default: if (logger != null) { logger.info("Loading state backend via factory {}", factoryClassName); } StateBackendFactory factory; try { @SuppressWarnings("rawtypes") Class clazz = Class.forName(factoryClassName, false, classLoader) .asSubclass(StateBackendFactory.class); factory = clazz.newInstance(); } catch (ClassNotFoundException e) { throw new DynamicCodeLoadingException( "Cannot find configured state backend factory class: " + backendName, e); } catch (ClassCastException | InstantiationException | IllegalAccessException e) { throw new DynamicCodeLoadingException("The class configured under '" + CoreOptions.STATE_BACKEND.key() + "' is not a valid state backend factory (" + backendName + ')', e); } return factory.createFromConfig(config); } } /** * Loads the state backend from the configuration, from the parameter 'state.backend', as defined * in {@link CoreOptions#STATE_BACKEND}. If no state backend is configures, this instantiates the * default state backend (the {@link MemoryStateBackend}). * *

Refer to {@link #loadStateBackendFromConfig(Configuration, ClassLoader, Logger)} for details on * how the state backend is loaded from the configuration. * * @param config The configuration to load the state backend from * @param classLoader The class loader that should be used to load the state backend * @param logger Optionally, a logger to log actions to (may be null) * * @return The instantiated state backend. * * @throws DynamicCodeLoadingException * Thrown if a state backend factory is configured and the factory class was not * found or the factory could not be instantiated * @throws IllegalConfigurationException * May be thrown by the StateBackendFactory when creating / configuring the state * backend in the factory * @throws IOException * May be thrown by the StateBackendFactory when instantiating the state backend */ public static StateBackend loadStateBackendFromConfigOrCreateDefault( Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException { final StateBackend fromConfig = loadStateBackendFromConfig(config, classLoader, logger); if (fromConfig != null) { return fromConfig; } else { if (logger != null) { logger.info("No state backend has been configured, using default state backend (Memory / JobManager)"); } return new MemoryStateBackend(); } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy