org.apache.flink.runtime.state.gemini.GeminiStateBackend Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ConfigurationUtils;
import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.core.fs.Path;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.AbstractInternalStateBackend;
import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.AbstractStateBackend;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
import org.apache.flink.runtime.state.ConfigurableStateBackend;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.LocalRecoveryConfig;
import org.apache.flink.runtime.state.OperatorStateBackend;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.filesystem.AbstractFileStateBackend;
import org.apache.flink.runtime.state.filesystem.AbstractFsCheckpointStorage;
import org.apache.flink.runtime.state.filesystem.FsCheckpointStorage;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.gemini.engine.GConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* A State Backend that stores its state in {@code GeminiDB}.
*/
@PublicEvolving
public class GeminiStateBackend extends AbstractStateBackend implements ConfigurableStateBackend {
private static final long serialVersionUID = -8191916350224044011L;
private static final Logger LOG = LoggerFactory.getLogger(GeminiStateBackend.class);
// ------------------------------------------------------------------------
// -- configuration values, set in the application / configuration
/** The state backend that we use for creating checkpoint streams. */
private final AbstractFileStateBackend checkpointStreamBackend;
private Configuration configuration = new Configuration();
/** Base paths for GeminiDB directory, as configured.
* Null if not yet set, in which case the configuration values will be used.
* The configuration defaults to the TaskManager's temp directories. */
@Nullable
private File[] localGeminiDbDirectories;
/** Base paths for GeminiDB directory, as initialized. */
private transient File[] initializedDbBasePaths;
/** JobID for uniquifying backup paths. */
private transient JobID jobId;
/** The index of the next directory to be used from {@link #initializedDbBasePaths}.*/
private transient int nextDirectory;
/** Whether we already lazily initialized our local storage directories. */
private transient boolean isInitialized;
// -----------------------------------------------------------------------
// ------------------------------------------------------------------------
/**
* Creates a new {@code GeminiStateBackend} that stores its checkpoint data in the
* file system and location defined by the given URI.
*
* A state backend that stores checkpoints in HDFS or S3 must specify the file system
* host and port in the URI, or have the Hadoop configuration that describes the file system
* (host / high-availability group / possibly credentials) either referenced from the Flink
* config, or included in the classpath.
*
* @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory.
* @throws IOException Thrown, if no file system can be found for the scheme in the URI.
*/
public GeminiStateBackend(String checkpointDataUri) throws IOException {
this(new Path(checkpointDataUri).toUri());
}
/**
* Creates a new {@code GeminiStateBackend} that stores its checkpoint data in the
* file system and location defined by the given URI.
*
*
A state backend that stores checkpoints in HDFS or S3 must specify the file system
* host and port in the URI, or have the Hadoop configuration that describes the file system
* (host / high-availability group / possibly credentials) either referenced from the Flink
* config, or included in the classpath.
*
* @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory.
* @throws IOException Thrown, if no file system can be found for the scheme in the URI.
*/
@SuppressWarnings("deprecation")
public GeminiStateBackend(URI checkpointDataUri) throws IOException {
this(new FsStateBackend(checkpointDataUri));
}
/**
* Creates a new {@code GeminiStateBackend} that uses the given state backend to store its
* checkpoint data streams. Typically, one would supply a filesystem or database state backend
* here where the snapshots from RocksDB would be stored.
*
*
The snapshots of the RocksDB state will be stored using the given backend's
* {@link StateBackend#createCheckpointStorage(JobID)}.
*
* @param checkpointStreamBackend The backend write the checkpoint streams to.
*/
public GeminiStateBackend(AbstractFileStateBackend checkpointStreamBackend) {
this.checkpointStreamBackend = checkNotNull(checkpointStreamBackend);
}
/**
* Private constructor that creates a re-configured copy of the state backend.
*
* @param original The state backend to re-configure
* @param configuration The configuration
*/
private GeminiStateBackend(GeminiStateBackend original, Configuration configuration) {
// reconfigure the state backend backing the streams
final AbstractFileStateBackend originalStreamBackend = original.checkpointStreamBackend;
this.checkpointStreamBackend = (AbstractFileStateBackend) (originalStreamBackend instanceof ConfigurableStateBackend
? ((ConfigurableStateBackend) originalStreamBackend).configure(configuration)
: originalStreamBackend);
// configure local directories
if (original.localGeminiDbDirectories != null) {
this.localGeminiDbDirectories = original.localGeminiDbDirectories;
} else {
final String geminiLocalPaths = configuration.getString(GeminiOptions.GEMINIDB_LOCAL_DIRECTORIES);
if (geminiLocalPaths != null) {
String[] directories = geminiLocalPaths.split(",|" + File.pathSeparator);
try {
setDbStoragePaths(directories);
}
catch (IllegalArgumentException e) {
throw new IllegalConfigurationException("Invalid configuration for GeminiDB state " +
"backend's local storage directories: " + e.getMessage(), e);
}
}
}
this.configuration = new Configuration(configuration);
}
// ------------------------------------------------------------------------
// Reconfiguration
// ------------------------------------------------------------------------
/**
* Creates a copy of this state backend that uses the values defined in the configuration
* for fields where that were not specified in this state backend.
*
* @param config the configuration
* @return The re-configured variant of the state backend
*/
@Override
public GeminiStateBackend configure(Configuration config) {
return new GeminiStateBackend(this, config);
}
// ------------------------------------------------------------------------
// Checkpoint initialization and persistent storage
// ------------------------------------------------------------------------
@Override
public CompletedCheckpointStorageLocation resolveCheckpoint(String pointer) throws IOException {
return checkpointStreamBackend.resolveCheckpoint(pointer);
}
@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
return checkpointStreamBackend.createCheckpointStorage(jobId);
}
// ------------------------------------------------------------------------
// state holding structures
// ------------------------------------------------------------------------
@Override
public AbstractKeyedStateBackend createKeyedStateBackend(
Environment env,
JobID jobID,
String operatorIdentifier,
TypeSerializer keySerializer,
int numberOfKeyGroups,
KeyGroupRange keyGroupRange,
TaskKvStateRegistry kvStateRegistry) {
throw new UnsupportedOperationException();
}
@Override
public OperatorStateBackend createOperatorStateBackend(
Environment env,
String operatorIdentifier) {
final boolean asyncSnapshots = true;
return new DefaultOperatorStateBackend(
env.getUserClassLoader(),
env.getExecutionConfig(),
asyncSnapshots);
}
@Override
public AbstractInternalStateBackend createInternalStateBackend(
Environment env,
String operatorIdentifier,
int numberOfGroups,
KeyGroupRange keyGroupRange,
MetricGroup metricGroup) throws Exception {
lazyInitializeForJob(env);
GConfiguration gConfiguration = getGConfiguration();
// replace all characters that are not legal for filenames with underscore
String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");
File instanceBasePath = new File(
getNextStoragePath(),
"job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());
gConfiguration.setLocalPath(instanceBasePath.getAbsolutePath());
String dfsPath = configuration.getString(GeminiOptions.DFS_PATH);
if (dfsPath == null) {
Path basePath = ((FsCheckpointStorage) checkpointStreamBackend.createCheckpointStorage(jobId)).getCheckpointsDirectory();
dfsPath = new Path(basePath, AbstractFsCheckpointStorage.CHECKPOINT_SHARED_STATE_DIR).toUri().toString();
}
gConfiguration.setDfsPath(new Path(dfsPath, fileCompatibleIdentifier).toUri().toString());
gConfiguration.setSubTaskIndex(env.getTaskInfo().getIndexOfThisSubtask());
gConfiguration.setNumParallelSubtasks(env.getTaskInfo().getNumberOfParallelSubtasks());
gConfiguration.setBackendUID(env.getExecutionId().toString());
gConfiguration.setDBNumberPerJVM(env.getTaskManagerInfo().getNumberSlots());
gConfiguration.setOperatorNameWithSubtask(getOperatorNameWithSubtask(operatorIdentifier));
LocalRecoveryConfig localRecoveryConfig = env.getTaskStateManager().createLocalRecoveryConfig();
gConfiguration.setLocalSnapshot(localRecoveryConfig.isLocalRecoveryEnabled());
LOG.info("Create internal state backend, dfs path ({}), local path ({}) TM slots{}",
dfsPath,
instanceBasePath,
env.getTaskManagerInfo().getNumberSlots());
return new GeminiInternalStateBackend(
numberOfGroups,
keyGroupRange,
env.getUserClassLoader(),
localRecoveryConfig,
env.getTaskKvStateRegistry(),
fileCompatibleIdentifier,
env.getExecutionConfig(),
gConfiguration,
metricGroup
);
}
// ------------------------------------------------------------------------
// utilities
// ------------------------------------------------------------------------
public void setDbStoragePaths(String... paths) {
if (paths == null) {
localGeminiDbDirectories = null;
}
else if (paths.length == 0) {
throw new IllegalArgumentException("empty paths");
}
else {
File[] pp = new File[paths.length];
for (int i = 0; i < paths.length; i++) {
final String rawPath = paths[i];
final String path;
if (rawPath == null) {
throw new IllegalArgumentException("null path");
}
else {
// we need this for backwards compatibility, to allow URIs like 'file:///'...
URI uri = null;
try {
uri = new Path(rawPath).toUri();
}
catch (Exception e) {
// cannot parse as a path
}
if (uri != null && uri.getScheme() != null) {
if ("file".equalsIgnoreCase(uri.getScheme())) {
path = uri.getPath();
}
else {
throw new IllegalArgumentException("Path " + rawPath + " has a non-local scheme");
}
}
else {
path = rawPath;
}
}
pp[i] = new File(path);
if (!pp[i].isAbsolute()) {
throw new IllegalArgumentException("Relative paths are not supported");
}
}
localGeminiDbDirectories = pp;
}
}
private void lazyInitializeForJob(Environment env) throws IOException {
if (isInitialized) {
return;
}
this.jobId = env.getJobID();
// initialize the paths where the local GeminiDB files should be stored
if (localGeminiDbDirectories == null) {
String[] workingDirs = ConfigurationUtils.parseWorkingDirectories(env.getTaskManagerInfo().getConfiguration());
if (workingDirs.length == 0) {
// initialize from the temp directories
initializedDbBasePaths = env.getIOManager().getSpillingDirectories();
nextDirectory = ThreadLocalRandom.current().nextInt(initializedDbBasePaths.length);
isInitialized = true;
return;
} else {
setDbStoragePaths(workingDirs);
}
}
List dirs = new ArrayList<>(localGeminiDbDirectories.length);
StringBuilder errorMessage = new StringBuilder();
for (File f : localGeminiDbDirectories) {
File testDir = new File(f, UUID.randomUUID().toString());
if (!testDir.mkdirs()) {
String msg = "Local DB files directory '" + f
+ "' does not exist and cannot be created. ";
LOG.error(msg);
errorMessage.append(msg);
} else {
dirs.add(f);
}
//noinspection ResultOfMethodCallIgnored
testDir.delete();
}
if (dirs.isEmpty()) {
throw new IOException("No local storage directories available. " + errorMessage);
} else {
initializedDbBasePaths = dirs.toArray(new File[dirs.size()]);
}
nextDirectory = ThreadLocalRandom.current().nextInt(initializedDbBasePaths.length);
isInitialized = true;
}
private File getNextStoragePath() {
int ni = nextDirectory + 1;
ni = ni >= initializedDbBasePaths.length ? 0 : ni;
nextDirectory = ni;
return initializedDbBasePaths[ni];
}
private GConfiguration getGConfiguration() {
return new GConfiguration(configuration);
}
private String getOperatorNameWithSubtask(String operatorIdentifier) {
String[] splits = operatorIdentifier.split("_");
int len = splits.length;
StringBuilder sb = new StringBuilder();
for (int i = 0; i < len - 2; i++) {
sb.append(splits[i]);
sb.append("_");
}
sb.append(splits[len-1]);
return sb.toString();
}
@Override
public String toString() {
return "GeminiStateBackend{" + "checkpointStreamBackend=" + checkpointStreamBackend +
", localGeminiDbDirectories=" + Arrays.toString(localGeminiDbDirectories);
}
}