Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.state.CheckpointListener;
import org.apache.flink.api.common.state.State;
import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.heap.InternalKeyContext;
import org.apache.flink.runtime.state.internal.InternalKvState;
import org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig;
import org.apache.flink.runtime.state.metrics.LatencyTrackingStateFactory;
import org.apache.flink.runtime.state.ttl.TtlStateFactory;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.util.IOUtils;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.stream.Stream;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Base implementation of KeyedStateBackend. The state can be checkpointed to streams using {@link
* #snapshot(long, long, CheckpointStreamFactory, CheckpointOptions)}.
*
* @param Type of the key by which state is keyed.
*/
public abstract class AbstractKeyedStateBackend
implements CheckpointableKeyedStateBackend,
CheckpointListener,
TestableKeyedStateBackend {
/** The key serializer. */
protected final TypeSerializer keySerializer;
/** Listeners to changes of ({@link #keyContext}). */
private final ArrayList> keySelectionListeners;
/** So that we can give out state when the user uses the same key. */
private final HashMap> keyValueStatesByName;
/** For caching the last accessed partitioned state. */
private String lastName;
@SuppressWarnings("rawtypes")
private InternalKvState lastState;
/** The number of key-groups aka max parallelism. */
protected final int numberOfKeyGroups;
/** Range of key-groups for which this backend is responsible. */
protected final KeyGroupRange keyGroupRange;
/** KvStateRegistry helper for this task. */
protected final TaskKvStateRegistry kvStateRegistry;
/**
* Registry for all opened streams, so they can be closed if the task using this backend is
* closed.
*/
protected CloseableRegistry cancelStreamRegistry;
protected final ClassLoader userCodeClassLoader;
private final ExecutionConfig executionConfig;
protected final TtlTimeProvider ttlTimeProvider;
protected final LatencyTrackingStateConfig latencyTrackingStateConfig;
/** Decorates the input and output streams to write key-groups compressed. */
protected final StreamCompressionDecorator keyGroupCompressionDecorator;
/** The key context for this backend. */
protected final InternalKeyContext keyContext;
public AbstractKeyedStateBackend(
TaskKvStateRegistry kvStateRegistry,
TypeSerializer keySerializer,
ClassLoader userCodeClassLoader,
ExecutionConfig executionConfig,
TtlTimeProvider ttlTimeProvider,
LatencyTrackingStateConfig latencyTrackingStateConfig,
CloseableRegistry cancelStreamRegistry,
InternalKeyContext keyContext) {
this(
kvStateRegistry,
keySerializer,
userCodeClassLoader,
executionConfig,
ttlTimeProvider,
latencyTrackingStateConfig,
cancelStreamRegistry,
determineStreamCompression(executionConfig),
keyContext);
}
public AbstractKeyedStateBackend(
TaskKvStateRegistry kvStateRegistry,
TypeSerializer keySerializer,
ClassLoader userCodeClassLoader,
ExecutionConfig executionConfig,
TtlTimeProvider ttlTimeProvider,
LatencyTrackingStateConfig latencyTrackingStateConfig,
CloseableRegistry cancelStreamRegistry,
StreamCompressionDecorator keyGroupCompressionDecorator,
InternalKeyContext keyContext) {
this.keyContext = Preconditions.checkNotNull(keyContext);
this.numberOfKeyGroups = keyContext.getNumberOfKeyGroups();
this.keyGroupRange = Preconditions.checkNotNull(keyContext.getKeyGroupRange());
Preconditions.checkArgument(
numberOfKeyGroups >= 1, "NumberOfKeyGroups must be a positive number");
Preconditions.checkArgument(
numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(),
"The total number of key groups must be at least the number in the key group range assigned to this backend. "
+ "The total number of key groups: %s, the number in key groups in range: %s",
numberOfKeyGroups,
keyGroupRange.getNumberOfKeyGroups());
this.kvStateRegistry = kvStateRegistry;
this.keySerializer = keySerializer;
this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader);
this.cancelStreamRegistry = cancelStreamRegistry;
this.keyValueStatesByName = new HashMap<>();
this.executionConfig = executionConfig;
this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
this.ttlTimeProvider = Preconditions.checkNotNull(ttlTimeProvider);
this.latencyTrackingStateConfig = Preconditions.checkNotNull(latencyTrackingStateConfig);
this.keySelectionListeners = new ArrayList<>(1);
}
private static StreamCompressionDecorator determineStreamCompression(
ExecutionConfig executionConfig) {
if (executionConfig != null && executionConfig.isUseSnapshotCompression()) {
return SnappyStreamCompressionDecorator.INSTANCE;
} else {
return UncompressedStreamCompressionDecorator.INSTANCE;
}
}
/**
* Closes the state backend, releasing all internal resources, but does not delete any
* persistent checkpoint data.
*/
@Override
public void dispose() {
IOUtils.closeQuietly(cancelStreamRegistry);
if (kvStateRegistry != null) {
kvStateRegistry.unregisterAll();
}
lastName = null;
lastState = null;
keyValueStatesByName.clear();
}
/** @see KeyedStateBackend */
@Override
public void setCurrentKey(K newKey) {
notifyKeySelected(newKey);
this.keyContext.setCurrentKey(newKey);
this.keyContext.setCurrentKeyGroupIndex(
KeyGroupRangeAssignment.assignToKeyGroup(newKey, numberOfKeyGroups));
}
private void notifyKeySelected(K newKey) {
// we prefer a for-loop over other iteration schemes for performance reasons here.
for (int i = 0; i < keySelectionListeners.size(); ++i) {
keySelectionListeners.get(i).keySelected(newKey);
}
}
@Override
public void registerKeySelectionListener(KeySelectionListener listener) {
keySelectionListeners.add(listener);
}
@Override
public boolean deregisterKeySelectionListener(KeySelectionListener listener) {
return keySelectionListeners.remove(listener);
}
/** @see KeyedStateBackend */
@Override
public TypeSerializer getKeySerializer() {
return keySerializer;
}
/** @see KeyedStateBackend */
@Override
public K getCurrentKey() {
return this.keyContext.getCurrentKey();
}
/** @see KeyedStateBackend */
public int getCurrentKeyGroupIndex() {
return this.keyContext.getCurrentKeyGroupIndex();
}
/** @see KeyedStateBackend */
public int getNumberOfKeyGroups() {
return numberOfKeyGroups;
}
/** @see KeyedStateBackend */
@Override
public KeyGroupRange getKeyGroupRange() {
return keyGroupRange;
}
/** @see KeyedStateBackend */
@Override
public void applyToAllKeys(
final N namespace,
final TypeSerializer namespaceSerializer,
final StateDescriptor stateDescriptor,
final KeyedStateFunction function)
throws Exception {
applyToAllKeys(
namespace,
namespaceSerializer,
stateDescriptor,
function,
this::getPartitionedState);
}
public void applyToAllKeys(
final N namespace,
final TypeSerializer namespaceSerializer,
final StateDescriptor stateDescriptor,
final KeyedStateFunction function,
final PartitionStateFactory partitionStateFactory)
throws Exception {
try (Stream keyStream = getKeys(stateDescriptor.getName(), namespace)) {
final S state =
partitionStateFactory.get(namespace, namespaceSerializer, stateDescriptor);
keyStream.forEach(
(K key) -> {
setCurrentKey(key);
try {
function.process(key, state);
} catch (Throwable e) {
// we wrap the checked exception in an unchecked
// one and catch it (and re-throw it) later.
throw new RuntimeException(e);
}
});
}
}
/** @see KeyedStateBackend */
@Override
@SuppressWarnings("unchecked")
public S getOrCreateKeyedState(
final TypeSerializer namespaceSerializer, StateDescriptor stateDescriptor)
throws Exception {
checkNotNull(namespaceSerializer, "Namespace serializer");
checkNotNull(
keySerializer,
"State key serializer has not been configured in the config. "
+ "This operation cannot use partitioned state.");
InternalKvState kvState = keyValueStatesByName.get(stateDescriptor.getName());
if (kvState == null) {
if (!stateDescriptor.isSerializerInitialized()) {
stateDescriptor.initializeSerializerUnlessSet(executionConfig);
}
kvState =
LatencyTrackingStateFactory.createStateAndWrapWithLatencyTrackingIfEnabled(
TtlStateFactory.createStateAndWrapWithTtlIfEnabled(
namespaceSerializer, stateDescriptor, this, ttlTimeProvider),
stateDescriptor,
latencyTrackingStateConfig);
keyValueStatesByName.put(stateDescriptor.getName(), kvState);
publishQueryableStateIfEnabled(stateDescriptor, kvState);
}
return (S) kvState;
}
public void publishQueryableStateIfEnabled(
StateDescriptor stateDescriptor, InternalKvState kvState) {
if (stateDescriptor.isQueryable()) {
if (kvStateRegistry == null) {
throw new IllegalStateException("State backend has not been initialized for job.");
}
String name = stateDescriptor.getQueryableStateName();
kvStateRegistry.registerKvState(keyGroupRange, name, kvState, userCodeClassLoader);
}
}
/**
* TODO: NOTE: This method does a lot of work caching / retrieving states just to update the
* namespace. This method should be removed for the sake of namespaces being lazily fetched from
* the keyed state backend, or being set on the state directly.
*
* @see KeyedStateBackend
*/
@SuppressWarnings("unchecked")
@Override
public S getPartitionedState(
final N namespace,
final TypeSerializer namespaceSerializer,
final StateDescriptor stateDescriptor)
throws Exception {
checkNotNull(namespace, "Namespace");
if (lastName != null && lastName.equals(stateDescriptor.getName())) {
lastState.setCurrentNamespace(namespace);
return (S) lastState;
}
InternalKvState previous = keyValueStatesByName.get(stateDescriptor.getName());
if (previous != null) {
lastState = previous;
lastState.setCurrentNamespace(namespace);
lastName = stateDescriptor.getName();
return (S) previous;
}
final S state = getOrCreateKeyedState(namespaceSerializer, stateDescriptor);
final InternalKvState kvState = (InternalKvState) state;
lastName = stateDescriptor.getName();
lastState = kvState;
kvState.setCurrentNamespace(namespace);
return state;
}
@Override
public void close() throws IOException {
cancelStreamRegistry.close();
}
public LatencyTrackingStateConfig getLatencyTrackingStateConfig() {
return latencyTrackingStateConfig;
}
@VisibleForTesting
public StreamCompressionDecorator getKeyGroupCompressionDecorator() {
return keyGroupCompressionDecorator;
}
@VisibleForTesting
public int numKeyValueStatesByName() {
return keyValueStatesByName.size();
}
// TODO remove this once heap-based timers are working with RocksDB incremental snapshots!
public boolean requiresLegacySynchronousTimerSnapshots(CheckpointType checkpointType) {
return false;
}
public interface PartitionStateFactory {
S get(
final N namespace,
final TypeSerializer namespaceSerializer,
final StateDescriptor stateDescriptor)
throws Exception;
}
}
