org.apache.flink.state.changelog.ChangelogKeyedStateBackend Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.state.changelog;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.state.CheckpointListener;
import org.apache.flink.api.common.state.State;
import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.CheckpointableKeyedStateBackend;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KeyGroupedInternalPriorityQueue;
import org.apache.flink.runtime.state.Keyed;
import org.apache.flink.runtime.state.KeyedStateBackend;
import org.apache.flink.runtime.state.KeyedStateFunction;
import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.PriorityComparable;
import org.apache.flink.runtime.state.SavepointResources;
import org.apache.flink.runtime.state.SnapshotResult;
import org.apache.flink.runtime.state.StateSnapshotTransformer;
import org.apache.flink.runtime.state.TestableKeyedStateBackend;
import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement;
import org.apache.flink.runtime.state.internal.InternalKvState;
import org.apache.flink.runtime.state.metrics.LatencyTrackingStateFactory;
import org.apache.flink.runtime.state.ttl.TtlStateFactory;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.util.FlinkRuntimeException;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.RunnableFuture;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* A {@link KeyedStateBackend} that keeps state on the underlying delegated keyed state backend as
* well as on the state change log.
*
* @param The key by which state is keyed.
*/
@Internal
class ChangelogKeyedStateBackend
implements CheckpointableKeyedStateBackend,
CheckpointListener,
TestableKeyedStateBackend {
private static final Map STATE_FACTORIES =
Stream.of(
Tuple2.of(
StateDescriptor.Type.VALUE,
(StateFactory) ChangelogValueState::create),
Tuple2.of(
StateDescriptor.Type.LIST,
(StateFactory) ChangelogListState::create),
Tuple2.of(
StateDescriptor.Type.REDUCING,
(StateFactory) ChangelogReducingState::create),
Tuple2.of(
StateDescriptor.Type.AGGREGATING,
(StateFactory) ChangelogAggregatingState::create),
Tuple2.of(
StateDescriptor.Type.MAP,
(StateFactory) ChangelogMapState::create))
.collect(Collectors.toMap(t -> t.f0, t -> t.f1));
/** delegated keyedStateBackend. */
private final AbstractKeyedStateBackend keyedStateBackend;
/**
* This is the cache maintained by the DelegateKeyedStateBackend itself. It is not the same as
* the underlying delegated keyedStateBackend. InternalKvState is a delegated state.
*/
private final HashMap> keyValueStatesByName;
private final ExecutionConfig executionConfig;
private final TtlTimeProvider ttlTimeProvider;
/** last accessed partitioned state. */
@SuppressWarnings("rawtypes")
private InternalKvState lastState;
/** For caching the last accessed partitioned state. */
private String lastName;
public ChangelogKeyedStateBackend(
AbstractKeyedStateBackend keyedStateBackend,
ExecutionConfig executionConfig,
TtlTimeProvider ttlTimeProvider) {
this.keyedStateBackend = keyedStateBackend;
this.executionConfig = executionConfig;
this.ttlTimeProvider = ttlTimeProvider;
this.keyValueStatesByName = new HashMap<>();
}
// -------------------- CheckpointableKeyedStateBackend --------------------------------
@Override
public KeyGroupRange getKeyGroupRange() {
return keyedStateBackend.getKeyGroupRange();
}
@Override
public void close() throws IOException {
keyedStateBackend.close();
}
@Override
public void setCurrentKey(K newKey) {
keyedStateBackend.setCurrentKey(newKey);
}
@Override
public K getCurrentKey() {
return keyedStateBackend.getCurrentKey();
}
@Override
public TypeSerializer getKeySerializer() {
return keyedStateBackend.getKeySerializer();
}
@Override
public Stream getKeys(String state, N namespace) {
return keyedStateBackend.getKeys(state, namespace);
}
@Override
public Stream> getKeysAndNamespaces(String state) {
return keyedStateBackend.getKeysAndNamespaces(state);
}
@Override
public void dispose() {
keyedStateBackend.dispose();
lastName = null;
lastState = null;
keyValueStatesByName.clear();
}
@Override
public void registerKeySelectionListener(KeySelectionListener listener) {
keyedStateBackend.registerKeySelectionListener(listener);
}
@Override
public boolean deregisterKeySelectionListener(KeySelectionListener listener) {
return keyedStateBackend.deregisterKeySelectionListener(listener);
}
@Override
public void applyToAllKeys(
N namespace,
TypeSerializer namespaceSerializer,
StateDescriptor stateDescriptor,
KeyedStateFunction function)
throws Exception {
keyedStateBackend.applyToAllKeys(
namespace,
namespaceSerializer,
stateDescriptor,
function,
this::getPartitionedState);
}
@Override
@SuppressWarnings("unchecked")
public S getPartitionedState(
N namespace,
TypeSerializer namespaceSerializer,
StateDescriptor stateDescriptor)
throws Exception {
checkNotNull(namespace, "Namespace");
if (lastName != null && lastName.equals(stateDescriptor.getName())) {
lastState.setCurrentNamespace(namespace);
return (S) lastState;
}
final InternalKvState previous =
keyValueStatesByName.get(stateDescriptor.getName());
if (previous != null) {
lastState = previous;
lastState.setCurrentNamespace(namespace);
lastName = stateDescriptor.getName();
return (S) previous;
}
final S state = getOrCreateKeyedState(namespaceSerializer, stateDescriptor);
final InternalKvState kvState = (InternalKvState) state;
lastName = stateDescriptor.getName();
lastState = kvState;
kvState.setCurrentNamespace(namespace);
return state;
}
@Nonnull
@Override
public RunnableFuture> snapshot(
long checkpointId,
long timestamp,
@Nonnull CheckpointStreamFactory streamFactory,
@Nonnull CheckpointOptions checkpointOptions)
throws Exception {
return keyedStateBackend.snapshot(
checkpointId, timestamp, streamFactory, checkpointOptions);
}
@Nonnull
@Override
public & Keyed>
KeyGroupedInternalPriorityQueue create(
@Nonnull String stateName,
@Nonnull TypeSerializer byteOrderedElementSerializer) {
return new ChangelogKeyGroupedPriorityQueue(
keyedStateBackend.create(stateName, byteOrderedElementSerializer));
}
@VisibleForTesting
@Override
public int numKeyValueStateEntries() {
return keyedStateBackend.numKeyValueStateEntries();
}
@Override
public boolean isStateImmutableInStateBackend(CheckpointType checkpointOptions) {
return keyedStateBackend.isStateImmutableInStateBackend(checkpointOptions);
}
@Nonnull
@Override
public SavepointResources savepoint() throws Exception {
return keyedStateBackend.savepoint();
}
// -------------------- CheckpointListener --------------------------------
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
keyedStateBackend.notifyCheckpointComplete(checkpointId);
}
@Override
public void notifyCheckpointAborted(long checkpointId) throws Exception {
keyedStateBackend.notifyCheckpointAborted(checkpointId);
}
// -------- Methods not simply delegating to wrapped state backend ---------
@Override
@SuppressWarnings("unchecked")
public S getOrCreateKeyedState(
TypeSerializer namespaceSerializer, StateDescriptor stateDescriptor)
throws Exception {
checkNotNull(namespaceSerializer, "Namespace serializer");
checkNotNull(
getKeySerializer(),
"State key serializer has not been configured in the config. "
+ "This operation cannot use partitioned state.");
InternalKvState kvState = keyValueStatesByName.get(stateDescriptor.getName());
if (kvState == null) {
if (!stateDescriptor.isSerializerInitialized()) {
stateDescriptor.initializeSerializerUnlessSet(executionConfig);
}
kvState =
LatencyTrackingStateFactory.createStateAndWrapWithLatencyTrackingIfEnabled(
TtlStateFactory.createStateAndWrapWithTtlIfEnabled(
namespaceSerializer, stateDescriptor, this, ttlTimeProvider),
stateDescriptor,
keyedStateBackend.getLatencyTrackingStateConfig());
keyValueStatesByName.put(stateDescriptor.getName(), kvState);
keyedStateBackend.publishQueryableStateIfEnabled(stateDescriptor, kvState);
}
return (S) kvState;
}
@Nonnull
@Override
@SuppressWarnings("unchecked")
public IS createInternalState(
@Nonnull TypeSerializer namespaceSerializer,
@Nonnull StateDescriptor stateDesc,
@Nonnull
StateSnapshotTransformer.StateSnapshotTransformFactory
snapshotTransformFactory)
throws Exception {
StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
if (stateFactory == null) {
String message =
String.format(
"State %s is not supported by %s",
stateDesc.getClass(), this.getClass());
throw new FlinkRuntimeException(message);
}
return stateFactory.create(
keyedStateBackend.createInternalState(
namespaceSerializer, stateDesc, snapshotTransformFactory));
}
// Factory function interface
private interface StateFactory {
IS create(InternalKvState kvState)
throws Exception;
}
}