Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.heap;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo;
import org.apache.flink.runtime.state.StateEntry;
import org.apache.flink.runtime.state.StateSnapshotKeyGroupReader;
import org.apache.flink.runtime.state.StateSnapshotRestore;
import org.apache.flink.runtime.state.StateTransformationFunction;
import org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor;
import org.apache.flink.util.Preconditions;
import javax.annotation.Nonnull;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.Objects;
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/**
* Base class for state tables. Accesses to state are typically scoped by the currently active key, as provided
* through the {@link InternalKeyContext}.
*
* @param type of key
* @param type of namespace
* @param type of state
*/
public abstract class StateTable
implements StateSnapshotRestore, Iterable> {
/**
* The key context view on the backend. This provides information, such as the currently active key.
*/
protected final InternalKeyContext keyContext;
/**
* Combined meta information such as name and serializers for this state.
*/
protected RegisteredKeyValueStateBackendMetaInfo metaInfo;
/**
* The serializer of the key.
*/
protected final TypeSerializer keySerializer;
/**
* The offset to the contiguous key groups.
*/
protected final int keyGroupOffset;
/**
* Map for holding the actual state objects. The outer array represents the key-groups.
* All array positions will be initialized with an empty state map.
*/
protected final StateMap[] keyGroupedStateMaps;
/**
* @param keyContext the key context provides the key scope for all put/get/delete operations.
* @param metaInfo the meta information, including the type serializer for state copy-on-write.
* @param keySerializer the serializer of the key.
*/
public StateTable(
InternalKeyContext keyContext,
RegisteredKeyValueStateBackendMetaInfo metaInfo,
TypeSerializer keySerializer) {
this.keyContext = Preconditions.checkNotNull(keyContext);
this.metaInfo = Preconditions.checkNotNull(metaInfo);
this.keySerializer = Preconditions.checkNotNull(keySerializer);
this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();
@SuppressWarnings("unchecked")
StateMap[] state = (StateMap[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
this.keyGroupedStateMaps = state;
for (int i = 0; i < this.keyGroupedStateMaps.length; i++) {
this.keyGroupedStateMaps[i] = createStateMap();
}
}
protected abstract StateMap createStateMap();
// Main interface methods of StateTable -------------------------------------------------------
/**
* Returns whether this {@link StateTable} is empty.
*
* @return {@code true} if this {@link StateTable} has no elements, {@code false}
* otherwise.
* @see #size()
*/
public boolean isEmpty() {
return size() == 0;
}
/**
* Returns the total number of entries in this {@link StateTable}. This is the sum of both sub-tables.
*
* @return the number of entries in this {@link StateTable}.
*/
public int size() {
int count = 0;
for (StateMap stateMap : keyGroupedStateMaps) {
count += stateMap.size();
}
return count;
}
/**
* Returns the state of the mapping for the composite of active key and given namespace.
*
* @param namespace the namespace. Not null.
* @return the states of the mapping with the specified key/namespace composite key, or {@code null}
* if no mapping for the specified key is found.
*/
public S get(N namespace) {
return get(keyContext.getCurrentKey(), keyContext.getCurrentKeyGroupIndex(), namespace);
}
/**
* Returns whether this table contains a mapping for the composite of active key and given namespace.
*
* @param namespace the namespace in the composite key to search for. Not null.
* @return {@code true} if this map contains the specified key/namespace composite key,
* {@code false} otherwise.
*/
public boolean containsKey(N namespace) {
return containsKey(keyContext.getCurrentKey(), keyContext.getCurrentKeyGroupIndex(), namespace);
}
/**
* Maps the composite of active key and given namespace to the specified state.
*
* @param namespace the namespace. Not null.
* @param state the state. Can be null.
*/
public void put(N namespace, S state) {
put(keyContext.getCurrentKey(), keyContext.getCurrentKeyGroupIndex(), namespace, state);
}
/**
* Removes the mapping for the composite of active key and given namespace. This method should be preferred
* over {@link #removeAndGetOld(N)} when the caller is not interested in the old state.
*
* @param namespace the namespace of the mapping to remove. Not null.
*/
public void remove(N namespace) {
remove(keyContext.getCurrentKey(), keyContext.getCurrentKeyGroupIndex(), namespace);
}
/**
* Removes the mapping for the composite of active key and given namespace, returning the state that was
* found under the entry.
*
* @param namespace the namespace of the mapping to remove. Not null.
* @return the state of the removed mapping or {@code null} if no mapping
* for the specified key was found.
*/
public S removeAndGetOld(N namespace) {
return removeAndGetOld(keyContext.getCurrentKey(), keyContext.getCurrentKeyGroupIndex(), namespace);
}
/**
* Applies the given {@link StateTransformationFunction} to the state (1st input argument), using the given value as
* second input argument. The result of {@link StateTransformationFunction#apply(Object, Object)} is then stored as
* the new state. This function is basically an optimization for get-update-put pattern.
*
* @param namespace the namespace. Not null.
* @param value the value to use in transforming the state. Can be null.
* @param transformation the transformation function.
* @throws Exception if some exception happens in the transformation function.
*/
public void transform(
N namespace,
T value,
StateTransformationFunction transformation) throws Exception {
K key = keyContext.getCurrentKey();
checkKeyNamespacePreconditions(key, namespace);
int keyGroup = keyContext.getCurrentKeyGroupIndex();
StateMap stateMap = getMapForKeyGroup(keyGroup);
stateMap.transform(key, namespace, value, transformation);
}
// For queryable state ------------------------------------------------------------------------
/**
* Returns the state for the composite of active key and given namespace. This is typically used by
* queryable state.
*
* @param key the key. Not null.
* @param namespace the namespace. Not null.
* @return the state of the mapping with the specified key/namespace composite key, or {@code null}
* if no mapping for the specified key is found.
*/
public S get(K key, N namespace) {
int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(key, keyContext.getNumberOfKeyGroups());
return get(key, keyGroup, namespace);
}
public Stream getKeys(N namespace) {
return Arrays.stream(keyGroupedStateMaps)
.flatMap(stateMap -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(stateMap.iterator(), 0), false))
.filter(entry -> entry.getNamespace().equals(namespace))
.map(StateEntry::getKey);
}
public StateIncrementalVisitor getStateIncrementalVisitor(int recommendedMaxNumberOfReturnedRecords) {
return new StateEntryIterator(recommendedMaxNumberOfReturnedRecords);
}
// ------------------------------------------------------------------------
private S get(K key, int keyGroupIndex, N namespace) {
checkKeyNamespacePreconditions(key, namespace);
StateMap stateMap = getMapForKeyGroup(keyGroupIndex);
if (stateMap == null) {
return null;
}
return stateMap.get(key, namespace);
}
private boolean containsKey(K key, int keyGroupIndex, N namespace) {
checkKeyNamespacePreconditions(key, namespace);
StateMap stateMap = getMapForKeyGroup(keyGroupIndex);
return stateMap != null && stateMap.containsKey(key, namespace);
}
private void checkKeyNamespacePreconditions(K key, N namespace) {
Preconditions.checkNotNull(key, "No key set. This method should not be called outside of a keyed context.");
Preconditions.checkNotNull(namespace, "Provided namespace is null.");
}
private void remove(K key, int keyGroupIndex, N namespace) {
checkKeyNamespacePreconditions(key, namespace);
StateMap stateMap = getMapForKeyGroup(keyGroupIndex);
stateMap.remove(key, namespace);
}
private S removeAndGetOld(K key, int keyGroupIndex, N namespace) {
checkKeyNamespacePreconditions(key, namespace);
StateMap stateMap = getMapForKeyGroup(keyGroupIndex);
return stateMap.removeAndGetOld(key, namespace);
}
// ------------------------------------------------------------------------
// access to maps
// ------------------------------------------------------------------------
/**
* Returns the internal data structure.
*/
@VisibleForTesting
public StateMap[] getState() {
return keyGroupedStateMaps;
}
public int getKeyGroupOffset() {
return keyGroupOffset;
}
@VisibleForTesting
StateMap getMapForKeyGroup(int keyGroupIndex) {
final int pos = indexToOffset(keyGroupIndex);
if (pos >= 0 && pos < keyGroupedStateMaps.length) {
return keyGroupedStateMaps[pos];
} else {
return null;
}
}
/**
* Translates a key-group id to the internal array offset.
*/
private int indexToOffset(int index) {
return index - keyGroupOffset;
}
// Meta data setter / getter and toString -----------------------------------------------------
public TypeSerializer getKeySerializer() {
return keySerializer;
}
public TypeSerializer getStateSerializer() {
return metaInfo.getStateSerializer();
}
public TypeSerializer getNamespaceSerializer() {
return metaInfo.getNamespaceSerializer();
}
public RegisteredKeyValueStateBackendMetaInfo getMetaInfo() {
return metaInfo;
}
public void setMetaInfo(RegisteredKeyValueStateBackendMetaInfo metaInfo) {
this.metaInfo = metaInfo;
}
// Snapshot / Restore -------------------------------------------------------------------------
public void put(K key, int keyGroup, N namespace, S state) {
checkKeyNamespacePreconditions(key, namespace);
StateMap stateMap = getMapForKeyGroup(keyGroup);
stateMap.put(key, namespace, state);
}
@Override
public Iterator> iterator() {
return Arrays.stream(keyGroupedStateMaps)
.filter(Objects::nonNull)
.flatMap(stateMap -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(stateMap.iterator(), 0), false))
.iterator();
}
// For testing --------------------------------------------------------------------------------
@VisibleForTesting
public int sizeOfNamespace(Object namespace) {
int count = 0;
for (StateMap stateMap : keyGroupedStateMaps) {
count += stateMap.sizeOfNamespace(namespace);
}
return count;
}
@Nonnull
@Override
public StateSnapshotKeyGroupReader keyGroupReader(int readVersion) {
return StateTableByKeyGroupReaders.readerForVersion(this, readVersion);
}
// StateEntryIterator ---------------------------------------------------------------------------------------------
class StateEntryIterator implements StateIncrementalVisitor {
final int recommendedMaxNumberOfReturnedRecords;
int keyGroupIndex;
StateIncrementalVisitor stateIncrementalVisitor;
StateEntryIterator(int recommendedMaxNumberOfReturnedRecords) {
this.recommendedMaxNumberOfReturnedRecords = recommendedMaxNumberOfReturnedRecords;
this.keyGroupIndex = 0;
next();
}
private void next() {
while (keyGroupIndex < keyGroupedStateMaps.length) {
StateMap stateMap = keyGroupedStateMaps[keyGroupIndex++];
StateIncrementalVisitor visitor =
stateMap.getStateIncrementalVisitor(recommendedMaxNumberOfReturnedRecords);
if (visitor.hasNext()) {
stateIncrementalVisitor = visitor;
return;
}
}
}
@Override
public boolean hasNext() {
while (stateIncrementalVisitor == null || !stateIncrementalVisitor.hasNext()) {
if (keyGroupIndex == keyGroupedStateMaps.length) {
return false;
}
StateIncrementalVisitor visitor =
keyGroupedStateMaps[keyGroupIndex++].getStateIncrementalVisitor(recommendedMaxNumberOfReturnedRecords);
if (visitor.hasNext()) {
stateIncrementalVisitor = visitor;
break;
}
}
return true;
}
@Override
public Collection> nextEntries() {
if (!hasNext()) {
return null;
}
return stateIncrementalVisitor.nextEntries();
}
@Override
public void remove(StateEntry stateEntry) {
keyGroupedStateMaps[keyGroupIndex - 1].remove(stateEntry.getKey(), stateEntry.getNamespace());
}
@Override
public void update(StateEntry stateEntry, S newValue) {
keyGroupedStateMaps[keyGroupIndex - 1].put(stateEntry.getKey(), stateEntry.getNamespace(), newValue);
}
}
}
