Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.subkeyed;
import org.apache.flink.api.common.functions.HashPartitioner;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.StateAccessException;
import org.apache.flink.runtime.state.gemini.engine.GRegion;
import org.apache.flink.runtime.state.gemini.engine.GeminiPKey2;
import org.apache.flink.runtime.state.gemini.engine.hashtable.AbstractGRegionKMapImpl;
import org.apache.flink.runtime.state.gemini.engine.hashtable.AbstractGTableSubKeyedMapImpl;
import org.apache.flink.runtime.state.gemini.engine.hashtable.GRegionKMapImpl;
import org.apache.flink.runtime.state.subkeyed.AbstractSubKeyedMapState;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* The base implementation of {@link AbstractSubKeyedMapState} backed by an a state storage.
* The pairs in the storage are formatted as {(K, N, MK) -> MV}. Because the pairs are
* partitioned by K, the mappings under the same key will be assigned to the same group and
* can be easily retrieved with the prefix iterator on the key.
*
* @param Type of the keys in the state.
* @param Type of the namespaces in the state.
* @param Type of the map keys in the state.
* @param Type of the map values in the state.
* @param Type of the maps in the state.
*/
abstract class AbstractGeminiSubKeyedMapStateImpl> implements AbstractSubKeyedMapState {
protected final AbstractGTableSubKeyedMapImpl table;
private final Byte dummyByte = new Byte((byte) 0);
/**
* The key serializer of current state.
*/
protected TypeSerializer keySerializer;
/**
* Serializer for map key of current state.
*/
protected TypeSerializer mapKeySerializer;
/**
* Serializer for map value of current state.
*/
protected TypeSerializer mapValueSerializer;
/**
* Serializer for namespace of current state.
*/
protected TypeSerializer namespaceSerializer;
private final Map emptyMap = new HashMap<>(0);
protected ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
protected DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
/** partitioner used to get key group.**/
protected final HashPartitioner partitioner = HashPartitioner.INSTANCE;
//--------------------------------------------------------------------------
AbstractGeminiSubKeyedMapStateImpl(AbstractGTableSubKeyedMapImpl table) {
this.table = checkNotNull(table);
}
/**
* Creates a map under a key.
*
* @return A map under a key.
*/
abstract M createMap();
//--------------------------------------------------------------------------
@Override
public boolean contains(K key, N namespace, MK mapKey) {
if (key == null || namespace == null) {
return false;
}
try {
return getRegion(key, namespace).contains(table.getGeminiPKey2(key, namespace), mapKey);
} catch (Exception e) {
throw new StateAccessException(e.getCause());
}
}
@Override
public boolean contains(K key, N namespace) {
if (key == null || namespace == null) {
return false;
}
try {
return getRegion(key, namespace).contains(table.getGeminiPKey2(key, namespace));
} catch (Exception e) {
throw new StateAccessException(e.getCause());
}
}
@Override
public MV get(K key, N namespace, MK mapKey) {
return getOrDefault(key, namespace, mapKey, null);
}
@SuppressWarnings("unchecked")
@Override
public MV getOrDefault(K key, N namespace, MK mapKey, MV defaultMapValue) {
if (key == null || namespace == null) {
return defaultMapValue;
}
MV result = getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace), mapKey);
if (result == null) {
result = defaultMapValue;
}
return result;
}
@Override
public M get(K key, N namespace) {
return getOrDefault(key, namespace, null);
}
@Override
public M getOrDefault(K key, N namespace, M defaultMap) {
if (key == null || namespace == null) {
return defaultMap;
}
// we can not use region#getOrDefault because the default value has incompatible type.
M result = (M) getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace));
return (result == null || result.isEmpty()) ? defaultMap : result;
}
@SuppressWarnings("unchecked")
@Override
public M getAll(K key, N namespace, Collection mapKeys) {
if (key == null || namespace == null || mapKeys == null || mapKeys.isEmpty()) {
return null;
}
M result = createMap();
for (MK mapKey : mapKeys) {
if (mapKey != null) {
MV value = get(key, namespace, mapKey);
if (value != null) {
result.put(mapKey, value);
}
}
}
return result.isEmpty() ? null : result;
}
@SuppressWarnings("unchecked")
@Override
public Map getAll(K key) {
if (key == null) {
return Collections.emptyMap();
}
Iterator namespaceIter = table.getSecondaryKeyByFirstKey(key);
Map result = new HashMap<>();
Set visitedNamespace = new HashSet<>();
while (namespaceIter.hasNext()) {
N namespace = namespaceIter.next();
if (visitedNamespace.contains(namespace)) {
continue;
}
M innerMap = get(key, namespace);
visitedNamespace.add(namespace);
if (innerMap != null) {
result.put(namespace, get(key, namespace));
}
}
return result;
}
@Override
public void add(K key, N namespace, MK mapKey, MV mapValue) {
checkNotNull(key);
checkNotNull(namespace);
checkNotNull(mapKey);
checkNotNull(mapValue);
getRegion(key, namespace).add(table.getGeminiPKey2(key, namespace), mapKey, mapValue);
getIndexRegion(key).add(key, namespace, dummyByte);
}
@Override
public void addAll(K key, N namespace, Map mappings) {
checkNotNull(key);
checkNotNull(namespace);
if (mappings == null || mappings.isEmpty()) {
return;
}
getRegion(key, namespace).addAll(table.getGeminiPKey2(key, namespace), mappings);
getIndexRegion(key).add(key, namespace, dummyByte);
}
@Override
public void remove(K key, N namespace) {
if (key == null || namespace == null) {
return;
}
getRegion(key, namespace).remove(table.getGeminiPKey2(key, namespace));
getIndexRegion(key).remove(key, namespace);
}
@Override
public void remove(K key, N namespace, MK mapKey) {
if (key == null || namespace == null) {
return;
}
getRegion(key, namespace).remove(table.getGeminiPKey2(key, namespace), mapKey);
// do not update index here.
}
@Override
public void removeAll(K key, N namespace, Collection mapKeys) {
if (key == null || namespace == null || mapKeys.isEmpty()) {
return;
}
for (MK mapKey : mapKeys) {
remove(key, namespace, mapKey);
}
// do not update index here for now.
}
@Override
public void removeAll(K key) {
if (key == null) {
return;
}
Iterator iterator = table.getSecondaryKeyByFirstKey(key);
while (iterator.hasNext()) {
remove(key, iterator.next());
}
}
@Override
public Iterator> iterator(K key, N namespace) {
checkNotNull(key);
return getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace)).entrySet().iterator();
}
@Override
public Iterable> entries(K key, N namespace) {
checkNotNull(key);
checkNotNull(namespace);
Map values = getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace));
return values == null ? emptyMap.entrySet() : values.entrySet();
}
@Override
public Iterable keys(K key, N namespace) {
checkNotNull(key);
checkNotNull(namespace);
Map result = getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace));
return result == null ? emptyMap.keySet() : result.keySet();
}
@Override
public Iterable values(K key, N namespace) {
checkNotNull(key);
checkNotNull(namespace);
Map result = getRegion(key, namespace).get(table.getGeminiPKey2(key, namespace));
return result == null ? emptyMap.values() : result.values();
}
@Override
public Iterator iterator(K key) {
checkNotNull(key);
Iterator namespaceIter = table.getSecondaryKeyByFirstKey(key);
return new Iterator() {
N nextNamespace = null;
N lastNamespace = null;
@Override
public boolean hasNext() {
while (nextNamespace == null && namespaceIter.hasNext()) {
nextNamespace = namespaceIter.next();
if (!contains(key, nextNamespace)) {
// set the flag to move on.
nextNamespace = null;
}
}
return nextNamespace != null;
}
@Override
public N next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
lastNamespace = nextNamespace;
nextNamespace = null;
return lastNamespace;
}
@Override
public void remove() {
AbstractGeminiSubKeyedMapStateImpl.this.remove(key, lastNamespace);
}
};
}
@Override
public Iterable keys(N namespace) {
Iterator iterator = table.regionIterator();
Set result = new HashSet<>();
while (iterator.hasNext()) {
AbstractGRegionKMapImpl, MK, MV, ?> cur = (AbstractGRegionKMapImpl, MK, MV, ?>) iterator.next();
result.addAll(
cur.getAll().entrySet().stream()
.map(a -> a.getKey())
.filter(x -> namespace.equals(x.getSecondKey()))
.map(y -> y.getFirstKey())
.collect(Collectors.toSet()));
}
return result.isEmpty() ? null : result;
}
@Override
public byte[] getSerializedValue(
byte[] serializedKeyAndNamespace,
TypeSerializer safeKeySerializer,
TypeSerializer safeNamespaceSerializer,
TypeSerializer safeValueSerializer) throws Exception {
// TODO
throw new UnsupportedOperationException();
}
@SuppressWarnings("unchecked")
protected AbstractGRegionKMapImpl, MK, MV, ? extends Map> getRegion(K key, N namespace) {
return (AbstractGRegionKMapImpl, MK, MV, ? extends Map>) table.getRegion(table.getGeminiPKey2(key, namespace));
}
protected GRegionKMapImpl getIndexRegion(K key) {
return table.getIndexRegion(key);
}
}
