Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.flink.runtime.state.gemini.engine.page.DataPageKMapImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.filter.StateFilter;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryKey;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryValue;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryValueForSplit;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryValueImpl;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.ByteBufferDataInputView;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinarySplitHashMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBufferAddressMapping;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GHashRoutingValue;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.SplitHashMapValueHelper;
import org.apache.flink.runtime.state.gemini.engine.rm.Allocator;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.rm.GUnPooledByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.utils.SeqIDUtils;
import org.apache.flink.util.MathUtils;
import org.apache.flink.util.Preconditions;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap.EMPTY_G_BINARY_HASHMAP;
/**
* DataPageKMapImpl.
*/
public class DataPageKMapImpl extends DataPageKVImpl>> implements DataPageKMap {
protected final TypeSerializer mkTypeSerializer;
protected final TypeSerializer mvTypeSerializer;
public DataPageKMapImpl(
GBinaryHashMap gBinaryHashMap,
TypeSerializer mkTypeSerializer,
TypeSerializer mvTypeSerializer,
TypeSerializer>> gMapTypeSerializer) {
super(gBinaryHashMap, gMapTypeSerializer);
this.mkTypeSerializer = mkTypeSerializer;
this.mvTypeSerializer = mvTypeSerializer;
}
@Override
public GSValue get(K key, MK mapKey) {
try {
BinaryValue binaryValue = this.gBinaryHashMap.get(key);
if (binaryValue == null) {
return null;
}
// the map related to K is deleted, means all the mapKey is Deleted.
if (binaryValue.getGValueType() == GValueType.Delete) {
return new GSValue<>(null, binaryValue.getGValueType(), binaryValue.getSeqID());
}
GSValue result = getForMapBinaryValue(getBinaryMapByBinaryValue(binaryValue, mapKey), mapKey);
//if this map's type is PutMap, means pages which are prior to this are useless
if (result == null && binaryValue.getGValueType() == GValueType.PutMap) {
return new GSValue<>(null, binaryValue.getGValueType(), binaryValue.getSeqID());
}
return result;
} catch (Exception e) {
throw new GeminiRuntimeException("get exception: " + e.getMessage(), e);
}
}
protected Map getBinaryMapByBinaryValue(BinaryValue binaryValue, MK mapKey) {
Map binaryMap;
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
Preconditions.checkState(this.gBinaryHashMap instanceof GBinarySplitHashMap, "Internal bug.");
binaryMap = GHashRoutingValue.getSubGBinaryHashMap(binaryValue, mapKey, this.mkTypeSerializer);
} else {
binaryMap = getBinaryMap(getDuplicateBB(binaryValue));
}
return binaryMap;
}
@Override
public GSValue>> get(K key) {
try {
BinaryValue binaryValue = this.gBinaryHashMap.get(key);
if (binaryValue == null) {
return null;
}
if (binaryValue.getGValueType() == GValueType.Delete) {
return new GSValue<>(null, GValueType.Delete, binaryValue.getSeqID());
}
Map> value = getMap(binaryValue);
return new GSValue<>(value, binaryValue.getGValueType(), binaryValue.getSeqID());
} catch (Exception e) {
throw new GeminiRuntimeException("get exception: " + e.getMessage(), e);
}
}
protected Map> getMap(BinaryValue binaryValue) throws IOException {
Map> value = new HashMap<>();
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
List> subMapList = GHashRoutingValue.getAllSubGBinaryHashMap(binaryValue, mkTypeSerializer);
for (GBinaryHashMap subMap : subMapList) {
DataInputView byteBufferDataInputView = new ByteBufferDataInputView(subMap.getData(),
0,
subMap.bytesSize());
value.putAll(valueTypeSerializer.deserialize(byteBufferDataInputView));
}
} else {
DataInputView byteBufferDataInputView = new ByteBufferDataInputView(binaryValue.getBb(),
binaryValue.getValueOffset(),
binaryValue.getValueLen());
value = valueTypeSerializer.deserialize(byteBufferDataInputView);
}
return value;
}
protected GSValue getForMapBinaryValue(Map mapValueMap, MK mapKey) {
if (mapValueMap == null) {
return null;
}
return getForBinaryValue(mapValueMap.get(mapKey));
}
protected GSValue getForBinaryValue(BinaryValue mvBinary) {
try {
if (mvBinary == null) {
return null;
}
if (mvBinary.getGValueType() == GValueType.Delete) {
return new GSValue<>(null, mvBinary.getGValueType(), mvBinary.getSeqID());
}
DataInputView byteBufferDataInputView = new ByteBufferDataInputView(mvBinary.getBb(),
mvBinary.getValueOffset(),
mvBinary.getValueLen());
MV value = mvTypeSerializer.deserialize(byteBufferDataInputView);
return new GSValue<>(value, mvBinary.getGValueType(), mvBinary.getSeqID());
} catch (Exception e) {
throw new GeminiRuntimeException("getForBinaryValue has Exception:" + e.getMessage(), e);
}
}
protected static GByteBuffer getDuplicateBB(BinaryValue binaryValue) {
if (binaryValue.getValueLen() == 0) {
return null;
}
ByteBuffer duplicateByteBuffer = binaryValue.getBb().duplicate();
duplicateByteBuffer.limit(binaryValue.getValueLen() + binaryValue.getValueOffset());
duplicateByteBuffer.position(binaryValue.getValueOffset());
return new GUnPooledByteBuffer(duplicateByteBuffer.slice());
}
protected Map getBinaryMap(GByteBuffer valueBB) {
return new GBinaryHashMap<>(valueBB, mkTypeSerializer);
}
@Override
public boolean contains(K key, MK mapKey) {
GSValue mvResult = get(key, mapKey);
return mvResult != null && mvResult.getValue() != null;
}
@Override
public DataPageType getDataPageType() {
return DataPageType.KHashMap;
}
public static BinaryValue doCompactionMapValue(
List valueByOrder,
TypeSerializer mkTypeSerializer,
boolean isMajor,
long version,
int logicPageId,
Allocator allocator,
@Nullable StateFilter stateFilter,
@Nullable GRegionContext gRegionContext,
GBufferAddressMapping pageMapping,
MapSplitConfig mapSplitConfig) {
//NOTICE: this variable provides a protection for unstable environment. This variable should not be modified while the job is running
if (mapSplitConfig.isMapSplitEnabled()) {
int maxSplitPartNum = computeMaxSplitPartNum(valueByOrder, mapSplitConfig);
//If the binaryValue is GRoutingValue before (it was split before), the maxSplitPartNum must be larger than one.
if (maxSplitPartNum > 1) {
return doCompactionMapValueForSplit(valueByOrder,
mkTypeSerializer,
isMajor,
version,
logicPageId,
allocator,
stateFilter,
gRegionContext,
pageMapping,
maxSplitPartNum);
}
}
return doCompactionMapValueNormal(valueByOrder,
mkTypeSerializer,
isMajor,
version,
logicPageId,
allocator,
stateFilter,
gRegionContext);
}
@VisibleForTesting
static int computeMaxSplitPartNum(List valueByOrder, MapSplitConfig mapSplitConfig) {
//no split
int maxSplitPartNum = 1;
int mapSplitSizeThreshold = mapSplitConfig.getMapSplitSizeThreshold();
int mapSplitSubMapSize = mapSplitConfig.getSubMapSize();
for (BinaryValue binaryValue : valueByOrder) {
if (binaryValue.getGValueType() == GValueType.Delete) {
continue;
}
int estimateSubMapNum = 1;
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
estimateSubMapNum = GHashRoutingValue.getSubMapCount(binaryValue);
int subMapMaxSize = GHashRoutingValue.getSubMapMaxSize(binaryValue);
if (subMapMaxSize > mapSplitSizeThreshold) {
//need next split
estimateSubMapNum <<= 1;
}
} else {
//need first split
if (binaryValue.getValueLen() > mapSplitSizeThreshold) {
int splitNum = (int) Math.ceil(binaryValue.getValueLen() * 1.0D / mapSplitSubMapSize);
estimateSubMapNum = MathUtils.roundUpToPowerOfTwo(splitNum);
}
}
maxSplitPartNum = Math.max(Math.min(estimateSubMapNum, mapSplitConfig.getMaxSubMapNum()), maxSplitPartNum);
}
return maxSplitPartNum;
}
private static BinaryValue doCompactionMapValueNormal(
List valueByOrder,
TypeSerializer mkTypeSerializer,
boolean isMajor,
long version,
int logicPageId,
Allocator allocator,
@Nullable StateFilter stateFilter,
@Nullable GRegionContext gRegionContext) {
try {
if (valueByOrder.size() == 1 && !isMajor) {
return valueByOrder.get(0);
}
List> listByOrder = new ArrayList<>();
long seqID = SeqIDUtils.INVALID_SEQID;
GValueType firstValueType = null;
for (BinaryValue binaryValue : valueByOrder) {
if (binaryValue.getGValueType() == GValueType.Delete) {
firstValueType = GValueType.Delete;
continue;
}
GBinaryHashMap mapValue = new GBinaryHashMap<>(getDuplicateBB(binaryValue), mkTypeSerializer);
//pick up newest page's seqID.
seqID = Math.max(seqID, binaryValue.getSeqID());
listByOrder.add(mapValue);
if (firstValueType == null) {
firstValueType = binaryValue.getGValueType();
}
}
GBinaryHashMap gBinaryHashMap;
if (listByOrder.size() == 0) {
gBinaryHashMap = EMPTY_G_BINARY_HASHMAP;
} else {
int index = 0;
//just for not to create a new map.
Map newMap = listByOrder.get(index).getBinaryMap();
long compactionCount = listByOrder.get(index).getCompactionCount();
index++;
while (index < listByOrder.size()) {
newMap.putAll(listByOrder.get(index).getBinaryMap());
compactionCount += listByOrder.get(index).getCompactionCount();
index++;
}
gBinaryHashMap = GBinaryHashMap.ofBinaryMap(DataPageType.KV,
isMajor,
version,
logicPageId,
mkTypeSerializer,
allocator,
newMap,
compactionCount,
stateFilter,
gRegionContext);
}
ByteBuffer bb = gBinaryHashMap == EMPTY_G_BINARY_HASHMAP ? null : gBinaryHashMap.getData();
GValueType gValueType = judgeFinalValueType(bb, firstValueType, isMajor);
return new BinaryValueImpl(bb, gValueType, seqID, 0, gBinaryHashMap.bytesSize());
} catch (Exception e) {
throw new GeminiRuntimeException("Internal BUG " + e.getMessage(), e);
}
}
@VisibleForTesting
public static BinaryValue doCompactionMapValueForSplit(
List valueByOrder,
TypeSerializer mkTypeSerializer,
boolean isMajor,
long version,
int logicPageId,
Allocator allocator,
@Nullable StateFilter stateFilter,
@Nullable GRegionContext gRegionContext,
GBufferAddressMapping pageMapping,
int maxSplitPartNum) {
Preconditions.checkState(maxSplitPartNum > 1);
Preconditions.checkArgument(MathUtils.roundDownToPowerOf2(maxSplitPartNum) == maxSplitPartNum);
if (valueByOrder.size() == 1 && !isMajor) {
BinaryValue binaryValue = valueByOrder.get(0);
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
GByteBuffer gByteBuffer = SplitHashMapValueHelper.replaceBinaryValueIdList((BinaryValueForSplit) binaryValue, pageMapping);
return new BinaryValueImpl(gByteBuffer.getByteBuffer(), binaryValue.getGValueType(), binaryValue.getSeqID(), 0, gByteBuffer.capacity());
}
return valueByOrder.get(0);
}
List listByOrder = new ArrayList<>();
long seqID = SeqIDUtils.INVALID_SEQID;
GValueType firstValueType = null;
for (BinaryValue binaryValue : valueByOrder) {
if (binaryValue.getGValueType() == GValueType.Delete) {
firstValueType = GValueType.Delete;
continue;
}
if (binaryValue.getBb() == null) {
continue;
}
//pick up newest page's seqID.
seqID = Math.max(seqID, binaryValue.getSeqID());
listByOrder.add(binaryValue);
if (firstValueType == null) {
firstValueType = binaryValue.getGValueType();
}
}
GByteBuffer finalByteBuffer;
if (listByOrder.size() == 0) {
finalByteBuffer = null;
} else {
Map[][] buckets = new Map[listByOrder.size()][maxSplitPartNum];
int[] subMapIdList = new int[maxSplitPartNum];
int subMapMaxSize = 0;
List indexList = new LinkedList<>();
indexList.add(0); //start from index 0
int firstSubMapSize = mergeSubList(0,
listByOrder,
buckets,
mkTypeSerializer,
isMajor,
version,
logicPageId,
allocator,
stateFilter,
gRegionContext,
pageMapping,
subMapIdList);
subMapMaxSize = Math.max(firstSubMapSize, subMapMaxSize);
int stepSize = maxSplitPartNum >> 1;
while (stepSize > 0 && indexList.size() < maxSplitPartNum) {
List newIndex = new LinkedList<>();
for (Integer index : indexList) {
int subMapSize = mergeSubList(index + stepSize,
listByOrder,
buckets,
mkTypeSerializer,
isMajor,
version,
logicPageId,
allocator,
stateFilter,
gRegionContext,
pageMapping,
subMapIdList);
subMapMaxSize = Math.max(subMapSize, subMapMaxSize);
newIndex.add(index + stepSize);
}
indexList.addAll(newIndex);
stepSize >>= 1;
}
finalByteBuffer = SplitHashMapValueHelper.genRoutingValueForSplitHashMap(subMapIdList, logicPageId, allocator, subMapMaxSize);
}
ByteBuffer bb = finalByteBuffer == null ? null : finalByteBuffer.getByteBuffer();
GValueType gValueType = judgeFinalValueType(bb, firstValueType, isMajor);
return new BinaryValueImpl(bb, gValueType, seqID, 0, bb == null ? 0 : bb.capacity());
}
private static int mergeSubList(
int partIndex,
List listByOrder,
Map[][] buckets,
TypeSerializer mkTypeSerializer,
boolean isMajor,
long version,
int logicPageId,
Allocator allocator,
@Nullable StateFilter stateFilter,
@Nullable GRegionContext gRegionContext,
GBufferAddressMapping mapping,
int[] subMapIdList) {
int compactionCount = 0;
Map subCompactionMap = new HashMap<>();
PageAddress tmpPageAddress = null;
GBufferAddressMapping tmpMapping = null;
for (int i = 0; i < listByOrder.size(); i++) {
BinaryValue binaryValue = listByOrder.get(i);
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
int maxSplitNum = subMapIdList.length;
int subMapCount = GHashRoutingValue.getSubMapCount(binaryValue);
//record the first subMap address when subMapCount equals maxSplitNum
if (subMapCount == maxSplitNum && tmpPageAddress == null && subCompactionMap.isEmpty()) {
tmpPageAddress = GHashRoutingValue.getSubMapPageAddress(partIndex, binaryValue);
tmpMapping = binaryValue.getPageMapping();
continue;
} else {
if (partIndex < GHashRoutingValue.getSubMapCount(binaryValue)) { //need fill the bucket
GBinaryHashMap subMap = GHashRoutingValue.getSubGBinaryHashMapWithKey(null, partIndex, binaryValue, mkTypeSerializer);
compactionCount += subMap.getCompactionCount();
divideBinaryMapToBuckets(subMap.getBinaryMap(), buckets[i]);
}
}
} else {
//need fill the bucket
if (partIndex == 0) {
GBinaryHashMap gBinaryHashMap = new GBinaryHashMap<>(getDuplicateBB(binaryValue), mkTypeSerializer);
compactionCount += gBinaryHashMap.getCompactionCount();
Map normalMap = gBinaryHashMap.getBinaryMap();
divideBinaryMapToBuckets(normalMap, buckets[i]);
}
}
if (buckets[i][partIndex] != null) {
if (tmpPageAddress != null && subCompactionMap.isEmpty()) {
subCompactionMap.putAll(getBinaryMap(tmpPageAddress, tmpMapping, mkTypeSerializer));
}
subCompactionMap.putAll(buckets[i][partIndex]);
}
}
int subMapId;
int subMapSize;
//If only one sub map need to be merge, just return the sub map;
if (subCompactionMap.isEmpty() && tmpPageAddress != null) {
subMapId = mapping.putGByteBufferAddress(tmpPageAddress);
subMapSize = tmpPageAddress.getDataLen();
} else {
GBinaryHashMap subMap = GBinaryHashMap.ofBinaryMap(DataPageType.KV,
isMajor,
version,
logicPageId,
mkTypeSerializer,
allocator,
subCompactionMap,
compactionCount,
stateFilter,
gRegionContext);
subMapId = mapping.putGByteBufferAddress(new DataPageHashSubPageImpl(subMap));
subMapSize = subMap.bytesSize();
}
subMapIdList[partIndex] = subMapId;
//help gc
for (int i = 0; i < listByOrder.size(); i++) {
buckets[i][partIndex] = null;
}
return subMapSize;
}
private static Map getBinaryMap(
PageAddress subMapPageAddress,
GBufferAddressMapping pageMapping,
TypeSerializer mkTypeSerializer) {
GByteBuffer gByteBuffer = pageMapping.getGByteBuffer(subMapPageAddress, null);
if (gByteBuffer == null) {
return new HashMap<>();
}
GBinaryHashMap gBinaryHashMap = new GBinaryHashMap<>(gByteBuffer, mkTypeSerializer);
return gBinaryHashMap.getBinaryMap();
}
private static void divideBinaryMapToBuckets(
Map binaryMap, Map[] bucket) {
for (Map.Entry entry : binaryMap.entrySet()) {
int index = entry.getKey().hashCode() & (bucket.length - 1);
if (bucket[index] == null) {
bucket[index] = new HashMap<>();
}
bucket[index].put(entry.getKey(), entry.getValue());
}
}
static GValueType judgeFinalValueType(ByteBuffer bb, GValueType firstValueType, boolean isMajor) {
//judge a compacted map value's type is a little complicated.
// Delete -> AddMap -> AddMap, it only happens when it's minor, and final type can be PutMap
// PutMap -> AddMap -> AddMap, when it's major, it can be PutMap or AddMap, when it's minor, it only can be PutMap
// AddMap -> AddMap -> AddMap, when it's major, it can be PutMap or AddMap, when it's minor, it only can be AddMap
// when compacted map value is null, it only happens when it's major, it can be any one.
if (bb == null) {
return GValueType.Delete;
}
if (firstValueType == GValueType.Delete) {
return GValueType.PutMap;
}
return firstValueType;
}
protected static Map doCompactValueToBinaryMap(
List binaryValueReversedOrderList, TypeSerializer mkTypeSerializer) {
try {
Map newMap = new HashMap<>();
for (int i = binaryValueReversedOrderList.size() - 1; i >= 0; i--) {
BinaryValue binaryValue = binaryValueReversedOrderList.get(i);
if (GHashRoutingValue.isGHashRoutingValue(binaryValue)) {
List> subMapList = GHashRoutingValue.getAllSubGBinaryHashMap(binaryValue, mkTypeSerializer);
for (GBinaryHashMap subMap : subMapList) {
newMap.putAll(subMap.getBinaryMap());
}
} else {
GBinaryHashMap mapValue = new GBinaryHashMap<>(getDuplicateBB(binaryValue), mkTypeSerializer);
newMap.putAll(mapValue.getBinaryMap());
}
}
return newMap;
} catch (Exception e) {
throw new GeminiRuntimeException("Internal BUG " + e.getMessage(), e);
}
}
public static DataPageKMapImpl readKMapPageFrom(
PageSerdeFlink2Key pageSerdeFlink, GByteBuffer dataPage, int crc) {
GBinaryHashMap gBinaryHashMap = new GBinaryHashMap<>(dataPage, pageSerdeFlink.getKeySerde(), crc);
return new DataPageKMapImpl<>(gBinaryHashMap,
pageSerdeFlink.getKey2Serde(),
pageSerdeFlink.getValueSerde(),
pageSerdeFlink.getMapValueTypeSerializer());
}
@Override
public Tuple2 getSplitDataByGBinaryMap(
GBinaryHashMap gBinaryHashMap1, GBinaryHashMap gBinaryHashMap2) {
DataPageKMapImpl dataPage1 = gBinaryHashMap1 == EMPTY_G_BINARY_HASHMAP
? null
: new DataPageKMapImpl<>(gBinaryHashMap1, mkTypeSerializer, mvTypeSerializer, valueTypeSerializer);
DataPageKMapImpl dataPage2 = gBinaryHashMap2 == EMPTY_G_BINARY_HASHMAP
? null
: new DataPageKMapImpl<>(gBinaryHashMap2, mkTypeSerializer, mvTypeSerializer, valueTypeSerializer);
return Tuple2.of(dataPage1, dataPage2);
}
}
