Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.flink.runtime.state.gemini.engine.page.DataPageKSortedMapImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.filter.StateFilter;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValueMap;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValueMapEntry;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryKey;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.BinaryValue;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinarySortedMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GComparator;
import org.apache.flink.runtime.state.gemini.engine.rm.Allocator;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.utils.SeqIDUtils;
import org.apache.flink.shaded.guava18.com.google.common.collect.Maps;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import static org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap.EMPTY_G_BINARY_HASHMAP;
import static org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinarySortedMap.EMPTY_G_BINARY_SORTEDMAP;
/**
* DataPageKSortedMapImpl.
*/
public class DataPageKSortedMapImpl extends DataPageKMapImpl implements DataPageKSortedMap {
private final GComparator gComparator;
public DataPageKSortedMapImpl(
GBinaryHashMap gBinaryHashMap,
TypeSerializer mkTypeSerializer,
TypeSerializer mvTypeSerializer,
AbstractGMapValueTypeSerializer gSortedMapValueTypeSerialiZer,
GComparator gComparator) {
super(gBinaryHashMap, mkTypeSerializer, mvTypeSerializer, gSortedMapValueTypeSerialiZer);
this.gComparator = gComparator;
}
@Override
protected GBinarySortedMap getBinaryMap(GByteBuffer valueBB) {
return new GBinarySortedMap<>(valueBB.getByteBuffer(), mkTypeSerializer, gComparator);
}
@Override
public GSValueMapEntry firstEntry(K key) {
return getFirstOrLastEntry(key, true);
}
@Override
public GSValueMapEntry lastEntry(K key) {
return getFirstOrLastEntry(key, false);
}
private GSValueMapEntry getFirstOrLastEntry(K key, boolean first) {
try {
BinaryValue binaryValue = this.gBinaryHashMap.get(key);
if (binaryValue == null) {
return null;
}
GValueType mapType = binaryValue.getgValueType();
// the map related to K is deleted, means all the mapKey is Deleted.
if (binaryValue.getgValueType() == GValueType.Delete) {
return new GSValueMapEntry<>(null, mapType, binaryValue.getSeqID());
}
GBinarySortedMap sortedMap = getBinaryMap(getDuplicateBB(binaryValue));
MK mkey = first ? sortedMap.firstKey() : sortedMap.lastKey();
//TODO, sortedMap directly support firstEntry future.
if (mkey == null) {
return new GSValueMapEntry<>(null, mapType, binaryValue.getSeqID());
}
GSValue gsValue = getForMapBinaryValue(sortedMap, mkey);
return new GSValueMapEntry<>(mkey, gsValue, mapType, binaryValue.getSeqID());
} catch (Exception e) {
throw new GeminiRuntimeException("get exception: " + e.getMessage(), e);
}
}
@Override
public GSValueMap head(K key, MK endMapKey) {
return subMap(key, null, endMapKey);
}
@Override
public GSValueMap tail(K key, MK startMapKey) {
return subMap(key, startMapKey, null);
}
@Override
public GSValueMap subMap(K key, MK startMapKey, MK endMapKey) {
try {
BinaryValue binaryValue = this.gBinaryHashMap.get(key);
if (binaryValue == null) {
return null;
}
GValueType mapType = binaryValue.getgValueType();
// the map related to K is deleted, means all the mapKey is Deleted.
if (binaryValue.getgValueType() == GValueType.Delete) {
return new GSValueMap<>(null, GValueType.Delete, binaryValue.getSeqID());
}
GBinarySortedMap sortedMap = getBinaryMap(getDuplicateBB(binaryValue));
SortedMap subSortedMap = sortedMap.subMap(startMapKey, endMapKey);
if (subSortedMap == null || subSortedMap.size() == 0) {
return new GSValueMap<>(null, mapType, binaryValue.getSeqID());
}
SortedMap> result = Maps.transformEntries(subSortedMap, (mk, mv) -> getForBinaryValue(mv));
return new GSValueMap<>(result, mapType, binaryValue.getSeqID());
} catch (Exception e) {
throw new GeminiRuntimeException("get exception: " + e.getMessage(), e);
}
}
@Override
public DataPageType getDataPageType() {
return DataPageType.KSortedMap;
}
public static BinaryValue doCompactionSortedMapValue(
List valueByOrder,
TypeSerializer mkTypeSerializer,
GComparator gComparator,
boolean isMajor,
long version,
int logicPageId,
Allocator allocator,
@Nullable StateFilter stateFilter,
@Nullable GRegionContext gRegionContext) {
try {
List> listByOrder = new ArrayList<>();
long seqID = SeqIDUtils.INVALID_SEQID;
GValueType firstValueType = null;
for (BinaryValue binaryValue : valueByOrder) {
if (binaryValue.getgValueType() == GValueType.Delete) {
firstValueType = GValueType.Delete;
continue;
}
GBinarySortedMap mapValue = new GBinarySortedMap<>(getDuplicateBB(binaryValue).getByteBuffer(),
mkTypeSerializer,
gComparator);
//pick up newest page's seqID.
seqID = Math.max(seqID, binaryValue.getSeqID());
listByOrder.add(mapValue);
if (firstValueType == null) {
firstValueType = binaryValue.getgValueType();
}
}
int index = 0;
Map newMap = listByOrder.get(index).getBinaryMap();
long compactionCount = listByOrder.get(index).getCompactionCount();
index++;
while (index < listByOrder.size()) {
newMap.putAll(listByOrder.get(index).getBinaryMap());
compactionCount += listByOrder.get(index).getCompactionCount();
index++;
}
GBinarySortedMap gBinarySortedMap = GBinarySortedMap.ofBinaryList(DataPageType.KV,
isMajor,
version,
logicPageId,
mkTypeSerializer,
gComparator,
allocator,
newMap,
compactionCount,
stateFilter,
gRegionContext);
ByteBuffer bb = gBinarySortedMap == EMPTY_G_BINARY_SORTEDMAP ? null : gBinarySortedMap.getData();
GValueType gValueType = judgeFinalValueType(bb, firstValueType, isMajor);
return new BinaryValue(bb, gValueType, seqID, 0, gBinarySortedMap.bytesSize());
} catch (Exception e) {
throw new GeminiRuntimeException("Internal BUG " + e.getMessage(), e);
}
}
public static DataPageKSortedMapImpl readKSortedMapPageFrom(
PageSerdeFlink2Key pageSerdeFlink, GByteBuffer dataPage, int crc) {
GBinaryHashMap gBinaryHashMap = new GBinaryHashMap<>(dataPage, pageSerdeFlink.getKeySerde(), crc);
return new DataPageKSortedMapImpl<>(gBinaryHashMap,
pageSerdeFlink.getKey2Serde(),
pageSerdeFlink.getValueSerde(),
pageSerdeFlink.getMapValueTypeSerializer(),
pageSerdeFlink.getMapComparator());
}
@Override
public Tuple2 getSplitDataByGBinaryMap(
GBinaryHashMap gBinaryHashMap1, GBinaryHashMap gBinaryHashMap2) {
DataPageKSortedMapImpl dataPage1 = gBinaryHashMap1 == EMPTY_G_BINARY_HASHMAP
? null
: new DataPageKSortedMapImpl<>(gBinaryHashMap1,
mkTypeSerializer,
mvTypeSerializer,
(AbstractGMapValueTypeSerializer) valueTypeSerializer,
gComparator);
DataPageKSortedMapImpl dataPage2 = gBinaryHashMap2 == EMPTY_G_BINARY_HASHMAP
? null
: new DataPageKSortedMapImpl<>(gBinaryHashMap2,
mkTypeSerializer,
mvTypeSerializer,
(AbstractGMapValueTypeSerializer) valueTypeSerializer,
gComparator);
return Tuple2.of(dataPage1, dataPage2);
}
}
