All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.flink.runtime.state.gemini.engine.memstore.AbstractWriteBufferKMapHashImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.memstore;
import org.apache.flink.runtime.state.gemini.engine.GRegion;
import org.apache.flink.runtime.state.gemini.engine.handler.PageHandler;
import org.apache.flink.runtime.state.gemini.engine.handler.PageKMapHandlerImpl;
import org.apache.flink.runtime.state.gemini.engine.page.GValueType;
import org.apache.flink.runtime.state.gemini.engine.page.PageStore;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreKMap;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedDeque;
/**
* AbstractWriteBufferKMapHashImpl.
*/
public abstract class AbstractWriteBufferKMapHashImpl extends AbstractWriteBuffer>> implements WriteBufferKMap {
private static final Logger LOG = LoggerFactory.getLogger(AbstractWriteBufferKMapHashImpl.class);
protected SegmentKMap active;
protected ConcurrentLinkedDeque> snapshotQueue = new ConcurrentLinkedDeque<>();
public AbstractWriteBufferKMapHashImpl(
GRegion gRegion, EventExecutor eventExecutor, PageStore pageStore) {
super(gRegion, eventExecutor, pageStore);
}
abstract void initActive();
abstract Map> createPOJOMap();
@Override
PageHandler createPageHandler(Segment segment, boolean onlyEstimatedSize) {
return new PageKMapHandlerImpl<>(gRegionContext,
(PageStoreKMap) gRegion.getPageStore(),
(SegmentKMapImpl) segment,
onlyEstimatedSize);
}
@Override
public Segment getActiveSegment() {
return active;
}
@Override
Segment pollFlushingSegment() {
return snapshotQueue.poll();
}
@Override
public void add(K key, MK mkey, MV mvalue) {
active.add(key, mkey, mvalue);
checkResource();
}
@Override
public void add(K key, Map map) {
active.add(key, map);
checkResource();
}
@Override
public void remove(K key, MK mapKey) {
active.remove(key, mapKey);
checkResource();
}
@Override
public GValueType contains(K key, MK mapKey) {
GSValueMap mapValue = active.get(key);
if (mapValue != null) {
mapValue.requestCount++;
if (mapValue.valueType == GValueType.Delete) {
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return GValueType.Delete;
}
GSValue value = mapValue.getValue().get(mapKey);
if (value != null) {
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return value.valueType;
}
}
Iterator> iterator = snapshotQueue.descendingIterator();
while (iterator.hasNext()) {
SegmentKMap inactive = iterator.next();
mapValue = inactive.get(key);
if (mapValue != null) {
mapValue.requestCount++;
if (mapValue.valueType == GValueType.Delete) {
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return GValueType.Delete;
}
GSValue value = mapValue.getValue().get(mapKey);
if (value != null) {
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return value.valueType;
}
}
}
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferMissCount();
return null;
}
@Override
public void put(K key, Map> value) {
active.put(key, value);
checkResource();
}
@Override
public GSValue get(K key, MK mapKey) {
GSValue gsValue = active.get(key, mapKey);
if (gsValue != null) {
gsValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return gsValue;
}
Iterator> iterator = snapshotQueue.descendingIterator();
while (iterator.hasNext()) {
SegmentKMap inactive = iterator.next();
gsValue = inactive.get(key, mapKey);
if (gsValue != null) {
gsValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return gsValue;
}
}
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferMissCount();
return null;
}
@Override
public GSValueMap get(K key) {
List> reverseOrderList = new ArrayList<>();
GSValueMap mapValue = active.get(key);
if (mapValue != null) {
mapValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
if (mapValue.valueType == GValueType.PutMap) {
return mapValue;
} else if (mapValue.valueType == GValueType.Delete) {
return mapValue;
}
reverseOrderList.add(mapValue);
}
Iterator> iterator = snapshotQueue.descendingIterator();
while (iterator.hasNext()) {
SegmentKMap inactive = iterator.next();
mapValue = inactive.get(key);
if (mapValue != null) {
mapValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
reverseOrderList.add(mapValue);
if (mapValue.valueType == GValueType.PutMap || mapValue.valueType == GValueType.Delete) {
break;
}
}
}
return reverseOrderList.size() == 0 ? null : mergeGSValueMap(reverseOrderList);
}
@Override
public void getAll(Map>>> container) {
// TODO
}
@Override
public void allKeysIncludeDeleted(Set container) {
Segment activeMap = getActiveSegment();
container.addAll(activeMap.getData().keySet());
Iterator> iterator = snapshotQueue.descendingIterator();
while (iterator.hasNext()) {
SegmentKMap inactive = iterator.next();
container.addAll(inactive.getData().keySet());
}
}
@Override
public void removeKey(K key) {
active.removeKey(key);
checkResource();
}
@Override
public void reset() {
initActive();
}
@Override
public GValueType contains(K key) {
// TODO: may be return wrong value, by pass to region.
GSValueMap mapValue = active.get(key);
if (mapValue != null) {
mapValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return mapValue.valueType;
}
Iterator> iterator = snapshotQueue.descendingIterator();
while (iterator.hasNext()) {
SegmentKMap inactive = iterator.next();
mapValue = inactive.get(key);
if (mapValue != null) {
mapValue.requestCount++;
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferHitCount();
return mapValue.valueType;
}
}
gRegionContext.getGContext().getSupervisor().getCacheManager().getCacheStats().addWriteBufferMissCount();
return null;
}
private GSValueMap mergeGSValueMap(List> reverseOrderList) {
if (reverseOrderList == null || reverseOrderList.size() == 0) {
return null;
}
//TODO check TTL
int index = reverseOrderList.size() - 1;
//Value list is right order.
Map> newMap = createPOJOMap();
//judge a compacted map value's type is a little complicated.
// any -> Delete -> AddMap -> AddMap, final type must be PutMap
// any -> PutMap -> AddMap -> AddMap, final type must be PutMap
// AddMap -> AddMap -> AddMap, final type must be AddMap
GValueType finalGValueType = GValueType.AddMap;
//map value's seqID is newest. that will be more confused.
long seqID = reverseOrderList.get(0).getSeqID();
while (index >= 0) {
GSValueMap currentMap = reverseOrderList.get(index);
if (currentMap.getValueType() == GValueType.Delete) {
newMap.clear();
finalGValueType = GValueType.PutMap;
} else if (currentMap.getValueType() == GValueType.PutMap) {
newMap.clear();
newMap.putAll(currentMap.value);
finalGValueType = GValueType.PutMap;
} else {
newMap.putAll(currentMap.value);
}
index--;
}
return new GSValueMap<>(newMap, finalGValueType, seqID);
}
}
