org.apache.flink.runtime.state.gemini.engine.handler.PageKMapHandlerImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.handler;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValueMap;
import org.apache.flink.runtime.state.gemini.engine.memstore.SegmentKMapImpl;
import org.apache.flink.runtime.state.gemini.engine.page.PageIndex;
import org.apache.flink.runtime.state.gemini.engine.page.PageIndexContext;
import org.apache.flink.runtime.state.gemini.engine.page.PageSerdeFlink2KeyImpl;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreKMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* PageKMapHandlerImpl.
*/
public class PageKMapHandlerImpl extends AbstractRegionHandler implements PageHandler {
private final SegmentKMapImpl segment;
private final PageStoreKMap pageStore;
private final boolean onlyEstimatedSize;
public PageKMapHandlerImpl(
GRegionContext gRegionContext,
PageStoreKMap pageStore,
SegmentKMapImpl segment,
boolean onlyEstimatedSize) {
super(gRegionContext);
this.segment = segment;
this.pageStore = pageStore;
this.onlyEstimatedSize = onlyEstimatedSize;
}
@Override
public void handle() {
if (onlyEstimatedSize) {
int keySize = estimateKeySize(this.segment);
if (keySize > 0) {
this.gRegionContext.getWriteBufferStats().setAverageKeyLen(keySize);
this.gRegionContext.getWriteBufferStats().setAverageValueLen(estimateValueSize(this.segment));
} else {
//next time, run estimate handler again.
this.gRegionContext.getWriteBufferStats().setAverageKeyLen(-2);
}
return;
}
PageIndex pageIndex = this.pageStore.getPageIndex();
Map>>>>> organizedData = new HashMap<>();
//step.1 divide records to different page.
segment.getDataMap().forEach((k, v) -> organizedData.computeIfAbsent(pageIndex.getPageIndexContext(k, true),
(nothing) -> new ArrayList<>()).add(Tuple2.of(k, v)));
Set needSplitPageList = new HashSet<>();
//step.2 write page
Iterator>>>>>> iterator = organizedData.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry>>>>> entry = iterator.next();
if (this.pageStore == null) {
System.out.println("KK,Null PageStore.");
}
if (entry.getKey().isNeedSplit()) {
needSplitPageList.add(entry.getKey());
}
this.pageStore.addPage(entry.getKey(), entry.getValue(), segment.getVersion());
//for fast gc.
iterator.remove();
}
if (needSplitPageList.size() > 0) {
needSplitPageList.stream().forEach(needSplitPage -> this.pageStore.splitPage(needSplitPage));
}
//if need, launch expand index handler.
this.pageStore.checkResource();
}
private int estimateKeySize(SegmentKMapImpl segment) {
if (segment.getDataMap() == null || segment.getDataMap().isEmpty()) {
return 0;
}
try {
PageSerdeFlink2KeyImpl pageSerdeFlink = (PageSerdeFlink2KeyImpl) gRegionContext.getPageSerdeFlink();
ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream);
for (K key : segment.getDataMap().keySet()) {
pageSerdeFlink.getKeySerde().serialize(key, outputView);
}
return outputStream.getPosition() / segment.getDataMap().size();
} catch (Exception e) {
throw new GeminiRuntimeException("estimateKeySize has exception=" + e.getMessage(), e);
}
}
private int estimateValueSize(SegmentKMapImpl segment) {
if (segment.getDataMap() == null || segment.getDataMap().isEmpty()) {
return 0;
}
try {
PageSerdeFlink2KeyImpl pageSerdeFlink = (PageSerdeFlink2KeyImpl) gRegionContext.getPageSerdeFlink();
ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream);
int totalCount = 0;
for (GSValueMap value : segment.getDataMap().values()) {
if (value.getValue() == null) {
continue;
}
//estimate size of the element of this map.
totalCount += value.getValue().size();
pageSerdeFlink.getMapValueTypeSerializer().serialize(value.getValue(), outputView);
}
return totalCount == 0 ? 0 : (outputStream.getPosition() / totalCount) + 9;
} catch (Exception e) {
throw new GeminiRuntimeException("estimateValueSize has exception=" + e.getMessage(), e);
}
}
}