org.apache.flink.runtime.state.gemini.engine.handler.PageKListHandlerImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.handler;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.hashtable.GRegionKListImpl;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValueList;
import org.apache.flink.runtime.state.gemini.engine.memstore.SegmentKListImpl;
import org.apache.flink.runtime.state.gemini.engine.page.PageIndex;
import org.apache.flink.runtime.state.gemini.engine.page.PageIndexContext;
import org.apache.flink.runtime.state.gemini.engine.page.PageSerdeFlink;
import org.apache.flink.runtime.state.gemini.engine.page.PageSerdeFlinkListImpl;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreKList;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* PageKListHandlerImpl.
*/
public class PageKListHandlerImpl extends AbstractRegionHandler implements PageHandler {
private final SegmentKListImpl segment;
private final GRegionKListImpl gRegion;
private final PageStoreKList pageStore;
private final boolean onlyEstimatedSize;
public PageKListHandlerImpl(
GRegionKListImpl gRegion, SegmentKListImpl segment, boolean onlyEstimatedSize) {
super(gRegion.getGRegionContext());
this.gRegion = gRegion;
this.segment = segment;
this.pageStore = gRegion.getPageStore();
this.onlyEstimatedSize = onlyEstimatedSize;
}
@Override
public void handle() {
if (onlyEstimatedSize) {
int keySize = estimateKeySize(this.segment);
if (keySize > 0) {
this.gRegion.getGRegionContext().getWriteBufferStats().setAverageKeyLen(keySize);
this.gRegion.getGRegionContext().getWriteBufferStats().setAverageValueLen(estimateValueSize(this.segment));
} else {
//next time, run estimate handler again.
this.gRegion.getGRegionContext().getWriteBufferStats().setAverageKeyLen(-2);
}
return;
}
PageIndex pageIndex = this.pageStore.getPageIndex();
Map>>>>> organizedData = new HashMap<>();
//step.1 divide records to different page.
segment.getData().forEach((k, v) -> organizedData.computeIfAbsent(pageIndex.getPageIndexContext(k, true),
(nothing) -> new ArrayList<>()).add(Tuple2.of(k, v)));
//step.2 write page
Iterator>>>>>> iterator = organizedData.entrySet().iterator();
Set needSplitPageList = new HashSet<>();
while (iterator.hasNext()) {
Map.Entry>>>>> entry = iterator.next();
if (entry.getKey().isNeedSplit()) {
needSplitPageList.add(entry.getKey());
}
this.pageStore.addPage(entry.getKey(), entry.getValue(), segment.getVersion());
//for fast gc.
iterator.remove();
}
if (needSplitPageList.size() > 0) {
needSplitPageList.stream().forEach(needSplitPage -> this.pageStore.splitPage(needSplitPage));
}
//if need, launch expand index handler.
this.pageStore.checkResource();
}
private int estimateKeySize(SegmentKListImpl segment) {
if (segment.getData() == null || segment.getData().isEmpty()) {
return 0;
}
try {
PageSerdeFlink pageSerdeFlink = this.gRegionContext.getPageSerdeFlink();
ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream);
for (K key : segment.getData().keySet()) {
pageSerdeFlink.getKeySerde().serialize(key, outputView);
}
return outputStream.getPosition() / segment.getData().size();
} catch (Exception e) {
throw new GeminiRuntimeException("estimateKeySize has exception=" + e.getMessage(), e);
}
}
private int estimateValueSize(SegmentKListImpl segment) {
if (segment.getData() == null || segment.getData().isEmpty()) {
return 0;
}
try {
PageSerdeFlinkListImpl pageSerdeFlink = (PageSerdeFlinkListImpl) this.gRegionContext.getPageSerdeFlink();
ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream);
int totalCount = 0;
for (GSValueList value : segment.getData().values()) {
if (value.getValue() == null) {
continue;
}
//estimate size of the element of this list.
totalCount += value.getValue().size();
pageSerdeFlink.getgListValueTypeSerializer().serialize(value.getValue(), outputView);
}
return totalCount == 0 ? 0 : (outputStream.getPosition() / totalCount) + 9;
} catch (Exception e) {
throw new GeminiRuntimeException("estimateValueSize has exception=" + e.getMessage(), e);
}
}
}