org.apache.flink.runtime.state.gemini.engine.GRegionContext Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.filter.StateFilter;
import org.apache.flink.runtime.state.gemini.engine.memstore.WriteBufferStats;
import org.apache.flink.runtime.state.gemini.engine.page.DataPage;
import org.apache.flink.runtime.state.gemini.engine.page.PageSerdeFlink;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreStats;
import org.apache.flink.runtime.state.gemini.engine.utils.SeqIDUtils;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/**
* GRegionContext. writeBuffer/PageStore, etc
*/
public class GRegionContext {
// TODO #SR use region name instead of tableName
private final String tableName;
private final GRegionID regionId;
private final WriteBufferStats writeBufferStats;
private final PageStoreStats pageStoreStats;
private GContext gContext;
private long writeBufferWaterMark;
private final PageSerdeFlink pageSerdeFlink;
private volatile long lastSeqId;
private volatile long removeAllSeqId;
//for skew Page
private final int hugePageSizeThreshold;
private final Map hugePageMap = new ConcurrentHashMap<>();
private final AtomicInteger hugePageTotalSize = new AtomicInteger(0);
public GRegionContext(GContext gContext, String tableName, GRegionID regionId, PageSerdeFlink pageSerdeFlink) {
this(gContext, tableName, regionId, pageSerdeFlink, SeqIDUtils.INVALID_SEQID, SeqIDUtils.INVALID_SEQID);
}
public GRegionContext(
GContext gContext,
String tableName,
GRegionID regionId,
PageSerdeFlink pageSerdeFlink,
long lastSeqId,
long removeAllSeqId) {
this.tableName = tableName;
this.gContext = gContext;
this.pageStoreStats = new PageStoreStats(tableName + regionId, gContext.getSupervisor().getCacheManager());
this.regionId = regionId;
this.pageSerdeFlink = pageSerdeFlink;
this.writeBufferWaterMark = gContext.getGConfiguration().getWriteBufferSize();
this.lastSeqId = lastSeqId;
this.removeAllSeqId = removeAllSeqId;
this.writeBufferStats = new WriteBufferStats(gContext.getSupervisor().getWriteBufferManager());
this.hugePageSizeThreshold = gContext.getGConfiguration().getHugePageSizeThreshold();
}
public WriteBufferStats getWriteBufferStats() {
return writeBufferStats;
}
public PageStoreStats getPageStoreStats() {
return pageStoreStats;
}
public GContext getGContext() {
return gContext;
}
public void setWriteBufferWaterMark(long writeBufferWaterMark) {
//set by supervisor, GConfiguration.getWriteBufferWaterMark divided by region num.
//that means each region have same fixed write buffer size.
this.writeBufferWaterMark = writeBufferWaterMark;
}
public long getWriteBufferWaterMark() {
return writeBufferWaterMark;
}
public PageSerdeFlink getPageSerdeFlink() {
return pageSerdeFlink;
}
public GRegionID getRegionId() {
return regionId;
}
public String getTableName() {
return tableName;
}
public long getLastSeqId() {
return lastSeqId;
}
public long getNextSeqId() {
long ts = gContext.getTimeProvider().currentTimestamp();
lastSeqId = SeqIDUtils.nextSeqID(lastSeqId, ts);
return lastSeqId;
}
public void setRemoveAllSeqId(long removeAllSeqId) {
this.removeAllSeqId = removeAllSeqId;
}
public long getRemoveAllSeqId() {
return removeAllSeqId;
}
/**
* Whether to filter a state when it's accessed.
*/
public boolean filterState(long seqId) {
StateFilter stateFilter = gContext.getStateFilter();
return stateFilter != null && gContext.getStateFilter().filter(this, seqId);
}
public boolean isHugePage(DataPage dataPage) {
//TODO if we support Map split, we should not regard it as a huge page.
return dataPage.getSize() >= this.hugePageSizeThreshold;
}
public void addHugePage(DataPage dataPage) {
if (this.hugePageMap.put(dataPage, dataPage) == null) {
hugePageTotalSize.addAndGet(dataPage.getSize());
gContext.getSupervisor().getCacheManager().getCacheStats().addTotalHugePageSize(dataPage.getSize());
gContext.getSupervisor().getCacheManager().getCacheStats().addTotalHugePageCount(1);
}
}
public int getHugePageMapCount() {
return hugePageMap.size();
}
public int getHugePageTotalSize() {
return hugePageTotalSize.get();
}
public void tryToDelHugePage(DataPage dataPage) {
if (this.hugePageMap.remove(dataPage) != null) {
hugePageTotalSize.addAndGet(-dataPage.getSize());
gContext.getSupervisor().getCacheManager().getCacheStats().addTotalHugePageSize(-dataPage.getSize());
gContext.getSupervisor().getCacheManager().getCacheStats().addTotalHugePageCount(-1);
}
}
}