Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.flink.runtime.state.gemini.engine.vm.EvictPolicySepImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.vm;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.state.gemini.engine.GRegion;
import org.apache.flink.runtime.state.gemini.engine.GTable;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiShutDownException;
import org.apache.flink.runtime.state.gemini.engine.page.LogicalPageChain;
import org.apache.flink.runtime.state.gemini.engine.page.PageAddress;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
/**
* EvictPolicy maintains EvictHandlerSepImpls.
*/
public class EvictPolicySepImpl implements EvictPolicy {
private static final Logger LOG = LoggerFactory.getLogger(EvictPolicySepImpl.class);
private final CacheManager cacheManager;
private final GContext gContext;
private final Map evictHandlerMap = new HashMap<>();
private final Object printObject = new Object();
private final long loopInterval = 1000;
private final int printTick;
private final boolean printAuditInfo;
private final int maxChainLen;
private final DeltaAndTotalAverage cacheHitRatio = new DeltaAndTotalAverage();
private final DeltaAndTotalAverage subPageCacheHitRatio = new DeltaAndTotalAverage();
private final DeltaAndTotalAverage fillPoolLatency = new DeltaAndTotalAverage();
public EvictPolicySepImpl(GContext gContext, CacheManager cacheManager) {
this.gContext = gContext;
this.cacheManager = cacheManager;
this.maxChainLen = gContext.getGConfiguration().getMaxCompactionThreshold();
this.printTick = gContext.getGConfiguration().geVmPrintTick();
this.printAuditInfo = gContext.getGConfiguration().geVmPrintAuditInfo();
}
@Override
public void run() {
int tick = 0;
while (this.gContext.isDBNormal()) {
try {
if (tick++ >= printTick) {
cacheHitRatio.calc(cacheManager.getCacheStats().getPageCacheHitCount(),
cacheManager.getCacheStats().getPageCacheRequestCount());
subPageCacheHitRatio.calc(cacheManager.getCacheStats().getSubPageCacheHitCount(),
cacheManager.getCacheStats().getSubPageCacheRequestCount());
fillPoolLatency.calc(cacheManager.getCacheStats().getTotalFillPoolTime(),
cacheManager.getCacheStats().getTotalFillPoolCount());
evictHandlerMap.values().forEach(handler -> LOG.info("Thread Mem WM {}", handler));
LOG.info(
"Mem WM:{}, Index WM:{} {} {} CacheHitRatio=[{}/{},{}/{}] fillPool[{} {}/{}] ReadPageLRUSize={}/{} fileSize={}/{} Allocator1={} Allocator2={} {} {} {}",
this.cacheManager.getMemWaterMark(0),
this.cacheManager.getIndexCapacityWaterMark(),
this.cacheManager.getCacheStats(),
this.gContext.getSupervisor().getWriteBufferManager(),
String.format("%.2f", cacheHitRatio.deltaRatio),
String.format("%.2f", cacheHitRatio.totalRatio),
String.format("%.2f", subPageCacheHitRatio.deltaRatio),
String.format("%.2f", subPageCacheHitRatio.totalRatio),
cacheManager.getCacheStats().getTotalFillPoolCount(),
String.format("%.2f", fillPoolLatency.deltaRatio),
String.format("%.2f", fillPoolLatency.totalRatio),
this.gContext.getSupervisor().getFetchPolicy().getDataPageLRU().getTotalSize(),
this.gContext.getSupervisor().getFetchPolicy().getDataPageLRU().size(),
this.gContext.getSupervisor().getFileCache().getFileCacheStat().totalLocalWriteSize,
this.gContext.getSupervisor().getFileCache().getFileCacheStat().totalLocalOriDataSize,
gContext.getSupervisor().getAllocator(),
gContext.getSupervisor().getForReadAllocator(),
this.gContext.getSupervisor().getPersistencyStrategy().toString(),
this.gContext.getSupervisor().getBloomFilterManager(),
debugInfo());
LOG.info("NoCriticalEvent count {}, lastTimestamp {}.",
gContext.getNoCriticalEvent().getCount(),
gContext.getNoCriticalEvent().getLastTimestamp(),
gContext.getNoCriticalEvent().getLastException());
if (LOG.isDebugEnabled()) {
LOG.debug("ExecutorGroup region {} compaction {} flush {} lru {} snapshot {}",
gContext.getSupervisor().getRegionExecutorGroup().toString(),
gContext.getSupervisor().getCompactionExecutorGroup().toString(),
gContext.getSupervisor().getFlushExecutorGroup().toString(),
gContext.getSupervisor().getLruIntoMainCacheExecutorGroup().toString(),
gContext.getSupervisor().getSnapshotExecutorGroup().toString());
}
tick = 0;
}
synchronized (printObject) {
printObject.wait(loopInterval);
}
} catch (GeminiShutDownException ie) {
LOG.debug("DB is in abnormal status " + gContext.getDBStatus().name());
} catch (Exception e) {
LOG.error("EvictPolicy has Exception={}", e.getMessage(), e);
}
}
}
private String debugInfo() {
if (!printAuditInfo) {
return "";
}
long totalPageSize = 0;
long totalPageInMemSize = 0;
long totalBiggerThanMaxChainLenMemSize = 0;
int longestChainLen = 0;
for (GTable gTable : gContext.getSupervisor().getAllTables().values()) {
Iterator regionIterator = gTable.regionIterator();
while (regionIterator.hasNext()) {
GRegion gRegion = regionIterator.next();
LogicalPageChain[] logicalPageChains = gRegion.getPageStore().getPageIndex().getPageIndex();
for (LogicalPageChain logicalPageChain : logicalPageChains) {
if (logicalPageChain == null) {
continue;
}
if (longestChainLen < logicalPageChain.getCurrentPageChainIndex() + 1) {
longestChainLen = logicalPageChain.getCurrentPageChainIndex() + 1;
}
for (int i = 0; i <= logicalPageChain.getCurrentPageChainIndex(); i++) {
Iterator pageAddressIterator = logicalPageChain.getPageAddress(i).pageIterator();
while (pageAddressIterator.hasNext()) {
PageAddress pageAddress = pageAddressIterator.next();
totalPageSize += pageAddress.getDataLen();
totalPageInMemSize += pageAddress.getMemorySize();
if (i >= this.maxChainLen) {
totalBiggerThanMaxChainLenMemSize += pageAddress.getMemorySize();
}
}
}
}
}
}
return String.format(
"debug:totalPageSize=%s/totalPageInMemSize=%s/totalBiggerThanMaxChainLenMemSize=%s/longestChainLen=%s",
totalPageSize,
totalPageInMemSize,
totalBiggerThanMaxChainLenMemSize,
longestChainLen);
}
@Override
public EvictHandlerSepImpl getEvictHandlerSepImpl(GRegion gRegion) {
return evictHandlerMap.get(gRegion.getExecutor());
}
@Override
public void addPage(
GRegion gRegion, PageAddress pageAddress) {
evictHandlerMap.get(gRegion.getExecutor()).addPage(pageAddress, gRegion);
}
@Override
public int removeInvalidPage(
GRegion gRegion, PageAddress pageAddress) {
return evictHandlerMap.get(gRegion.getExecutor()).removeInvalidPage(pageAddress);
}
@Override
public void addPageUsedMemory(
GRegion gRegion, EventExecutor regionEventExecutor, int logicPageSize, boolean needEvict) {
evictHandlerMap.get(regionEventExecutor).addPageUsedMemory(logicPageSize, needEvict);
}
@Override
public void tryPrepareFlush(GRegion gRegion, int minSize) {
evictHandlerMap.get(gRegion.getExecutor()).tryPrepareFlush(minSize);
}
@Override
public MemoryUsedWaterMark getMemoryUsedWaterMark(GRegion gRegion, int addSize) {
return evictHandlerMap.get(gRegion.getExecutor()).getMemoryUsedWaterMark(addSize);
}
@Override
public void addRegion(GRegion gRegion) {
evictHandlerMap.computeIfAbsent(gRegion.getExecutor(),
nothing -> new EvictHandlerSepImpl(gRegion.getExecutor().toString(), gContext));
evictHandlerMap.get(gRegion.getExecutor()).addRegion(gRegion);
}
@VisibleForTesting
public void addRegionEventExecutor(EventExecutor regionEventExecutor, EvictHandlerSepImpl evictHandlerSep) {
evictHandlerMap.putIfAbsent(regionEventExecutor, evictHandlerSep);
}
@Override
public void shutdown() {
evictHandlerMap.values().forEach(EvictHandlerSepImpl::shutdown);
}
private static class DeltaAndTotalAverage {
long lastValue = 0;
long lastTotal = 0;
float deltaRatio;
float totalRatio;
void calc(long curValue, long curTotal) {
long deltaValue = curValue - this.lastValue;
long delaTotal = curTotal - this.lastTotal;
deltaRatio = delaTotal == 0 ? -1 : ((float) deltaValue) / delaTotal;
totalRatio = curTotal == 0 ? -1 : ((float) curValue) / curTotal;
this.lastValue = curValue;
this.lastTotal = curTotal;
}
}
}
