Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.vm;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.runtime.state.gemini.engine.GTable;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.dbms.Supervisor;
import org.apache.flink.shaded.guava18.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.runtime.state.gemini.engine.metrics.CacheMetrics.TOTAL_MEM_HIGH_MARK;
import static org.apache.flink.runtime.state.gemini.engine.metrics.CacheMetrics.TOTAL_MEM_LOW_MARK;
import static org.apache.flink.runtime.state.gemini.engine.metrics.CacheMetrics.TOTAL_MEM_MIDDLE_MARK;
/**
* CacheManagerImpl.
* 1. check Memory used.
* 2. depended on Policy to decide pages to flush or fetch.
*/
public class CacheManagerImpl implements CacheManager {
private static final Logger LOG = LoggerFactory.getLogger(CacheManagerImpl.class);
private final GContext gContext;
private final Supervisor supervisor;
private final long totalHeapMemSize;
private final long totalOffheapMemSize;
private final long totalMemSize;
private final long totalMemLowMark;
private final long totalMemMiddleMark;
private final long totalMemHighMark;
private final long totalIndexCountHighMark;
private final long totalIndexCountLowMark;
private EvictPolicy evictPolicy;
private final CacheStats cacheStats = new CacheStats();
private final ExecutorService cacheManagerExecutor;
private final int readPageCacheLRUSize;
public CacheManagerImpl(GContext gContext) {
this.gContext = gContext;
this.supervisor = gContext.getSupervisor();
this.totalHeapMemSize = gContext.getGConfiguration().getTotalHeapMemSize();
this.totalOffheapMemSize = gContext.getGConfiguration().getTotalOffheapMemSize();
long max = gContext.getGConfiguration().getUseOffheap() ? totalOffheapMemSize : totalHeapMemSize;
this.totalMemSize = (long) (max * gContext.getGConfiguration().getTotalHeapRate());
this.totalMemLowMark = (long) (this.totalMemSize * gContext.getGConfiguration().getTotalHeapLowMarkRate());
this.totalMemMiddleMark = (long) (this.totalMemSize * gContext.getGConfiguration().getTotalHeapMiddleMarkRate());
this.totalMemHighMark = (long) (this.totalMemSize * gContext.getGConfiguration().getTotalHeapHighMarkRate());
this.totalIndexCountHighMark = gContext.getGConfiguration().getTotalIndexCountHighMark();
this.totalIndexCountLowMark = gContext.getGConfiguration().getTotalIndexCountLowMark();
String prefix = gContext.getGConfiguration().getExcetorPrefixName();
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat(prefix + "GeminiCacheManagerImpl-%d").build();
this.cacheManagerExecutor = new ThreadPoolExecutor(1,
1,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue(Short.MAX_VALUE),
namedThreadFactory);
//Bug fix: if use offheap, readPageCacheLRUSize can't be zero because handling reference can't work correctly when LRU is 0.
int configReadPageCacheLRUSize = Math.min(20000, gContext.getGConfiguration().getTotalReadPageLRUNum());
if (gContext.getGConfiguration().getUseOffheap() || gContext.getGConfiguration().getForceReadUseOffheap()) {
readPageCacheLRUSize = Math.max(128, configReadPageCacheLRUSize);
} else {
readPageCacheLRUSize = configReadPageCacheLRUSize;
}
this.evictPolicy = new EvictPolicySepImpl(this.gContext, this);
LOG.info(
"CacheManagerImpl{}, offheap={}, maxHeapSize={}, maxDirectMemorySize={}, totalMemSize={},lowMark={},MiddleMark={},HighMark={},indexLowMark={},indexHighMark={}, dbSlots={}, readPageCacheLRUSize={}, checksumEnable={}",
this,
gContext.getGConfiguration().getUseOffheap(),
totalHeapMemSize,
totalOffheapMemSize,
totalMemSize,
totalMemLowMark,
totalMemMiddleMark,
totalMemHighMark,
totalIndexCountLowMark,
totalIndexCountHighMark,
gContext.getGConfiguration().getDBNumberPerJVM(),
readPageCacheLRUSize,
gContext.getGConfiguration().isChecksumEnable());
}
@Override
public void start() {
this.cacheManagerExecutor.submit(evictPolicy);
// Register the metrics for guage.
if (gContext.getCacheMetric() != null) {
gContext.getCacheMetric().getMetricGroup().gauge(TOTAL_MEM_HIGH_MARK, (Gauge) () -> totalMemHighMark);
gContext.getCacheMetric().getMetricGroup().gauge(TOTAL_MEM_MIDDLE_MARK,
(Gauge) () -> totalMemMiddleMark);
gContext.getCacheMetric().getMetricGroup().gauge(TOTAL_MEM_LOW_MARK, (Gauge) () -> totalMemLowMark);
gContext.getCacheMetric().registerMetricsCacheStat(cacheStats);
}
LOG.info("CacheManager is initialized.");
}
@Override
public void addIndexCapacity(int indexCapacity) {
this.cacheStats.addIndexCapacity(indexCapacity);
}
@Override
public long getPageUsedMemory() {
return this.cacheStats.getTotalPageUsedMemory();
}
@Override
public boolean forbidIndexExpand() {
return this.cacheStats.getTotalIndexCapacity() >= this.totalIndexCountHighMark * 2;
}
@Override
public WaterMark getIndexCapacityWaterMark() {
if (this.cacheStats.getTotalIndexCapacity() >= this.totalIndexCountHighMark) {
return WaterMark.High;
} else if (this.cacheStats.getTotalIndexCapacity() >= this.totalIndexCountLowMark) {
return WaterMark.Low;
}
return WaterMark.Normal;
}
@Override
public WaterMark getMemWaterMark(int addEstimatedSize) {
if (this.cacheStats.getTotalPageUsedMemory() + addEstimatedSize >= this.totalMemHighMark) {
return WaterMark.High;
} else if (this.cacheStats.getTotalPageUsedMemory() + addEstimatedSize >= this.totalMemMiddleMark) {
return WaterMark.Middle;
} else if (this.cacheStats.getTotalPageUsedMemory() + addEstimatedSize >= this.totalMemLowMark) {
return WaterMark.Low;
}
return WaterMark.Normal;
}
@Override
public CacheStats getCacheStats() {
return this.cacheStats;
}
@Override
public void close() {
cacheManagerExecutor.shutdownNow();
}
@Override
public void addTable(GTable gTable) {
//do nothing now
}
@Override
public EvictPolicy getEvictPolicy() {
return this.evictPolicy;
}
@Override
public void addRegionEventExecutor(EventExecutor regionEventExecutor) {
this.evictPolicy.addRegionEventExecutor(regionEventExecutor);
}
@Override
public long getTotalTargetEvictSize() {
return this.cacheStats.getTotalLogicPageSize() < totalMemMiddleMark
? 1
: this.cacheStats.getTotalLogicPageSize() - totalMemMiddleMark;
}
@Override
public long getMemLowMark() {
return this.totalMemLowMark;
}
@Override
public long getMemMidMark() {
return this.totalMemMiddleMark;
}
@Override
public long getMemHighMark() {
return this.totalMemHighMark;
}
@Override
public int getReadPageCacheLRUSize() {
return readPageCacheLRUSize;
}
}
