com.gemstone.gemfire.internal.cache.lru.HeapEvictor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of gemfire-core Show documentation
Show all versions of gemfire-core Show documentation
TIBCO ComputeDB store based off Pivotal GemFireXD
/*
* Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.gemstone.gemfire.internal.cache.lru;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import com.gemstone.gemfire.LogWriter;
import com.gemstone.gemfire.cache.Cache;
import com.gemstone.gemfire.cache.RegionDestroyedException;
import com.gemstone.gemfire.cache.control.ResourceManager;
import com.gemstone.gemfire.distributed.internal.OverflowQueueWithDMStats;
import com.gemstone.gemfire.i18n.LogWriterI18n;
import com.gemstone.gemfire.internal.LogWriterImpl;
import com.gemstone.gemfire.internal.cache.BucketRegion;
import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
import com.gemstone.gemfire.internal.cache.LocalRegion;
import com.gemstone.gemfire.internal.cache.PartitionedRegion;
import com.gemstone.gemfire.internal.cache.RegionEvictorTask;
import com.gemstone.gemfire.internal.cache.control.HeapMemoryMonitor;
import com.gemstone.gemfire.internal.cache.control.InternalResourceManager;
import com.gemstone.gemfire.internal.cache.control.InternalResourceManager.ResourceType;
import com.gemstone.gemfire.internal.cache.control.MemoryEvent;
import com.gemstone.gemfire.internal.cache.control.MemoryThresholds;
import com.gemstone.gemfire.internal.cache.control.ResourceListener;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.shared.LauncherBase;
import com.gemstone.gemfire.internal.snappy.CallbackFactoryProvider;
import com.gemstone.gemfire.internal.snappy.StoreCallbacks;
import com.gemstone.gnu.trove.THashSet;
import com.gemstone.gnu.trove.TObjectLongHashMap;
/**
* Triggers centralized eviction(asynchronously) when the ResourceManager sends
* an eviction event for on-heap regions. This is registered with the ResourceManager.
*
* @author Yogesh, Suranjan, Amardeep, rholmes
* @since 6.0
*
*/
public class HeapEvictor implements ResourceListener {
// Add 1 for the management task that's putting more eviction tasks on the queue
public static final int MAX_EVICTOR_THREADS = Integer.getInteger(
"gemfire.HeapLRUCapacityController.MAX_EVICTOR_THREADS", (Runtime.getRuntime().availableProcessors()*4)) + 1;
public static final boolean DISABLE_HEAP_EVICTIOR_THREAD_POOL = Boolean
.getBoolean("gemfire.HeapLRUCapacityController.DISABLE_HEAP_EVICTIOR_THREAD_POOL");
public static final String EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_PROP =
LauncherBase.EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_PROP;
public static final boolean EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST = Boolean.valueOf(
System.getProperty(EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_PROP,
"true")).booleanValue();
public static final String EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_FOR_EVICTOR_PROP =
LauncherBase.EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_FOR_EVICTOR_PROP;
public static final boolean EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_FOR_EVICTOR =
Boolean.valueOf(System.getProperty(EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST_FOR_EVICTOR_PROP,
Boolean.toString(EVICT_HIGH_ENTRY_COUNT_BUCKETS_FIRST))).booleanValue();
public static final int MINIMUM_ENTRIES_PER_BUCKET = Integer
.getInteger("gemfire.HeapLRUCapacityController.inlineEvictionThreshold",0);
public static final long TOTAL_BYTES_TO_EVICT_FROM_HEAP;
public static final int BUCKET_SORTING_INTERVAL = Integer.getInteger(
"gemfire.HeapLRUCapacityController.higherEntryCountBucketCalculationInterval",
100).intValue();
private static final String EVICTOR_THREAD_GROUP_NAME = "EvictorThreadGroup";
private static final String EVICTOR_THREAD_NAME = "EvictorThread";
static {
float evictionBurstPercentage = Float.parseFloat(System.getProperty(
ResourceManager.EVICTION_BURST_PERCENT_PROP, "0.4"));
long maxTenuredBytes = HeapMemoryMonitor.getTenuredPoolMaxMemory();
TOTAL_BYTES_TO_EVICT_FROM_HEAP = (long)(maxTenuredBytes * 0.01 * evictionBurstPercentage);
}
private ThreadPoolExecutor evictorThreadPool = null;
private final LogWriterI18n logger;
private final AtomicBoolean mustEvict = new AtomicBoolean(false);
protected final Cache cache;
private final ArrayList testTaskSetSizes = new ArrayList();
public volatile int testAbortAfterLoopCount = Integer.MAX_VALUE;
private BlockingQueue poolQueue;
private final AtomicBoolean isRunning = new AtomicBoolean(true);
public HeapEvictor(Cache gemFireCache) {
this.cache = gemFireCache;
this.logger = cache.getLoggerI18n();
initializeEvictorThreadPool();
}
protected boolean includePartitionedRegion(PartitionedRegion region) {
return (region.getEvictionAttributes().getAlgorithm().isLRUHeap()
&& (region.getDataStore() != null)
&& !region.getAttributes().getEnableOffHeapMemory());
}
protected boolean includeLocalRegion(LocalRegion region) {
return (region.getEvictionAttributes().getAlgorithm().isLRUHeap()
&& !region.getAttributes().getEnableOffHeapMemory());
}
private List getAllRegionList() {
List allRegionList = new ArrayList();
InternalResourceManager irm = (InternalResourceManager)cache
.getResourceManager();
for (ResourceListener listener : irm.getResourceListeners(getResourceType())) {
if (listener instanceof PartitionedRegion) {
PartitionedRegion pr = (PartitionedRegion)listener;
if (includePartitionedRegion(pr)) {
allRegionList.addAll(pr.getDataStore().getAllLocalBucketRegions());
}
}
else if (listener instanceof LocalRegion) {
LocalRegion lr = (LocalRegion)listener;
if (includeLocalRegion(lr)) {
allRegionList.add(lr);
}
}
}
if (HeapEvictor.MINIMUM_ENTRIES_PER_BUCKET > 0) {
Iterator iter = allRegionList.iterator();
while(iter.hasNext()){
LocalRegion lr = iter.next();
if (lr instanceof BucketRegion) {
if (((BucketRegion)lr).getNumEntriesInVM() <= HeapEvictor.MINIMUM_ENTRIES_PER_BUCKET) {
iter.remove();
}
}
}
}
return allRegionList;
}
private List getAllSortedRegionList(){
List allRegionList = getAllRegionList();
//Capture the sizes so that they do not change while sorting
final TObjectLongHashMap sizes = new TObjectLongHashMap(allRegionList.size());
for(LocalRegion r : allRegionList) {
long size = r instanceof BucketRegion ?((BucketRegion)r).getSizeForEviction() : r.size();
sizes.put(r, size);
}
//Sort with respect to other PR buckets also in case of multiple PRs
Collections.sort(allRegionList, new Comparator() {
public int compare(LocalRegion r1, LocalRegion r2) {
long numEntries1 = sizes.get(r1);
long numEntries2 = sizes.get(r2);
if (numEntries1 > numEntries2) {
return -1;
}
else if (numEntries1 < numEntries2) {
return 1;
}
return 0;
}
});
return allRegionList;
}
public GemFireCacheImpl getGemFireCache() {
return (GemFireCacheImpl)this.cache;
}
public LogWriterI18n getLoggerI18n() {
return this.logger;
}
private void initializeEvictorThreadPool() {
final ThreadGroup evictorThreadGroup = LogWriterImpl.createThreadGroup(
getEvictorThreadGroupName(), getLoggerI18n());
ThreadFactory evictorThreadFactory = new ThreadFactory() {
private int next = 0;
public Thread newThread(Runnable command) {
Thread t = new Thread(evictorThreadGroup, command, getEvictorThreadName()
+ next++);
t.setDaemon(true);
return t;
}
};
if (!DISABLE_HEAP_EVICTIOR_THREAD_POOL) {
this.poolQueue = new OverflowQueueWithDMStats(getGemFireCache().getCachePerfStats().getEvictionQueueStatHelper());
this.evictorThreadPool = new ThreadPoolExecutor(MAX_EVICTOR_THREADS, MAX_EVICTOR_THREADS,
15, TimeUnit.SECONDS, this.poolQueue, evictorThreadFactory);
}
}
/**
* The task(i.e the region on which eviction needs to be performed) is
* assigned to the threadpool.
*/
private void submitRegionEvictionTask(Callable