org.apache.flink.runtime.state.gemini.engine.GConfiguration Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of flink-runtime_2.11 Show documentation
There is a newer version: 1.5.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state.gemini.engine;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.runtime.state.gemini.GeminiOptions;
import org.apache.flink.runtime.state.gemini.engine.filecache.FileCache;
import org.apache.flink.runtime.state.gemini.engine.fs.PersistenceStrategyFactory.PersistenceType;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GComparatorType;
import org.apache.flink.runtime.state.gemini.engine.page.compress.GCompressAlgorithm;
import org.apache.flink.util.Preconditions;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Objects;

/**
 * GConfiguration.
 */
public class GConfiguration {

	private static final Logger LOG = LoggerFactory.getLogger(GConfiguration.class);

	private String dfsPath;

	private String localPath;

	private int metricSampleCount;

	private int metricHistogramWindowSize;

	private int flushThreadNum;

	private int snapshotThreadNum;

	private int regionThreadNum;

	private int compactionThreadNum;

	private long maxFileSize;

	private long writeBufferSize;

	private int maxCompactionThreshold;

	private int inMemoryCompactionThreshold;

	private int maxRunningMajorCompaction;

	private int maxRunningMinorCompaction;

	private float totalWriteBufferRate;
	private int totalReadPageLRUNum;

	private int numFlushingSegment;

	private float totalNumFlushingSegmentRatio;

	private int bucketNum;

	private int splitPageThreshold;

	private int hugePageThreshold;

	private long ttl;

	private GComparatorType comparatorType = GComparatorType.bytes;

	private int subTaskIndex;

	private int numParallelSubtasks;

	private String backendUID;

	private int fileCleanThreadNum;

	/** The interval for file cleaner. */
	private long fileCleanInterval;

	private long fileAliveTimeAfterNoDataReference;

	private long fileDeletionCheckInterval;

	private boolean readCopy;

	private boolean writeCopy;

	private int writerFailCountThreshold;

	private int fileManagerFailCountThreshold;

	private int fileManagerCreateFileWriterRetryInterval;

	private boolean localSnapshotEnabled;

	private long preFetchThreadSleepTimeNs;

	private long commonThreadSleepTimeNs;

	private GCompressAlgorithm inPageCompressAlgorithm;
	private GCompressAlgorithm pageFlushLocalCompressAlgorithm;
	private GCompressAlgorithm pageFlushDfsCompressAlgorithm;

	private boolean prefetchEnable = true;

	private int allocatorDirectArena;

	private int logicTableDefaultChainLen;

	private float totalHeapLowMarkRate;

	private float totalHeapMiddleMarkRate;

	private float totalHeapHighMarkRate;

	private float pageSizeRateBetweenPOJOAndHeap;

	private long totalIndexCountHighMark;

	private long totalIndexCountLowMark;

	private boolean checksumEnable;

	private String operatorNameWithSubtask = "";

	private int minSortedListCountForFlush;

	private int maxPreparedFlushSize;

	private int evictPoolFactor;

	private int timePerTick;

	private int wholePageCompressThreshold;

	// configuration for split map

	private boolean dataPageMapSplitEnabled;

	private int mapSplitSizeThreshold;

	private int mapSplitSubMapSize;

	private int mapSplitMaxSubMapNum;

	// configuration for memory

	/** Whether to use offheap. */
	private boolean useOffheap;

	/** Whether to use offheap when reading from file. */
	private boolean useOffheapForRead;

	/** Heap size used by DB. A non-positive value indicates it's not configured. */
	private long heapSize;

	/** Offheap size used by DB. A non-positive value indicates it's not configured. */
	private long offheapSize;

	/** Ratio of memory usage used by DB relative to JVM's heap/offheap. */
	private float memoryRatio;

	/** Number of slots in this JVM. */
	private int numberSlots = 1;

	// configuration for file cache

	private FileCache.FileCacheType fileCacheType;

	private long fileCacheCapacity;

	// configuration for file compaction

	private boolean enableFileCompaction;

	private float compactionTriggerRatio;

	private float compactionTargetRatio;

	private long amplificationCheckInterval;

	// configuration for snapshot

	private boolean enableSnapshotCompaction;

	private float snapshotCompactionTargetRatio;

	private int snapshotFlushBatchNumPage;

	private int snapshotFlushBatchDataSize;

	private boolean snapshotSyncWhenBatchFlush;

	// configuration for restore

	private boolean enableRestorePreFetch;

	private int fetchFilesThreadNum;

	private int vmPrintTick;

	private boolean vmPrintAuditInfo;

	private boolean evictBaseOnPageAddressComposite;
	private int batchPersistenceSize;
	private int forceSyncToCacheSize;
	private int maxPersistenceRunningTask;
	private PersistenceType persistenceType;

	private int lruIntoMainCacheThreadNum;

	private int lruIntoMainCacheSleepMs;

	private boolean enableAddIntoMainWhenSplitting;

	private boolean enableLoadPageFromLRUIntoMain;

	private boolean enableLruAccessMode;

	private boolean enableEvictRegionEven;

	private float bloomFilterMemRate;

	private boolean enableBloomFilter;

	public GConfiguration() {

	}

	public GConfiguration(Configuration configuration) {
		load(configuration);
	}

	public boolean isEnableEvictRegionEven() {
		return enableEvictRegionEven;
	}

	public boolean isEnableLruAccessMode() {
		return enableLruAccessMode;
	}

	public void setLogicTableDefaultChainLen(int logicTableDefaultChainLen) {
		this.logicTableDefaultChainLen = logicTableDefaultChainLen;
	}

	public int getLogicTableDefaultChainLen() {
		return logicTableDefaultChainLen;
	}

	public int getPageIndexBucketLenDefault() {
		return bucketNum;
	}

	public void setPageIndexBucketLenDefault(int bucketNum) {
		this.bucketNum = bucketNum;
	}

	public float getTotalWriteBufferRate() {
		return totalWriteBufferRate;
	}

	public int getTotalReadPageLRUNum() {
		return totalReadPageLRUNum;
	}

	public void setTotalHeapLowMarkRate(float totalHeapLowMarkRate) {
		this.totalHeapLowMarkRate = totalHeapLowMarkRate;
	}

	public float getTotalHeapLowMarkRate() {
		return totalHeapLowMarkRate;
	}

	public void setTotalHeapMiddleMarkRate(float totalHeapMiddleMarkRate) {
		this.totalHeapMiddleMarkRate = totalHeapMiddleMarkRate;
	}

	public float getTotalHeapMiddleMarkRate() {
		return totalHeapMiddleMarkRate;
	}

	public void setTotalHeapHighMarkRate(float totalHeapHighMarkRate) {
		this.totalHeapHighMarkRate = totalHeapHighMarkRate;
	}

	public float getTotalHeapHighMarkRate() {
		return totalHeapHighMarkRate;
	}

	public void setPageSizeRateBetweenPOJOAndHeap(float pageSizeRateBetweenPOJOAndHeap) {
		this.pageSizeRateBetweenPOJOAndHeap = pageSizeRateBetweenPOJOAndHeap;
	}

	public float getPageSizeRateBetweenPOJOAndHeap() {
		return pageSizeRateBetweenPOJOAndHeap;
	}

	public int getSplitPageSizeThreshold() {
		return splitPageThreshold;
	}

	public int getHugePageSizeThreshold() {
		return hugePageThreshold;
	}

	public long getWriteBufferSize() {
		return writeBufferSize;
	}

	public void setWriteBUfferSize(long writeBufferSize) {
		Preconditions.checkArgument(writeBufferSize > 0,
			"Write buffer size should be positive, actual is " + writeBufferSize);
		this.writeBufferSize = writeBufferSize;
	}

	public void setTotalIndexCountHighMark(long totalIndexCountHighMark) {
		this.totalIndexCountHighMark = totalIndexCountHighMark;
	}

	public long getTotalIndexCountHighMark() {
		return totalIndexCountHighMark;
	}

	public void setTotalIndexCountLowMark(long totalIndexCountLowMark) {
		this.totalIndexCountLowMark = totalIndexCountLowMark;
	}

	public long getTotalIndexCountLowMark() {
		return totalIndexCountLowMark;
	}

	public int getMaxCompactionThreshold() {
		return maxCompactionThreshold;
	}

	public void setDfsPath(String dfsPath) {
		this.dfsPath = Preconditions.checkNotNull(dfsPath);
	}

	public String getDfsPath() {
		return dfsPath;
	}

	public void setLocalPath(String localPath) {
		this.localPath = Preconditions.checkNotNull(localPath);
	}

	public String getLocalPath() {
		return localPath;
	}

	public void setMetricSampleCount(int count) {
		this.metricSampleCount = count;
	}

	public int getMetricSampleCount() {
		return metricSampleCount;
	}

	public void setMetricHistogramWindowSize(int metricHistogramWindowSize) {
		this.metricHistogramWindowSize = metricHistogramWindowSize;
	}

	public String getBackendUID() {
		return backendUID;
	}

	public void setBackendUID(String backendUID) {
		this.backendUID = backendUID;
	}

	public void setFlushThreadNum(int flushThreadNum) {
		if (checkNonnegativeArgument(flushThreadNum,
			"The number of threads used to flush data should be larger than zero.")) {
			this.flushThreadNum = flushThreadNum;
		}
	}

	public int getFlushThreadNum() {
		return flushThreadNum;
	}

	public int getSnapshotThreadNum() {
		return snapshotThreadNum;
	}

	public void setSnapshotThreadNum(int snapshotThreadNum) {
		if (checkNonnegativeArgument(snapshotThreadNum,
			"The number of threads used to each region should be larger than zero.")) {
			this.snapshotThreadNum = snapshotThreadNum;
		}
	}

	public void setRegionThreadNum(int regionThreadNum) {
		if (checkNonnegativeArgument(snapshotThreadNum,
			"The number of threads used to snapshot data should be larger than zero.")) {
			this.regionThreadNum = regionThreadNum;
		}
	}

	public int getRegionThreadNum() {
		return regionThreadNum;
	}

	public int getMetricHistogramWindowSize() {
		return metricHistogramWindowSize;
	}

	public void setMaxFileSize(long maxFileSize) {
		Preconditions.checkArgument(maxFileSize > 0,
			"Max file size should be positive, actual is " + maxFileSize);
		this.maxFileSize = maxFileSize;
	}

	public long getMaxFileSize() {
		return maxFileSize;
	}

	public int getInMemoryCompactionThreshold() {
		return inMemoryCompactionThreshold;
	}

	public int getMaxRunningMajorCompaction() {
		return maxRunningMajorCompaction;
	}

	public int getMaxRunningMinorCompaction() {
		return maxRunningMinorCompaction;
	}

	public void setInMemoryCompactionThreshold(int threshold) {
		this.inMemoryCompactionThreshold = threshold;
	}

	public int getNumFlushingSegment() {
		return numFlushingSegment;
	}

	public float getTotalNumFlushingSegmentRatio() {
		return totalNumFlushingSegmentRatio;
	}

	public void setTtl(long ttl) {
		this.ttl = ttl;
	}

	public long getTtl() {
		return ttl;
	}

	public GComparatorType getComparatorType() {
		return comparatorType;
	}

	public void setSubTaskIndex(int subTaskIndex) {
		this.subTaskIndex = subTaskIndex;
	}

	public void setNumParallelSubtasks(int numParallelSubtasks) {
		this.numParallelSubtasks = numParallelSubtasks;
	}

	public int getSubTaskIndex() {
		return subTaskIndex;
	}

	public int getNumParallelSubtasks() {
		return numParallelSubtasks;
	}

	public void setFileCleanThreadNum(int threadNum) {
		if (checkNonnegativeArgument(threadNum,
			"The number of threads used to clean files should be larger than zero.")) {
			this.fileCleanThreadNum = threadNum;
		}
	}

	public int getFileCleanThreadNum() {
		return fileCleanThreadNum;
	}

	public void setFileCleanInterval(long interval) {
		this.fileCleanInterval = interval;
	}

	public long getFileCleanInterval() {
		return fileCleanInterval;
	}

	public void setFileAliveTimeAfterNoDataReference(long aliveTime) {
		this.fileAliveTimeAfterNoDataReference = aliveTime;
	}

	public long getFileAliveTimeAfterNoDataReference() {
		return fileAliveTimeAfterNoDataReference;
	}

	public void setFileDeletionCheckInterval(long interval) {
		fileDeletionCheckInterval = interval;
	}

	public long getFileDeletionCheckInterval() {
		return fileDeletionCheckInterval;
	}

	public boolean isReadCopy() {
		return readCopy;
	}

	public boolean isWriteCopy() {
		return writeCopy;
	}

	public int getWriterFailCountThreshold() {
		return writerFailCountThreshold;
	}

	public void setWriterFailCountThreshold(int writerFailCountThreshold) {
		this.writerFailCountThreshold = writerFailCountThreshold;
	}

	public int getFileManagerFailCountThreshold() {
		return fileManagerFailCountThreshold;
	}

	public void setFileManagerFailCountThreshold(int fileManagerFailCountThreshold) {
		this.fileManagerFailCountThreshold = fileManagerFailCountThreshold;
	}

	public int getFileManagerCreateFileWriterRetryInterval() {
		return fileManagerCreateFileWriterRetryInterval;
	}

	public void setFileManagerCreateFileWriterRetryInterval(int fileManagerCreateFileWriterRetryInterval) {
		this.fileManagerCreateFileWriterRetryInterval = fileManagerCreateFileWriterRetryInterval;
	}

	private void load(Configuration configuration) {
		this.metricSampleCount = configuration.getInteger(GeminiOptions.SAMPLE_COUNT);
		this.metricHistogramWindowSize = configuration.getInteger(GeminiOptions.HISTOGRAM_WINDOW_SIZE);
		this.flushThreadNum = configuration.getInteger(GeminiOptions.FLUSH_THREAD_NUM);
		this.snapshotThreadNum = configuration.getInteger(GeminiOptions.SNAPSHOT_THREAD_NUM);
		this.regionThreadNum = configuration.getInteger(GeminiOptions.REGION_THREAD_NUM);

		long fileSize = MemorySize.parseBytes(configuration.getString(GeminiOptions.MAX_FILE_SIZE));
		setMaxFileSize(fileSize);

		long writeBufferSize = MemorySize.parseBytes(configuration.getString(GeminiOptions.WRITE_BUFFER_SIZE));
		setWriteBUfferSize(writeBufferSize);

		this.compactionThreadNum = configuration.getInteger(GeminiOptions.COMPACTION_THREAD_NUM);
		this.maxCompactionThreshold = configuration.getInteger(GeminiOptions.MAJOR_COMPACTION_THRESHOLD);
		this.inMemoryCompactionThreshold = configuration.getInteger(GeminiOptions.MINOR_COMPACTION_THRESHOLD);
		this.maxRunningMajorCompaction = configuration.getInteger(GeminiOptions.MAJOR_COMPACTION_MAX_RUNNING);
		this.maxRunningMinorCompaction = configuration.getInteger(GeminiOptions.MINOR_COMPACTION_MAX_RUNNING);
		this.useOffheap = configuration.getBoolean(GeminiOptions.USE_OFFHEAP);
		this.totalWriteBufferRate = configuration.getFloat(GeminiOptions.TOTAL_WRITEBUFFER_RATE);
		this.totalReadPageLRUNum = configuration.getInteger(GeminiOptions.READ_LRU_SIZE);
		this.numFlushingSegment = configuration.getInteger(GeminiOptions.FLUSHING_SEGMENT);
		this.totalNumFlushingSegmentRatio = configuration.getFloat(GeminiOptions.FLUSHING_SEGMENT_RATIO);
		this.bucketNum = configuration.getInteger(GeminiOptions.BUCKET_INIT_NUM);
		this.splitPageThreshold = configuration.getInteger(GeminiOptions.SPLIT_SIZE_THRESHOLD);
		int hugeThreshold = configuration.getInteger(GeminiOptions.HUGE_PAGE_THRESHOLD);
		if (hugeThreshold < 10 * splitPageThreshold) {
			hugeThreshold = 10 * splitPageThreshold;
		}
		this.hugePageThreshold = hugeThreshold;
		this.ttl = configuration.getLong(GeminiOptions.TTL);
		this.fileCleanInterval = configuration.getLong(GeminiOptions.FILE_CLEAN_CHECK_INTERVAL);
		this.fileCleanThreadNum = configuration.getInteger(GeminiOptions.FILE_CLEAN_THREAD_NUM);
		this.fileAliveTimeAfterNoDataReference = configuration.getLong(GeminiOptions.FILE_ALIVE_TIME_AFTER_NO_DATA_REFERENCE);
		this.fileDeletionCheckInterval = configuration.getLong(GeminiOptions.FILE_DELETION_CHECK_INTERVAL);
		this.comparatorType  = GComparatorType.getComparatorType(configuration.getString(GeminiOptions.COMPARATOR_TYPE));
		this.readCopy = configuration.getBoolean(GeminiOptions.READ_COPY);
		this.writeCopy = configuration.getBoolean(GeminiOptions.WRITE_COPY);
		this.writerFailCountThreshold = configuration.getInteger(GeminiOptions.WRITER_FAIL_COUNT_THRESHOLD);
		this.fileManagerFailCountThreshold = configuration.getInteger(GeminiOptions.FILE_FAIL_COUNT_THRESHOLD);
		this.fileManagerCreateFileWriterRetryInterval = configuration.getInteger(GeminiOptions.FILE_RETRY_INTERVAL);
		this.preFetchThreadSleepTimeNs = configuration.getLong(GeminiOptions.PREFETCH_THREAD_SLEEP_TIME_NS);
		this.commonThreadSleepTimeNs = configuration.getLong(GeminiOptions.THREAD_SLEEP_TIME_NS);

		this.wholePageCompressThreshold = configuration.getInteger(GeminiOptions.WHOLE_PAGE_COMPRESS_THRESHOLD);
		this.prefetchEnable = configuration.getBoolean(GeminiOptions.PREFETCH_ENABLE);
		this.allocatorDirectArena = configuration.getInteger(GeminiOptions.ALLOCATOR_DIRECT_ARENA);
		this.logicTableDefaultChainLen = configuration.getInteger(GeminiOptions.LOGIC_CHAIN_INIT_LEN);
		this.totalHeapLowMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_LOW_MARK_RATE);
		this.totalHeapMiddleMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_MIDDLE_MARK_RATE);
		this.totalHeapHighMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_HIGH_MARK_RATE);
		this.pageSizeRateBetweenPOJOAndHeap = configuration.getFloat(GeminiOptions.PAGE_SIZE_RATE_BETWEEN_POJO_HEAP);
		this.totalIndexCountHighMark = configuration.getInteger(GeminiOptions.INDEX_COUNT_HIGH_MARK);
		this.totalIndexCountLowMark = configuration.getInteger(GeminiOptions.INDEX_COUNT_LOW_MARK);
		this.checksumEnable = configuration.getBoolean(GeminiOptions.CHECKSUM_ENABLE);
		this.minSortedListCountForFlush = configuration.getInteger(GeminiOptions.SORTED_LIST_COUNT_FOR_FLUSH);
		this.maxPreparedFlushSize = configuration.getInteger(GeminiOptions.MAX_PREPARED_FLUSH_SIZE);
		int factor = configuration.getInteger(GeminiOptions.EVICT_POOL_FACTOR);
		if (factor < 2) {
			factor = 2;
		}
		this.evictPoolFactor = factor;
		this.timePerTick = configuration.getInteger(GeminiOptions.CACHE_TIME_PER_TICK);

		// load configuration for memory
		setUseOffheap(configuration.getBoolean(GeminiOptions.USE_OFFHEAP));
		setUseOffheapForRead(configuration.getBoolean(GeminiOptions.USE_OFFHEAP_FOR_READ));

		String heapSizeStr = configuration.getString(GeminiOptions.HEAP_SIZE);
		setHeapSize(heapSizeStr != null ? parsePositiveBytes(heapSizeStr) : -1);

		String offHeapSizeStr = configuration.getString(GeminiOptions.OFFHEAP_SIZE);
		setOffheapSize(offHeapSizeStr != null ? parsePositiveBytes(offHeapSizeStr) : -1);

		setMemoryRatio(configuration.getFloat(GeminiOptions.MEMORY_RATIO));

		// load configuration for compression
		this.pageFlushDfsCompressAlgorithm = GCompressAlgorithm.getCompressAlgorithm(
			configuration.getString(GeminiOptions.PAGE_FLUSH_DFS_COMPRESSION), configuration);
		this.pageFlushLocalCompressAlgorithm = GCompressAlgorithm.getCompressAlgorithm(
			configuration.getString(GeminiOptions.PAGE_FLUSH_LOCAL_COMPRESSION), configuration);
		this.inPageCompressAlgorithm = GCompressAlgorithm.getCompressAlgorithm(
			configuration.getString(GeminiOptions.IN_PAGE_COMPRESSION), configuration);

		// load configuration for file cache
		FileCache.FileCacheType fileCacheType =
			FileCache.FileCacheType.valueOf(configuration.getString(GeminiOptions.FILE_CACHE_TYPE));
		setFileCacheType(fileCacheType);
		if (fileCacheType == FileCache.FileCacheType.LIMITED) {
			long fileCacheCapacity = MemorySize.parseBytes(configuration.getString(GeminiOptions.FILE_CACHE_CAPACITY));
			setFileCacheCapacity(fileCacheCapacity);
		}

		// load configuration for map split
		this.dataPageMapSplitEnabled = configuration.getBoolean(GeminiOptions.DATA_PAGE_MAP_SPLIT_ENABLED);
		this.mapSplitSizeThreshold = (int) parsePositiveBytes(configuration.getString(GeminiOptions.MAP_SPLIT_THRESHOLD_SIZE));
		int confMapSplitSubMapSize = configuration.getInteger(GeminiOptions.MAP_SPLIT_SUB_MAP_SIZE);
		this.mapSplitSubMapSize = confMapSplitSubMapSize == -1 ? mapSplitSizeThreshold / 2 : confMapSplitSubMapSize;
		this.mapSplitMaxSubMapNum = configuration.getInteger(GeminiOptions.MAP_SPLIT_MAX_SUB_MAP_NUM);

		// load configuration for file compaction
		this.enableFileCompaction = configuration.getBoolean(GeminiOptions.FILE_COMPACTION);
		this.compactionTriggerRatio = configuration.getFloat(GeminiOptions.FILE_COMPACTION_TRIGGER_RATIO);
		this.compactionTargetRatio = configuration.getFloat(GeminiOptions.FILE_COMPACTION_TARGET_RATIO);
		this.amplificationCheckInterval = configuration.getLong(GeminiOptions.FILE_AMPLIFICATION_CHECK_INTERVAL_MS);

		// load configuration for snapshot
		this.enableSnapshotCompaction = configuration.getBoolean(GeminiOptions.SNAPSHOT_COMPACTION);
		this.snapshotCompactionTargetRatio = configuration.getFloat(GeminiOptions.SNAPSHOT_COMPACTION_TARGET_RATIO);
		this.snapshotFlushBatchNumPage = configuration.getInteger(GeminiOptions.SNAPSHOT_BATCH_FLUSH_NUM_PAGE);
		this.snapshotFlushBatchDataSize = (int) parsePositiveBytes(configuration.getString(GeminiOptions.SNAPSHOT_BATCH_FLUSH_DATA_SIZE));
		this.snapshotSyncWhenBatchFlush = configuration.getBoolean(GeminiOptions.SNAPSHOT_SYNC_WHEN_BATCH_FLUSH);

		this.enableRestorePreFetch = configuration.getBoolean(GeminiOptions.RESTORE_FETCH_FILES);
		this.fetchFilesThreadNum = configuration.getInteger(GeminiOptions.FETCH_FILES_THREAD_NUM);
		this.vmPrintTick = configuration.getInteger(GeminiOptions.VM_PRINT_TICK);
		if (this.vmPrintTick < 30) {
			this.vmPrintTick = 30;
		}
		this.vmPrintAuditInfo = configuration.getBoolean(GeminiOptions.VM_PRINT_AUDIT_INFO);
		this.evictBaseOnPageAddressComposite = configuration.getBoolean(GeminiOptions.VM_EVICT_BASE_ON_COMPOSITE_PAGE_ADDRESS);
		this.batchPersistenceSize = configuration.getInteger(GeminiOptions.PERSISTENCE_BATCH_SIZE);
		this.forceSyncToCacheSize = configuration.getInteger(GeminiOptions.PERSISTENCE_FORCE_CACHE_SIZE);
		this.maxPersistenceRunningTask = configuration.getInteger(GeminiOptions.PERSISTENCE_MAX_RUNNING_TASK);
		this.persistenceType = PersistenceType.getPersistenceType(configuration.getString(GeminiOptions.PERSISTENCE_TYPE));
		this.lruIntoMainCacheThreadNum = configuration.getInteger(GeminiOptions.LRU_INTO_MAIN_CACHE_THREAD_NUM);
		this.lruIntoMainCacheSleepMs = configuration.getInteger(GeminiOptions.LRU_INTO_MAIN_CACHE_SLEEP_MS);
		this.enableAddIntoMainWhenSplitting = configuration.getBoolean(GeminiOptions.LRU_ADD_INTO_MAIN_WHEN_SPLITTING);
		this.enableLoadPageFromLRUIntoMain = configuration.getBoolean(GeminiOptions.ENABLE_LRU_INTO_MAIN_CACHE);
		this.enableLruAccessMode = configuration.getBoolean(GeminiOptions.ENABLE_LRU_ACCESS_MODE);
		this.enableEvictRegionEven = configuration.getBoolean(GeminiOptions.LRU_CACHE_EVEN_EVICT);
		this.bloomFilterMemRate = configuration.getFloat(GeminiOptions.VM_BLOOM_FILTER_MEM_RATE);
		this.enableBloomFilter = configuration.getBoolean(GeminiOptions.VM_BLOOM_FILTER_ENABLE);
	}

	public boolean isEnableLoadPageFromLRUIntoMain() {
		return enableLoadPageFromLRUIntoMain;
	}

	public boolean isEnableAddIntoMainWhenSplitting() {
		return enableAddIntoMainWhenSplitting;
	}

	public int getLruIntoMainCacheSleepMs() {
		return lruIntoMainCacheSleepMs;
	}

	public int getLruIntoMainCacheThreadNum() {
		return lruIntoMainCacheThreadNum;
	}

	public int getCompactionThreadNum() {
		return compactionThreadNum;
	}

	public void setLocalSnapshot(boolean enableLocalSnapshot) {
		localSnapshotEnabled = enableLocalSnapshot;
	}

	public boolean isLocalSnapshotEnabled() {
		return localSnapshotEnabled;
	}

	public long getFetchThreadSleepTimeNS() {
		return this.preFetchThreadSleepTimeNs;
	}

	public long getCommonThreadSleepTimeNs() {
		return commonThreadSleepTimeNs;
	}

	public GCompressAlgorithm getInPageGCompressAlgorithm() {
		return inPageCompressAlgorithm;
	}

	public GCompressAlgorithm getPageFlushLocalCompressAlgorithm() {
		return pageFlushLocalCompressAlgorithm;
	}

	public int getWholePageCompressThreshold() {
		return wholePageCompressThreshold;
	}

	public GCompressAlgorithm getPageFlushDfsCompressAlgorithm() {
		return pageFlushDfsCompressAlgorithm;
	}

	public boolean getEnablePrefetch() {
		return prefetchEnable;
	}

	public int getAllocatorDirectArena() {
		return allocatorDirectArena;
	}

	public boolean isChecksumEnable() {
		return checksumEnable;
	}

	public void setOperatorNameWithSubtask(String operatorNameWithSubtask) {
		this.operatorNameWithSubtask = operatorNameWithSubtask;
	}

	public String getOperatorNameWithSubtask() {
		return operatorNameWithSubtask;
	}

	public String getExecutorPrefixName() {
		return operatorNameWithSubtask == null || operatorNameWithSubtask.isEmpty() ? "" : operatorNameWithSubtask + "-";
	}

	public int getMinSortedListCountForFlush() {
		return minSortedListCountForFlush;
	}

	public int getTimePerTick(){
		return timePerTick;
	}

	public int getMaxPreparedFlushSize(){
		return maxPreparedFlushSize;
	}

	public int getEvictPoolFactor() {
		return evictPoolFactor;
	}

	// getter/setter for memory configuration ======================================

	public boolean isUseOffheap() {
		return useOffheap;
	}

	public void setUseOffheap(boolean useOffheap) {
		this.useOffheap = useOffheap;
	}

	public boolean isUseOffheapForRead() {
		return useOffheapForRead;
	}

	public void setUseOffheapForRead(boolean useOffheapForRead) {
		this.useOffheapForRead = useOffheapForRead;
	}

	public long getHeapSize() {
		return heapSize;
	}

	public void setHeapSize(long heapSize) {
		this.heapSize = heapSize;
	}

	public long getOffheapSize() {
		return offheapSize;
	}

	public void setOffheapSize(long offheapSize) {
		this.offheapSize = offheapSize;
	}

	public float getMemoryRatio() {
		return memoryRatio;
	}

	public void setMemoryRatio(float memoryRatio) {
		Preconditions.checkArgument(memoryRatio >= 0.0f && memoryRatio <= 1.0f,
			"Memory ratio must be in [0.0, 1.0], but actual is " + memoryRatio);
		this.memoryRatio = memoryRatio;
	}

	public void setNumberSlots(int numberSlots) {
		if (numberSlots <= 1) {
			numberSlots = 1;
		}
		this.numberSlots = numberSlots;
	}

	public int getNumberSlots() {
		return numberSlots;
	}

	// getter/setter for map split configuration ===================================

	public boolean isMapSplitEnabled() {
		return dataPageMapSplitEnabled;
	}

	public void setMapSplitEnabled(boolean allow) {
		dataPageMapSplitEnabled = allow;
	}

	public int getMapSplitSizeThreshold() {
		return mapSplitSizeThreshold;
	}

	public void setMapSplitSizeThreshold(int threshold) {
		mapSplitSizeThreshold = threshold;
	}

	public int getMapSplitSubMapSize() {
		return mapSplitSubMapSize;
	}

	public void setMapSplitSubMapSize(int subMapSize) {
		mapSplitSubMapSize = subMapSize;
	}

	public int getMapSplitMaxSubMapNum() {
		return mapSplitMaxSubMapNum;
	}

	public void setMapSplitMaxSubMapNum(int maxSubMapNum) {
		mapSplitMaxSubMapNum = maxSubMapNum;
	}

	// getter/setter for file cache ============================

	public void setFileCacheType(FileCache.FileCacheType type) {
		this.fileCacheType = type;
	}

	public FileCache.FileCacheType getFileCacheType() {
		return fileCacheType;
	}

	public void setFileCacheCapacity(long fileCacheCapacity) {
		Preconditions.checkArgument(fileCacheCapacity >= 0,
			"Capacity of file cache should be non-negative, actual is " + fileCacheCapacity);
		this.fileCacheCapacity = fileCacheCapacity;
	}

	public long getFileCacheCapacity() {
		return fileCacheCapacity;
	}

	// getter/setter for file compaction configuration ============================

	public void setFileCompaction(boolean enable) {
		this.enableFileCompaction = enable;
	}

	public boolean isFileCompactionEnabled() {
		return enableFileCompaction;
	}

	public void setCompactionTriggerRatio(float ratio) {
		this.compactionTriggerRatio = ratio;
	}

	public float getCompactionTriggerRatio() {
		return compactionTriggerRatio;
	}

	public void setCompactionTargetRatio(float ratio) {
		this.compactionTargetRatio = ratio;
	}

	public float getCompactionTargetRatio() {
		return compactionTargetRatio;
	}

	public void setAmplificationCheckInterval(long interval) {
		this.amplificationCheckInterval = interval;
	}

	public long getAmplificationCheckInterval() {
		return amplificationCheckInterval;
	}

	// getter/setter for file compaction configuration ============================

	public void setSnapshotCompaction(boolean enable) {
		this.enableSnapshotCompaction = enable;
	}

	public boolean isSnapshotCompactionEnabled() {
		return enableSnapshotCompaction;
	}

	public void setSnapshotCompactionTargetRatio(float ratio) {
		this.snapshotCompactionTargetRatio = ratio;
	}

	public float getSnapshotCompactionTargetRatio() {
		return snapshotCompactionTargetRatio;
	}

	public boolean isEnableRestorePreFetch() {
		return enableRestorePreFetch;
	}

	// getter/setter for snapshot

	public int getSnapshotFlushBatchNumPage() {
		return snapshotFlushBatchNumPage;
	}

	public void setSnapshotFlushBatchNumPage(int snapshotFlushBatchNumPage) {
		this.snapshotFlushBatchNumPage = snapshotFlushBatchNumPage;
	}

	public int getSnapshotFlushBatchDataSize() {
		return snapshotFlushBatchDataSize;
	}

	public void setSnapshotFlushBatchDataSize(int snapshotFlushBatchDataSize) {
		this.snapshotFlushBatchDataSize = snapshotFlushBatchDataSize;
	}

	public boolean isSnapshotSyncWhenBatchFlush() {
		return snapshotSyncWhenBatchFlush;
	}

	public void setSnapshotSyncWhenBatchFlush(boolean snapshotSyncWhenBatchFlush) {
		this.snapshotSyncWhenBatchFlush = snapshotSyncWhenBatchFlush;
	}

	public void setEnableRestorePreFetch(boolean enableRestorePreFetch) {
		this.enableRestorePreFetch = enableRestorePreFetch;
	}

	public void setFetchFilesThreadNum(int fetchFilesThreadNum) {
		Preconditions.checkArgument(fetchFilesThreadNum > 0,
			"The number of threads used to download files from DFS should be larger than zero.");
		this.fetchFilesThreadNum = fetchFilesThreadNum;
	}

	public int getFetchFilesThreadNum() {
		return fetchFilesThreadNum;
	}

	private boolean checkNonnegativeArgument(int argumentValue, String errorMessage) {
		if (argumentValue <= 0) {
			LOG.warn(errorMessage + ", the field would still stay as previous value.");
			return false;
		}
		return true;
	}

	private long parsePositiveBytes(String value) {
		long size = MemorySize.parseBytes(value);
		Preconditions.checkArgument(size > 0);
		return size;
	}

	@Override
	public String toString() {
		// only output some important configurations
		return "GConfiguration={" +
			"ttl=" + ttl +
			", dfsPath=" + Objects.toString(dfsPath) +
			", localPath=" + Objects.toString(localPath) +
			", flushThreadNum=" + flushThreadNum +
			", snapshotThreadNum=" + snapshotThreadNum +
			", regionThreadNum=" + regionThreadNum +
			", compactionThreadNum=" + compactionThreadNum +
			", maxFileSize=" + maxFileSize +
			", writeBufferSize=" + writeBufferSize +
			", maxCompactionThreshold=" + maxCompactionThreshold +
			", inMemoryCompactionThreshold=" + inMemoryCompactionThreshold +
			", useOffheap=" + useOffheap +
			", totalWriteBufferRate=" + totalWriteBufferRate +
			", numFlushingSegment=" + numFlushingSegment +
			", totalNumFlushingSegmentRatio=" + totalNumFlushingSegmentRatio +
			", bucketNum=" + bucketNum +
			", splitPageThreshold=" + splitPageThreshold +
			", subTaskIndex=" + subTaskIndex +
			", numParallelSubtasks=" + numParallelSubtasks +
			", comparatorType=" + comparatorType +
			", numberSlots=" + numberSlots +
			", readCopy=" + readCopy +
			", writeCopy=" + writeCopy +
			", inPageCompressAlgorithm=" + inPageCompressAlgorithm +
			", pageFlushLocalCompressAlgorithm=" + pageFlushLocalCompressAlgorithm +
			", pageFlushDfsCompressAlgorithm=" + pageFlushDfsCompressAlgorithm +
			", enableFileCompaction=" + enableFileCompaction +
			", enableSnapshotCompaction=" + enableSnapshotCompaction +
			", enableRestorePreFetch=" + enableRestorePreFetch +
			", fileCacheType=" + fileCacheType +
			"}";
		}

	public int geVmPrintTick() {
		return vmPrintTick;
	}

	public boolean geVmPrintAuditInfo() {
		return vmPrintAuditInfo;
	}

	public void setEvictPoolFactor(int evictPoolFactor) {
		this.evictPoolFactor = evictPoolFactor;
	}

	public boolean getEvictBaseOnPageAddressComposite() {
		return evictBaseOnPageAddressComposite;
	}

	public void setEvictBaseOnPageAddressComposite(boolean evictBaseOnPageAddressComposite) {
		this.evictBaseOnPageAddressComposite = evictBaseOnPageAddressComposite;
	}

	public int getBatchPersistenceSize() {
		return batchPersistenceSize;
	}

	public int getForceSyncToCacheSize() {
		return forceSyncToCacheSize;
	}

	public int getMaxPersistenceRunningTask() {
		return maxPersistenceRunningTask;
	}

	public PersistenceType getPersistenceType() {
		return persistenceType;
	}

	public float getBloomFilterMemRate() {
		return bloomFilterMemRate;
	}

	public boolean isEnableBloomFilter() {
		return enableBloomFilter && bloomFilterMemRate > 0;
	}
}