org.apache.flink.runtime.state.gemini.engine.GConfiguration Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state.gemini.engine;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.gemini.GeminiOptions;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GComparatorType;
import org.apache.flink.runtime.state.gemini.engine.page.compress.GCompressAlgorithm;
import org.apache.flink.util.Preconditions;

import java.lang.management.ManagementFactory;
import java.lang.reflect.Field;

/**
 * GConfiguration.
 */
public class GConfiguration {

	private long fileCacheCapacity;

	private String dfsPath;

	private String localPath;

	private int metricSampleCount;

	private int metricHistogramWindowSize;

	private int flushThreadNum;

	private int snapshotThreadNum;

	private int regionThreadNum;

	private int compactionThreadNum;

	private long maxLogStructureFileSize;

	private long writeBufferSize;

	private int maxCompactionThreshold;

	private int inMemoryCompactionThreshold;

	private int maxRunningMajorCompaction;

	private int maxRunningMinorCompaction;

	private boolean useOffheap;

	private boolean forceReadUseOffheap;

	private float totalWriteBufferRate;
	private int totalReadPageLRUNum;
	private float heapRate;

	private int numFlushingSegment;

	private float totalNumFlushingSegmentRatio;

	private int bucketNum;

	private int spillHighWaterMark;

	private int spillLowWaterMark;

	private int spillMiddleWaterMark;

	private long ttl;

	private GComparatorType comparatorType = GComparatorType.bytes;

	private int subTaskIndex;

	private int numParallelSubtasks;

	private String backendUID;

	private int fileCleanThreadNum;

	private long fileCleanInterval;

	private long fileAliveTimeAfterNoDataReference;

	private long fileDeletionCheckInterval;

	private int dbNumberPerJVM = 1;

	private long totalHeapMemSize;
	private long totalOffheapMemSize;

	private boolean readCopy;

	private boolean writeCopy;

	private int writerFailCountThreshold;

	private int fileManagerFailCountThreshold;

	private int fileManagerCreateFileWriterRetryInterval;

	private boolean localSnapshotEnabled;

	private long preFetchThreadSleepTimeNs;

	private long commonThreadSleepTimeNs;

	private GCompressAlgorithm inPageCompressAlgorithm;
	private GCompressAlgorithm flushWholePageCompressAlgorithm;

	private boolean prefetchEnable = true;

	private int allocatorDirectArena;

	private int logicTableDefaultChainLen;

	private float totalHeapLowMarkRate;

	private float totalHeapMiddleMarkRate;

	private float totalHeapHighMarkRate;

	private float pageSizeRateBetweenPOJOAndHeap;

	private float totalPOJOPageUsedRate;

	private long totalIndexCountHighMark;

	private long totalIndexCountLowMark;

	private boolean checksumEnable;

	private String operatorNameWithSubtask = "";

	private int batchSortCount;

	public GConfiguration() {

	}

	public GConfiguration(Configuration configuration) {
		load(configuration);
	}

	public void setLogicTableDefaultChainLen(int logicTableDefaultChainLen) {
		this.logicTableDefaultChainLen = logicTableDefaultChainLen;
	}

	public int getLogicTableDefaultChainLen() {
		return logicTableDefaultChainLen;
	}

	public int getPageIndexBucketLenDefault() {
		return bucketNum;
	}

	public void setPageIndexBucketLenDefault(int bucketNum) {
		this.bucketNum = bucketNum;
	}

	public float getTotalWriteBufferRate() {
		return totalWriteBufferRate;
	}

	public int getTotalReadPageLRUNum() {
		return totalReadPageLRUNum;
	}

	public float getTotalHeapRate() {
		return heapRate;
	}

	public void setTotalHeapRate(float heapRate) {
		this.heapRate = heapRate;
	}

	public void setTotalHeapLowMarkRate(float totalHeapLowMarkRate) {
		this.totalHeapLowMarkRate = totalHeapLowMarkRate;
	}

	public float getTotalHeapLowMarkRate() {
		return totalHeapLowMarkRate;
	}

	public void setTotalHeapMiddleMarkRate(float totalHeapMiddleMarkRate) {
		this.totalHeapMiddleMarkRate = totalHeapMiddleMarkRate;
	}

	public float getTotalHeapMiddleMarkRate() {
		return totalHeapMiddleMarkRate;
	}

	public void setTotalHeapHighMarkRate(float totalHeapHighMarkRate) {
		this.totalHeapHighMarkRate = totalHeapHighMarkRate;
	}

	public float getTotalHeapHighMarkRate() {
		return totalHeapHighMarkRate;
	}

	public void setPageSizeRateBetweenPOJOAndHeap(float pageSizeRateBetweenPOJOAndHeap) {
		this.pageSizeRateBetweenPOJOAndHeap = pageSizeRateBetweenPOJOAndHeap;
	}

	public float getPageSizeRateBetweenPOJOAndHeap() {
		return pageSizeRateBetweenPOJOAndHeap;
	}

	public void setTotalPOJOPageUsedRate(float pojoPageUsedRate) {
		this.totalPOJOPageUsedRate = pojoPageUsedRate;
	}

	public float getTotalPOJOPageUsedRate() {
		return totalPOJOPageUsedRate;
	}

	public int getSpilledPageSizeThresholdUnderHighMark() {
		return spillHighWaterMark;
	}

	public int getSpilledPageSizeThresholdUnderLowMark() {
		return spillLowWaterMark;
	}

	public int getSpilledPageSizeThresholdUnderMiddleMark() {
		return spillMiddleWaterMark;
	}

	public long getWriteBufferWaterMark() {
		return writeBufferSize;
	}

	public void setWriteBUfferWaterMark(long writeBufferSize) {
		this.writeBufferSize = writeBufferSize;
	}

	public void setTotalIndexCountHighMark(long totalIndexCountHighMark) {
		this.totalIndexCountHighMark = totalIndexCountHighMark;
	}

	public long getTotalIndexCountHighMark() {
		return totalIndexCountHighMark;
	}

	public void setTotalIndexCountLowMark(long totalIndexCountLowMark) {
		this.totalIndexCountLowMark = totalIndexCountLowMark;
	}

	public long getTotalIndexCountLowMark() {
		return totalIndexCountLowMark;
	}

	public int getMaxCompactionChainThreshold() {
		return maxCompactionThreshold;
	}

	public void setDfsPath(String dfsPath) {
		this.dfsPath = Preconditions.checkNotNull(dfsPath);
	}

	public String getDfsPath() {
		return dfsPath;
	}

	public void setLocalPath(String localPath) {
		this.localPath = Preconditions.checkNotNull(localPath);
	}

	public String getLocalPath() {
		return localPath;
	}

	public void setMetricSampleCount(int count) {
		this.metricSampleCount = count;
	}

	public int getMetricSampleCount() {
		return metricSampleCount;
	}

	public void setMetricHistogramWindowSize(int metricHistogramWindowSize) {
		this.metricHistogramWindowSize = metricHistogramWindowSize;
	}

	public String getBackendUID() {
		return backendUID;
	}

	public void setBackendUID(String backendUID) {
		this.backendUID = backendUID;
	}

	public void setFlushThreadNum(int flushThreadNum) {
		this.flushThreadNum = flushThreadNum;
	}

	public int getFlushThreadNum() {
		return flushThreadNum;
	}

	public int getSnapshotThreadNum() {
		return snapshotThreadNum;
	}

	public void setSnapshotThreadNum(int snapshotThreadNum) {
		this.snapshotThreadNum = snapshotThreadNum;
	}

	public void setRegionThreadNum(int regionThreadNum) {
		this.regionThreadNum = regionThreadNum;
	}

	public int getRegionThreadNum() {
		return regionThreadNum;
	}

	public int getMetricHistogramWindowSize() {
		return metricHistogramWindowSize;
	}

	public void setMaxLogStructureFileSize(long maxLogStructureFileSize) {
		this.maxLogStructureFileSize = maxLogStructureFileSize;
	}

	public long getMaxLogStructureFileSize() {
		return maxLogStructureFileSize;
	}

	public int getInMemoryCompactionThreshold() {
		return inMemoryCompactionThreshold;
	}

	public int getMaxRunningMajorCompaction() {
		return maxRunningMajorCompaction;
	}

	public int getMaxRunningMinorCompaction() {
		return maxRunningMinorCompaction;
	}

	public void setInMemoryCompactionThreshold(int threshold) {
		this.inMemoryCompactionThreshold = threshold;
	}

	public void setUseOffheap() {
		this.useOffheap = true;
	}

	public boolean getUseOffheap() {
		return useOffheap;
	}

	public boolean getForceReadUseOffheap() {
		return forceReadUseOffheap;
	}

	public int getNumFlushingSegment() {
		return numFlushingSegment;
	}

	public float getTotalNumFlushingSegmentRatio() {
		return totalNumFlushingSegmentRatio;
	}

	public void setTtl(long ttl) {
		this.ttl = ttl;
	}

	public long getTtl() {
		return ttl;
	}

	public GComparatorType getComparatorType() {
		return comparatorType;
	}

	public void setSubTaskIndex(int subTaskIndex) {
		this.subTaskIndex = subTaskIndex;
	}

	public void setNumParallelSubtasks(int numParallelSubtasks) {
		this.numParallelSubtasks = numParallelSubtasks;
	}

	public int getSubTaskIndex() {
		return subTaskIndex;
	}

	public int getNumParallelSubtasks() {
		return numParallelSubtasks;
	}

	public long getFileCacheCapacity() {
		return fileCacheCapacity;
	}

	public void setFileCacheCapacity(long fileCacheCapacity) {
		this.fileCacheCapacity = fileCacheCapacity;
	}

	public void setFileCleanThreadNum(int threadNum) {
		this.fileCleanThreadNum = threadNum;
	}

	public int getFileCleanThreadNum() {
		return fileCleanThreadNum;
	}

	public void setFileCleanInterval(long interval) {
		this.fileCleanInterval = interval;
	}

	public long getFileCleanInterval() {
		return fileCleanInterval;
	}

	public void setFileAliveTimeAfterNoDataReference(long aliveTime) {
		this.fileAliveTimeAfterNoDataReference = aliveTime;
	}

	public long getFileAliveTimeAfterNoDataReference() {
		return fileAliveTimeAfterNoDataReference;
	}

	public void setFileDeletionCheckInterval(long interval) {
		fileDeletionCheckInterval = interval;
	}

	public long getFileDeletionCheckInterval() {
		return fileDeletionCheckInterval;
	}

	public boolean isReadCopy() {
		return readCopy;
	}

	public boolean isWriteCopy() {
		return writeCopy;
	}

	public int getWriterFailCountThreshold() {
		return writerFailCountThreshold;
	}

	public void setWriterFailCountThreshold(int writerFailCountThreshold) {
		this.writerFailCountThreshold = writerFailCountThreshold;
	}

	public int getFileManagerFailCountThreshold() {
		return fileManagerFailCountThreshold;
	}

	public void setFileManagerFailCountThreshold(int fileManagerFailCountThreshold) {
		this.fileManagerFailCountThreshold = fileManagerFailCountThreshold;
	}

	public int getFileManagerCreateFileWriterRetryInterval() {
		return fileManagerCreateFileWriterRetryInterval;
	}

	public void setFileManagerCreateFileWriterRetryInterval(int fileManagerCreateFileWriterRetryInterval) {
		this.fileManagerCreateFileWriterRetryInterval = fileManagerCreateFileWriterRetryInterval;
	}

	private void load(Configuration configuration) {
		this.metricSampleCount = configuration.getInteger(GeminiOptions.SAMPLE_COUNT);
		this.metricHistogramWindowSize = configuration.getInteger(GeminiOptions.HISTOGRAM_WINDOW_SIZE);
		this.flushThreadNum = configuration.getInteger(GeminiOptions.FLUSH_THREAD_NUM);
		this.snapshotThreadNum = configuration.getInteger(GeminiOptions.SNAPSHOT_THREAD_NUM);
		this.regionThreadNum = configuration.getInteger(GeminiOptions.REGION_THREAD_NUM);
		this.maxLogStructureFileSize = configuration.getLong(GeminiOptions.MAX_LOG_STRUCTURE_FILE_SIZE);
		this.writeBufferSize = configuration.getLong(GeminiOptions.GEMINIDB_WRITE_BUFFER_SIZE);
		this.compactionThreadNum = configuration.getInteger(GeminiOptions.COMPACTION_THREAD_NUM);
		this.maxCompactionThreshold = configuration.getInteger(GeminiOptions.GEMINIDB_MAX_COMPACTION_THRESHOLD);
		this.inMemoryCompactionThreshold = configuration.getInteger(GeminiOptions.GEMINIDB_IN_MEMORY_COMPACTION_THRESHOLD);
		this.maxRunningMajorCompaction = configuration.getInteger(GeminiOptions.GEMINIDB_MAX_RUNNING_MAJOR_COMPACTION_THRESHOLD);
		this.maxRunningMinorCompaction = configuration.getInteger(GeminiOptions.GEMINIDB_MAX_RUNNING_MINOR_COMPACTION_THRESHOLD);
		this.useOffheap = configuration.getBoolean(GeminiOptions.GEMINIDB_OFFHEAP_MEMORY);
		this.forceReadUseOffheap = configuration.getBoolean(GeminiOptions.GEMINIDB_FORCE_READ_OFFHEAP_MEMORY);
		this.totalWriteBufferRate = configuration.getFloat(GeminiOptions.GEMINIDB_TOTAL_WRITEBUFFER_RATE);
		this.totalReadPageLRUNum = configuration.getInteger(GeminiOptions.GEMINIDB_TOTAL_READPAGELRU_NUM);
		this.heapRate = configuration.getFloat(GeminiOptions.GEMINIDB_HEAP_RATE);
		this.numFlushingSegment = configuration.getInteger(GeminiOptions.GEMINIDB_FLUSING_SEGMENT);
		this.totalNumFlushingSegmentRatio = configuration.getFloat(GeminiOptions.GEMINIDB_FLUSING_SEGMENT_RATIO);
		this.bucketNum = configuration.getInteger(GeminiOptions.GEMINIDB_BUCKET_NUM);
		this.spillHighWaterMark = configuration.getInteger(GeminiOptions.GEMINIDB_SPILL_HIGH_WATERMARK);
		this.spillLowWaterMark = configuration.getInteger(GeminiOptions.GEMINIDB_SPILL_LOW_WATERMARK);
		this.spillMiddleWaterMark = configuration.getInteger(GeminiOptions.GEMINIDB_SPILL_MIDDLE_WATERMARK);
		this.ttl = configuration.getLong(GeminiOptions.GEMINIDB_TTL);
		this.fileCacheCapacity = configuration.getLong(GeminiOptions.FILE_CACHE_CAPACITY);
		this.fileCleanInterval = configuration.getLong(GeminiOptions.FILE_CLEAN_INTERVAL);
		this.fileCleanThreadNum = configuration.getInteger(GeminiOptions.FILE_CLEAN_THREAD_NUM);
		this.fileAliveTimeAfterNoDataReference = configuration.getLong(GeminiOptions.FILE_ALIVE_TIME_AFTER_NO_DATA_REFERENCE);
		this.fileDeletionCheckInterval = configuration.getLong(GeminiOptions.FILE_DELETION_CHECK_INTERVAL);
		this.comparatorType  = GComparatorType.getComparatorType(configuration.getString(GeminiOptions.GEMINI_COMPARATOR_TYPE));
		this.readCopy = configuration.getBoolean(GeminiOptions.READ_COPY);
		this.writeCopy = configuration.getBoolean(GeminiOptions.WRITE_COPY);
		this.writerFailCountThreshold = configuration.getInteger(GeminiOptions.WRITER_FAILCOUNT_THESHOLD);
		this.fileManagerFailCountThreshold = configuration.getInteger(GeminiOptions.FILEMANAGER_FAILCOUNT_THESHOLD);
		this.fileManagerCreateFileWriterRetryInterval = configuration.getInteger(GeminiOptions.FILEMANAGER_RETRY_INTERVAL);
		this.preFetchThreadSleepTimeNs = configuration.getLong(GeminiOptions.PREFETCH_THREAD_SLEEP_TIME_NS);
		this.commonThreadSleepTimeNs = configuration.getLong(GeminiOptions.COMMON_THREAD_SLEEP_TIME_NS);
		//TODO now close compression.
//		this.flushWholePageCompressAlgorithm = GCompressAlgorithm.valueOf(configuration.getString(GeminiOptions.FLUSH_PAGE_COMPRESSION));
		this.flushWholePageCompressAlgorithm = GCompressAlgorithm.None;

		//TODO inPageCompress only support Lz4, Snappy not support HeapByteBuffer now.
		if (GCompressAlgorithm.valueOf(configuration.getString(GeminiOptions.IN_PAGE_COMPRESSION)) != GCompressAlgorithm.None) {
			this.inPageCompressAlgorithm = GCompressAlgorithm.Lz4;
		} else {
			this.inPageCompressAlgorithm = GCompressAlgorithm.None;
		}
		this.prefetchEnable = configuration.getBoolean(GeminiOptions.PREFETCH_ENABLE);
		this.allocatorDirectArena = configuration.getInteger(GeminiOptions.ALLOCATOR_DIRECT_ARENA);
		this.logicTableDefaultChainLen = configuration.getInteger(GeminiOptions.LOGIC_CHAIN_INIT_LEN);
		this.totalHeapLowMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_LOW_MARK_RATE);
		this.totalHeapMiddleMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_MIDDLE_MARK_RATE);
		this.totalHeapHighMarkRate = configuration.getFloat(GeminiOptions.TOTAL_HEAP_HIGH_MARK_RATE);
		this.pageSizeRateBetweenPOJOAndHeap = configuration.getFloat(GeminiOptions.PAGE_SIZE_RATE_BETWEEN_POJO_HEAP);
		this.totalPOJOPageUsedRate = configuration.getFloat(GeminiOptions.TOTAL_POJO_PAGE_USED_RATE);
		this.totalIndexCountHighMark = configuration.getInteger(GeminiOptions.INDEX_COUNT_HIGH_MARK);
		this.totalIndexCountLowMark = configuration.getInteger(GeminiOptions.INDEX_COUNT_LOW_MARK);
		this.checksumEnable = configuration.getBoolean(GeminiOptions.CHECKSUM_ENABLE);
		this.batchSortCount = configuration.getInteger(GeminiOptions.BATCH_SORT_COUNT);
		updateTotalHeapMemSize();
		updateTotalOffHeapMemSize();
	}

	public int getCompactionThreadNum() {
		return compactionThreadNum;
	}

	public void setDBNumberPerJVM(int dbNumberPerJVM) {
		if (dbNumberPerJVM <= 1) {
			dbNumberPerJVM = 1;
		}
		this.dbNumberPerJVM = dbNumberPerJVM;
		updateTotalHeapMemSize();
		updateTotalOffHeapMemSize();
	}

	public int getDBNumberPerJVM() {
		return dbNumberPerJVM;
	}

	private void updateTotalHeapMemSize() {
		//dbSlots is the same as TM slots number. so this DB only use 1/dbSlots memory.
		totalHeapMemSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() / dbNumberPerJVM;
	}

	public long getTotalHeapMemSize() {
		return totalHeapMemSize;
	}

	private void updateTotalOffHeapMemSize() {
		//dbSlots is the same as TM slots number. so this DB only use 1/dbSlots memory.
		this.totalOffheapMemSize = getDirectMemorySize() / dbNumberPerJVM;
	}

	public long getTotalOffheapMemSize() {
		return totalOffheapMemSize;
	}

	private static long getDirectMemorySize() {
		try {
			Class c = Class.forName("java.nio.Bits");
			Field maxMemory = c.getDeclaredField("maxMemory");
			maxMemory.setAccessible(true);
			Field reservedMemory = c.getDeclaredField("reservedMemory");
			reservedMemory.setAccessible(true);
			return (Long) maxMemory.get(null);
		} catch (Exception e) {
			throw new GeminiRuntimeException(e);
		}
	}

	public void setLocalSnapshot(boolean enableLocalSnapshot) {
		localSnapshotEnabled = enableLocalSnapshot;
	}

	public boolean isLocalSnapshotEnabled() {
		return localSnapshotEnabled;
	}

	public long getFetchThreadSleepTimeNS() {
		return this.preFetchThreadSleepTimeNs;
	}

	public long getCommonThreadSleepTimeNs() {
		return commonThreadSleepTimeNs;
	}

	public GCompressAlgorithm getInPageGCompressAlgorithm() {
		return inPageCompressAlgorithm;
	}

	public GCompressAlgorithm getFlushWholePageGCompressAlgorithm() {
		return flushWholePageCompressAlgorithm;
	}

	public boolean getEnablePrefetch() {
		return prefetchEnable;
	}

	public int getAllocatorDirectArena() {
		return allocatorDirectArena;
	}

	public boolean isChecksumEnable() {
		return checksumEnable;
	}

	public void setOperatorNameWithSubtask(String operatorNameWithSubtask) {
		this.operatorNameWithSubtask = operatorNameWithSubtask;
	}

	public String getOperatorNameWithSubtask() {
		return operatorNameWithSubtask;
	}

	public String getExcetorPrefixName() {
		return operatorNameWithSubtask == null || operatorNameWithSubtask.isEmpty() ? "" : operatorNameWithSubtask + "-";
	}

	public int getBatchSortCount() {
		return batchSortCount;
	}
}