org.apache.flink.runtime.state.gemini.GeminiOptions Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.runtime.state.gemini.engine.filecache.FileCache;
import org.apache.flink.runtime.state.gemini.engine.page.PageConstants;
import static org.apache.flink.configuration.ConfigOptions.key;
/**
* A collection of all configuration options that relate to {@link GeminiStateBackend}.
*/
public class GeminiOptions {
public static final ConfigOption DFS_PATH = key("state.backend.gemini.dfs.dir").noDefaultValue().withDescription(
"The dfs path where GeminiDB puts its files.");
/** The sampling rate of state related metrics based on action count. */
public static final ConfigOption SAMPLE_COUNT = ConfigOptions.key(
"state.backend.gemini.metric.sample.count").defaultValue(100);
/** The window size used for state histogram metrics. */
public static final ConfigOption HISTOGRAM_WINDOW_SIZE = ConfigOptions.key(
"state.backend.gemini.metric.histogram.window").defaultValue(5);
public static final ConfigOption FLUSH_THREAD_NUM = ConfigOptions.key(
"state.backend.gemini.flush.thread.num").defaultValue(4);
public static final ConfigOption SNAPSHOT_THREAD_NUM = ConfigOptions.key(
"state.backend.gemini.snapshot.thread.num").defaultValue(3);
public static final ConfigOption REGION_THREAD_NUM = ConfigOptions.key(
"state.backend.gemini.region.thread.num").defaultValue(8);
public static final ConfigOption COMPACTION_THREAD_NUM = ConfigOptions.key(
"state.backend.gemini.compaction.thread.num").defaultValue(8);
public static final ConfigOption MAX_LOG_STRUCTURE_FILE_SIZE = ConfigOptions.key(
"state.backend.gemini.log.structure.file.size").defaultValue(512 * 1024 * 1024L);
/** The local directory (on the TaskManager) where GeminiDB puts its files. */
public static final ConfigOption GEMINIDB_LOCAL_DIRECTORIES = key("state.backend.gemini.local.dir").noDefaultValue().withDescription(
"The local directory (on the TaskManager) where GeminiDB puts its files.");
public static final ConfigOption GEMINIDB_WRITE_BUFFER_SIZE = ConfigOptions.key(
"state.backend.gemini.wirtebuffer.size").defaultValue(640 * 1024L);
public static final ConfigOption GEMINIDB_IN_MEMORY_COMPACTION_THRESHOLD = ConfigOptions.key(
"state.backend.gemini.in-memory.compaction.threshold").defaultValue(3);
public static final ConfigOption GEMINIDB_MAX_RUNNING_MAJOR_COMPACTION_THRESHOLD = ConfigOptions.key(
"state.backend.gemini.running.major.compaction.threshold").defaultValue(8);
public static final ConfigOption GEMINIDB_MAX_RUNNING_MINOR_COMPACTION_THRESHOLD = ConfigOptions.key(
"state.backend.gemini.running.minor.compaction.threshold").defaultValue(128);
public static final ConfigOption GEMINIDB_MAX_COMPACTION_THRESHOLD = ConfigOptions.key(
"state.backend.gemini.max.compaction.threshold").defaultValue(10);
public static final ConfigOption GEMINIDB_OFFHEAP_MEMORY = ConfigOptions.key(
"state.backend.gemini.off-heap.memory").defaultValue(false);
public static final ConfigOption GEMINIDB_FORCE_READ_OFFHEAP_MEMORY = ConfigOptions.key(
"state.backend.gemini.force.read.use.off-heap").defaultValue(false);
public static final ConfigOption GEMINIDB_TOTAL_WRITEBUFFER_RATE = ConfigOptions.key(
"state.backend.gemini.total.writebuffer.rate").defaultValue(PageConstants.DEFAULT_TOTAL_WRITEBUFFER_RATE);
//Notice: For ourself Allocator in future, the data read from Disk will directly enter the cache. so 1024 page for LRU
//is totally enough. before that, we can increase this num to improve performance, for example 10000.
public static final ConfigOption GEMINIDB_TOTAL_READPAGELRU_NUM = ConfigOptions.key(
"state.backend.gemini.total.read.page.lru.num").defaultValue(1024);
public static final ConfigOption GEMINIDB_HEAP_RATE = ConfigOptions.key("state.backend.gemini.heap.rate").defaultValue(
PageConstants.DEFAULT_TOTAL_HEAP_RATE);
public static final ConfigOption GEMINIDB_FLUSING_SEGMENT = ConfigOptions.key(
"state.backend.gemini.flushing.segment").defaultValue(3);
public static final ConfigOption GEMINIDB_FLUSING_SEGMENT_RATIO = ConfigOptions.key(
"state.backend.gemini.flushing.segment.total.ratio").defaultValue(1.0f);
public static final ConfigOption GEMINIDB_BUCKET_NUM = ConfigOptions.key(
"state.backend.gemini.bucket.index.num").defaultValue(PageConstants.DEFAULT_BUCKET_NUM);
public static final ConfigOption GEMINIDB_SPILL_HIGH_WATERMARK = ConfigOptions.key(
"state.backend.gemini.spill.high.watermark").defaultValue(PageConstants.DEFAULT_SPILLED_PAGE_SIZE_HIGH_THRESHOLD);
public static final ConfigOption GEMINIDB_SPILL_LOW_WATERMARK = ConfigOptions.key(
"state.backend.gemini.spill.low.watermark").defaultValue(PageConstants.DEFAULT_SPILLED_PAGE_SIZE_LOW_THRESHOLD);
public static final ConfigOption GEMINIDB_SPILL_MIDDLE_WATERMARK = ConfigOptions.key(
"state.backend.gemini.spill.middle.watermark").defaultValue(PageConstants.DEFAULT_SPILLED_PAGE_SIZE_MIDDLE_THRESHOLD);
public static final ConfigOption GEMINIDB_TTL = ConfigOptions.key(
"state.backend.gemini.tll").defaultValue(-1L);
public static final ConfigOption FILE_CACHE_CAPACITY = ConfigOptions.key(
"state.backend.gemini.file.cache.capacity").defaultValue(FileCache.INFINITE_CAPACITY);
public static final ConfigOption FILE_CLEAN_THREAD_NUM = ConfigOptions.key(
"state.backend.gemini.file.clean.thread.num").defaultValue(1);
public static final ConfigOption FILE_CLEAN_INTERVAL = ConfigOptions.key(
"state.backend.gemini.file.clean.interval").defaultValue(60000L);
public static final ConfigOption FILE_ALIVE_TIME_AFTER_NO_DATA_REFERENCE = ConfigOptions.key(
"state.backend.gemini.file.alive.time.after.no.data.reference").defaultValue(30000L);
public static final ConfigOption FILE_DELETION_CHECK_INTERVAL = ConfigOptions.key(
"state.backend.gemini.file.deletion.check.interval").defaultValue(20000L);
public static final ConfigOption GEMINI_COMPARATOR_TYPE = ConfigOptions.key(
"state.backend.gemini.comparator.type").defaultValue("bytes");
public static final ConfigOption READ_COPY = ConfigOptions.key(
"state.backend.gemini.read.copy").defaultValue(true);
public static final ConfigOption WRITE_COPY = ConfigOptions.key(
"state.backend.gemini.write.copy").defaultValue(true);
public static final ConfigOption WRITER_FAILCOUNT_THESHOLD = ConfigOptions.key(
"state.backend.gemini.writer.failcount.threshold")
.defaultValue(5);
// TODO: #SR.
// 1. Currently, we will disable creating new file writer after *all writer* failed only.
// If we have 2 filewriter in snapshotEventExecutorGroup, and one is good, the other is bad, we will still
// create new fileWriter.
// 2. Do not support caching on DFS now.
public static final ConfigOption FILEMANAGER_FAILCOUNT_THESHOLD = ConfigOptions.key(
"state.backend.gemini.filemanager.failcount.threshold")
.defaultValue(3)
.withDescription("Threshold of all filewriters failed count, after exceeding the threshold we will not create " +
"new filewriter in the following ${FILEMANAGER_RETRY_INTERVAL} second.");
public static final ConfigOption FILEMANAGER_RETRY_INTERVAL = ConfigOptions.key(
"state.backend.gemini.filemanager.retry.s")
.defaultValue(3600)
.withDescription("Interval will retry to create new filewriter after disabling create new filewriter because of" +
"exceed the threshold of ${FILEMANAGER_FAILCOUNT_THESHOLD}");
public static final ConfigOption PREFETCH_THREAD_SLEEP_TIME_NS = ConfigOptions.key(
"state.backend.gemini.prefetch.sleep.ns").defaultValue(50L);
public static final ConfigOption COMMON_THREAD_SLEEP_TIME_NS = ConfigOptions.key(
"state.backend.gemini.common.sleep.ns").defaultValue(1000_000L);
//TODO NOTICE: we can't support changing compression.
public static final ConfigOption FLUSH_PAGE_COMPRESSION = ConfigOptions.key(
"state.backend.gemini.compression.flush.page").defaultValue("None");
//TODO NOTICE: in page compression only support None or LZ4
public static final ConfigOption IN_PAGE_COMPRESSION = ConfigOptions.key(
"state.backend.gemini.compression.in.page").defaultValue("None");
public static final ConfigOption PREFETCH_ENABLE = ConfigOptions.key(
"state.backend.gemini.prefetch.enable").defaultValue(true);
public static final ConfigOption ALLOCATOR_DIRECT_ARENA = ConfigOptions.key(
"state.backend.gemini.allocator.direct.arena.num").defaultValue(8);
public static final ConfigOption LOGIC_CHAIN_INIT_LEN = ConfigOptions.key(
"state.backend.gemini.logic.chain.init.len").defaultValue(PageConstants.DEFAULT_LOGIC_TABLE_CHAIN_LEN);
public static final ConfigOption TOTAL_HEAP_LOW_MARK_RATE = ConfigOptions.key(
"state.backend.gemini.total.heap.low_mark.rate").defaultValue(PageConstants.DEFAULT_TOTAL_HEAP_LOW_MARK_RATE);
public static final ConfigOption TOTAL_HEAP_MIDDLE_MARK_RATE = ConfigOptions.key(
"state.backend.gemini.total.heap.middle_mark.rate").defaultValue(PageConstants.DEFAULT_TOTAL_HEAP_MIDDLE_MARK_RATE);
public static final ConfigOption TOTAL_HEAP_HIGH_MARK_RATE = ConfigOptions.key(
"state.backend.gemini.total.heap.high_mark.rate").defaultValue(PageConstants.DEFAULT_TOTAL_HEAP_HIGH_MARK_RATE);
public static final ConfigOption PAGE_SIZE_RATE_BETWEEN_POJO_HEAP = ConfigOptions.key(
"state.backend.gemini.page.size.rate.between.pojo.heap").defaultValue(PageConstants.DEFAULT_PAGE_SIZE_RATE_BETWEEN_POJO_HEAP);
public static final ConfigOption TOTAL_POJO_PAGE_USED_RATE = ConfigOptions.key(
"state.backend.gemini.pojo.page.used.rate").defaultValue(PageConstants.DEFAULT_TOTAL_POJO_PAGE_USED_RATE);
public static final ConfigOption INDEX_COUNT_HIGH_MARK = ConfigOptions.key(
"state.backend.gemini.index.count.high.mark").defaultValue(PageConstants.DEFAULT_INDEX_COUNT_HIGH_MARK);
public static final ConfigOption INDEX_COUNT_LOW_MARK = ConfigOptions.key(
"state.backend.gemini.index.count.low.mark").defaultValue(PageConstants.DEFAULT_INDEX_COUNT_LOW_MARK);
public static final ConfigOption CHECKSUM_ENABLE = ConfigOptions.key(
"state.backend.gemini.checksum.enable").defaultValue(true);
public static final ConfigOption BATCH_SORT_COUNT = ConfigOptions.key(
"state.backend.gemini.page.pool.batch.sort.count").defaultValue(16);
}