org.apache.flink.configuration.ResourceManagerOptions Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flink-core Show documentation
There is a newer version: 1.5.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.configuration;

import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.docs.ConfigGroup;
import org.apache.flink.annotation.docs.ConfigGroups;

/**
 * The set of configuration options relating to the ResourceManager.
 */
@PublicEvolving
@ConfigGroups(groups = {
	@ConfigGroup(name = "SlotManager", keyPrefix = "slotmanager")
})
public class ResourceManagerOptions {

	/**
	 * Timeout for jobs which don't have a job manager as leader assigned.
	 */
	public static final ConfigOption JOB_TIMEOUT = ConfigOptions
		.key("resourcemanager.job.timeout")
		.defaultValue("5 minutes")
		.withDescription("Timeout for jobs which don't have a job manager as leader assigned.");

	public static final ConfigOption LOCAL_NUMBER_RESOURCE_MANAGER = ConfigOptions
		.key("local.number-resourcemanager")
		.defaultValue(1);

	public static final ConfigOption IPC_PORT = ConfigOptions
		.key("resourcemanager.rpc.port")
		.defaultValue(0)
		.withDescription("Defines the network port to connect to for communication with the resource manager. By" +
			" default, the port of the JobManager, because the same ActorSystem is used." +
			" Its not possible to use this configuration key to define port ranges.");

	/**
	 * Percentage of heap space to remove from containers (YARN / Mesos), to compensate
	 * for other JVM memory usage.
	 */
	public static final ConfigOption CONTAINERIZED_HEAP_CUTOFF_RATIO = ConfigOptions
		.key("containerized.heap-cutoff-ratio")
		.defaultValue(0.25f)
		.withDeprecatedKeys("yarn.heap-cutoff-ratio")
		.withDescription("Percentage of heap space to remove from containers (YARN / Mesos), to compensate for other JVM memory usage. " +
			"This config option will not take effect in all session mode of Yarn/Kubernetes/Standalone. " +
			"Please use fine-grained config option instead. (" + TaskManagerOptions.TASK_MANAGER_PROCESS_HEAP_MEMORY.key() + "," +
			TaskManagerOptions.TASK_MANAGER_PROCESS_NETTY_MEMORY.key() + "," +
			TaskManagerOptions.TASK_MANAGER_PROCESS_NATIVE_MEMORY.key() + ")");

	/**
	 * Minimum amount of heap memory to remove in containers, as a safety margin.
	 */
	public static final ConfigOption CONTAINERIZED_HEAP_CUTOFF_MIN = ConfigOptions
		.key("containerized.heap-cutoff-min")
		.defaultValue(600)
		.withDeprecatedKeys("yarn.heap-cutoff-min")
		.withDescription("Minimum amount of heap memory to remove in containers, as a safety margin. " +
			"This config option will not take effect in all session mode of Yarn/Kubernetes/Standalone. " +
			"Please use fine-grained config option instead. (" + TaskManagerOptions.TASK_MANAGER_PROCESS_HEAP_MEMORY.key() + "," +
			TaskManagerOptions.TASK_MANAGER_PROCESS_NETTY_MEMORY.key() + "," +
			TaskManagerOptions.TASK_MANAGER_PROCESS_NATIVE_MEMORY.key() + ")");

	/**
	 * The timeout for a slot request to be discarded, in milliseconds.
	 */
	public static final ConfigOption SLOT_REQUEST_TIMEOUT = ConfigOptions
		.key("slotmanager.request-timeout")
		.defaultValue(600000L)
		.withDescription("The timeout for a slot request to be discarded.");

	/**
	 * The timeout for an idle task manager to be released, in milliseconds.
	 */
	public static final ConfigOption TASK_MANAGER_TIMEOUT = ConfigOptions
		.key("slotmanager.taskmanager-timeout")
		.defaultValue(30000L)
		.withDescription("The timeout for an idle task manager to be released.");

	/**
	 * The timeout for an idle task manager to be released if it blocks the starting of another, in millisecond.
	 */
	public static final ConfigOption TASK_MANAGER_FAST_TIMEOUT = ConfigOptions
		.key("slotmanager.taskmanager-fast-timeout")
		.defaultValue(10000L);

	/**
	 * The timeout for a result partition to be released after the corresponding job master disconnected, in millisecond.
	 */
	public static final ConfigOption RESULT_PARTITION_DISCONNECT_JM_TIMEOUT = ConfigOptions
		.key("resultpartition.disconnect-jm-timeout")
		.defaultValue(180000L)
		.withDescription("The timeout for a result partition to be released " +
			"after the corresponding job master disconnected in millisecond.");

	/**
	 * The timeout for an idle task manager to be released when slot manager starts, in milliseconds.
	 */
	public static final ConfigOption TASK_MANAGER_CHECKER_INITIAL_DELAY = ConfigOptions
		.key("slotmanager.taskmanager.checker-initial-delay")
		.defaultValue(180000L);

	/**
	 * The timeout for an unused pre-stop task manager to be released, in milliseconds.
	 */
	public static final ConfigOption TASK_MANAGER_PRESTOP_TIMEOUT = ConfigOptions
		.key("slotmanager.taskmanager.pre-stop-timeout")
		.defaultValue(30000L)
		.withDescription("The timeout for an unused pre-stop task manager to be released, in milliseconds.");

	/**
	 * Enable auto-scale mechanism. Only valid in session mode.
	 */
	public static final ConfigOption ENABLE_SESSION_AUTO_SCALE = ConfigOptions
		.key("resourcemanager.session-auto-scale.enabled")
		.defaultValue(false)
		.withDescription("Enable auto-scale mechanism. Only valid in yarn and k8s session mode.");

	public static final ConfigOption ENABLE_CONFIG_HOTUPDATE = ConfigOptions
		.key("session-auto-scale.hot-update-config.enabled")
		.defaultValue(false)
		.withDescription("Enable configuration hot-update in auto-scale mechanism.");

	public static final ConfigOption SLOT_PLACEMENT_POLICY = ConfigOptions
		.key("slotmanager.slot-placement-policy")
		.defaultValue("RANDOM");

	public static final ConfigOption USING_STRICT_FIFO = ConfigOptions
		.key("slotmanager.strict-fifo-mode.enabled")
		.defaultValue(false)
		.withDescription("Flag to activate/deactivate strict fifo mode. If this option is set to \"true\", " +
			"successive job will be blocked until former jobs fulfilled. This is applicable only in session mode");

	public static final ConfigOption MAX_TOTAL_RESOURCE_LIMIT_CPU_CORE = ConfigOptions
		.key("max.total-resource.limit.cpu.core")
		.defaultValue(Double.MAX_VALUE);

	public static final ConfigOption MAX_TOTAL_RESOURCE_LIMIT_MEMORY_MB = ConfigOptions
		.key("max.total-resource.limit.memory.mb")
		.defaultValue(Integer.MAX_VALUE);

	public static final ConfigOption TM_EXCEPTION_CACHE_SIZE = ConfigOptions
		.key("resourcemanager.tm-exception.cache.size")
		.defaultValue(128);

	public static final ConfigOption ALLOCATE_RESOURCES_ON_DEMAND = ConfigOptions
		.key("resourcemanager.allocate.resources.on-demand")
		.defaultValue(false);

	/**
	 * Prefix for passing custom environment variables to Flink's master process.
	 * For example for passing LD_LIBRARY_PATH as an env variable to the AppMaster, set:
	 * containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
	 * in the flink-conf.yaml.
	 */
	public static final String CONTAINERIZED_MASTER_ENV_PREFIX = "containerized.master.env.";

	/**
	 * Similar to the {@see CONTAINERIZED_MASTER_ENV_PREFIX}, this configuration prefix allows
	 * setting custom environment variables for the workers (TaskManagers).
	 */
	public static final String CONTAINERIZED_TASK_MANAGER_ENV_PREFIX = "containerized.taskmanager.env.";

	// ---------------------------------------------------------------------------------------------

	/** Not intended to be instantiated. */
	private ResourceManagerOptions() {}
}