templates.flink.0.10.conf.flink-conf.yaml.mustache Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of peel-extensions Show documentation
There is a newer version: 1.1.8
################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

#==============================================================================
# Common Options
#==============================================================================

# The default degree of parallelism to use for programs that have no degree of
# parallelism specified.
#
{{#parallelism.default}}parallelism.default: {{parallelism.default}}{{/parallelism.default}}{{^parallelism.default}}parallelism.default: 1{{/parallelism.default}}

#==============================================================================
# JAVA Options
#==============================================================================

# The path to the Java installation to use.
#
{{#env.java.home}}env.java.home: {{env.java.home}}{{/env.java.home}}{{^env.java.home}}env.java.home: $JAVA_HOME{{/env.java.home}}

# The path to the Java installation to use.
#
{{#env.java.opts}}env.java.opts: {{env.java.opts}}{{/env.java.opts}}{{^env.java.opts}}# env.java.opts: {{/env.java.opts}}

#==============================================================================
# Environment Options
#==============================================================================

# The directory where the PID files are stored
{{#env.pid.dir}}env.pid.dir: {{env.pid.dir}}{{/env.pid.dir}}{{^env.pid.dir}}env.pid.dir: /tmp{{/env.pid.dir}}

#==============================================================================
# JobManager & TaskManager
#==============================================================================

# The IP address of the JobManager, which is the master/coordinator of the
# distributed system.
#
{{#jobmanager.rpc.address}}jobmanager.rpc.address: {{jobmanager.rpc.address}}{{/jobmanager.rpc.address}}{{^jobmanager.rpc.address}}jobmanager.rpc.address: localhost{{/jobmanager.rpc.address}}

# The port number of the JobManager.
#
{{#jobmanager.rpc.port}}jobmanager.rpc.port: {{jobmanager.rpc.port}}{{/jobmanager.rpc.port}}{{^jobmanager.rpc.port}}jobmanager.rpc.port: 6123{{/jobmanager.rpc.port}}

# The task manager’s RPC port.
#
{{#taskmanager.rpc.port}}taskmanager.rpc.port: {{taskmanager.rpc.port}}{{/taskmanager.rpc.port}}{{^taskmanager.rpc.port}}taskmanager.rpc.port: 6122{{/taskmanager.rpc.port}}

# The task manager’s port used for data exchange operations.
#
{{#taskmanager.data.port}}taskmanager.data.port: {{taskmanager.data.port}}{{/taskmanager.data.port}}{{^taskmanager.data.port}}taskmanager.data.port: 6121{{/taskmanager.data.port}}

# The task manager’s RPC port.
#
{{#jobmanager.heap.mb}}jobmanager.heap.mb: {{jobmanager.heap.mb}}{{/jobmanager.heap.mb}}{{^jobmanager.heap.mb}}jobmanager.heap.mb: 256{{/jobmanager.heap.mb}}

# The task manager’s port used for data exchange operations.
#
{{#taskmanager.heap.mb}}taskmanager.heap.mb: {{taskmanager.heap.mb}}{{/taskmanager.heap.mb}}{{^taskmanager.heap.mb}}taskmanager.heap.mb: 512{{/taskmanager.heap.mb}}

#  The number of parallel operator or user function instances that a single
# TaskManager can run.
#
{{#taskmanager.numberOfTaskSlots}}taskmanager.numberOfTaskSlots: {{taskmanager.numberOfTaskSlots}}{{/taskmanager.numberOfTaskSlots}}{{^taskmanager.numberOfTaskSlots}}taskmanager.numberOfTaskSlots: -1{{/taskmanager.numberOfTaskSlots}}

# Directories for temporary files.
#
# Add a delimited list for multiple directories, using the system directory
# delimiter (colon ':' on unix) or a comma, e.g.:
#     /data1/tmp:/data2/tmp:/data3/tmp
#
# Note: Each directory entry is read from and written to by a different I/O
# thread. You can include the same directory multiple times in order to create
# multiple I/O threads against that directory. This is for example relevant for
# high-throughput RAIDs.
#
# If not specified, the system-specific Java temporary directory (java.io.tmpdir
# property) is taken.
#
{{#taskmanager.tmp.dirs}}taskmanager.tmp.dirs: {{taskmanager.tmp.dirs}}{{/taskmanager.tmp.dirs}}{{^taskmanager.tmp.dirs}}# taskmanager.tmp.dirs: /tmp{{/taskmanager.tmp.dirs}}

# The number of buffers for the network stack.
#
{{#taskmanager.network.numberOfBuffers}}taskmanager.network.numberOfBuffers: {{taskmanager.network.numberOfBuffers}}{{/taskmanager.network.numberOfBuffers}}{{^taskmanager.network.numberOfBuffers}}# taskmanager.network.numberOfBuffers: 2048{{/taskmanager.network.numberOfBuffers}}

# The size of the network buffers, in bytes.
#
{{#taskmanager.network.bufferSizeInBytes}}taskmanager.network.bufferSizeInBytes: {{taskmanager.network.bufferSizeInBytes}}{{/taskmanager.network.bufferSizeInBytes}}{{^taskmanager.network.bufferSizeInBytes}}# taskmanager.network.bufferSizeInBytes: 32768{{/taskmanager.network.bufferSizeInBytes}}

# The amount of memory (in megabytes) that the task manager reserves on the
# JVM’s heap space for sorting, hash tables, and caching of intermediate
# results. If unspecified (-1), the memory manager will take a fixed ratio of
# the heap memory available to the JVM, as specified by
# taskmanager.memory.fraction.
#
{{#taskmanager.memory.size}}taskmanager.memory.size: {{taskmanager.memory.size}}{{/taskmanager.memory.size}}{{^taskmanager.memory.size}}# taskmanager.memory.size: -1{{/taskmanager.memory.size}}

# The relative amount of memory that the task manager reserves for sorting,
# hash tables, and caching of intermediate results.
#
{{#taskmanager.memory.fraction}}taskmanager.memory.fraction: {{taskmanager.memory.fraction}}{{/taskmanager.memory.fraction}}{{^taskmanager.memory.fraction}}# taskmanager.memory.fraction: 0.7{{/taskmanager.memory.fraction}}

# The interval (in seconds) in which the client polls the JobManager for the
# status of its job (DEFAULT: 2).
#
{{#jobclient.polling.interval}}jobclient.polling.interval: {{jobclient.polling.interval}}{{/jobclient.polling.interval}}{{^jobclient.polling.interval}}# jobclient.polling.interval: 2{{/jobclient.polling.interval}}

# The maximal fan-in for external merge joins and fan-out for spilling hash
# tables. Limits the number of file handles per operator, but may cause
# intermediate merging/partitioning, if set too small.
#
{{#taskmanager.runtime.max-fan}}taskmanager.runtime.max-fan: {{taskmanager.runtime.max-fan}}{{/taskmanager.runtime.max-fan}}{{^taskmanager.runtime.max-fan}}# taskmanager.runtime.max-fan: 128{{/taskmanager.runtime.max-fan}}

# A sort operation starts spilling when this fraction of its memory budget is
# full.
#
{{#taskmanager.runtime.sort-spilling-threshold}}taskmanager.runtime.sort-spilling-threshold: {{taskmanager.runtime.sort-spilling-threshold}}{{/taskmanager.runtime.sort-spilling-threshold}}{{^taskmanager.runtime.sort-spilling-threshold}}# taskmanager.runtime.sort-spilling-threshold: 0.8{{/taskmanager.runtime.sort-spilling-threshold}}

# The interval in which the TaskManager sends heartbeats to the JobManager.
#
{{#taskmanager.heartbeat-interval}}taskmanager.heartbeat-interval: {{taskmanager.heartbeat-interval}}{{/taskmanager.heartbeat-interval}}{{^taskmanager.heartbeat-interval}}# taskmanager.heartbeat-interval: 5000{{/taskmanager.heartbeat-interval}}

# The maximum time that a TaskManager hearbeat may be missing before the
# TaskManager is considered failed.
#
{{#jobmanager.max-heartbeat-delay-before-failure.msecs}}jobmanager.max-heartbeat-delay-before-failure.msecs: {{jobmanager.max-heartbeat-delay-before-failure.msecs}}{{/jobmanager.max-heartbeat-delay-before-failure.msecs}}{{^jobmanager.max-heartbeat-delay-before-failure.msecs}}# jobmanager.max-heartbeat-delay-before-failure.msecs: 30000{{/jobmanager.max-heartbeat-delay-before-failure.msecs}}

# Start a thread that periodically logs the memory usage in the TaskManager.
#
{{#taskmanager.debug.memory.startLogThread}}taskmanager.debug.memory.startLogThread: {{taskmanager.debug.memory.startLogThread}}{{/taskmanager.debug.memory.startLogThread}}{{^taskmanager.debug.memory.startLogThread}}# taskmanager.debug.memory.startLogThread: false{{/taskmanager.debug.memory.startLogThread}}

# The reporting interval for logging memory usage in the TaskManager.
#
{{#taskmanager.debug.memory.logIntervalMs}}taskmanager.debug.memory.logIntervalMs: {{taskmanager.debug.memory.logIntervalMs}}{{/taskmanager.debug.memory.logIntervalMs}}{{^taskmanager.debug.memory.logIntervalMs}}# taskmanager.debug.memory.logIntervalMs: 5000L{{/taskmanager.debug.memory.logIntervalMs}}

#==============================================================================
# BLOB / Cache
#==============================================================================

# The storage directory to be used by the BLOB server.
#
{{#blob.storage.directory}}blob.storage.directory: {{blob.storage.directory}}{{/blob.storage.directory}}{{^blob.storage.directory}}# blob.storage.directory: /tmp{{/blob.storage.directory}}

# The number of retires for failed BLOB fetches.
#
{{#blob.fetch.retries}}blob.fetch.retries: {{blob.fetch.retries}}{{/blob.fetch.retries}}{{^blob.fetch.retries}}# blob.fetch.retries: 5{{/blob.fetch.retries}}

# The maximum number of concurrent BLOB fetches that the JobManager serves.
#
{{#blob.fetch.num-concurrent}}blob.fetch.num-concurrent: {{blob.fetch.num-concurrent}}{{/blob.fetch.num-concurrent}}{{^blob.fetch.num-concurrent}}# blob.fetch.num-concurrent: 50{{/blob.fetch.num-concurrent}}

# The backlog of BLOB fetches on the JobManager.
#
{{#blob.fetch.backlog}}blob.fetch.backlog: {{blob.fetch.backlog}}{{/blob.fetch.backlog}}{{^blob.fetch.backlog}}# blob.fetch.backlog: 1000{{/blob.fetch.backlog}}

# The cleanup interval of the library cache manager.
#
{{#library-cache-manager.cleanup.interval}}library-cache-manager.cleanup.interval: {{library-cache-manager.cleanup.interval}}{{/library-cache-manager.cleanup.interval}}{{^library-cache-manager.cleanup.interval}}# library-cache-manager.cleanup.interval: 3600{{/library-cache-manager.cleanup.interval}}

#==============================================================================
# JobManager Web Frontend
#==============================================================================

# Port of the JobManager’s web interface that displays status of running jobs
# and execution time breakdowns of finished jobs.
#
{{#jobmanager.web.port}}jobmanager.web.port: {{jobmanager.web.port}}{{/jobmanager.web.port}}{{^jobmanager.web.port}}# jobmanager.web.port: 8081{{/jobmanager.web.port}}

# The number of latest jobs that the JobManager’s web front-end in its history.
#
{{#jobmanager.web.history}}jobmanager.web.history: {{jobmanager.web.history}}{{/jobmanager.web.history}}{{^jobmanager.web.history}}# jobmanager.web.history: 5{{/jobmanager.web.history}}

#==============================================================================
# Webclient
#==============================================================================

# The port of the webclient server.
#
{{#webclient.port}}webclient.port: {{webclient.port}}{{/webclient.port}}{{^webclient.port}}# webclient.port: 8080{{/webclient.port}}

# The temp directory for the web server. Used for example for caching file
# fragments during file-uploads.
#
{{#webclient.tempdir}}webclient.tempdir: {{webclient.tempdir}}{{/webclient.tempdir}}{{^webclient.tempdir}}# webclient.tempdir: ${java.io.tmpdir}{{/webclient.tempdir}}

# The directory into which the web server will store uploaded programs.
#
{{#webclient.uploaddir}}webclient.uploaddir: {{webclient.uploaddir}}{{/webclient.uploaddir}}{{^webclient.uploaddir}}# webclient.uploaddir: ${webclient.tempdir}/webclient-jobs/{{/webclient.uploaddir}}

# The directory into which the web server will dump temporary JSON files
# describing the execution plans.
#
{{#webclient.plandump}}webclient.plandump: {{webclient.plandump}}{{/webclient.plandump}}{{^webclient.plandump}}# webclient.plandump: ${webclient.tempdir}/webclient-plans/{{/webclient.plandump}}

#==============================================================================
# HDFS
#==============================================================================

# Path to the Hadoop configuration directory.
#
# This configuration is used when writing into HDFS. Unless specified otherwise,
# HDFS file creation will use HDFS default settings with respect to block-size,
# replication factor, etc.
#
# You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml
# via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'.
#
{{#fs.hdfs.hadoopconf}}fs.hdfs.hadoopconf: {{fs.hdfs.hadoopconf}}{{/fs.hdfs.hadoopconf}}{{^fs.hdfs.hadoopconf}}# fs.hdfs.hadoopconf: /path/to/hadoop/conf/{{/fs.hdfs.hadoopconf}}


# The absolute path of Hadoop’s own configuration file “hdfs-default.xml”.
#
{{#fs.hdfs.hdfsdefault}}fs.hdfs.hdfsdefault: {{fs.hdfs.hdfsdefault}}{{/fs.hdfs.hdfsdefault}}{{^fs.hdfs.hdfsdefault}}# fs.hdfs.hdfsdefault: null{{/fs.hdfs.hdfsdefault}}

# The absolute path of Hadoop’s own configuration file “hdfs-site.xml”.
#
{{#fs.hdfs.hdfssite}}fs.hdfs.hdfssite: {{fs.hdfs.hdfssite}}{{/fs.hdfs.hdfssite}}{{^fs.hdfs.hdfssite}}# fs.hdfs.hdfssite: null{{/fs.hdfs.hdfssite}}

#==============================================================================
# File Systems
#==============================================================================

# Specifies whether file output writers should overwrite existing files by
# default.
#
{{#fs.overwrite-files}}fs.overwrite-files: {{fs.overwrite-files}}{{/fs.overwrite-files}}{{^fs.overwrite-files}}# fs.overwrite-files: false{{/fs.overwrite-files}}

# File writers running with a parallelism larger than one create a directory
# for the output file path and put the different result files (one per parallel
# writer task) into that directory. If this option is set to true, writers with
# a parallelism of 1 will also create a directory and place a single result
# file into it.
#
{{#fs.output.always-create-directory}}fs.output.always-create-directory: {{fs.output.always-create-directory}}{{/fs.output.always-create-directory}}{{^fs.output.always-create-directory}}# fs.output.always-create-directory: false{{/fs.output.always-create-directory}}

#==============================================================================
# Akka
#==============================================================================

# Timeout used for all futures and blocking Akka calls. If Flink fails due to timeouts then you should try to increase this value. Timeouts can be caused by slow machines or a congested network.
# The timeout value requires a time-unit specifier (ms/s/min/h/d) (DEFAULT: 10 s).
#
{{#akka.ask.timeout}}akka.ask.timeout: {{akka.ask.timeout}}{{/akka.ask.timeout}}{{^akka.ask.timeout}}# akka.ask.timeout: 10s{{/akka.ask.timeout}}

# Timeout used for the lookup of the JobManager.
# The timeout value has to contain a time-unit specifier (ms/s/min/h/d) (DEFAULT: 10 s).
#
{{#akka.lookup.timeout}}akka.lookup.timeout: {{akka.lookup.timeout}}{{/akka.lookup.timeout}}{{^akka.lookup.timeout}}# akka.lookup.timeout: 10s{{/akka.lookup.timeout}}

# Maximum size of messages which are sent between the JobManager and the TaskManagers. If Flink fails because messages exceed this limit, then you should increase it.
# The message size requires a size-unit specifier (DEFAULT: 10485760b).
#
{{#akka.framesize}}akka.framesize: {{akka.framesize}}{{/akka.framesize}}{{^akka.framesize}}# akka.framesize: 10485760b{{/akka.framesize}}

# Heartbeat interval for Akka’s DeathWatch mechanism to detect dead TaskManagers. If TaskManagers are wrongly marked dead because of lost or delayed heartbeat messages, then you should increase this value.
#
{{#akka.watch.heartbeat.interval}}akka.watch.heartbeat.interval: {{akka.watch.heartbeat.interval}}{{/akka.watch.heartbeat.interval}}{{^akka.watch.heartbeat.interval}}# akka.watch.heartbeat.interval: akka.ask.timeout/10{{/akka.watch.heartbeat.interval}}

# Acceptable heartbeat pause for Akka’s DeathWatch mechanism. A low value does not allow a irregular heartbeat.
#
{{#akka.watch.heartbeat.pause}}akka.watch.heartbeat.pause: {{akka.watch.heartbeat.pause}}{{/akka.watch.heartbeat.pause}}{{^akka.watch.heartbeat.pause}}# akka.watch.heartbeat.pause: akka.ask.timeout{{/akka.watch.heartbeat.pause}}

# Threshold for the DeathWatch failure detector. A low value is prone to false positives whereas a high value increases the time to detect a dead TaskManager.
#
{{#akka.watch.threshold}}akka.watch.threshold: {{akka.watch.threshold}}{{/akka.watch.threshold}}{{^akka.watch.threshold}}# akka.watch.threshold: 12{{/akka.watch.threshold}}

# Heartbeat interval for Akka’s transport failure detector. Since Flink uses TCP, the detector is not necessary. Therefore, the detector is disabled by setting the interval to a very high value.
# In case you should need the transport failure detector, set the interval to some reasonable value. The interval value requires a time-unit specifier (ms/s/min/h/d) (DEFAULT: 1000 s).
#
{{#akka.transport.heartbeat.interval}}akka.transport.heartbeat.interval: {{akka.transport.heartbeat.interval}}{{/akka.transport.heartbeat.interval}}{{^akka.transport.heartbeat.interval}}# akka.transport.heartbeat.interval: 1000s{{/akka.transport.heartbeat.interval}}

# Acceptable heartbeat pause for Akka’s transport failure detector. Since Flink uses TCP, the detector is not necessary. Therefore, the detector is disabled by setting the pause to a very high value.
# In case you should need the transport failure detector, set the pause to some reasonable value. The pause value requires a time-unit specifier (ms/s/min/h/d) (DEFAULT: 6000 s).
#
{{#akka.transport.heartbeat.pause}}akka.transport.heartbeat.pause: {{akka.transport.heartbeat.pause}}{{/akka.transport.heartbeat.pause}}{{^akka.transport.heartbeat.pause}}# akka.transport.heartbeat.pause: 6000s{{/akka.transport.heartbeat.pause}}

# Threshold for the transport failure detector. Since Flink uses TCP, the detector is not necessary and, thus, the threshold is set to a high value (DEFAULT: 300).
#
{{#akka.transport.threshold}}akka.transport.threshold: {{akka.transport.threshold}}{{/akka.transport.threshold}}{{^akka.transport.threshold}}# akka.transport.threshold: 300s{{/akka.transport.threshold}}

# Timeout for all outbound connections. If you should experience problems with connecting to a TaskManager due to a slow network, you should increase this value (DEFAULT: akka.ask.timeout).
#
{{#akka.tcp.timeout}}akka.tcp.timeout: {{akka.tcp.timeout}}{{/akka.tcp.timeout}}{{^akka.tcp.timeout}}# akka.tcp.timeout: akka.ask.timeout{{/akka.tcp.timeout}}

# Number of messages that are processed in a batch before returning the thread to the pool. Low values denote a fair scheduling whereas high values can increase the performance at the cost of unfairness (DEFAULT: 15).
#
{{#akka.throughput}}akka.throughput: {{akka.throughput}}{{/akka.throughput}}{{^akka.throughput}}# akka.throughput: 15{{/akka.throughput}}

# Turns on the Akka’s remote logging of events. Set this value to ‘on’ in case of debugging (DEFAULT: off).
#
{{#akka.log.lifecycle.events}}akka.log.lifecycle.events: {{akka.log.lifecycle.events}}{{/akka.log.lifecycle.events}}{{^akka.log.lifecycle.events}}# akka.log.lifecycle.events: off{{/akka.log.lifecycle.events}}

# Timeout after which the startup of a remote component is considered being failed (DEFAULT: akka.ask.timeout).
#
{{#akka.startup-timeout}}akka.startup-timeout: {{akka.startup-timeout}}{{/akka.startup-timeout}}{{^akka.startup-timeout}}# akka.startup-timeout: akka.ask.timeout{{/akka.startup-timeout}}