org.apache.flink.table.client.config.entries.ExecutionEntry Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.table.client.config.entries;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.table.client.config.ConfigUtil;
import org.apache.flink.table.client.config.Environment;
import org.apache.flink.table.descriptors.DescriptorProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import static org.apache.flink.table.client.config.Environment.EXECUTION_ENTRY;
/**
* Configuration of a table program execution. This class parses the `execution` part
* in an environment file. The execution describes properties that would usually be defined in the
* ExecutionEnvironment/StreamExecutionEnvironment/TableEnvironment or as code in a Flink job.
*
* All properties of this entry are optional and evaluated lazily.
*/
public class ExecutionEntry extends ConfigEntry {
private static final Logger LOG = LoggerFactory.getLogger(ExecutionEntry.class);
public static final ExecutionEntry DEFAULT_INSTANCE =
new ExecutionEntry(new DescriptorProperties(true));
private static final String EXECUTION_TYPE = "type";
private static final String EXECUTION_TYPE_VALUE_STREAMING = "streaming";
private static final String EXECUTION_TYPE_VALUE_BATCH = "batch";
private static final String EXECUTION_TIME_CHARACTERISTIC = "time-characteristic";
private static final String EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME = "event-time";
private static final String EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME = "processing-time";
private static final String EXECUTION_PERIODIC_WATERMARKS_INTERVAL = "periodic-watermarks-interval";
private static final String EXECUTION_MIN_STATE_RETENTION = "min-idle-state-retention";
private static final String EXECUTION_MAX_STATE_RETENTION = "max-idle-state-retention";
private static final String EXECUTION_PARALLELISM = "parallelism";
private static final String EXECUTION_MAX_PARALLELISM = "max-parallelism";
private static final String EXECUTION_RESULT_MODE = "result-mode";
private static final String EXECUTION_RESULT_MODE_VALUE_CHANGELOG = "changelog";
private static final String EXECUTION_RESULT_MODE_VALUE_TABLE = "table";
private static final String EXECUTION_MAX_TABLE_RESULT_ROWS = "max-table-result-rows";
private static final String EXECUTION_RESTART_STRATEGY_TYPE = "restart-strategy.type";
private static final String EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FALLBACK = "fallback";
private static final String EXECUTION_RESTART_STRATEGY_TYPE_VALUE_NONE = "none";
private static final String EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FIXED_DELAY = "fixed-delay";
private static final String EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FAILURE_RATE = "failure-rate";
private static final String EXECUTION_RESTART_STRATEGY_ATTEMPTS = "restart-strategy.attempts";
private static final String EXECUTION_RESTART_STRATEGY_DELAY = "restart-strategy.delay";
private static final String EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL = "restart-strategy.failure-rate-interval";
private static final String EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL = "restart-strategy.max-failures-per-interval";
private static final String EXECUTION_VIEW_MODE = "view-mode";
private static final String EXECUTION_VIEW_MODE_VALUE_INTERACTIVE = "interactive";
private static final String EXECUTION_VIEW_MODE_VALUE_NON_INTERACTIVE = "non-interactive";
private static final String EXECUTION_MAX_QUERY_TIMEOUT_MS = "max-query-timeout-ms";
private ExecutionEntry(DescriptorProperties properties) {
super(properties);
}
@Override
protected void validate(DescriptorProperties properties) {
properties.validateEnumValues(
EXECUTION_TYPE,
true,
Arrays.asList(
EXECUTION_TYPE_VALUE_BATCH,
EXECUTION_TYPE_VALUE_STREAMING));
properties.validateEnumValues(
EXECUTION_TIME_CHARACTERISTIC,
true,
Arrays.asList(
EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME,
EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME));
properties.validateLong(EXECUTION_PERIODIC_WATERMARKS_INTERVAL, true, 1);
properties.validateLong(EXECUTION_MIN_STATE_RETENTION, true, 0);
properties.validateLong(EXECUTION_MAX_STATE_RETENTION, true, 0);
properties.validateInt(EXECUTION_PARALLELISM, true, 1);
properties.validateInt(EXECUTION_MAX_PARALLELISM, true, 1);
properties.validateInt(EXECUTION_MAX_TABLE_RESULT_ROWS, true, 1);
properties.validateEnumValues(
EXECUTION_RESTART_STRATEGY_TYPE,
true,
Arrays.asList(
EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FALLBACK,
EXECUTION_RESTART_STRATEGY_TYPE_VALUE_NONE,
EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FIXED_DELAY,
EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FAILURE_RATE));
properties.validateInt(EXECUTION_RESTART_STRATEGY_ATTEMPTS, true, 1);
properties.validateLong(EXECUTION_RESTART_STRATEGY_DELAY, true, 0);
properties.validateLong(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL, true, 1);
properties.validateInt(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL, true, 1);
properties.validateEnumValues(
EXECUTION_VIEW_MODE,
true,
Arrays.asList(
EXECUTION_VIEW_MODE_VALUE_INTERACTIVE,
EXECUTION_VIEW_MODE_VALUE_NON_INTERACTIVE));
properties.validateInt(EXECUTION_MAX_QUERY_TIMEOUT_MS, true, 1);
}
public boolean isStreamingExecution() {
return properties.getOptionalString(EXECUTION_TYPE)
.map((v) -> v.equals(EXECUTION_TYPE_VALUE_STREAMING))
.orElse(false);
}
public boolean isBatchExecution() {
return properties.getOptionalString(EXECUTION_TYPE)
.map((v) -> v.equals(EXECUTION_TYPE_VALUE_BATCH))
.orElse(false);
}
public TimeCharacteristic getTimeCharacteristic() {
return properties.getOptionalString(EXECUTION_TIME_CHARACTERISTIC)
.flatMap((v) -> {
switch (v) {
case EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME:
return Optional.of(TimeCharacteristic.EventTime);
case EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME:
return Optional.of(TimeCharacteristic.ProcessingTime);
default:
return Optional.empty();
}
})
.orElseGet(() ->
useDefaultValue(
EXECUTION_TIME_CHARACTERISTIC,
TimeCharacteristic.EventTime,
EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME));
}
public long getPeriodicWatermarksInterval() {
return properties.getOptionalLong(EXECUTION_PERIODIC_WATERMARKS_INTERVAL)
.orElseGet(() -> useDefaultValue(EXECUTION_PERIODIC_WATERMARKS_INTERVAL, 200L));
}
public long getMinStateRetention() {
return properties.getOptionalLong(EXECUTION_MIN_STATE_RETENTION)
.orElseGet(() -> useDefaultValue(EXECUTION_MIN_STATE_RETENTION, 0L));
}
public long getMaxStateRetention() {
return properties.getOptionalLong(EXECUTION_MAX_STATE_RETENTION)
.orElseGet(() -> useDefaultValue(EXECUTION_MAX_STATE_RETENTION, 0L));
}
public int getParallelism() {
return properties.getOptionalInt(EXECUTION_PARALLELISM)
.orElseGet(() -> useDefaultValue(EXECUTION_PARALLELISM, 1));
}
public int getMaxParallelism() {
return properties.getOptionalInt(EXECUTION_MAX_PARALLELISM)
.orElseGet(() -> useDefaultValue(EXECUTION_MAX_PARALLELISM, 128));
}
public int getMaxTableResultRows() {
return properties.getOptionalInt(EXECUTION_MAX_TABLE_RESULT_ROWS)
.orElseGet(() -> useDefaultValue(EXECUTION_MAX_TABLE_RESULT_ROWS, 1_000_000));
}
public int getMaxQueryTimeoutMs() {
return properties.getOptionalInt(EXECUTION_MAX_QUERY_TIMEOUT_MS)
.orElseGet(() -> useDefaultValue(EXECUTION_MAX_QUERY_TIMEOUT_MS, 1000 * 60 * 10)); // 10 minutes
}
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
return properties.getOptionalString(EXECUTION_RESTART_STRATEGY_TYPE)
.flatMap((v) -> {
switch (v) {
case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_NONE:
return Optional.of(RestartStrategies.noRestart());
case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FIXED_DELAY:
final int attempts = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_ATTEMPTS)
.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_ATTEMPTS, Integer.MAX_VALUE));
final long fixedDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
return Optional.of(RestartStrategies.fixedDelayRestart(attempts, fixedDelay));
case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FAILURE_RATE:
final int failureRate = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL)
.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL, 1));
final long failureInterval = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL)
.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL, 60_000L));
final long attemptDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
return Optional.of(RestartStrategies.failureRateRestart(
failureRate,
Time.milliseconds(failureInterval),
Time.milliseconds(attemptDelay)));
default:
return Optional.empty();
}
})
.orElseGet(() ->
useDefaultValue(
EXECUTION_RESTART_STRATEGY_TYPE,
RestartStrategies.fallBackRestart(),
EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FALLBACK));
}
public boolean isChangelogMode() {
return properties.getOptionalString(EXECUTION_RESULT_MODE)
.map((v) -> v.equals(EXECUTION_RESULT_MODE_VALUE_CHANGELOG))
.orElse(false);
}
public boolean isTableMode() {
return properties.getOptionalString(EXECUTION_RESULT_MODE)
.map((v) -> v.equals(EXECUTION_RESULT_MODE_VALUE_TABLE))
.orElse(false);
}
public boolean isInteractiveViewMode() {
return properties.getOptionalString(EXECUTION_VIEW_MODE)
.map((v) -> v.equals(EXECUTION_VIEW_MODE_VALUE_INTERACTIVE))
.orElse(false);
}
public boolean isNonInteractiveViewMode() {
return properties.getOptionalString(EXECUTION_VIEW_MODE)
.map((v) -> v.equals(EXECUTION_VIEW_MODE_VALUE_NON_INTERACTIVE))
.orElse(false);
}
public Map asTopLevelMap() {
return properties.asPrefixedMap(EXECUTION_ENTRY + '.');
}
private V useDefaultValue(String key, V defaultValue) {
return useDefaultValue(key, defaultValue, defaultValue.toString());
}
private V useDefaultValue(String key, V defaultValue, String defaultString) {
LOG.info("Property '{}.{}' not specified. Using default value: {}", EXECUTION_ENTRY, key, defaultString);
return defaultValue;
}
// --------------------------------------------------------------------------------------------
public static ExecutionEntry create(Map config) {
return new ExecutionEntry(ConfigUtil.normalizeYaml(config));
}
/**
* Merges two execution entries. The properties of the first execution entry might be
* overwritten by the second one.
*/
public static ExecutionEntry merge(ExecutionEntry execution1, ExecutionEntry execution2) {
final Map mergedProperties = new HashMap<>(execution1.asMap());
mergedProperties.putAll(execution2.asMap());
final DescriptorProperties properties = new DescriptorProperties(true);
properties.putProperties(mergedProperties);
return new ExecutionEntry(properties);
}
/**
* Creates a new execution entry enriched with additional properties that are prefixed with
* {@link Environment#EXECUTION_ENTRY}.
*/
public static ExecutionEntry enrich(ExecutionEntry execution, Map prefixedProperties) {
final Map enrichedProperties = new HashMap<>(execution.asMap());
prefixedProperties.forEach((k, v) -> {
final String normalizedKey = k.toLowerCase();
if (k.startsWith(EXECUTION_ENTRY + '.')) {
enrichedProperties.put(normalizedKey.substring(EXECUTION_ENTRY.length() + 1), v);
}
});
final DescriptorProperties properties = new DescriptorProperties(true);
properties.putProperties(enrichedProperties);
return new ExecutionEntry(properties);
}
}