org.apache.flink.runtime.healthmanager.plugins.detectors.RpsUnsatisfiedDetector Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.healthmanager.plugins.detectors;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.healthmanager.HealthMonitor;
import org.apache.flink.runtime.healthmanager.metrics.MetricAggType;
import org.apache.flink.runtime.healthmanager.metrics.TaskMetricSubscription;
import org.apache.flink.runtime.healthmanager.metrics.timeline.TimelineAggType;
import org.apache.flink.runtime.healthmanager.plugins.Detector;
import org.apache.flink.runtime.healthmanager.plugins.symptoms.JobVertexRpsUnsatisfied;
import org.apache.flink.runtime.healthmanager.plugins.utils.HealthMonitorOptions;
import org.apache.flink.runtime.healthmanager.plugins.utils.MetricNames;
import org.apache.flink.runtime.healthmanager.plugins.utils.MetricUtils;
import org.apache.flink.runtime.healthmanager.plugins.utils.TaskMetrics;
import org.apache.flink.runtime.healthmanager.plugins.utils.TaskMetricsSubscriber;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Detectors which check actual rps lower than target rps.
*/
public class RpsUnsatisfiedDetector implements Detector {
private static final Logger LOGGER = LoggerFactory.getLogger(RpsUnsatisfiedDetector.class);
private TaskMetricsSubscriber subscriber;
private HealthMonitor monitor;
private boolean enabled;
private long interval;
private Map sourceTargetRps;
private Map> allRpsSubscriptions;
@Override
public void open(HealthMonitor monitor) {
this.interval = monitor.getConfig().getLong(HealthMonitorOptions.PARALLELISM_SCALE_INTERVAL);
this.subscriber = monitor.subscribeTaskMetrics(interval);
this.monitor = monitor;
if (monitor.getConfig().contains(HealthMonitorOptions.TARGET_TPS_VALUES)) {
this.enabled = true;
this.sourceTargetRps = new HashMap<>();
List values =
Arrays.stream(monitor.getConfig().getString(HealthMonitorOptions.TARGET_TPS_VALUES).split(","))
.map(e -> Double.valueOf(e.trim())).collect(Collectors.toList());
List ids;
if (monitor.getConfig().contains(HealthMonitorOptions.TARGET_TPS_IDS)) {
ids =
Arrays.stream(monitor.getConfig().getString(HealthMonitorOptions.TARGET_TPS_IDS).split(","))
.map(e -> Integer.valueOf(e.trim()))
.map(e ->
monitor.getJobConfig().getVertexConfigs().entrySet().stream()
.filter(config -> config.getValue().getOperatorIds().contains(e))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not find operator with id:" + e))
.getKey())
.collect(Collectors.toList());
} else if (monitor.getConfig().contains(HealthMonitorOptions.TARGET_TPS_NAMES)) {
ids =
Arrays.stream(monitor.getConfig().getString(HealthMonitorOptions.TARGET_TPS_NAMES).split(","))
.map(e -> getVertexIdBySourceName(e.trim()))
.collect(Collectors.toList());
} else {
this.enabled = false;
throw new IllegalArgumentException("Config of target source names or ids missing");
}
Preconditions.checkArgument(values.size() == ids.size());
Iterator valueIterator = values.iterator();
ids.stream().forEach(id -> sourceTargetRps.put(id, valueIterator.next()));
} else {
this.enabled = false;
}
if (this.enabled) {
Set subDagRoots = monitor.getJobTopologyAnalyzer().getAllSubDagRoots();
allRpsSubscriptions = new HashMap<>();
// subscribe rps metrics.
for (JobVertexID rootId : subDagRoots) {
if (sourceTargetRps.containsKey(rootId)) {
allRpsSubscriptions.put(rootId, new LinkedList<>());
for (JobVertexID vertexID : monitor.getJobTopologyAnalyzer().getSubDagVertices(rootId)) {
TaskMetricSubscription subscription = monitor.getMetricProvider().subscribeTaskMetric(
monitor.getJobID(),
vertexID,
MetricNames.PARSER_TPS,
MetricAggType.SUM,
interval,
TimelineAggType.AVG);
allRpsSubscriptions.get(rootId).add(subscription);
}
}
}
}
}
@Override
public void close() {
if (allRpsSubscriptions != null) {
for (List subscriptionList : allRpsSubscriptions.values()) {
for (TaskMetricSubscription subscription : subscriptionList) {
this.monitor.getMetricProvider().unsubscribe(subscription);
}
}
}
}
@Override
public JobVertexRpsUnsatisfied detect() throws Exception {
LOGGER.debug("Start detecting.");
if (!enabled) {
LOGGER.debug("Skip because detector disabled.");
return null;
}
Map sourceCurrentRps = getCurrentRPS();
if (sourceCurrentRps == null) {
LOGGER.debug("Skip because source rps missing.");
return null;
}
Map allTaskMetrics = subscriber.getTaskMetrics();
if (allTaskMetrics == null) {
LOGGER.debug("Skip because task metrics missing.");
return null;
}
LOGGER.debug("current rps:" + sourceCurrentRps);
LOGGER.debug("target rps:" + sourceTargetRps);
JobVertexRpsUnsatisfied jobVertexRpsUnsatisfied = new JobVertexRpsUnsatisfied();
for (JobVertexID vertexID : sourceCurrentRps.keySet()) {
double targetRps = sourceTargetRps.get(vertexID);
double currentRps = sourceCurrentRps.get(vertexID);
if (targetRps > currentRps) {
jobVertexRpsUnsatisfied.addVertex(vertexID, targetRps, currentRps);
}
}
if (jobVertexRpsUnsatisfied.getCurrentRps().isEmpty()) {
return null;
}
return jobVertexRpsUnsatisfied;
}
public Map getCurrentRPS() {
if (allRpsSubscriptions == null) {
return null;
}
Map allRps = new HashMap<>();
for (Map.Entry> entry : allRpsSubscriptions.entrySet()) {
Tuple2 rps = null;
for (TaskMetricSubscription subscription : entry.getValue()) {
if (MetricUtils.validateTaskMetric(monitor, interval * 2, subscription) && subscription.getValue().f1 > 0) {
rps = subscription.getValue();
break;
}
}
if (rps != null) {
allRps.put(entry.getKey(), rps.f1);
} else {
LOGGER.debug("{} rps metrics missing", entry.getKey());
return null;
}
}
return allRps;
}
/**
* Get the source vertex id by sourceName from vertexProfiles. Throws exception if unfound.
*
* @param tableName source table name that user specified in Blink SQL
* @return id of the source vertex that contains the table name
* @throws IllegalArgumentException if no source vertex has the table name
*/
public JobVertexID getVertexIdBySourceName(String tableName) {
// TODO: add more generic name matching to find the source for given name.
// Matching pattern: "Source: -tableName -> ...."
String filter = String.format("-%s-Stream ", tableName);
// Matching pattern: "Source: -tableName"
String filter2 = String.format("-%s-Stream", tableName);
return monitor.getJobConfig().getVertexConfigs().entrySet().stream()
.filter(e -> monitor.getJobTopologyAnalyzer().isSource(e.getKey()))
.filter(e -> e.getValue().getName().contains(filter) || e.getValue().getName().contains(filter2))
.findFirst()
.map(e -> e.getKey())
.orElseThrow(() -> new IllegalArgumentException(
String.format("Cannot find corresponding vertex of source '%s'. Please enter a valid source name",
tableName)
));
}
}