org.apache.flink.runtime.io.network.metrics.NettyShuffleMetricFactory Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.io.network.metrics;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.io.network.partition.ResultPartition;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
import java.util.Arrays;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Factory for netty shuffle service metrics.
*/
public class NettyShuffleMetricFactory {
// deprecated metric groups
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
private static final String METRIC_GROUP_NETWORK_DEPRECATED = "Network";
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
private static final String METRIC_GROUP_BUFFERS_DEPRECATED = "buffers";
// shuffle environment level metrics: Shuffle.Netty.*
private static final String METRIC_TOTAL_MEMORY_SEGMENT = "TotalMemorySegments";
private static final String METRIC_AVAILABLE_MEMORY_SEGMENT = "AvailableMemorySegments";
// task level metric group structure: Shuffle.Netty..Buffers
private static final String METRIC_GROUP_SHUFFLE = "Shuffle";
private static final String METRIC_GROUP_NETTY = "Netty";
public static final String METRIC_GROUP_OUTPUT = "Output";
public static final String METRIC_GROUP_INPUT = "Input";
private static final String METRIC_GROUP_BUFFERS = "Buffers";
// task level output metrics: Shuffle.Netty.Output.*
private static final String METRIC_OUTPUT_QUEUE_LENGTH = "outputQueueLength";
private static final String METRIC_OUTPUT_POOL_USAGE = "outPoolUsage";
// task level input metrics: Shuffle.Netty.Input.*
private static final String METRIC_INPUT_QUEUE_LENGTH = "inputQueueLength";
private static final String METRIC_INPUT_POOL_USAGE = "inPoolUsage";
private static final String METRIC_INPUT_FLOATING_BUFFERS_USAGE = "inputFloatingBuffersUsage";
private static final String METRIC_INPUT_EXCLUSIVE_BUFFERS_USAGE = "inputExclusiveBuffersUsage";
private NettyShuffleMetricFactory() {
}
public static void registerShuffleMetrics(
MetricGroup metricGroup,
NetworkBufferPool networkBufferPool) {
checkNotNull(metricGroup);
checkNotNull(networkBufferPool);
//noinspection deprecation
registerShuffleMetrics(METRIC_GROUP_NETWORK_DEPRECATED, metricGroup, networkBufferPool);
registerShuffleMetrics(METRIC_GROUP_NETTY, metricGroup.addGroup(METRIC_GROUP_SHUFFLE), networkBufferPool);
}
private static void registerShuffleMetrics(
String groupName,
MetricGroup metricGroup,
NetworkBufferPool networkBufferPool) {
MetricGroup networkGroup = metricGroup.addGroup(groupName);
networkGroup.>gauge(METRIC_TOTAL_MEMORY_SEGMENT,
networkBufferPool::getTotalNumberOfMemorySegments);
networkGroup.>gauge(METRIC_AVAILABLE_MEMORY_SEGMENT,
networkBufferPool::getNumberOfAvailableMemorySegments);
}
public static MetricGroup createShuffleIOOwnerMetricGroup(MetricGroup parentGroup) {
return parentGroup.addGroup(METRIC_GROUP_SHUFFLE).addGroup(METRIC_GROUP_NETTY);
}
/**
* Registers legacy network metric groups before shuffle service refactoring.
*
* Registers legacy metric groups if shuffle service implementation is original default one.
*
* @deprecated should be removed in future
*/
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
public static void registerLegacyNetworkMetrics(
boolean isDetailedMetrics,
boolean isCreditBased,
MetricGroup metricGroup,
ResultPartitionWriter[] producedPartitions,
InputGate[] inputGates) {
checkNotNull(metricGroup);
checkNotNull(producedPartitions);
checkNotNull(inputGates);
// add metrics for buffers
final MetricGroup buffersGroup = metricGroup.addGroup(METRIC_GROUP_BUFFERS_DEPRECATED);
// similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup (metricGroup)
final MetricGroup networkGroup = metricGroup.addGroup(METRIC_GROUP_NETWORK_DEPRECATED);
final MetricGroup outputGroup = networkGroup.addGroup(METRIC_GROUP_OUTPUT);
final MetricGroup inputGroup = networkGroup.addGroup(METRIC_GROUP_INPUT);
ResultPartition[] resultPartitions = Arrays.copyOf(producedPartitions, producedPartitions.length, ResultPartition[].class);
registerOutputMetrics(isDetailedMetrics, outputGroup, buffersGroup, resultPartitions);
SingleInputGate[] singleInputGates = Arrays.copyOf(inputGates, inputGates.length, SingleInputGate[].class);
registerInputMetrics(isDetailedMetrics, isCreditBased, inputGroup, buffersGroup, singleInputGates);
}
public static void registerOutputMetrics(
boolean isDetailedMetrics,
MetricGroup outputGroup,
ResultPartition[] resultPartitions) {
registerOutputMetrics(
isDetailedMetrics,
outputGroup,
outputGroup.addGroup(METRIC_GROUP_BUFFERS),
resultPartitions);
}
private static void registerOutputMetrics(
boolean isDetailedMetrics,
MetricGroup outputGroup,
MetricGroup buffersGroup,
ResultPartition[] resultPartitions) {
if (isDetailedMetrics) {
ResultPartitionMetrics.registerQueueLengthMetrics(outputGroup, resultPartitions);
}
buffersGroup.gauge(METRIC_OUTPUT_QUEUE_LENGTH, new OutputBuffersGauge(resultPartitions));
buffersGroup.gauge(METRIC_OUTPUT_POOL_USAGE, new OutputBufferPoolUsageGauge(resultPartitions));
}
public static void registerInputMetrics(
boolean isDetailedMetrics,
boolean isCreditBased,
MetricGroup inputGroup,
SingleInputGate[] inputGates) {
registerInputMetrics(
isDetailedMetrics,
isCreditBased,
inputGroup,
inputGroup.addGroup(METRIC_GROUP_BUFFERS),
inputGates);
}
private static void registerInputMetrics(
boolean isDetailedMetrics,
boolean isCreditBased,
MetricGroup inputGroup,
MetricGroup buffersGroup,
SingleInputGate[] inputGates) {
if (isDetailedMetrics) {
InputGateMetrics.registerQueueLengthMetrics(inputGroup, inputGates);
}
buffersGroup.gauge(METRIC_INPUT_QUEUE_LENGTH, new InputBuffersGauge(inputGates));
if (isCreditBased) {
FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
CreditBasedInputBuffersUsageGauge creditBasedInputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(
floatingBuffersUsageGauge,
exclusiveBuffersUsageGauge,
inputGates);
buffersGroup.gauge(METRIC_INPUT_EXCLUSIVE_BUFFERS_USAGE, exclusiveBuffersUsageGauge);
buffersGroup.gauge(METRIC_INPUT_FLOATING_BUFFERS_USAGE, floatingBuffersUsageGauge);
buffersGroup.gauge(METRIC_INPUT_POOL_USAGE, creditBasedInputBuffersUsageGauge);
} else {
buffersGroup.gauge(METRIC_INPUT_POOL_USAGE, new InputBufferPoolUsageGauge(inputGates));
}
}
}