Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.rest.messages;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptDetailsInfo;
import org.apache.flink.runtime.rest.messages.job.metrics.IOMetricsInfo;
import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.math3.stat.descriptive.rank.Percentile;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.BinaryOperator;
import java.util.stream.Collectors;
/** Aggregated info of a set of tasks. */
public class AggregatedTaskDetailsInfo implements ResponseBody {
public static final String FIELD_NAME_METRICS = "metrics";
public static final String FIELD_NAME_STATUS_DURATION = "status-duration";
private static final String[] metricsNames = {
IOMetricsInfo.FIELD_NAME_BYTES_READ,
IOMetricsInfo.FIELD_NAME_BYTES_WRITTEN,
IOMetricsInfo.FIELD_NAME_RECORDS_READ,
IOMetricsInfo.FIELD_NAME_RECORDS_WRITTEN,
IOMetricsInfo.FIELD_NAME_ACC_BACK_PRESSURE,
IOMetricsInfo.FIELD_NAME_ACC_BUSY,
IOMetricsInfo.FIELD_NAME_ACC_IDLE
};
private static final String[] statusNames = {
ExecutionState.CREATED.name(),
ExecutionState.SCHEDULED.name(),
ExecutionState.INITIALIZING.name(),
ExecutionState.DEPLOYING.name(),
ExecutionState.RUNNING.name()
};
@JsonProperty(FIELD_NAME_METRICS)
private final Map> metrics;
@JsonProperty(FIELD_NAME_STATUS_DURATION)
private final Map> statusDuration;
@JsonCreator
public AggregatedTaskDetailsInfo(
@JsonProperty(FIELD_NAME_METRICS) Map> metrics,
@JsonProperty(FIELD_NAME_STATUS_DURATION)
Map> statusDuration) {
this.metrics = Preconditions.checkNotNull(metrics);
this.statusDuration = Preconditions.checkNotNull(statusDuration);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
AggregatedTaskDetailsInfo that = (AggregatedTaskDetailsInfo) o;
return Objects.equals(metrics, that.metrics)
&& Objects.equals(statusDuration, that.statusDuration);
}
@Override
public int hashCode() {
return Objects.hash(metrics, statusDuration);
}
public static AggregatedTaskDetailsInfo create(
List subtaskInfo) {
return create(
subtaskInfo.stream()
.map(SubtaskExecutionAttemptDetailsInfo::getIoMetricsInfo)
.collect(Collectors.toList()),
subtaskInfo.stream()
.map(SubtaskExecutionAttemptDetailsInfo::getStatusDuration)
.collect(Collectors.toList()));
}
public static AggregatedTaskDetailsInfo create(
List ioMetricsInfos, List> statusDurations) {
Map metrics = new HashMap<>();
Map statusDuration = new HashMap<>();
Arrays.stream(metricsNames).forEach(name -> metrics.put(name, new MetricsStatistics(name)));
Arrays.stream(statusNames)
.forEach(name -> statusDuration.put(name, new MetricsStatistics(name)));
ioMetricsInfos.forEach(
ioMetricsInfo -> {
metrics.get(IOMetricsInfo.FIELD_NAME_BYTES_READ)
.addValue(ioMetricsInfo.getBytesRead());
metrics.get(IOMetricsInfo.FIELD_NAME_BYTES_WRITTEN)
.addValue(ioMetricsInfo.getBytesWritten());
metrics.get(IOMetricsInfo.FIELD_NAME_RECORDS_READ)
.addValue(ioMetricsInfo.getRecordsRead());
metrics.get(IOMetricsInfo.FIELD_NAME_RECORDS_WRITTEN)
.addValue(ioMetricsInfo.getRecordsWritten());
metrics.get(IOMetricsInfo.FIELD_NAME_ACC_IDLE)
.addValue(ioMetricsInfo.getAccumulatedIdle());
metrics.get(IOMetricsInfo.FIELD_NAME_ACC_BACK_PRESSURE)
.addValue(ioMetricsInfo.getAccumulatedBackpressured());
if (!Double.isNaN(ioMetricsInfo.getAccumulatedBusy())) {
metrics.get(IOMetricsInfo.FIELD_NAME_ACC_BUSY)
.addValue((long) ioMetricsInfo.getAccumulatedBusy());
} else {
metrics.get(IOMetricsInfo.FIELD_NAME_ACC_BUSY).addValue(-1);
}
});
statusDurations.forEach(
status -> {
statusDuration
.get(ExecutionState.CREATED.name())
.addValue(status.get(ExecutionState.CREATED));
statusDuration
.get(ExecutionState.SCHEDULED.name())
.addValue(status.get(ExecutionState.SCHEDULED));
statusDuration
.get(ExecutionState.INITIALIZING.name())
.addValue(status.get(ExecutionState.INITIALIZING));
statusDuration
.get(ExecutionState.DEPLOYING.name())
.addValue(status.get(ExecutionState.DEPLOYING));
statusDuration
.get(ExecutionState.RUNNING.name())
.addValue(status.get(ExecutionState.RUNNING));
});
return new AggregatedTaskDetailsInfo(
metrics.values().stream()
.collect(
Collectors.toMap(
MetricsStatistics::getName, MetricsStatistics::toMap)),
statusDuration.values().stream()
.collect(
Collectors.toMap(
MetricsStatistics::getName, MetricsStatistics::toMap)));
}
@VisibleForTesting
static class MetricsStatistics {
private final List values = new ArrayList<>();
private final String name;
private long sum = 0;
private Percentile percentile = null;
MetricsStatistics(String name) {
this.name = name;
}
void addValue(long value) {
values.add(value);
sum += value;
}
private String getName() {
return name;
}
private Map toMap() {
Map result = new HashMap<>();
result.put("min", getMin());
result.put("max", getMax());
result.put("avg", getAvg());
result.put("sum", getSum());
result.put("median", getPercentile(50));
result.put("p25", getPercentile(25));
result.put("p75", getPercentile(75));
result.put("p95", getPercentile(95));
return result;
}
long getMin() {
return values.stream()
.reduce(BinaryOperator.minBy(Comparator.naturalOrder()))
.orElse(0L);
}
long getMax() {
return values.stream()
.reduce(BinaryOperator.maxBy(Comparator.naturalOrder()))
.orElse(0L);
}
long getSum() {
return sum;
}
long getAvg() {
return values.isEmpty() ? 0 : sum / values.size();
}
long getPercentile(int percent) {
if (percentile == null) {
percentile = new Percentile();
percentile.setData(values.stream().mapToDouble(Long::doubleValue).toArray());
}
return (long) percentile.evaluate(percent);
}
}
}
