All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.python.metric.process.FlinkMetricContainer Maven / Gradle / Ivy

There is a newer version: 2.0-preview1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.python.metric.process;

import org.apache.flink.annotation.Internal;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.configuration.MetricOptions;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.Meter;
import org.apache.flink.metrics.MeterView;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.util.jackson.JacksonMapperFactory;

import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;

import org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo;
import org.apache.beam.runners.core.metrics.MetricsContainerImpl;
import org.apache.beam.runners.core.metrics.MetricsContainerStepMap;
import org.apache.beam.runners.core.metrics.MonitoringInfoMetricName;
import org.apache.beam.sdk.metrics.DistributionResult;
import org.apache.beam.sdk.metrics.GaugeResult;
import org.apache.beam.sdk.metrics.MetricKey;
import org.apache.beam.sdk.metrics.MetricName;
import org.apache.beam.sdk.metrics.MetricQueryResults;
import org.apache.beam.sdk.metrics.MetricResult;
import org.apache.beam.sdk.metrics.MetricResults;
import org.apache.beam.sdk.metrics.MetricsFilter;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.apache.beam.runners.core.metrics.MetricsContainerStepMap.asAttemptedOnlyMetricResults;
import static org.apache.flink.util.Preconditions.checkNotNull;

/** Helper class for forwarding Python metrics to Java accumulators and metrics. */
@Internal
public final class FlinkMetricContainer {

    private static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper();

    private static final String METRIC_KEY_SEPARATOR =
            GlobalConfiguration.loadConfiguration().get(MetricOptions.SCOPE_DELIMITER);

    private final MetricsContainerStepMap metricsContainers;
    private final MetricGroup baseMetricGroup;
    private final Map flinkCounterCache;
    private final Map flinkMeterCache;
    private final Map flinkDistributionGaugeCache;
    private final Map flinkGaugeCache;

    public FlinkMetricContainer(MetricGroup metricGroup) {
        this.baseMetricGroup = checkNotNull(metricGroup);
        this.flinkCounterCache = new HashMap<>();
        this.flinkMeterCache = new HashMap<>();
        this.flinkDistributionGaugeCache = new HashMap<>();
        this.flinkGaugeCache = new HashMap<>();
        this.metricsContainers = new MetricsContainerStepMap();
    }

    private MetricsContainerImpl getMetricsContainer(String stepName) {
        return metricsContainers.getContainer(stepName);
    }

    /**
     * Update this container with metrics from the passed {@link MonitoringInfo}s, and send updates
     * along to Flink's internal metrics framework.
     */
    public void updateMetrics(String stepName, List monitoringInfos) {
        getMetricsContainer(stepName).update(monitoringInfos);
        updateMetrics(stepName);
    }

    /**
     * Update Flink's internal metrics ({@link #flinkCounterCache}) with the latest metrics for a
     * given step.
     */
    private void updateMetrics(String stepName) {
        MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainers);
        MetricQueryResults metricQueryResults =
                metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build());
        updateCounterOrMeter(metricQueryResults.getCounters());
        updateDistributions(metricQueryResults.getDistributions());
        updateGauge(metricQueryResults.getGauges());
    }

    private boolean isUserMetric(MetricResult metricResult) {
        MetricName metricName = metricResult.getKey().metricName();
        if (metricName instanceof MonitoringInfoMetricName) {
            String urn = ((MonitoringInfoMetricName) metricName).getUrn();
            return urn.startsWith("beam:metric:user");
        }
        return false;
    }

    private void updateCounterOrMeter(Iterable> counters) {
        for (MetricResult metricResult : counters) {
            if (!isUserMetric(metricResult)) {
                continue;
            }
            // get identifier
            String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());

            // get metric type
            ArrayList scopeComponents = getNameSpaceArray(metricResult.getKey());
            if ((scopeComponents.size() % 2) != 0) {
                Meter meter = flinkMeterCache.get(flinkMetricIdentifier);
                if (null == meter) {
                    int timeSpanInSeconds =
                            Integer.parseInt(scopeComponents.get(scopeComponents.size() - 1));
                    MetricGroup metricGroup =
                            registerMetricGroup(metricResult.getKey(), baseMetricGroup);
                    meter =
                            metricGroup.meter(
                                    metricResult.getKey().metricName().getName(),
                                    new MeterView(timeSpanInSeconds));
                    flinkMeterCache.put(flinkMetricIdentifier, meter);
                }

                Long update = metricResult.getAttempted();
                meter.markEvent(update - meter.getCount());
            } else {
                Counter counter = flinkCounterCache.get(flinkMetricIdentifier);
                if (null == counter) {
                    MetricGroup metricGroup =
                            registerMetricGroup(metricResult.getKey(), baseMetricGroup);
                    counter = metricGroup.counter(metricResult.getKey().metricName().getName());
                    flinkCounterCache.put(flinkMetricIdentifier, counter);
                }

                Long update = metricResult.getAttempted();
                counter.inc(update - counter.getCount());
            }
        }
    }

    private void updateDistributions(Iterable> distributions) {
        for (MetricResult metricResult : distributions) {
            if (!isUserMetric(metricResult)) {
                continue;
            }
            // get identifier
            String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());
            DistributionResult update = metricResult.getAttempted();

            // update flink metric
            FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricIdentifier);
            if (gauge == null) {
                MetricGroup metricGroup =
                        registerMetricGroup(metricResult.getKey(), baseMetricGroup);
                gauge =
                        metricGroup.gauge(
                                metricResult.getKey().metricName().getName(),
                                new FlinkDistributionGauge(update));
                flinkDistributionGaugeCache.put(flinkMetricIdentifier, gauge);
            } else {
                gauge.update(update);
            }
        }
    }

    private void updateGauge(Iterable> gauges) {
        for (MetricResult metricResult : gauges) {
            if (!isUserMetric(metricResult)) {
                continue;
            }
            // get identifier
            String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());

            GaugeResult update = metricResult.getAttempted();

            // update flink metric
            FlinkGauge gauge = flinkGaugeCache.get(flinkMetricIdentifier);
            if (gauge == null) {
                MetricGroup metricGroup =
                        registerMetricGroup(metricResult.getKey(), baseMetricGroup);
                gauge =
                        metricGroup.gauge(
                                metricResult.getKey().metricName().getName(),
                                new FlinkGauge(update));
                flinkGaugeCache.put(flinkMetricIdentifier, gauge);
            } else {
                gauge.update(update);
            }
        }
    }

    @VisibleForTesting
    static ArrayList getNameSpaceArray(MetricKey metricKey) {
        MetricName metricName = metricKey.metricName();
        try {
            return OBJECT_MAPPER.readValue(metricName.getNamespace(), ArrayList.class);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(
                    String.format("Parse namespace[%s] error. ", metricName.getNamespace()), e);
        }
    }

    @VisibleForTesting
    static String getFlinkMetricIdentifierString(MetricKey metricKey) {
        MetricName metricName = metricKey.metricName();
        ArrayList scopeComponents = getNameSpaceArray(metricKey);
        List results = scopeComponents.subList(0, scopeComponents.size() / 2);
        results.add(metricName.getName());
        return String.join(METRIC_KEY_SEPARATOR, results);
    }

    @VisibleForTesting
    static MetricGroup registerMetricGroup(MetricKey metricKey, MetricGroup metricGroup) {
        ArrayList scopeComponents = getNameSpaceArray(metricKey);
        int size = scopeComponents.size();
        List metricGroupNames = scopeComponents.subList(0, size / 2);
        List metricGroupTypes = scopeComponents.subList(size / 2, size);
        for (int i = 0; i < metricGroupNames.size(); ++i) {
            if (metricGroupTypes.get(i).equals("MetricGroupType.generic")) {
                metricGroup = metricGroup.addGroup(metricGroupNames.get(i));
            } else if (metricGroupTypes.get(i).equals("MetricGroupType.key")) {
                metricGroup =
                        metricGroup.addGroup(metricGroupNames.get(i), metricGroupNames.get(++i));
            }
        }
        return metricGroup;
    }

    /** Flink {@link Gauge} for {@link DistributionResult}. */
    public static class FlinkDistributionGauge implements Gauge {

        private DistributionResult data;

        FlinkDistributionGauge(DistributionResult data) {
            this.data = data;
        }

        void update(DistributionResult data) {
            this.data = data;
        }

        @Override
        public DistributionResult getValue() {
            return data;
        }
    }

    /** Flink {@link Gauge} for {@link GaugeResult}. */
    public static class FlinkGauge implements Gauge {

        private GaugeResult data;

        FlinkGauge(GaugeResult data) {
            this.data = data;
        }

        void update(GaugeResult update) {
            this.data = update;
        }

        @Override
        public Long getValue() {
            return data.getValue();
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy