org.apache.flink.runtime.metrics.DescriptiveStatisticsHistogramStatistics Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flink-runtime_2.11 Show documentation
There is a newer version: 1.13.6
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.metrics;

import org.apache.flink.metrics.HistogramStatistics;

import org.apache.commons.math3.exception.MathIllegalArgumentException;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.apache.commons.math3.stat.descriptive.UnivariateStatistic;
import org.apache.commons.math3.stat.descriptive.moment.SecondMoment;
import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
import org.apache.commons.math3.stat.descriptive.rank.Percentile;
import org.apache.commons.math3.stat.ranking.NaNStrategy;

import java.util.Arrays;

/**
 * DescriptiveStatistics histogram statistics implementation returned by {@link
 * DescriptiveStatisticsHistogram}.
 *
 * The statistics takes a point-in-time snapshot of a {@link DescriptiveStatistics} instance and
 * allows optimised metrics retrieval from this.
 */
public class DescriptiveStatisticsHistogramStatistics extends HistogramStatistics {
    private final CommonMetricsSnapshot statisticsSummary = new CommonMetricsSnapshot();

    public DescriptiveStatisticsHistogramStatistics(
            DescriptiveStatisticsHistogram.CircularDoubleArray histogramValues) {
        statisticsSummary.evaluate(histogramValues.toUnsortedArray());
    }

    @Override
    public double getQuantile(double quantile) {
        return statisticsSummary.getPercentile(quantile * 100);
    }

    @Override
    public long[] getValues() {
        return Arrays.stream(statisticsSummary.getValues()).mapToLong(i -> (long) i).toArray();
    }

    @Override
    public int size() {
        return (int) statisticsSummary.getCount();
    }

    @Override
    public double getMean() {
        return statisticsSummary.getMean();
    }

    @Override
    public double getStdDev() {
        return statisticsSummary.getStandardDeviation();
    }

    @Override
    public long getMax() {
        return (long) statisticsSummary.getMax();
    }

    @Override
    public long getMin() {
        return (long) statisticsSummary.getMin();
    }

    /**
     * Function to extract several commonly used metrics in an optimised way, i.e. with as few runs
     * over the data / calculations as possible.
     *
     * Note that calls to {@link #evaluate(double[])} or {@link #evaluate(double[], int, int)}
     * will not return a value but instead populate this class so that further values can be
     * retrieved from it.
     */
    private static class CommonMetricsSnapshot implements UnivariateStatistic {
        private long count = 0;
        private double min = Double.NaN;
        private double max = Double.NaN;
        private double mean = Double.NaN;
        private double stddev = Double.NaN;
        private Percentile percentilesImpl = new Percentile().withNaNStrategy(NaNStrategy.FIXED);

        @Override
        public double evaluate(final double[] values) throws MathIllegalArgumentException {
            return evaluate(values, 0, values.length);
        }

        @Override
        public double evaluate(double[] values, int begin, int length)
                throws MathIllegalArgumentException {
            this.count = length;
            percentilesImpl.setData(values, begin, length);

            SimpleStats secondMoment = new SimpleStats();
            secondMoment.evaluate(values, begin, length);
            this.mean = secondMoment.getMean();
            this.min = secondMoment.getMin();
            this.max = secondMoment.getMax();

            this.stddev = new StandardDeviation(secondMoment).getResult();

            return Double.NaN;
        }

        @Override
        public CommonMetricsSnapshot copy() {
            CommonMetricsSnapshot result = new CommonMetricsSnapshot();
            result.count = count;
            result.min = min;
            result.max = max;
            result.mean = mean;
            result.stddev = stddev;
            result.percentilesImpl = percentilesImpl.copy();
            return result;
        }

        long getCount() {
            return count;
        }

        double getMin() {
            return min;
        }

        double getMax() {
            return max;
        }

        double getMean() {
            return mean;
        }

        double getStandardDeviation() {
            return stddev;
        }

        double getPercentile(double p) {
            return percentilesImpl.evaluate(p);
        }

        double[] getValues() {
            return percentilesImpl.getData();
        }
    }

    /**
     * Calculates min, max, mean (first moment), as well as the second moment in one go over the
     * value array.
     */
    private static class SimpleStats extends SecondMoment {
        private static final long serialVersionUID = 1L;

        private double min = Double.NaN;
        private double max = Double.NaN;

        @Override
        public void increment(double d) {
            if (d < min || Double.isNaN(min)) {
                min = d;
            }
            if (d > max || Double.isNaN(max)) {
                max = d;
            }
            super.increment(d);
        }

        @Override
        public SecondMoment copy() {
            SimpleStats result = new SimpleStats();
            SecondMoment.copy(this, result);
            result.min = min;
            result.max = max;
            return result;
        }

        public double getMin() {
            return min;
        }

        public double getMax() {
            return max;
        }

        public double getMean() {
            return m1;
        }
    }
}