cz.cuni.mff.d3s.spl.interpretation.WelchTestWithEnlargedVariancesInterpretation Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of spl-evaluation-java Show documentation

Stochastice Performance Logic is a formalism for capturing performance assumptions. It is, for example, possible to capture assumption that newer version of a function bar is faster than the previous version or that library foobar is faster than library barfoo when rendering antialiased text. The purpose of this framework is to allow evaluation of SPL formulas inside Java applications.

There is a newer version: 1.0.4

Show newest version

/*
 * Copyright 2015 Charles University in Prague
 * Copyright 2015 Vojtech Horky
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cz.cuni.mff.d3s.spl.interpretation;

import cz.cuni.mff.d3s.spl.data.BenchmarkRun;
import cz.cuni.mff.d3s.spl.data.BenchmarkRunUtils;
import cz.cuni.mff.d3s.spl.data.DataSnapshot;
import cz.cuni.mff.d3s.spl.utils.ArrayUtils;
import cz.cuni.mff.d3s.spl.utils.StatisticsUtils;
import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.commons.math3.distribution.RealDistribution;

import java.util.Collection;

/** SPL interpretation based on Welch's t-test with enlarged variances.
 * 
 * See Automated
 * Detection of Performance Regressions: The Mono Experience by
 * Kalibera, Bulej and Tuma for details.
*
*/
public class WelchTestWithEnlargedVariancesInterpretation implements Interpretation {
	public WelchTestWithEnlargedVariancesInterpretation() {
	}

	/** {@inheritDoc} */
	@Override
	public ComparisonResult compare(DataSnapshot left, DataSnapshot right) {
		NeededStatistics leftStat = NeededStatistics.create(left, getHistoricalIfAvailable(left));
		NeededStatistics rightStat = NeededStatistics.create(right, getHistoricalIfAvailable(right));
		
		double stat = getStatistic(leftStat, rightStat);
		
		RealDistribution distribution = new NormalDistribution();
		
		return new DistributionBasedComparisonResult(stat, distribution);
	}

	/** {@inheritDoc} */
	@Override
	public ComparisonResult compare(DataSnapshot data, double value) {
		throw new UnsupportedOperationException("This is not yet implemented.");
	}
	
	private double getStatistic(NeededStatistics x, NeededStatistics y) {
		double numer = x.getMean() - y.getMean();
		double denom = Math.sqrt(x.getSigma2() + y.getSigma2());
		return numer / denom;
	}
	
	private DataSnapshot getHistoricalIfAvailable(DataSnapshot data) {
		try  {
			DataSnapshot result = data.getPreviousEpoch();
			if ((result == null) || (result.getRunCount() == 0)) {
				return data;
			} else {
				return result;
			}
		} catch (UnsupportedOperationException e) {
			return data;
		}
	}
	
	private static class NeededStatistics {
		private double mean;
		private double sigma2;
		
		public static NeededStatistics create(DataSnapshot data, DataSnapshot historical) {
			NeededStatistics result = new NeededStatistics();
			
			Collection meansCollection = BenchmarkRunUtils.reduce(data.getRuns(), BenchmarkRunUtils.MEAN);
			double[] means = ArrayUtils.makeArray(meansCollection);
			result.mean = StatisticsUtils.mean(means);
			
			double[] meansHistorical = ArrayUtils.makeArray(
					BenchmarkRunUtils.reduce(historical.getRuns(), BenchmarkRunUtils.MEAN));
			double varianceOfMeansHistorical = StatisticsUtils.variance(meansHistorical);
			
			Collection variancesCollection = BenchmarkRunUtils.reduce(
					data.getRuns(), BenchmarkRunUtils.VARIANCE_N);
			double meanOfVariances = StatisticsUtils.mean(ArrayUtils.makeArray(variancesCollection));
			
			long totalSampleCount = 0;
			for (BenchmarkRun run : data.getRuns()) {
				totalSampleCount += run.getSampleCount();
			}
			
			result.sigma2 = varianceOfMeansHistorical / data.getRunCount()
					+ meanOfVariances / totalSampleCount;
			
			return result;
		}
		
		public double getMean() {
			return mean;
		}
		
		public double getSigma2() {
			return sigma2;
		}
	}
}