• Home
• com.github.rawls238
• Scientist4JCore
• 1.0
• source code
• Experiment.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.github.rawls238.scientist4j.Experiment Maven / Gradle / Ivy

package com.github.rawls238.scientist4j;

import com.github.rawls238.scientist4j.exceptions.MismatchException;
import com.github.rawls238.scientist4j.metrics.MetricsProvider;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.BiFunction;

public class Experiment {

    private final ExecutorService executor;
    private static final String NAMESPACE_PREFIX = "scientist";
    private final MetricsProvider metricsProvider;
    private final String name;
    private final boolean raiseOnMismatch;
    private Map context;
    private final MetricsProvider.Timer controlTimer;
    private final MetricsProvider.Timer candidateTimer;
    private final MetricsProvider.Counter mismatchCount;
    private final MetricsProvider.Counter candidateExceptionCount;
    private final MetricsProvider.Counter totalCount;
    private final BiFunction comparator;

    public Experiment(MetricsProvider metricsProvider) {
        this("Experiment", metricsProvider);
    }

    public Experiment(String name, MetricsProvider metricsProvider) {
        this(name, false, metricsProvider);
    }

    public Experiment(String name, Map context, MetricsProvider metricsProvider) {
        this(name, context, false, metricsProvider);
    }

    public Experiment(String name, boolean raiseOnMismatch, MetricsProvider metricsProvider) {
        this(name, new HashMap<>(), raiseOnMismatch, metricsProvider);
    }

    public Experiment(String name, Map context, boolean raiseOnMismatch, MetricsProvider metricsProvider) {
        this(name, context, raiseOnMismatch, metricsProvider, Objects::equals);
    }

    public Experiment(String name, Map context, boolean raiseOnMismatch,
                      MetricsProvider metricsProvider, BiFunction comparator) {
        this(name, context, raiseOnMismatch, metricsProvider, comparator, Executors.newFixedThreadPool(2));
    }

    public Experiment(String name, Map context, boolean raiseOnMismatch,
                      MetricsProvider metricsProvider, BiFunction comparator,
                      ExecutorService executorService) {
        this.name = name;
        this.context = context;
        this.raiseOnMismatch = raiseOnMismatch;
        this.comparator = comparator;
        this.metricsProvider = metricsProvider;
        controlTimer = getMetricsProvider().timer(NAMESPACE_PREFIX, this.name, "control");
        candidateTimer = getMetricsProvider().timer(NAMESPACE_PREFIX, this.name, "candidate");
        mismatchCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "mismatch");
        candidateExceptionCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "candidate.exception");
        totalCount = getMetricsProvider().counter(NAMESPACE_PREFIX, this.name, "total");
        executor = executorService;
    }

    /**
     * Allow override here if extending the class
     */
    public MetricsProvider getMetricsProvider() {
        return this.metricsProvider;
    }

    /**
     * Note that if {@code raiseOnMismatch} is true, {@link #runAsync(Callable, Callable)} will block waiting for
     * the candidate function to complete before it can raise any resulting errors. In situations where the candidate
     * function may be significantly slower than the control, it is not recommended to raise on mismatch.
     */
    public boolean getRaiseOnMismatch() {
        return raiseOnMismatch;
    }

    public String getName() {
        return name;
    }

    public T run(Callable control, Callable candidate) throws Exception {
        if (isAsyncCandidateOnly()) {
            return runAsyncCandidateOnly(control, candidate);
        } else if (isAsync()) {
            return runAsync(control, candidate);
        } else {
            return runSync(control, candidate);
        }
    }

    private T runSync(Callable control, Callable candidate) throws Exception {
        Observation controlObservation;
        Optional> candidateObservation = Optional.empty();
        if (Math.random() < 0.5) {
            controlObservation = executeResult("control", controlTimer, control, true);
            if (runIf() && enabled()) {
                candidateObservation = Optional.of(executeResult("candidate", candidateTimer, candidate, false));
            }
        } else {
            if (runIf() && enabled()) {
                candidateObservation = Optional.of(executeResult("candidate", candidateTimer, candidate, false));
            }
            controlObservation = executeResult("control", controlTimer, control, true);
        }

        countExceptions(candidateObservation, candidateExceptionCount);
        Result result = new Result(this, controlObservation, candidateObservation, context);
        publish(result);
        return controlObservation.getValue();
    }

    public T runAsync(Callable control, Callable candidate) throws Exception {
        Future>> observationFutureCandidate;
        Future> observationFutureControl;

        if (runIf() && enabled()) {
            if (Math.random() < 0.5) {
                observationFutureControl = executor.submit(() -> executeResult("control", controlTimer, control, true));
                observationFutureCandidate = executor.submit(() -> Optional.of(executeResult("candidate", candidateTimer, candidate, false)));
            } else {
                observationFutureCandidate = executor.submit(() -> Optional.of(executeResult("candidate", candidateTimer, candidate, false)));
                observationFutureControl = executor.submit(() -> executeResult("control", controlTimer, control, true));
            }
        } else {
            observationFutureControl = executor.submit(() -> executeResult("control", controlTimer, control, true));
            observationFutureCandidate = null;
        }

        Observation controlObservation;
        try {
            controlObservation = observationFutureControl.get();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }

        Future publishedResult = executor.submit(() -> publishAsync(controlObservation, observationFutureCandidate));

        if (raiseOnMismatch) {
            try {
                publishedResult.get();
            } catch (ExecutionException e) {
                throw (Exception) e.getCause();
            }
        }

        return controlObservation.getValue();
    }

    public T runAsyncCandidateOnly(Callable control, Callable candidate) throws Exception {
        Future>> observationFutureCandidate;
        Observation controlObservation;

        if (runIf() && enabled()) {
            if (Math.random() < 0.5) {
                observationFutureCandidate = executor.submit(() -> Optional.of(executeResult("candidate", candidateTimer, candidate, false)));
                controlObservation = executeResult("control", controlTimer, control, true);
            } else {
                controlObservation = executeResult("control", controlTimer, control, true);
                observationFutureCandidate = executor.submit(() -> Optional.of(executeResult("candidate", candidateTimer, candidate, false)));
            }
        } else {
            controlObservation = executeResult("control", controlTimer, control, true);
            observationFutureCandidate = null;
        }

        Future publishedResult = executor.submit(() -> publishAsync(controlObservation, observationFutureCandidate));

        if (raiseOnMismatch) {
            try {
                publishedResult.get();
            } catch (ExecutionException e) {
                throw (Exception) e.getCause();
            }
        }

        return controlObservation.getValue();
    }

    private Void publishAsync(Observation controlObservation, Future>> observationFutureCandidate) throws Exception {
        Optional> candidateObservation = Optional.empty();
        if (observationFutureCandidate != null) {
            candidateObservation = observationFutureCandidate.get();
        }

        countExceptions(candidateObservation, candidateExceptionCount);
        Result result = new Result<>(this, controlObservation, candidateObservation, context);
        publish(result);
        return null;
    }

    private void countExceptions(Optional> observation, MetricsProvider.Counter exceptions) {
        if (observation.isPresent() && observation.get().getException().isPresent()) {
            exceptions.increment();
        }
    }

    public Observation executeResult(String name, MetricsProvider.Timer timer, Callable control, boolean shouldThrow) throws Exception {
        Observation observation = new Observation<>(name, timer);

        observation.time(() -> {
            try {
                observation.setValue(control.call());
            } catch (Exception e) {
                observation.setException(e);
            }
        });

        if (shouldThrow && observation.getException().isPresent()) {
            throw observation.getException().get();
        }

        return observation;
    }

    protected boolean compareResults(T controlVal, T candidateVal) {
        return comparator.apply(controlVal, candidateVal);
    }

    public boolean compare(Observation controlVal, Observation candidateVal) throws MismatchException {
        boolean resultsMatch = !candidateVal.getException().isPresent() && compareResults(controlVal.getValue(), candidateVal.getValue());
        totalCount.increment();
        if (!resultsMatch) {
            mismatchCount.increment();
            handleComparisonMismatch(controlVal, candidateVal);
        }
        return true;
    }

    protected void publish(Result r) {
    }

    protected boolean runIf() {
        return true;
    }

    protected boolean enabled() {
        return true;
    }

    protected boolean isAsync() {
        return false;
    }

    protected boolean isAsyncCandidateOnly() {
        return false;
    }

    private void handleComparisonMismatch(Observation controlVal, Observation candidateVal) throws MismatchException {
        String msg;
        if (candidateVal.getException().isPresent()) {
            String stackTrace = candidateVal.getException().get().getStackTrace().toString();
            String exceptionName = candidateVal.getException().get().getClass().getName();
            msg = new StringBuilder().append(candidateVal.getName()).append(" raised an exception: ")
                .append(exceptionName).append(" ").append(stackTrace).toString();
        } else {
            msg = new StringBuilder().append(candidateVal.getName()).append(" does not match control value (")
                .append(controlVal.getValue().toString()).append(" != ").append(candidateVal.getValue().toString()).append(")").toString();
        }
        throw new MismatchException(msg);
    }
}