netflix.ocelli.retrys.BackupRequestRetryStrategy Maven / Gradle / Ivy

Go to download
package netflix.ocelli.retrys;

import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import netflix.ocelli.functions.Metrics;
import netflix.ocelli.functions.Retrys;
import netflix.ocelli.functions.Stopwatches;
import netflix.ocelli.util.SingleMetric;
import netflix.ocelli.util.Stopwatch;
import rx.Observable;
import rx.Observable.OnSubscribe;
import rx.Observable.Transformer;
import rx.Scheduler;
import rx.Subscriber;
import rx.functions.Action1;
import rx.functions.Func0;
import rx.functions.Func1;
import rx.schedulers.Schedulers;

/**
 * Retry strategy that kicks off a second request if the first request does not
 * respond within an expected amount of time.  The original request remains in 
 * flight until either one responds.  The strategy tracks response latencies and 
 * feeds them into a SingleMetric that is used to determine the backup request
 * timeout.  A common metric to use is the 90th percentile response time. 
 * 
 * Note that the same BackupRequestRetryStrategy instance is stateful and should 
 * be used for all requests.  Multiple BackupRequestRetryStrategy instances may be
 * used for different request types known to have varying response latency 
 * distributions.
 * 
 * Usage,
 * 
 * {@code
 *  * 
 * BackupRequestRetryStrategy strategy = BackupRequestRetryStrategy.builder()
 *                  .withTimeoutMetric(Metrics.quantile(0.90))
 *                  .withIsRetriablePolicy(somePolicyThatReturnsTrueOnRetriableErrors)
 *                  .build();
 *                  
 * loadBalancer
 *      .flatMap(operation)
 *      .compose(strategy)
 *      .subscribe(responseHandler)
 * 
 * code}
 * 
 * @author elandau
 *
 * @param 
 */
public class BackupRequestRetryStrategy implements Transformer {
    public static Func0        DEFAULT_CLOCK   = Stopwatches.systemNano();
    
    private final Func0          sw;
    private final SingleMetric        metric;
    private final Func1 retriableError;
    private final Scheduler                 scheduler;
    
    public static class Builder {
        private Func0          sw             = DEFAULT_CLOCK;
        private SingleMetric        metric         = Metrics.memoize(10L);
        private Func1 retriableError = Retrys.ALWAYS;
        private Scheduler                 scheduler      = Schedulers.computation();

        /**
         * Function to determine if an exception is retriable or not.  A non 
         * retriable exception will result in an immediate error being returned
         * while the first retriable exception on either the primary or secondary
         * request will be ignored to allow the other request to complete.
         * @param retriableError
         */
        public Builder withIsRetriablePolicy(Func1 retriableError) {
            this.retriableError = retriableError;
            return this;
        }
        
        /**
         * Function to determine the backup request timeout for each operation.
         * @param func
         * @param units
         */
        public Builder withTimeoutMetric(SingleMetric metric) {
            this.metric = metric;
            return this;
        }
        
        /**
         * Provide an external scheduler to drive the backup timeout.  Use this
         * to test with a TestScheduler
         * 
         * @param scheduler
         */
        public Builder withScheduler(Scheduler scheduler) {
            this.scheduler = scheduler;
            return this;
        }
        
        /**
         * Factory for creating stopwatches.  A new stopwatch is created per operation.
         * @param clock
         */
        public Builder withStopwatch(Func0 sw) {
            this.sw = sw;
            return this;
        }
        
        public BackupRequestRetryStrategy build() {
            return new BackupRequestRetryStrategy(this);
        }
    }
    
    public static  Builder builder() {
        return new Builder();
    }
    
    private BackupRequestRetryStrategy(Builder builder) {
        this.metric         = builder.metric;
        this.retriableError = builder.retriableError;
        this.scheduler      = builder.scheduler;
        this.sw             = builder.sw;
    }
    
    @Override
    public Observable call(final Observable o) {
        Observable timedO = Observable.create(new OnSubscribe() {
            @Override
            public void call(Subscriber s) {
                final Stopwatch timer = sw.call();
                o.doOnNext(new Action1() {
                    @Override
                    public void call(T t1) {
                        metric.add(timer.elapsed(TimeUnit.MILLISECONDS));
                    }
                }).subscribe(s);
            }
        });
        
        return Observable
            .just(timedO, timedO.delaySubscription(metric.get(), TimeUnit.MILLISECONDS, scheduler))
            .flatMap(new Func1, Observable>() {
                final AtomicInteger counter = new AtomicInteger();
                
                @Override
                public Observable call(Observable t1) {
                    return t1.onErrorResumeNext(new Func1>() {
                        @Override
                        public Observable call(Throwable e) {
                            if (counter.incrementAndGet() == 2 || !retriableError.call(e)) {
                                return Observable.error(e);
                            }
                            return Observable.never();
                        }
                    });
                }
            })
            .take(1);
    }
}