tech.ytsaurus.client.RetryPolicy Maven / Gradle / Ivy

Go to download
package tech.ytsaurus.client;

import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.function.Predicate;

import javax.annotation.Nullable;

import tech.ytsaurus.client.rpc.RpcFailoverPolicy;
import tech.ytsaurus.client.rpc.RpcOptions;
import tech.ytsaurus.core.common.YTsaurusError;
import tech.ytsaurus.core.common.YTsaurusErrorCode;
import tech.ytsaurus.lang.NonNullApi;
import tech.ytsaurus.lang.NonNullFields;

/**
 * Class determines which errors must be retried.
 *
 * 
 * Users must not override this class, instead they should use factory methods.
 * 
 * 
 * Example below creates retry policy that retries codes 100 and 500 no more than 10 times:
 * 
 * RequestRetryPolicy.attemptLimited(10, RequestRetryPolicy.forCodes(100, 500))
 * 
 * 
 */
@NonNullApi
public abstract class RetryPolicy {
    private RetryPolicy() {
    }

    /**
     * Create retry policy that disallows all retries.
     */
    public static RetryPolicy noRetries() {
        return new NoRetryPolicy();
    }

    /**
     * Recommended default retry policy
     */
    public static RetryPolicy defaultPolicy() {
        return new DefaultRetryPolicy();
    }

    /**
     * Create retry policy that will retry all errors.
     */
    public static RetryPolicy retryAll(int attemptLimit) {
        return RetryPolicy.attemptLimited(attemptLimit, new RetryAllPolicy());
    }

    /**
     * Create retry policy that will retry given set of error codes.
     */
    public static RetryPolicy forCodes(Collection errorCodes) {
        return new YtErrorRetryPolicy(errorCodes);
    }

    /**
     * Create retry policy that will retry given set of error codes.
     */
    public static RetryPolicy forCodes(Integer... errorCodes) {
        return forCodes(Arrays.asList(errorCodes));
    }

    /**
     * Create retry policy that will retry `code`: isCodeForRetry(`code`) == true
     */
    public static RetryPolicy forCodes(Predicate isCodeForRetry) {
        return new YtErrorRetryPolicy(isCodeForRetry);
    }

    /**
     * Create retry policy from old RpcFailoverPolicy
     */
    public static RetryPolicy fromRpcFailoverPolicy(RpcFailoverPolicy oldPolicy) {
        return new OldFailoverRetryPolicy(oldPolicy);
    }

    /**
     * Wrap other retry policy to limit total number of attempts.
     */
    public static RetryPolicy attemptLimited(int attemptLimit, RetryPolicy inner) {
        return new AttemptLimitedRetryPolicy(attemptLimit, inner);
    }

    public static RetryPolicy either(RetryPolicy... retryPolicies) {
        return new EitherRetryPolicy(Arrays.asList(retryPolicies));
    }

    public abstract Optional getBackoffDuration(Throwable error, RpcOptions options);

    public void onNewAttempt() {
    }

    String getTotalRetryCountDescription() {
        return "";
    }

    @NonNullApi
    @NonNullFields
    static class OldFailoverRetryPolicy extends RetryPolicy {
        private final RpcFailoverPolicy oldPolicy;

        OldFailoverRetryPolicy(RpcFailoverPolicy oldPolicy) {
            this.oldPolicy = oldPolicy;
        }

        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            boolean isRetriable;

            if (error instanceof TimeoutException) {
                isRetriable = oldPolicy.onTimeout();
            } else {
                isRetriable = oldPolicy.onError(error);
            }

            if (isRetriable) {
                return Optional.of(Duration.ZERO);
            } else {
                return Optional.empty();
            }
        }
    }

    @NonNullApi
    @NonNullFields
    static class DefaultRetryPolicy extends RetryPolicy {
        private static final HashSet CODES_FOR_RETRY = new HashSet<>(Arrays.asList(
                YTsaurusErrorCode.TransactionLockConflict.getCode(),
                YTsaurusErrorCode.AllWritesDisabled.getCode(),
                YTsaurusErrorCode.TableMountInfoNotReady.getCode(),
                YTsaurusErrorCode.TooManyRequests.getCode(),
                YTsaurusErrorCode.RequestQueueSizeLimitExceeded.getCode(),
                YTsaurusErrorCode.RpcRequestQueueSizeLimitExceeded.getCode(),
                YTsaurusErrorCode.TooManyOperations.getCode(),
                YTsaurusErrorCode.TransportError.getCode(),
                YTsaurusErrorCode.OperationProgressOutdated.getCode(),
                YTsaurusErrorCode.Canceled.getCode()
        ));

        private static final HashSet CHUNK_NOT_RETRIABLE_CODES = new HashSet<>(Arrays.asList(
                YTsaurusErrorCode.SessionAlreadyExists.getCode(),
                YTsaurusErrorCode.ChunkAlreadyExists.getCode(),
                YTsaurusErrorCode.WindowError.getCode(),
                YTsaurusErrorCode.BlockContentMismatch.getCode(),
                YTsaurusErrorCode.InvalidBlockChecksum.getCode(),
                YTsaurusErrorCode.BlockOutOfRange.getCode(),
                YTsaurusErrorCode.MissingExtension.getCode(),
                YTsaurusErrorCode.NoSuchBlock.getCode(),
                YTsaurusErrorCode.NoSuchChunk.getCode(),
                YTsaurusErrorCode.NoSuchChunkList.getCode(),
                YTsaurusErrorCode.NoSuchChunkTree.getCode(),
                YTsaurusErrorCode.NoSuchChunkView.getCode(),
                YTsaurusErrorCode.NoSuchMedium.getCode()
        ));

        private final RetryPolicy inner = attemptLimited(3, RetryPolicy.forCodes(
                code -> CODES_FOR_RETRY.contains(code) || isChunkRetriableError(code)
        ));

        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            return inner.getBackoffDuration(error, options);
        }

        @Override
        public void onNewAttempt() {
            inner.onNewAttempt();
        }

        @Override
        public String getTotalRetryCountDescription() {
            return inner.getTotalRetryCountDescription();
        }

        private boolean isChunkRetriableError(Integer code) {
            if (CHUNK_NOT_RETRIABLE_CODES.contains(code)) {
                return false;
            }
            return code / 100 == 7;
        }
    }

    @NonNullApi
    @NonNullFields
    static class YtErrorRetryPolicy extends RetryPolicy {
        private final Predicate isCodeForRetry;
        private final BackoffProvider backoffProvider = new BackoffProvider();

        YtErrorRetryPolicy(Collection codesToRetry) {
            HashSet errorCodesToRetry = new HashSet<>(codesToRetry);
            this.isCodeForRetry = errorCodesToRetry::contains;
        }

        YtErrorRetryPolicy(Predicate isCodeForRetry) {
            this.isCodeForRetry = isCodeForRetry;
        }

        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            TimeoutException timeoutException = null;
            IOException ioException = null;
            YTsaurusError rpcError = null;
            while (error != null) {
                if (error instanceof TimeoutException) {
                    timeoutException = (TimeoutException) error;
                } else if (error instanceof IOException) {
                    ioException = (IOException) error;
                } else if (error instanceof YTsaurusError) {
                    rpcError = (YTsaurusError) error;
                }
                error = error.getCause();
            }

            if (rpcError != null) {
                if (rpcError.matches(isCodeForRetry)) {
                    return Optional.of(backoffProvider.getBackoffTime(rpcError, options));
                } else {
                    return Optional.empty();
                }
            }
            if (timeoutException != null) {
                return Optional.of(Duration.ZERO);
            }
            if (ioException != null) {
                return Optional.of(Duration.ZERO);
            }
            return Optional.empty();
        }
    }

    @NonNullApi
    static class AttemptLimitedRetryPolicy extends RetryPolicy {
        private final int attemptLimit;
        private final RetryPolicy inner;
        private int currentAttempt = 0;

        AttemptLimitedRetryPolicy(int attemptLimit, RetryPolicy inner) {
            this.attemptLimit = attemptLimit;
            this.inner = inner;
        }

        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            if (currentAttempt < attemptLimit) {
                return inner.getBackoffDuration(error, options);
            } else {
                return Optional.empty();
            }
        }

        @Override
        public void onNewAttempt() {
            currentAttempt += 1;
            inner.onNewAttempt();
        }

        @Override
        public String getTotalRetryCountDescription() {
            return Integer.toString(attemptLimit);
        }
    }

    static class NoRetryPolicy extends RetryPolicy {
        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            return Optional.empty();
        }
    }

    static class RetryAllPolicy extends RetryPolicy {
        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            return Optional.of(Duration.ZERO);
        }
    }

    static class EitherRetryPolicy extends RetryPolicy {
        private final List retryPolicies;

        EitherRetryPolicy(List retryPolicies) {
            this.retryPolicies = retryPolicies;
        }

        @Override
        public Optional getBackoffDuration(Throwable error, RpcOptions options) {
            for (RetryPolicy retryPolicy : retryPolicies) {
                Optional backoff = retryPolicy.getBackoffDuration(error, options);
                if (backoff.isPresent()) {
                    return backoff;
                }
            }
            return Optional.empty();
        }

        @Override
        public void onNewAttempt() {
            for (RetryPolicy retryPolicy : retryPolicies) {
                retryPolicy.onNewAttempt();
            }
        }
    }
}

@NonNullApi
@NonNullFields
class BackoffProvider {
    private @Nullable
    Duration currentExponentialBackoff = null;

    Duration getBackoffTime(YTsaurusError error, RpcOptions options) {
        Set errorCodes = error.getErrorCodes();
        if (errorCodes.stream().anyMatch(BackoffProvider::isExponentialBackoffError)) {
            if (currentExponentialBackoff == null) {
                currentExponentialBackoff = options.getMinBackoffTime();
                return currentExponentialBackoff;
            }

            currentExponentialBackoff = Collections.max(Arrays.asList(
                    currentExponentialBackoff.plus(Duration.ofSeconds(1)),
                    currentExponentialBackoff.multipliedBy(2)
            ));
            assert currentExponentialBackoff != null;

            Duration maxBackoffTime = options.getMaxBackoffTime();
            if (currentExponentialBackoff.compareTo(maxBackoffTime) > 0) {
                currentExponentialBackoff = options.getMaxBackoffTime();
            }
            return currentExponentialBackoff;
        }
        return options.getMinBackoffTime();
    }

    private static boolean isExponentialBackoffError(int errorCode) {
        return errorCode == YTsaurusErrorCode.RequestQueueSizeLimitExceeded.code ||
                errorCode == 904; // NSecurityClient.RequestQueueSizeLimitExceeded
    }
}