io.servicetalk.http.netty.RetryingServiceDiscoverer Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of servicetalk-http-netty Show documentation
A networking framework that evolves with your application
The newest version!
/*
 * Copyright © 2024 Apple Inc. and the ServiceTalk project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.servicetalk.http.netty;

import io.servicetalk.client.api.DelegatingServiceDiscoverer;
import io.servicetalk.client.api.ServiceDiscoverer;
import io.servicetalk.client.api.ServiceDiscovererEvent;
import io.servicetalk.concurrent.api.BiIntFunction;
import io.servicetalk.concurrent.api.Completable;
import io.servicetalk.concurrent.api.Publisher;
import io.servicetalk.transport.api.ExecutionContext;
import io.servicetalk.transport.api.ExecutionStrategy;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.UnaryOperator;
import javax.annotation.Nullable;

import static io.servicetalk.client.api.ServiceDiscovererEvent.Status.UNAVAILABLE;
import static io.servicetalk.concurrent.api.RetryStrategies.retryWithExponentialBackoffFullJitter;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.emptyMap;

final class RetryingServiceDiscoverer>
        extends DelegatingServiceDiscoverer {
    private static final Logger LOGGER = LoggerFactory.getLogger(RetryingServiceDiscoverer.class);

    private static final Duration SD_RETRY_STRATEGY_INIT_DURATION = ofSeconds(2);
    private static final Duration SD_RETRY_STRATEGY_MAX_DELAY = ofSeconds(128);

    private final String targetResource;
    private final BiIntFunction retryStrategy;
    private final UnaryOperator makeUnavailable;

    RetryingServiceDiscoverer(final String targetResource,
                              final ServiceDiscoverer delegate,
                              @Nullable BiIntFunction retryStrategy,
                              final ExecutionContext executionContext,
                              final UnaryOperator makeUnavailable) {
        super(delegate);
        this.targetResource = targetResource;
        if (retryStrategy == null) {
            retryStrategy = retryWithExponentialBackoffFullJitter(__ -> true, SD_RETRY_STRATEGY_INIT_DURATION,
                    SD_RETRY_STRATEGY_MAX_DELAY, executionContext.executor());
        }
        this.retryStrategy = retryStrategy;
        this.makeUnavailable = makeUnavailable;
    }

    @Override
    public Publisher> discover(final U address) {
        // The returned publisher is guaranteed to never fail because we retry all errors here. However, LoadBalancer
        // can still cancel and re-subscribe in attempt to recover from unhealthy state. In this case, we need to
        // re-initialize the ServiceDiscovererEventsCache and restart from an empty state.
        return Publisher.defer(() -> {
            final ServiceDiscovererEventsCache eventsCache =
                    new ServiceDiscovererEventsCache<>(targetResource, makeUnavailable);
            return delegate().discover(address)
                    .map(eventsCache::consumeAndFilter)
                    .beforeOnError(eventsCache::errorSeen)
                    // terminateOnNextException false -> LB is after this operator, if LB throws do best effort retry
                    .retryWhen(false, retryStrategy);
        });
    }

    private static final class ServiceDiscovererEventsCache> {
        @SuppressWarnings("rawtypes")
        private static final Map NONE_RETAINED = emptyMap();

        private final String targetResource;
        private final UnaryOperator makeUnavailable;
        private final Map currentState = new HashMap<>();
        private Map retainedState = noneRetained();

        private ServiceDiscovererEventsCache(final String targetResource, final UnaryOperator makeUnavailable) {
            this.targetResource = targetResource;
            this.makeUnavailable = makeUnavailable;
        }

        void errorSeen(final Throwable t) {
            if (retainedState == NONE_RETAINED) {
                retainedState = new HashMap<>(currentState);
                currentState.clear();
            }
            LOGGER.debug("{} observed an error from ServiceDiscoverer", targetResource, t);
        }

        Collection consumeAndFilter(final Collection events) {
            if (retainedState == NONE_RETAINED) {
                for (E event : events) {
                    if (UNAVAILABLE.equals(event.status())) {
                        currentState.remove(event.address());
                    } else {
                        currentState.put(event.address(), event);
                    }
                }
                return events;
            }

            // We have seen an error and re-subscribed upon retry. Based on the Publisher rule 1.10
            // (https://github.com/reactive-streams/reactive-streams-jvm?tab=readme-ov-file#1.10), each subscribe
            // expects a different Subscriber. Therefore, discovery Publisher suppose to start from a fresh state. We
            // should populate currentState with new addresses and deactivate the ones which are not present in the new
            // collection, but were left in retainedState. Original events are propagated as-is, even if they contain
            // duplicate events because retry strategy should not alter the original flow from ServiceDiscoverer.
            assert currentState.isEmpty();
            final List toReturn = new ArrayList<>(events.size() + retainedState.size());
            int unavailableCounter = 0;
            for (E event : events) {
                final R address = event.address();
                toReturn.add(event);
                retainedState.remove(address);
                if (!UNAVAILABLE.equals(event.status())) {
                    currentState.put(address, event);
                } else {
                    ++unavailableCounter;
                }
            }

            if (unavailableCounter > 0) {
                LOGGER.warn("{} received {} UNAVAILABLE events but expected a new 'state of the world'. This is an " +
                        "indicator of a buggy ServiceDiscoverer implementation that doesn't honor the API contract.",
                        targetResource, unavailableCounter);
            }

            for (E event : retainedState.values()) {
                assert event.status() != UNAVAILABLE;
                toReturn.add(makeUnavailable.apply(event));
            }

            retainedState = noneRetained();
            return toReturn;
        }

        @SuppressWarnings("unchecked")
        private static > Map noneRetained() {
            return NONE_RETAINED;
        }
    }
}