All Downloads are FREE. Search and download functionalities are using the official Maven repository.

net.ravendb.client.http.RequestExecutor Maven / Gradle / Ivy

package net.ravendb.client.http;

import net.ravendb.client.Constants;
import net.ravendb.client.documents.conventions.DocumentConventions;
import net.ravendb.client.documents.operations.DatabaseHealthCheckOperation;
import net.ravendb.client.documents.operations.GetStatisticsOperation;
import net.ravendb.client.documents.operations.configuration.GetClientConfigurationOperation;
import net.ravendb.client.documents.session.*;
import net.ravendb.client.exceptions.*;
import net.ravendb.client.exceptions.database.DatabaseDoesNotExistException;
import net.ravendb.client.exceptions.security.AuthorizationException;
import net.ravendb.client.extensions.HttpExtensions;
import net.ravendb.client.extensions.JsonExtensions;
import net.ravendb.client.primitives.*;
import net.ravendb.client.primitives.Timer;
import net.ravendb.client.serverwide.commands.GetDatabaseTopologyCommand;
import net.ravendb.client.util.CertificateUtils;
import net.ravendb.client.util.TimeUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpRequest;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.SSLContexts;

import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.InputStream;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.security.*;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.stream.Collectors;

@SuppressWarnings("SameParameterValue")
public class RequestExecutor implements CleanCloseable {

    private static UUID GLOBAL_APPLICATION_IDENTIFIER = UUID.randomUUID();

    private static final int INITIAL_TOPOLOGY_ETAG = -2;

    public static Consumer configureHttpClient = null;

    private static final GetStatisticsOperation backwardCompatibilityFailureCheckOperation = new GetStatisticsOperation("failure=check");

    private static final DatabaseHealthCheckOperation failureCheckOperation = new DatabaseHealthCheckOperation();
    private static Set _useOldFailureCheckOperation = ConcurrentHashMap.newKeySet();

    /**
     * Extension point to plug - in request post processing like adding proxy etc.
     */
    public static Consumer requestPostProcessor = null;

    public static final String CLIENT_VERSION = "5.2.0";

    private static final ConcurrentMap globalHttpClientWithCompression = new ConcurrentHashMap<>();
    private static final ConcurrentMap globalHttpClientWithoutCompression = new ConcurrentHashMap<>();

    private final Semaphore _updateDatabaseTopologySemaphore = new Semaphore(1);

    private final Semaphore _updateClientConfigurationSemaphore = new Semaphore(1);

    private final ConcurrentMap _failedNodesTimers = new ConcurrentHashMap<>();

    private final KeyStore certificate;
    private final char[] keyPassword;
    private final KeyStore trustStore;

    private final String _databaseName;

    private static final Log logger = LogFactory.getLog(RequestExecutor.class);
    private Date _lastReturnedResponse;

    protected final ExecutorService _executorService;

    private final HttpCache cache;

    private ServerNode _topologyTakenFromNode;

    public HttpCache getCache() {
        return cache;
    }

    public final ThreadLocal aggressiveCaching = new ThreadLocal<>();

    public Topology getTopology() {
        return _nodeSelector != null ? _nodeSelector.getTopology() : null;
    }

    private CloseableHttpClient _httpClient;

    public CloseableHttpClient getHttpClient() {
        CloseableHttpClient httpClient = _httpClient;
        if (httpClient != null) {
            return httpClient;
        }

        return _httpClient = createHttpClient();
    }

    public List getTopologyNodes() {
        return Optional.ofNullable(getTopology())
                .map(Topology::getNodes)
                .map(Collections::unmodifiableList)
                .orElse(null);
    }

    private volatile Timer _updateTopologyTimer;

    protected NodeSelector _nodeSelector;

    private Duration _defaultTimeout;

    public final AtomicLong numberOfServerRequests = new AtomicLong(0);

    public String getUrl() {
        if (_nodeSelector == null) {
            return null;
        }

        CurrentIndexAndNode preferredNode = _nodeSelector.getPreferredNode();

        return preferredNode != null ? preferredNode.currentNode.getUrl() : null;
    }

    protected long topologyEtag;

    public long getTopologyEtag() {
        return topologyEtag;
    }

    protected long clientConfigurationEtag;

    public long getClientConfigurationEtag() {
        return clientConfigurationEtag;
    }

    private final DocumentConventions conventions;

    protected boolean _disableTopologyUpdates;

    protected boolean _disableClientConfigurationUpdates;

    protected String lastServerVersion;

    public String getLastServerVersion() {
        return lastServerVersion;
    }

    public Duration getDefaultTimeout() {
        return _defaultTimeout;
    }

    public void setDefaultTimeout(Duration timeout) {
        _defaultTimeout = timeout;
    }

    private Duration _secondBroadcastAttemptTimeout;

    public Duration getSecondBroadcastAttemptTimeout() {
        return _secondBroadcastAttemptTimeout;
    }

    public void setSecondBroadcastAttemptTimeout(Duration secondBroadcastAttemptTimeout) {
        _secondBroadcastAttemptTimeout = secondBroadcastAttemptTimeout;
    }

    private Duration _firstBroadcastAttemptTimeout;

    public Duration getFirstBroadcastAttemptTimeout() {
        return _firstBroadcastAttemptTimeout;
    }

    public void setFirstBroadcastAttemptTimeout(Duration firstBroadcastAttemptTimeout) {
        _firstBroadcastAttemptTimeout = firstBroadcastAttemptTimeout;
    }

    private final List> _onFailedRequest = new ArrayList<>();

    public void addOnFailedRequestListener(EventHandler handler) {
        this._onFailedRequest.add(handler);
    }

    public void removeOnFailedRequestListener(EventHandler handler) {
        this._onFailedRequest.remove(handler);
    }

    private final List> _onBeforeRequest = new ArrayList<>();

    public void addOnBeforeRequestListener(EventHandler handler) {
        this._onBeforeRequest.add(handler);
    }

    public void removeOnBeforeRequestListener(EventHandler handler) {
        this._onBeforeRequest.remove(handler);
    }

    private final List> _onSucceedRequest = new ArrayList<>();

    public void addOnSucceedRequestListener(EventHandler handler) {
        this._onSucceedRequest.add(handler);
    }

    public void removeOnSucceedRequestListener(EventHandler handler) {
        this._onSucceedRequest.remove(handler);
    }

    private final List> _onTopologyUpdated = new ArrayList<>();

    public void addOnTopologyUpdatedListener(EventHandler handler) {
        _onTopologyUpdated.add(handler);
    }

    public void removeOnTopologyUpdatedListener(EventHandler handler) {
        _onTopologyUpdated.remove(handler);
    }

    private void onFailedRequestInvoke(String url, Exception e) {
        EventHelper.invoke(_onFailedRequest, this, new FailedRequestEventArgs(_databaseName, url, e));
    }

    private CloseableHttpClient createHttpClient() {
        ConcurrentMap httpClientCache = getHttpClientCache();

        String name = getHttpClientName();

        return httpClientCache.computeIfAbsent(name, n -> createClient());
    }

    private String getHttpClientName() {
        if (certificate != null) {
            return CertificateUtils.extractThumbprintFromCertificate(certificate);
        }
        return "";
    }

    private ConcurrentMap getHttpClientCache() {
        return conventions.isUseCompression() ? globalHttpClientWithCompression : globalHttpClientWithoutCompression;
    }

    public DocumentConventions getConventions() {
        return conventions;
    }

    public KeyStore getCertificate() {
        return certificate;
    }

    public char[] getKeyPassword() {
        return keyPassword;
    }

    public KeyStore getTrustStore() {
        return trustStore;
    }

    protected RequestExecutor(String databaseName, KeyStore certificate, char[] keyPassword, KeyStore trustStore, DocumentConventions conventions, ExecutorService executorService, String[] initialUrls) {
        cache = new HttpCache(conventions.getMaxHttpCacheSize());
        _executorService = executorService;
        _databaseName = databaseName;
        this.certificate = certificate;
        this.keyPassword = keyPassword;
        this.trustStore = trustStore;

        _lastReturnedResponse = new Date();
        this.conventions = conventions.clone();
        this._defaultTimeout = conventions.getRequestTimeout();
        this._secondBroadcastAttemptTimeout = conventions.getSecondBroadcastAttemptTimeout();
        this._firstBroadcastAttemptTimeout = conventions.getFirstBroadcastAttemptTimeout();
    }

    public static RequestExecutor create(String[] initialUrls, String databaseName, KeyStore certificate, char[] keyPassword, KeyStore trustStore, ExecutorService executorService, DocumentConventions conventions) {
        RequestExecutor executor = new RequestExecutor(databaseName, certificate, keyPassword, trustStore, conventions, executorService, initialUrls);
        executor._firstTopologyUpdate = executor.firstTopologyUpdate(initialUrls, GLOBAL_APPLICATION_IDENTIFIER);
        return executor;
    }

    public static RequestExecutor createForSingleNodeWithConfigurationUpdates(String url, String databaseName, KeyStore certificate, char[] keyPassword, KeyStore trustStore, ExecutorService executorService, DocumentConventions conventions) {
        RequestExecutor executor = createForSingleNodeWithoutConfigurationUpdates(url, databaseName, certificate, keyPassword, trustStore, executorService, conventions);
        executor._disableClientConfigurationUpdates = false;
        return executor;
    }

    public static RequestExecutor createForSingleNodeWithoutConfigurationUpdates(String url, String databaseName, KeyStore certificate, char[] keyPassword, KeyStore trustStore, ExecutorService executorService, DocumentConventions conventions) {
        final String[] initialUrls = validateUrls(new String[]{url}, certificate);

        RequestExecutor executor = new RequestExecutor(databaseName, certificate, keyPassword, trustStore, conventions, executorService, initialUrls);

        Topology topology = new Topology();
        topology.setEtag(-1L);

        ServerNode serverNode = new ServerNode();
        serverNode.setDatabase(databaseName);
        serverNode.setUrl(initialUrls[0]);
        topology.setNodes(Collections.singletonList(serverNode));

        executor._nodeSelector = new NodeSelector(topology, executorService);
        executor.topologyEtag = INITIAL_TOPOLOGY_ETAG;
        executor._disableTopologyUpdates = true;
        executor._disableClientConfigurationUpdates = true;

        return executor;
    }

    protected CompletableFuture updateClientConfigurationAsync(ServerNode serverNode) {
        if (_disposed) {
            return CompletableFuture.completedFuture(null);
        }

        return CompletableFuture.runAsync(() -> {
            try {
                _updateClientConfigurationSemaphore.acquire();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }

            boolean oldDisableClientConfigurationUpdates = _disableClientConfigurationUpdates;
            _disableClientConfigurationUpdates = true;

            try {
                if (_disposed) {
                    return;
                }

                GetClientConfigurationOperation.GetClientConfigurationCommand command = new GetClientConfigurationOperation.GetClientConfigurationCommand();
                execute(serverNode, null, command, false, null);

                GetClientConfigurationOperation.Result result = command.getResult();
                if (result == null) {
                    return;
                }

                conventions.updateFrom(result.getConfiguration());
                clientConfigurationEtag = result.getEtag();
            } finally {
                _disableClientConfigurationUpdates = oldDisableClientConfigurationUpdates;
                _updateClientConfigurationSemaphore.release();
            }
        }, _executorService);
    }

    public CompletableFuture updateTopologyAsync(UpdateTopologyParameters parameters) {
        if (parameters == null) {
            throw new IllegalArgumentException("Parameters cannot be null");
        }

        if (_disableTopologyUpdates) {
            return CompletableFuture.completedFuture(false);
        }

        if (_disposed) {
            return CompletableFuture.completedFuture(false);
        }

        return CompletableFuture.supplyAsync(() -> {

            //prevent double topology updates if execution takes too much time
            // --> in cases with transient issues
            try {
                boolean lockTaken = _updateDatabaseTopologySemaphore.tryAcquire(parameters.getTimeoutInMs(), TimeUnit.MILLISECONDS);
                if (!lockTaken) {
                    return false;
                }
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }

            try {

                if (_disposed) {
                    return false;
                }

                GetDatabaseTopologyCommand command = new GetDatabaseTopologyCommand(parameters.getDebugTag(),
                        getConventions().isSendApplicationIdentifier() ? parameters.getApplicationIdentifier() : null);
                execute(parameters.getNode(), null, command, false, null);
                Topology topology = command.getResult();

                if (_nodeSelector == null) {
                    _nodeSelector = new NodeSelector(topology, _executorService);

                    if (conventions.getReadBalanceBehavior() == ReadBalanceBehavior.FASTEST_NODE) {
                        _nodeSelector.scheduleSpeedTest();
                    }
                } else if (_nodeSelector.onUpdateTopology(topology, parameters.isForceUpdate())) {
                    disposeAllFailedNodesTimers();
                    if (conventions.getReadBalanceBehavior() == ReadBalanceBehavior.FASTEST_NODE) {
                        _nodeSelector.scheduleSpeedTest();
                    }
                }

                topologyEtag = _nodeSelector.getTopology().getEtag();

                onTopologyUpdatedInvoke(topology);
            } catch (Exception e) {
                if (!_disposed) {
                    throw e;
                }
            } finally {
                _updateDatabaseTopologySemaphore.release();
            }

            return true;
        }, _executorService);

    }

    protected void disposeAllFailedNodesTimers() {
        _failedNodesTimers.forEach((node, status) -> status.close());
        _failedNodesTimers.clear();
    }

    public  void execute(RavenCommand command) {
        execute(command, null);
    }

    public  void execute(RavenCommand command, SessionInfo sessionInfo) {
        CompletableFuture topologyUpdate = _firstTopologyUpdate;
        if (topologyUpdate != null &&
                (topologyUpdate.isDone() && !topologyUpdate.isCompletedExceptionally() && !topologyUpdate.isCancelled())
                || _disableTopologyUpdates) {
            CurrentIndexAndNode currentIndexAndNode = chooseNodeForRequest(command, sessionInfo);
            execute(currentIndexAndNode.currentNode, currentIndexAndNode.currentIndex, command, true, sessionInfo);
        } else {
            unlikelyExecute(command, topologyUpdate, sessionInfo);
        }
    }

    public  CurrentIndexAndNode chooseNodeForRequest(RavenCommand cmd, SessionInfo sessionInfo) {
        if (!_disableTopologyUpdates) {
            // when we disable topology updates we cannot rely on the node tag,
            // because the initial topology will not have them

            if (StringUtils.isNotBlank(cmd.getSelectedNodeTag())) {
                return _nodeSelector.getRequestedNode(cmd.getSelectedNodeTag());
            }
        }

        if (conventions.getLoadBalanceBehavior() == LoadBalanceBehavior.USE_SESSION_CONTEXT) {
            if (sessionInfo != null && sessionInfo.canUseLoadBalanceBehavior()) {
                return _nodeSelector.getNodeBySessionId(sessionInfo.getSessionId());
            }
        }

        if (!cmd.isReadRequest()) {
            return _nodeSelector.getPreferredNode();
        }

        switch (conventions.getReadBalanceBehavior()) {
            case NONE:
                return _nodeSelector.getPreferredNode();
            case ROUND_ROBIN:
                return _nodeSelector.getNodeBySessionId(sessionInfo != null ? sessionInfo.getSessionId() : 0);
            case FASTEST_NODE:
                return _nodeSelector.getFastestNode();
            default:
                throw new IllegalArgumentException();
        }
    }

    private  void unlikelyExecute(RavenCommand command, CompletableFuture topologyUpdate, SessionInfo sessionInfo) {
        waitForTopologyUpdate(topologyUpdate);

        CurrentIndexAndNode currentIndexAndNode = chooseNodeForRequest(command, sessionInfo);
        execute(currentIndexAndNode.currentNode, currentIndexAndNode.currentIndex, command, true, sessionInfo);
    }

    private void waitForTopologyUpdate(CompletableFuture topologyUpdate) {
        try {
            if (topologyUpdate == null || topologyUpdate.isCompletedExceptionally()) {
                synchronized (this) {
                    if (_firstTopologyUpdate == null || topologyUpdate == _firstTopologyUpdate) {
                        if (_lastKnownUrls == null) {
                            // shouldn't happen
                            throw new IllegalStateException("No known topology and no previously known one, cannot proceed, likely a bug");
                        }
                        _firstTopologyUpdate = firstTopologyUpdate(_lastKnownUrls, null);
                    }

                    topologyUpdate = _firstTopologyUpdate;
                }
            }

            topologyUpdate.get();
        } catch (InterruptedException | ExecutionException e) {
            synchronized (this) {
                if (_firstTopologyUpdate == topologyUpdate) {
                    _firstTopologyUpdate = null; // next request will raise it
                }
            }

            throw ExceptionsUtils.unwrapException(e);
        }
    }

    private void updateTopologyCallback() {
        Date time = new Date();
        if (time.getTime() - _lastReturnedResponse.getTime() <= Duration.ofMinutes(5).toMillis()) {
            return;
        }

        ServerNode serverNode;

        try {
            NodeSelector selector = _nodeSelector;
            if (selector == null) {
                return;
            }
            CurrentIndexAndNode preferredNode = selector.getPreferredNode();
            serverNode = preferredNode.currentNode;
        } catch (Exception e) {
            if (logger.isInfoEnabled()) {
                logger.info("Couldn't get preferred node Topology from _updateTopologyTimer", e);
            }
            return;
        }

        UpdateTopologyParameters updateParameters = new UpdateTopologyParameters(serverNode);
        updateParameters.setTimeoutInMs(0);
        updateParameters.setDebugTag("timer-callback");

        updateTopologyAsync(updateParameters)
                .exceptionally(ex -> {
                    if (logger.isInfoEnabled()) {
                        logger.info("Couldn't update topology from _updateTopologyTimer", ex);
                    }
                    return null;
                });
    }

    protected CompletableFuture firstTopologyUpdate(String[] inputUrls) {
        return firstTopologyUpdate(inputUrls, null);
    }

    @SuppressWarnings({"ConstantConditions"})
    protected CompletableFuture firstTopologyUpdate(String[] inputUrls, UUID applicationIdentifier) {
        final String[] initialUrls = validateUrls(inputUrls, certificate);

        ArrayList> list = new ArrayList<>();

        return CompletableFuture.runAsync(() -> {

            for (String url : initialUrls) {
                try {
                    ServerNode serverNode = new ServerNode();
                    serverNode.setUrl(url);
                    serverNode.setDatabase(_databaseName);

                    UpdateTopologyParameters updateParameters = new UpdateTopologyParameters(serverNode);
                    updateParameters.setTimeoutInMs(Integer.MAX_VALUE);
                    updateParameters.setDebugTag("first-topology-update");
                    updateParameters.setApplicationIdentifier(applicationIdentifier);

                    updateTopologyAsync(updateParameters).get();

                    initializeUpdateTopologyTimer();

                    _topologyTakenFromNode = serverNode;
                    return;
                } catch (Exception e) {

                    if (e instanceof ExecutionException && e.getCause() instanceof AuthorizationException) {
                        // auth exceptions will always happen, on all nodes
                        // so errors immediately
                        _lastKnownUrls = initialUrls;
                        throw (AuthorizationException) e.getCause();
                    }

                    if (e instanceof ExecutionException && e.getCause() instanceof DatabaseDoesNotExistException) {
                        // Will happen on all node in the cluster,
                        // so errors immediately
                        _lastKnownUrls = initialUrls;
                        throw (DatabaseDoesNotExistException) e.getCause();
                    }

                    list.add(Tuple.create(url, e));
                }
            }

            Topology topology = new Topology();
            topology.setEtag(topologyEtag);

            List topologyNodes = getTopologyNodes();
            if (topologyNodes == null) {
                topologyNodes = Arrays.stream(initialUrls)
                        .map(url -> {
                            ServerNode serverNode = new ServerNode();
                            serverNode.setUrl(url);
                            serverNode.setDatabase(_databaseName);
                            serverNode.setClusterTag("!");
                            return serverNode;
                        }).collect(Collectors.toList());
            }

            topology.setNodes(topologyNodes);

            _nodeSelector = new NodeSelector(topology, _executorService);

            if (initialUrls != null && initialUrls.length > 0) {
                initializeUpdateTopologyTimer();
                return;
            }

            _lastKnownUrls = initialUrls;
            String details = list.stream().map(x -> x.first + " -> " + Optional.ofNullable(x.second).map(Throwable::getMessage).orElse("")).collect(Collectors.joining(", "));
            throwExceptions(details);
        }, _executorService);
    }

    protected void throwExceptions(String details) {
        throw new IllegalStateException("Failed to retrieve database topology from all known nodes" + System.lineSeparator() + details);
    }

    public static String[] validateUrls(String[] initialUrls, KeyStore certificate) {
        String[] cleanUrls = new String[initialUrls.length];
        boolean requireHttps = certificate != null;
        for (int index = 0; index < initialUrls.length; index++) {
            String url = initialUrls[index];
            try {
                new URL(url);
            } catch (MalformedURLException e) {
                throw new IllegalArgumentException("'" + url + "' is not a valid url");
            }

            cleanUrls[index] = StringUtils.stripEnd(url, "/");
            requireHttps |= url.startsWith("https://");
        }

        if (!requireHttps) {
            return cleanUrls;
        }

        for (String url : initialUrls) {
            if (!url.startsWith("http://")) {
                continue;
            }

            if (certificate != null) {
                throw new IllegalStateException("The url " + url + " is using HTTP, but a certificate is specified, which require us to use HTTPS");
            }
            throw new IllegalStateException("The url " + url + " is using HTTP, but other urls are using HTTPS, and mixing of HTTP and HTTPS is not allowed.");
        }

        return cleanUrls;
    }

    private void initializeUpdateTopologyTimer() {
        if (_updateTopologyTimer != null) {
            return;
        }

        synchronized (this) {
            if (_updateTopologyTimer != null) {
                return;
            }

            _updateTopologyTimer = new Timer(this::updateTopologyCallback, Duration.ofMinutes(1), Duration.ofMinutes(1), _executorService);
        }
    }

    public  void execute(ServerNode chosenNode, Integer nodeIndex, RavenCommand command, boolean shouldRetry, SessionInfo sessionInfo) {
        execute(chosenNode, nodeIndex, command, shouldRetry, sessionInfo, null);
    }

    @SuppressWarnings({"ConstantConditions"})
    public  void execute(ServerNode chosenNode, Integer nodeIndex, RavenCommand command, boolean shouldRetry, SessionInfo sessionInfo, Reference requestRef) {
        if (command.failoverTopologyEtag == INITIAL_TOPOLOGY_ETAG) {
            command.failoverTopologyEtag = INITIAL_TOPOLOGY_ETAG;
            if (_nodeSelector != null && _nodeSelector.getTopology() != null) {
                Topology topology = _nodeSelector.getTopology();
                if (topology.getEtag() != null) {
                    command.failoverTopologyEtag = topology.getEtag();
                }
            }
        }

        Reference urlRef = new Reference<>();
        HttpRequestBase request = createRequest(chosenNode, command, urlRef);

        if (request == null) {
            return;
        }

        if (requestRef != null) {
            requestRef.value = request;
        }

        if (request == null) {
            return;
        }

        //noinspection SimplifiableConditionalExpression
        boolean noCaching = sessionInfo != null ? sessionInfo.isNoCaching() : false;

        Reference cachedChangeVectorRef = new Reference<>();
        Reference cachedValue = new Reference<>();

        try (HttpCache.ReleaseCacheItem cachedItem = getFromCache(command, !noCaching, urlRef.value, cachedChangeVectorRef, cachedValue)) {
            if (cachedChangeVectorRef.value != null) {
                if (tryGetFromCache(command, cachedItem, cachedValue.value)) {
                    return;
                }
            }

            setRequestHeaders(sessionInfo, cachedChangeVectorRef.value, request);

            command.numberOfAttempts = command.numberOfAttempts + 1;
            int attemptNum = command.numberOfAttempts;
            EventHelper.invoke(_onBeforeRequest, this, new BeforeRequestEventArgs(_databaseName, urlRef.value, request, attemptNum));

            CloseableHttpResponse response = sendRequestToServer(chosenNode, nodeIndex, command, shouldRetry, sessionInfo, request, urlRef.value);

            if (response == null) {
                return;
            }

            CompletableFuture refreshTask = refreshIfNeeded(chosenNode, response);

            command.statusCode = response.getStatusLine().getStatusCode();

            ResponseDisposeHandling responseDispose = ResponseDisposeHandling.AUTOMATIC;

            try {
                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
                    EventHelper.invoke(_onSucceedRequest, this, new SucceedRequestEventArgs(_databaseName, urlRef.value, response, request, attemptNum));

                    cachedItem.notModified();

                    try {
                        if (command.getResponseType() == RavenCommandResponseType.OBJECT) {
                            command.setResponse(cachedValue.value, true);
                        }
                    } catch (IOException e) {
                        throw ExceptionsUtils.unwrapException(e);
                    }

                    return;
                }

                if (response.getStatusLine().getStatusCode() >= 400) {
                    if (!handleUnsuccessfulResponse(chosenNode, nodeIndex, command, request, response, urlRef.value, sessionInfo, shouldRetry)) {
                        Header dbMissingHeader = response.getFirstHeader("Database-Missing");
                        if (dbMissingHeader != null && dbMissingHeader.getValue() != null) {
                            throw new DatabaseDoesNotExistException(dbMissingHeader.getValue());
                        }

                        throwFailedToContactAllNodes(command, request);
                    }
                    return; // we either handled this already in the unsuccessful response or we are throwing
                }

                EventHelper.invoke(_onSucceedRequest, this, new SucceedRequestEventArgs(_databaseName, urlRef.value, response, request, attemptNum));

                responseDispose = command.processResponse(cache, response, urlRef.value);
                _lastReturnedResponse = new Date();
            } finally {
                if (responseDispose == ResponseDisposeHandling.AUTOMATIC) {
                    IOUtils.closeQuietly(response, null);
                }

                try {
                    refreshTask.get();
                } catch (Exception e) {
                    //noinspection ThrowFromFinallyBlock
                    throw ExceptionsUtils.unwrapException(e);
                }
            }
        }
    }

    private CompletableFuture refreshIfNeeded(ServerNode chosenNode, CloseableHttpResponse response) {
        Boolean refreshTopology = Optional.ofNullable(HttpExtensions.getBooleanHeader(response, Constants.Headers.REFRESH_TOPOLOGY)).orElse(false);
        Boolean refreshClientConfiguration = Optional.ofNullable(HttpExtensions.getBooleanHeader(response, Constants.Headers.REFRESH_CLIENT_CONFIGURATION)).orElse(false);

        if (refreshTopology || refreshClientConfiguration) {
            ServerNode serverNode = new ServerNode();
            serverNode.setUrl(chosenNode.getUrl());
            serverNode.setDatabase(_databaseName);

            UpdateTopologyParameters updateParameters = new UpdateTopologyParameters(serverNode);
            updateParameters.setTimeoutInMs(0);
            updateParameters.setDebugTag("refresh-topology-header");

            CompletableFuture topologyTask = refreshTopology ? updateTopologyAsync(updateParameters) : CompletableFuture.completedFuture(false);
            CompletableFuture clientConfiguration = refreshClientConfiguration ? updateClientConfigurationAsync(serverNode) : CompletableFuture.completedFuture(null);

            return CompletableFuture.allOf(topologyTask, clientConfiguration);
        }

        return CompletableFuture.allOf();
    }

    private  CloseableHttpResponse sendRequestToServer(ServerNode chosenNode, Integer nodeIndex, RavenCommand command,
                                                                boolean shouldRetry, SessionInfo sessionInfo, HttpRequestBase request, String url) {
        try {
            numberOfServerRequests.incrementAndGet();

            Duration timeout = ObjectUtils.firstNonNull(command.getTimeout(), _defaultTimeout);
            if (timeout != null) {
                AggressiveCacheOptions callingTheadAggressiveCaching = aggressiveCaching.get();

                CompletableFuture sendTask = CompletableFuture.supplyAsync(() -> {
                    AggressiveCacheOptions aggressiveCacheOptionsToRestore = aggressiveCaching.get();

                    try {
                        aggressiveCaching.set(callingTheadAggressiveCaching);
                        return send(chosenNode, command, sessionInfo, request);
                    } catch (IOException e) {
                        throw ExceptionsUtils.unwrapException(e);
                    } finally {
                        aggressiveCaching.set(aggressiveCacheOptionsToRestore);
                    }
                }, _executorService);

                try {
                    return sendTask.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                    throw ExceptionsUtils.unwrapException(e);
                } catch (TimeoutException e) {
                    request.abort();

                    net.ravendb.client.exceptions.TimeoutException timeoutException = new net.ravendb.client.exceptions.TimeoutException("The request for " + request.getURI() + " failed with timeout after " + TimeUtils.durationToTimeSpan(timeout), e);
                    if (!shouldRetry) {
                        if (command.getFailedNodes() == null) {
                            command.setFailedNodes(new HashMap<>());
                        }

                        command.getFailedNodes().put(chosenNode, timeoutException);
                        throw timeoutException;
                    }

                    if (!handleServerDown(url, chosenNode, nodeIndex, command, request, null, timeoutException, sessionInfo, shouldRetry)) {
                        throwFailedToContactAllNodes(command, request);
                    }

                    return null;
                } catch (ExecutionException e) {
                    Throwable rootCause = ExceptionUtils.getRootCause(e);
                    if (rootCause instanceof IOException) {
                        throw (IOException) rootCause;
                    }

                    throw ExceptionsUtils.unwrapException(e);
                }
            } else {
                return send(chosenNode, command, sessionInfo, request);
            }
        } catch (IOException e) {
            if (!shouldRetry) {
                throw ExceptionsUtils.unwrapException(e);
            }

            if (!handleServerDown(url, chosenNode, nodeIndex, command, request, null, e, sessionInfo, shouldRetry)) {
                throwFailedToContactAllNodes(command, request);
            }

            return null;
        }
    }

    private  CloseableHttpResponse send(ServerNode chosenNode, RavenCommand command, SessionInfo sessionInfo, HttpRequestBase request) throws IOException {
        CloseableHttpResponse response = null;

        if (shouldExecuteOnAll(chosenNode, command)) {
            response = executeOnAllToFigureOutTheFastest(chosenNode, command);
        } else {
            response = command.send(getHttpClient(), request);
        }

        // PERF: The reason to avoid rechecking every time is that servers wont change so rapidly
        //       and therefore we dimish its cost by orders of magnitude just doing it
        //       once in a while. We dont care also about the potential race conditions that may happen
        //       here mainly because the idea is to have a lax mechanism to recheck that is at least
        //       orders of magnitude faster than currently.
        if (chosenNode.shouldUpdateServerVersion()) {
            String serverVersion = tryGetServerVersion(response);
            if (serverVersion != null) {
                chosenNode.updateServerVersion(serverVersion);

            }
        }

        lastServerVersion = chosenNode.getLastServerVersion();

        if (sessionInfo != null && sessionInfo.getLastClusterTransactionIndex() != null) {
            // if we reach here it means that sometime a cluster transaction has occurred against this database.
            // Since the current executed command can be dependent on that, we have to wait for the cluster transaction.
            // But we can't do that if the server is an old one.

            if (lastServerVersion == null || lastServerVersion.compareToIgnoreCase("4.1") < 0) {
                throw new ClientVersionMismatchException("The server on " + chosenNode.getUrl() + " has an old version and can't perform " +
                        "the command since this command dependent on a cluster transaction which this node doesn't support.");
            }
        }

        return response;
    }

    private void setRequestHeaders(SessionInfo sessionInfo, String cachedChangeVector, HttpRequest request) {
        if (cachedChangeVector != null) {
            request.addHeader("If-None-Match", "\"" + cachedChangeVector + "\"");
        }

        if (!_disableClientConfigurationUpdates) {
            request.addHeader(Constants.Headers.CLIENT_CONFIGURATION_ETAG, "\"" + clientConfigurationEtag + "\"");
        }

        if (sessionInfo != null && sessionInfo.getLastClusterTransactionIndex() != null) {
            request.addHeader(Constants.Headers.LAST_KNOWN_CLUSTER_TRANSACTION_INDEX, sessionInfo.getLastClusterTransactionIndex().toString());
        }

        if (!_disableTopologyUpdates) {
            request.addHeader(Constants.Headers.TOPOLOGY_ETAG, "\"" + topologyEtag + "\"");
        }

        if (request.getFirstHeader(Constants.Headers.CLIENT_VERSION) == null) {
            request.addHeader(Constants.Headers.CLIENT_VERSION, RequestExecutor.CLIENT_VERSION);
        }
    }

    private  boolean tryGetFromCache(RavenCommand command, HttpCache.ReleaseCacheItem cachedItem, String cachedValue) {
        AggressiveCacheOptions aggressiveCacheOptions = aggressiveCaching.get();
        if (aggressiveCacheOptions != null &&
                cachedItem.getAge().compareTo(aggressiveCacheOptions.getDuration()) < 0 &&
                (!cachedItem.getMightHaveBeenModified() || aggressiveCacheOptions.getMode() != AggressiveCacheMode.TRACK_CHANGES) &&
                command.canCacheAggressively()) {
            try {
                if (cachedItem.item.flags.contains(ItemFlags.NOT_FOUND)) {
                    // if this is a cached delete, we only respect it if it _came_ from an aggressively cached
                    // block, otherwise, we'll run the request again

                    if (cachedItem.item.flags.contains(ItemFlags.AGGRESSIVELY_CACHED)) {
                        command.setResponse(cachedValue, true);
                        return true;
                    }
                } else {
                    command.setResponse(cachedValue, true);
                    return true;
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        return false;
    }

    private static String tryGetServerVersion(CloseableHttpResponse response) {
        Header serverVersionHeader = response.getFirstHeader(Constants.Headers.SERVER_VERSION);

        if (serverVersionHeader != null) {
            return serverVersionHeader.getValue();
        }

        return null;
    }


    private  void throwFailedToContactAllNodes(RavenCommand command, HttpRequestBase request) {
        if (command.getFailedNodes() == null || command.getFailedNodes().size() == 0) { //precaution, should never happen at this point
            throw new IllegalStateException("Received unsuccessful response and couldn't recover from it. " +
                    "Also, no record of exceptions per failed nodes. This is weird and should not happen.");
        }

        if (command.getFailedNodes().size() == 1) {
            throw ExceptionsUtils.unwrapException(command.getFailedNodes().values().iterator().next());
        }

        String message = "Tried to send " + command.resultClass.getName() + " request via " + request.getMethod()
                + " " + request.getURI() + " to all configured nodes in the topology, none of the attempt succeeded." + System.lineSeparator();

        if (_topologyTakenFromNode != null) {
            message += "I was able to fetch " + _topologyTakenFromNode.getDatabase()
                    + " topology from " + _topologyTakenFromNode.getUrl() + "." + System.lineSeparator();
        }

        List nodes = null;
        if (_nodeSelector != null && _nodeSelector.getTopology() != null) {
            nodes = _nodeSelector.getTopology().getNodes();
        }

        if (nodes == null) {
            message += "Topology is empty.";
        } else {
            message += "Topology: ";

            for (ServerNode node : nodes) {
                Exception exception = command.getFailedNodes().get(node);
                message += System.lineSeparator() +
                        "[Url: " + node.getUrl() + ", " +
                        "ClusterTag: " + node.getClusterTag() + ", " +
                        "ServerRole: " + node.getServerRole() + ", " +
                        "Exception: " + (exception != null ? exception.getMessage() : "No exception") + "]";

            }
        }

        throw new AllTopologyNodesDownException(message);
    }

    public boolean inSpeedTestPhase() {
        return Optional.ofNullable(_nodeSelector).map(NodeSelector::inSpeedTestPhase).orElse(false);
    }

    private  boolean shouldExecuteOnAll(ServerNode chosenNode, RavenCommand command) {
        return conventions.getReadBalanceBehavior() == ReadBalanceBehavior.FASTEST_NODE &&
                _nodeSelector != null &&
                _nodeSelector.inSpeedTestPhase() &&
                Optional.ofNullable(_nodeSelector)
                        .map(NodeSelector::getTopology)
                        .map(Topology::getNodes)
                        .map(x -> x.size() > 1)
                        .orElse(false) &&
                command.isReadRequest() &&
                command.getResponseType() == RavenCommandResponseType.OBJECT &&
                chosenNode != null &&
                !(command instanceof IBroadcast);
    }

    @SuppressWarnings("ConstantConditions")
    private  CloseableHttpResponse executeOnAllToFigureOutTheFastest(ServerNode chosenNode, RavenCommand command) {
        AtomicInteger numberOfFailedTasks = new AtomicInteger();

        CompletableFuture preferredTask = null;

        List nodes = _nodeSelector.getTopology().getNodes();
        List> tasks = new ArrayList<>(Collections.nCopies(nodes.size(), null));

        for (int i = 0; i < nodes.size(); i++) {
            final int taskNumber = i;
            numberOfServerRequests.incrementAndGet();

            CompletableFuture task = CompletableFuture.supplyAsync(() -> {
                try {
                    Reference strRef = new Reference<>();
                    HttpRequestBase request = createRequest(nodes.get(taskNumber), command, strRef);
                    setRequestHeaders(null, null, request);
                    return new IndexAndResponse(taskNumber, command.send(getHttpClient(), request));
                } catch (Exception e){
                    numberOfFailedTasks.incrementAndGet();
                    tasks.set(taskNumber, null);
                    throw new RuntimeException("Request execution failed", e);
                }
            }, _executorService);

            if (nodes.get(i).getClusterTag().equals(chosenNode.getClusterTag())) {
                preferredTask = task;
            } else {
                task.thenAcceptAsync(result -> IOUtils.closeQuietly(result.response, null));
            }

            tasks.set(i, task);
        }

        while (numberOfFailedTasks.get() < tasks.size()) {
            try {
                IndexAndResponse fastest = (IndexAndResponse) CompletableFuture
                        .anyOf(tasks.stream().filter(Objects::nonNull)
                        .toArray(CompletableFuture[]::new))
                        .get();
                _nodeSelector.recordFastest(fastest.index, nodes.get(fastest.index));
                break;
            } catch (InterruptedException | ExecutionException e) {
                for (int i = 0; i < nodes.size(); i++) {
                    if (tasks.get(i).isCompletedExceptionally()) {
                        numberOfFailedTasks.incrementAndGet();
                        tasks.set(i, null);
                    }
                }
            }
        }

        // we can reach here if the number of failed task equal to the number
        // of the nodes, in which case we have nothing to do

        try {
            return preferredTask.get().response;
        } catch (InterruptedException | ExecutionException e) {
            throw ExceptionsUtils.unwrapException(e);
        }
    }

    private  HttpCache.ReleaseCacheItem getFromCache(RavenCommand command, boolean useCache, String url, Reference cachedChangeVector, Reference cachedValue) {
        if (useCache && command.canCache() && command.isReadRequest() && command.getResponseType() == RavenCommandResponseType.OBJECT) {
            return cache.get(url, cachedChangeVector, cachedValue);
        }

        cachedChangeVector.value = null;
        cachedValue.value = null;
        return new HttpCache.ReleaseCacheItem();
    }

    private  HttpRequestBase createRequest(ServerNode node, RavenCommand command, Reference url) {
        try {
            HttpRequestBase request = command.createRequest(node, url);
            if (request == null) {
                return null;
            }
            URI builder = new URI(url.value);

            if (requestPostProcessor != null) {
                requestPostProcessor.accept(request);
            }

            if (command instanceof IRaftCommand) {
                IRaftCommand raftCommand = (IRaftCommand) command;

                String raftRequestString = "raft-request-id=" + raftCommand.getRaftUniqueRequestId();

                builder = new URI(builder.getQuery() != null ? builder.toString() + "&" + raftRequestString : builder.toString() + "?" + raftRequestString);
            }

            if (shouldBroadcast(command)) {
                command.setTimeout(ObjectUtils.firstNonNull(command.getTimeout(), _firstBroadcastAttemptTimeout));
            }

            request.setURI(builder);

            return request;
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException("Unable to parse URL", e);
        }
    }

    private  boolean handleUnsuccessfulResponse(ServerNode chosenNode, Integer nodeIndex, RavenCommand command, HttpRequestBase request, CloseableHttpResponse response, String url, SessionInfo sessionInfo, boolean shouldRetry) {
        try {
            switch (response.getStatusLine().getStatusCode()) {
                case HttpStatus.SC_NOT_FOUND:
                    cache.setNotFound(url, aggressiveCaching.get() != null);
                    switch (command.getResponseType()) {
                        case EMPTY:
                            return true;
                        case OBJECT:
                            command.setResponse(null, false);
                            break;
                        default:
                            command.setResponseRaw(response, null);
                            break;
                    }
                    return true;

                case HttpStatus.SC_FORBIDDEN:
                    String msg = tryGetResponseOfError(response);
                    StringBuilder builder = new StringBuilder("Forbidden access to ");
                    builder.append(chosenNode.getDatabase())
                            .append("@")
                            .append(chosenNode.getUrl())
                            .append(", ");

                    if (certificate == null) {
                        builder.append("a certificate is required. ");
                    } else {
                        builder.append("certificate does not have permission to access it or is unknown. ");
                    }

                    builder.append(" Method: ")
                        .append(request.getMethod())
                            .append(", Request: ")
                            .append(request.getURI().toString())
                            .append(System.lineSeparator())
                            .append(msg);

                    throw new AuthorizationException(builder.toString());

                case HttpStatus.SC_GONE: // request not relevant for the chosen node - the database has been moved to a different one
                    if (!shouldRetry) {
                        return false;
                    }

                    if (nodeIndex != null) {
                        _nodeSelector.onFailedRequest(nodeIndex);
                    }

                    if (command.getFailedNodes() == null) {
                        command.setFailedNodes(new HashMap<>());
                    }

                    if (!command.isFailedWithNode(chosenNode)) {
                        command.getFailedNodes().put(chosenNode, new UnsuccessfulRequestException("Request to " + request.getURI() + " (" + request.getMethod() + ") is not relevant for this node anymore."));
                    }

                    CurrentIndexAndNode indexAndNode = chooseNodeForRequest(command, sessionInfo);

                    if (command.getFailedNodes().containsKey(indexAndNode.currentNode)) {
                        // we tried all the nodes, let's try to update topology and retry one more time
                        UpdateTopologyParameters updateParameters = new UpdateTopologyParameters(chosenNode);
                        updateParameters.setTimeoutInMs(60_000);
                        updateParameters.setForceUpdate(true);
                        updateParameters.setDebugTag("handle-unsuccessful-response");
                        Boolean success = updateTopologyAsync(updateParameters).get();
                        if (!success) {
                            return false;
                        }

                        command.getFailedNodes().clear(); // we just update the topology
                        indexAndNode = chooseNodeForRequest(command, sessionInfo);

                        execute(indexAndNode.currentNode, indexAndNode.currentIndex, command, false, sessionInfo);
                        return true;
                    }

                    execute(indexAndNode.currentNode, indexAndNode.currentIndex, command, false, sessionInfo);
                    return true;
                case HttpStatus.SC_GATEWAY_TIMEOUT:
                case HttpStatus.SC_REQUEST_TIMEOUT:
                case HttpStatus.SC_BAD_GATEWAY:
                case HttpStatus.SC_SERVICE_UNAVAILABLE:
                    return handleServerDown(url, chosenNode, nodeIndex, command, request, response, null, sessionInfo, shouldRetry);
                case HttpStatus.SC_CONFLICT:
                    handleConflict(response);
                    break;
                case 425: // TooEarly
                    if (!shouldRetry) {
                        return false;
                    }

                    if (nodeIndex != null) {
                        _nodeSelector.onFailedRequest(nodeIndex);
                    }

                    if (command.getFailedNodes() == null) {
                        command.setFailedNodes(new HashMap<>());
                    }

                    if (!command.isFailedWithNode(chosenNode)) {
                        command.getFailedNodes().put(chosenNode,
                                new UnsuccessfulRequestException("Request to '" + request.getURI() + "' ("  +request.getMethod() + ") is processing and not yet available on that node."));
                    }

                    CurrentIndexAndNode nextNode = chooseNodeForRequest(command, sessionInfo);
                    execute(nextNode.currentNode, nextNode.currentIndex, command, true, sessionInfo);

                    if (nodeIndex != null) {
                        _nodeSelector.restoreNodeIndex(nodeIndex);
                    }

                    return true;
                default:
                    command.onResponseFailure(response);
                    ExceptionDispatcher.throwException(response);
                    break;
            }
        } catch (IOException | ExecutionException | InterruptedException e) {
            throw ExceptionsUtils.unwrapException(e);
        }

        return false;
    }

    private static String tryGetResponseOfError(CloseableHttpResponse response) {
        try {
            return IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
        } catch (Exception e) {
            return "Could not read request: " + e.getMessage();
        }
    }

    private static void handleConflict(CloseableHttpResponse response) {
        ExceptionDispatcher.throwException(response);
    }

    public static InputStream readAsStream(CloseableHttpResponse response) throws IOException {
        return response.getEntity().getContent();
    }

    private  boolean handleServerDown(String url, ServerNode chosenNode, Integer nodeIndex,
                                               RavenCommand command, HttpRequestBase request,
                                               CloseableHttpResponse response, Exception e,
                                               SessionInfo sessionInfo, boolean shouldRetry) {
        if (command.getFailedNodes() == null) {
            command.setFailedNodes(new HashMap<>());
        }

        command.getFailedNodes().put(chosenNode, readExceptionFromServer(request, response, e));

        if (nodeIndex == null) {
            //We executed request over a node not in the topology. This means no failover...
            return false;
        }

        if (_nodeSelector == null) {
            spawnHealthChecks(chosenNode, nodeIndex);
            return false;
        }

        // As the server is down, we discard the server version to ensure we update when it goes up.
        chosenNode.discardServerVersion();

        _nodeSelector.onFailedRequest(nodeIndex);

        if (shouldBroadcast(command)) {
            command.setResult(broadcast(command, sessionInfo));
            return true;
        }

        spawnHealthChecks(chosenNode, nodeIndex);

        CurrentIndexAndNodeAndEtag indexAndNodeAndEtag = _nodeSelector.getPreferredNodeWithTopology();
        if (command.failoverTopologyEtag != topologyEtag) {
            command.getFailedNodes().clear();
            command.failoverTopologyEtag = topologyEtag;
        }

        if (command.getFailedNodes().containsKey(indexAndNodeAndEtag.currentNode)) {
            return false;
        }

        onFailedRequestInvoke(url, e);

        execute(indexAndNodeAndEtag.currentNode, indexAndNodeAndEtag.currentIndex, command, shouldRetry, sessionInfo);

        return true;
    }

    private  boolean shouldBroadcast(RavenCommand command) {
        if (!(command instanceof IBroadcast)) {
            return false;
        }

        List topologyNodes = getTopologyNodes();

        if (topologyNodes == null || topologyNodes.size() < 2) {
            return false;
        }

        return true;
    }

    public static class BroadcastState {
        private RavenCommand command;
        private int index;
        private ServerNode node;
        private HttpRequestBase request;

        public RavenCommand getCommand() {
            return command;
        }

        public void setCommand(RavenCommand command) {
            this.command = command;
        }

        public int getIndex() {
            return index;
        }

        public void setIndex(int index) {
            this.index = index;
        }

        public ServerNode getNode() {
            return node;
        }

        public void setNode(ServerNode node) {
            this.node = node;
        }

        public HttpRequestBase getRequest() {
            return request;
        }

        public void setRequest(HttpRequestBase request) {
            this.request = request;
        }
    }

    private  TResult broadcast(RavenCommand command, SessionInfo sessionInfo) {
        if (!(command instanceof IBroadcast)) {
            throw new IllegalStateException("You can broadcast only commands that implement 'IBroadcast'.");
        }

        IBroadcast broadcastCommand = (IBroadcast) command;
        final Map failedNodes = command.getFailedNodes();

        command.setFailedNodes(new HashMap<>()); // clean the current failures
        Map, BroadcastState> broadcastTasks = new HashMap<>();

        try {
            sendToAllNodes(broadcastTasks, sessionInfo, broadcastCommand);

            return waitForBroadcastResult(command, broadcastTasks);
        } finally {
            for (Map.Entry, BroadcastState> broadcastState : broadcastTasks.entrySet()) {
                CompletableFuture task = broadcastState.getKey();
                if (task != null) {
                    task.exceptionally(throwable -> {
                        int index = broadcastState.getValue().getIndex();
                        ServerNode node = _nodeSelector.getTopology().getNodes().get(index);
                        if (failedNodes.containsKey(node)) {
                            // if other node succeed in broadcast we need to send health checks to the original failed node
                            spawnHealthChecks(node, index);
                        }
                        return null;
                    });
                }
            }
        }
    }

    private  TResult waitForBroadcastResult(RavenCommand command, Map, BroadcastState> tasks) {
        while (!tasks.isEmpty()) {
            Exception error = null;
            try {
                CompletableFuture.anyOf(tasks.keySet().toArray(new CompletableFuture[0])).get();
            } catch (InterruptedException | ExecutionException e) {
                error = e;
            }

            CompletableFuture completed = tasks
                    .keySet()
                    .stream()
                    .filter(CompletableFuture::isDone)
                    .findFirst()
                    .orElse(null);

            if (error != null) {
                BroadcastState failed = tasks.get(completed);
                ServerNode node = _nodeSelector.getTopology().getNodes().get(failed.index);

                command.getFailedNodes().put(node, error.getCause() != null ? (Exception) error.getCause() : error);

                _nodeSelector.onFailedRequest(failed.getIndex());
                spawnHealthChecks(node, failed.getIndex());

                tasks.remove(completed);
                continue;
            }

            for (BroadcastState state : tasks.values()) {
                HttpRequestBase request = state.getRequest();
                if (request != null) {
                    request.abort();
                }
            }

            _nodeSelector.restoreNodeIndex(tasks.get(completed).getIndex());
            return tasks.get(completed).getCommand().getResult();
        }

        String exceptions = command
                .getFailedNodes()
                .entrySet()
                .stream()
                .map(x -> new UnsuccessfulRequestException(x.getKey().getUrl(), x.getValue()))
                .map(Throwable::toString)
                .collect(Collectors.joining(", "));

        throw new AllTopologyNodesDownException("Broadcasting " + command.getClass().getSimpleName() + " failed: " + exceptions);
    }

    @SuppressWarnings("unchecked")
    private  void sendToAllNodes(Map, BroadcastState> tasks, SessionInfo sessionInfo, IBroadcast command) {
        for (int index = 0; index < _nodeSelector.getTopology().getNodes().size(); index++) {
            BroadcastState state = new BroadcastState<>();
            state.setCommand((RavenCommand)command.prepareToBroadcast(getConventions()));
            state.setIndex(index);
            state.setNode(_nodeSelector.getTopology().getNodes().get(index));

            state.getCommand().setTimeout(_secondBroadcastAttemptTimeout);

            AggressiveCacheOptions callingTheadAggressiveCaching = aggressiveCaching.get();

            CompletableFuture task = CompletableFuture.runAsync(() -> {
                AggressiveCacheOptions aggressiveCacheOptionsToRestore = aggressiveCaching.get();

                aggressiveCaching.set(callingTheadAggressiveCaching);
                try {
                    Reference requestRef = new Reference<>();
                    execute(state.getNode(), null, state.getCommand(), false, sessionInfo, requestRef);
                    state.setRequest(requestRef.value);
                } finally {
                    aggressiveCaching.set(aggressiveCacheOptionsToRestore);
                }
            }, _executorService);
            tasks.put(task, state);
        }
    }

    public ServerNode handleServerNotResponsive(String url, ServerNode chosenNode, int nodeIndex, Exception e) {
        spawnHealthChecks(chosenNode, nodeIndex);
        if (_nodeSelector != null) {
            _nodeSelector.onFailedRequest(nodeIndex);
        }
        CurrentIndexAndNode preferredNode = getPreferredNode();

        if (_disableTopologyUpdates) {
            performHealthCheck(chosenNode, nodeIndex);
        } else {
            try {
                UpdateTopologyParameters updateParameters = new UpdateTopologyParameters(preferredNode.currentNode);
                updateParameters.setTimeoutInMs(0);
                updateParameters.setForceUpdate(true);
                updateParameters.setDebugTag("handle-server-not-responsive");
                updateTopologyAsync(updateParameters).get();
            } catch (InterruptedException | ExecutionException ee) {
                throw ExceptionsUtils.unwrapException(e);
            }
        }

        onFailedRequestInvoke(url, e);

        return preferredNode.currentNode;
    }

    private void spawnHealthChecks(ServerNode chosenNode, int nodeIndex) {
        if (_nodeSelector != null && _nodeSelector.getTopology().getNodes().size() < 1) {
            return;
        }

        NodeStatus nodeStatus = new NodeStatus(this, nodeIndex, chosenNode);

        if (_failedNodesTimers.putIfAbsent(chosenNode, nodeStatus) == null) {
            nodeStatus.startTimer();
        }
    }

    private void checkNodeStatusCallback(NodeStatus nodeStatus) {
        List copy = getTopologyNodes();

        if (nodeStatus.nodeIndex >= copy.size()) {
            return; // topology index changed / removed
        }

        ServerNode serverNode = copy.get(nodeStatus.nodeIndex);
        if (serverNode != nodeStatus.node) {
            return;  // topology changed, nothing to check
        }

        try {
            NodeStatus status;

            try {
                performHealthCheck(serverNode, nodeStatus.nodeIndex);
            } catch (Exception e) {
                if (logger.isInfoEnabled()) {
                    logger.info(serverNode.getClusterTag() + " is still down", e);
                }

                status = _failedNodesTimers.get(nodeStatus.node);
                if (status != null) {
                    status.updateTimer();
                }

                return; // will wait for the next timer call
            }

            status = _failedNodesTimers.get(nodeStatus.node);
            if (status != null) {
                _failedNodesTimers.remove(nodeStatus.node);
                status.close();
            }

            if (_nodeSelector != null) {
                _nodeSelector.restoreNodeIndex(nodeStatus.nodeIndex);
            }

        } catch (Exception e) {
            if (logger.isInfoEnabled()) {
                logger.info("Failed to check node topology, will ignore this node until next topology update", e);
            }
        }
    }

    protected void performHealthCheck(ServerNode serverNode, int nodeIndex) {
        try {
            if (!_useOldFailureCheckOperation.contains(serverNode.getUrl())) {
                execute(serverNode, nodeIndex, failureCheckOperation.getCommand(conventions), false, null);
            } else {
                executeOldHealthCheck(serverNode, nodeIndex);
            }
        } catch (Exception e) {
            if (e.getMessage().contains("RouteNotFoundException")) {
                _useOldFailureCheckOperation.add(serverNode.getUrl());
                executeOldHealthCheck(serverNode, nodeIndex);
                return;
            }

            throw ExceptionsUtils.unwrapException(e);
        }
    }

    private void executeOldHealthCheck(ServerNode serverNode, int nodeIndex) {
        execute(serverNode, nodeIndex, backwardCompatibilityFailureCheckOperation.getCommand(conventions), false, null);
    }

    private static  Exception readExceptionFromServer(HttpRequestBase request, CloseableHttpResponse response, Exception e) {
        if (response != null && response.getEntity() != null) {
            String responseJson = null;
            try {
                responseJson = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);

                return ExceptionDispatcher.get(JsonExtensions.getDefaultMapper().readValue(responseJson, ExceptionDispatcher.ExceptionSchema.class), response.getStatusLine().getStatusCode(), e);
            } catch (Exception __) {
                ExceptionDispatcher.ExceptionSchema exceptionSchema = new ExceptionDispatcher.ExceptionSchema();
                exceptionSchema.setUrl(request.getURI().toString());
                exceptionSchema.setMessage("Get unrecognized response from the server");
                exceptionSchema.setError(responseJson);
                exceptionSchema.setType("Unparsable Server Response");

                return ExceptionDispatcher.get(exceptionSchema, response.getStatusLine().getStatusCode(), e);
            }
        }

        // this would be connections that didn't have response, such as "couldn't connect to remote server"
        ExceptionDispatcher.ExceptionSchema exceptionSchema = new ExceptionDispatcher.ExceptionSchema();
        exceptionSchema.setUrl(request.getURI().toString());
        exceptionSchema.setMessage(e.getMessage());
        exceptionSchema.setError("An exception occurred while contacting " + request.getURI() + "." + System.lineSeparator() + e.toString());
        exceptionSchema.setType(e.getClass().getCanonicalName());

        return ExceptionDispatcher.get(exceptionSchema, HttpStatus.SC_SERVICE_UNAVAILABLE, e);
    }

    protected CompletableFuture _firstTopologyUpdate;
    protected String[] _lastKnownUrls;
    protected boolean _disposed;

    @Override
    public void close() {
        if (_disposed) {
            return;
        }

        _disposed = true;
        cache.close();

        if (_updateTopologyTimer != null) {
            _updateTopologyTimer.close();
        }
        
        disposeAllFailedNodesTimers();
    }

    private CloseableHttpClient createClient() {
        HttpClientBuilder httpClientBuilder = HttpClients
                .custom()
                .setMaxConnPerRoute(30)
                .setMaxConnTotal(40)
                .setDefaultRequestConfig(
                        RequestConfig.custom()
                                .setConnectionRequestTimeout(3000)
                                .build()
                );

        if (conventions.hasExplicitlySetCompressionUsage() && !conventions.isUseCompression()) {
            httpClientBuilder.disableContentCompression();
        }

        httpClientBuilder
                .setRetryHandler(new StandardHttpRequestRetryHandler(0, false))
                .setDefaultSocketConfig(SocketConfig.custom().setTcpNoDelay(true).build());

        if (certificate != null) {
            try {
                httpClientBuilder.setSSLHostnameVerifier((s, sslSession) -> {
                    // Here we are explicitly ignoring trust issues in the case of ClusterRequestExecutor.
                    // this is because we don't actually require trust, we just use the certificate
                    // as a way to authenticate. Either we encounter the same server certificate which we already
                    // trust, or the admin is going to tell us which specific certs we can trust.
                    return true;
                });

                httpClientBuilder.setSSLContext(createSSLContext());
            } catch ( Exception e) {
                throw new IllegalStateException("Unable to configure ssl context: " + e.getMessage(), e);
            }
        }

        if (configureHttpClient != null) {
            configureHttpClient.accept(httpClientBuilder);
        }

        return httpClientBuilder.build();
    }

    public SSLContext createSSLContext() throws UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
        SSLContextBuilder sslContextBuilder = SSLContexts.custom()
                .loadKeyMaterial(certificate, keyPassword);

        if (this.trustStore != null) {
            sslContextBuilder.loadTrustMaterial(trustStore, null);
        }

        return sslContextBuilder.build();
    }

    public static class NodeStatus implements CleanCloseable {

        private Duration _timerPeriod;
        private final RequestExecutor _requestExecutor;
        public final int nodeIndex;
        public final ServerNode node;
        private Timer _timer;

        public NodeStatus(RequestExecutor requestExecutor, int nodeIndex, ServerNode node) {
            _requestExecutor = requestExecutor;
            this.nodeIndex = nodeIndex;
            this.node = node;
            _timerPeriod = Duration.ofMillis(100);
        }

        private Duration nextTimerPeriod() {
            if (_timerPeriod.compareTo(Duration.ofSeconds(5)) >= 0) {
                return Duration.ofSeconds(5);
            }

            _timerPeriod = _timerPeriod.plus(Duration.ofMillis(100));

            return _timerPeriod;
        }

        public void startTimer() {
            _timer = new Timer(this::timerCallback, _timerPeriod, _requestExecutor._executorService);
        }

        private void timerCallback() {
            if (_requestExecutor._disposed) {
                close();
                return;
            }

            _requestExecutor.checkNodeStatusCallback(this);
        }

        public void updateTimer() {
            _timer.change(nextTimerPeriod());
        }

        @Override
        public void close() {
            _timer.close();
        }
    }

    public CurrentIndexAndNode getRequestedNode(String nodeTag) {
        ensureNodeSelector();

        return _nodeSelector.getRequestedNode(nodeTag);
    }

    public CurrentIndexAndNode getPreferredNode() {
        ensureNodeSelector();

        return _nodeSelector.getPreferredNode();
    }

    public CurrentIndexAndNode getNodeBySessionId(int sessionId) {
        ensureNodeSelector();

        return _nodeSelector.getNodeBySessionId(sessionId);
    }

    public CurrentIndexAndNode getFastestNode() {
        ensureNodeSelector();

        return _nodeSelector.getFastestNode();
    }

    private void ensureNodeSelector() {
        if (!_disableTopologyUpdates) {
            waitForTopologyUpdate(_firstTopologyUpdate);
        }

        if (_nodeSelector == null) {
            Topology topology = new Topology();

            topology.setNodes(new ArrayList<>(getTopologyNodes()));
            topology.setEtag(topologyEtag);

            _nodeSelector = new NodeSelector(topology, _executorService);
        }
    }

    protected void onTopologyUpdatedInvoke(Topology newTopology) {
        EventHelper.invoke(_onTopologyUpdated, this, new TopologyUpdatedEventArgs(newTopology));
    }

    public static class IndexAndResponse {
        public final int index;
        public final CloseableHttpResponse response;

        public IndexAndResponse(int index, CloseableHttpResponse response) {
            this.index = index;
            this.response = response;
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy