com.sap.hana.datalake.files.utils.http.HttpClientUtils Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of sap-hdlfs Show documentation
An implementation of org.apache.hadoop.fs.FileSystem targeting SAP HANA Data Lake Files.
There is a newer version: 3.0.27
// © 2023-2024 SAP SE or an SAP affiliate company. All rights reserved.
package com.sap.hana.datalake.files.utils.http;

import com.sap.hana.datalake.files.HdlfsConstants;
import com.sap.hana.datalake.files.classification.InterfaceAudience;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;

import java.io.Closeable;
import java.io.IOException;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.IntPredicate;

@InterfaceAudience.Private
public class HttpClientUtils {

  // Some status codes are not available on current httpcore version
  public static final int SC_PERMANENT_REDIRECT = 308;
  private static final int SC_TOO_MANY_REQUESTS = 429;

  private static final int SC_REQUEST_TIMEOUT = HttpStatus.SC_REQUEST_TIMEOUT;

  public static HttpClient createHttpClient(final Builder builder) {
    // Retry: 5xx, 408 Request timeout and 429 Too many requests
    // https://cloud.google.com/storage/docs/retry-strategy
    final IntPredicate statusCodeCheck = statusCode -> (statusCode >= 500 && statusCode <= 599) ||
            statusCode == SC_TOO_MANY_REQUESTS ||
            statusCode == SC_REQUEST_TIMEOUT;

    final ExponentialBackoffRetryStrategy retryStrategy = new ExponentialBackoffRetryStrategy(builder.retriesMaxCount, builder.retriesMinIntervalMs, builder.retriesMaxIntervalMs, statusCodeCheck);
    final HttpRequestRetryHandler retryHandler = new RequestRetryHandler(builder.retriesMaxCount, builder.retrySentRequests, builder.nonRetriableExceptions);
    final DefaultTimeConnectionKeepAliveStrategy keepAliveStrategy = new DefaultTimeConnectionKeepAliveStrategy(builder.connectionsKeepAliveSeconds);
    final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(builder.connectionsTtlSeconds, TimeUnit.SECONDS);

    connectionManager.setValidateAfterInactivity(builder.validateConnectionsAfterInactivityMs);
    connectionManager.setMaxTotal(builder.connectionsMaxCount);
    connectionManager.setDefaultMaxPerRoute(builder.connectionsMaxCount);

    return HttpClients.custom()
            .setServiceUnavailableRetryStrategy(retryStrategy)
            .setRetryHandler(retryHandler)
            .setKeepAliveStrategy(keepAliveStrategy)
            .setConnectionManager(connectionManager)
            .setDefaultRequestConfig(RequestConfig.custom()
                    .setConnectTimeout(builder.connectTimeoutMs)
                    .setSocketTimeout(builder.socketTimeoutMs)
                    .build())
            .build();
  }

  public static IOException getAndLogExceptionForUnsuccessfulResponse(final HttpResponse httpResponse, final String errMessagePrefix,
                                                                      final Logger logger) throws IOException {
    final int statusCode = httpResponse.getStatusLine().getStatusCode();
    final String errMessage = String.format("%s - status code: %d", errMessagePrefix, statusCode);

    if (logger.isDebugEnabled()) {
      logger.debug("{}, resp body: {}", errMessage, EntityUtils.toString(httpResponse.getEntity()));
    }

    return new IOException(errMessage);
  }

  public static void closeHttpResponseQuietly(final HttpResponse httpResponse) {
    if (httpResponse == null) {
      return;
    }

    if (httpResponse instanceof Closeable) {
      IOUtils.closeQuietly((Closeable) httpResponse);
    }

    EntityUtils.consumeQuietly(httpResponse.getEntity());
  }

  public static boolean isSuccessfulStatusCode(final int statusCode) {
    return statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
  }

  public static String getHeaderValue(final HttpResponse response, final String headerName) {
    return Optional.ofNullable(response.getFirstHeader(headerName))
            .map(Header::getValue)
            .orElse(null);
  }

  public static String getRequiredHeaderValue(final HttpResponse response, final String headerName) {
    final String headerValue = getHeaderValue(response, headerName);

    if (headerValue == null || headerValue.isEmpty()) {
      throw new IllegalStateException(String.format("Required header [%s] is absent", headerName));
    }

    return headerValue;
  }

  public static class Builder {

    // Connection pool
    private int connectionsMaxCount = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_CONNECTIONS_MAX_COUNT_DEFAULT;
    private int connectionsKeepAliveSeconds = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_CONNECTIONS_KEEP_ALIVE_SECONDS_DEFAULT;
    private int connectionsTtlSeconds = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_CONNECTIONS_TTL_SECONDS_DEFAULT;
    private int validateConnectionsAfterInactivityMs = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_CONNECTIONS_VALIDATE_AFTER_INACTIVITY_MS_DEFAULT;

    // Socket
    private int connectTimeoutMs = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_CONNECT_TIMEOUT_MS_DEFAULT;
    private int socketTimeoutMs = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_SOCKET_TIMEOUT_MS_DEFAULT;

    // Retry handling
    private int retriesMaxCount = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_RETRIES_MAX_COUNT_DEFAULT;
    private int retriesMaxIntervalMs = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_RETRIES_MAX_INTERVAL_MS_DEFAULT;
    private int retriesMinIntervalMs = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_RETRIES_MIN_INTERVAL_MS_DEFAULT;
    private boolean retrySentRequests = HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_RETRY_SENT_REQUESTS_DEFAULT;
    private Set> nonRetriableExceptions = toSetOfIOExceptionClasses(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_HTTP_CLIENT_NON_RETRIABLE_EXCEPTIONS_DEFAULT);

    @SuppressWarnings("unchecked")
    private static Set> toSetOfIOExceptionClasses(final Class[] arrayOfClasses) {
      final Set> set = new HashSet<>();

      for (final Class nonRetriableExceptionClass : arrayOfClasses) {
        if (!IOException.class.isAssignableFrom(nonRetriableExceptionClass)) {
          throw new IllegalArgumentException(String.format("Class [%s] is not a superclass of %s", nonRetriableExceptionClass, IOException.class));
        }

        set.add((Class) nonRetriableExceptionClass);
      }

      return set;
    }

    public Builder setConnectionsMaxCount(final int connectionsMaxCount) {
      this.connectionsMaxCount = connectionsMaxCount;
      return this;
    }

    public Builder setConnectionsTtlSeconds(final int connectionsTtlSeconds) {
      this.connectionsTtlSeconds = connectionsTtlSeconds;
      return this;
    }

    public Builder setValidateConnectionsAfterInactivityMs(final int validateConnectionsAfterInactivityMs) {
      this.validateConnectionsAfterInactivityMs = validateConnectionsAfterInactivityMs;
      return this;
    }

    public Builder setConnectionsKeepAliveSeconds(final int connectionsKeepAliveSeconds) {
      this.connectionsKeepAliveSeconds = connectionsKeepAliveSeconds;
      return this;
    }

    public Builder setConnectTimeoutMs(final int connectTimeoutMs) {
      this.connectTimeoutMs = connectTimeoutMs;
      return this;
    }

    public Builder setSocketTimeoutMs(final int socketTimeoutMs) {
      this.socketTimeoutMs = socketTimeoutMs;
      return this;
    }

    public Builder setRetriesMaxCount(final int retriesMaxCount) {
      this.retriesMaxCount = retriesMaxCount;
      return this;
    }

    public Builder setRetriesMaxIntervalMs(final int retriesMaxIntervalMs) {
      this.retriesMaxIntervalMs = retriesMaxIntervalMs;
      return this;
    }

    public Builder setRetriesMinIntervalMs(final int retriesMinIntervalMs) {
      this.retriesMinIntervalMs = retriesMinIntervalMs;
      return this;
    }

    public Builder setRetrySentRequests(final boolean retrySentRequests) {
      this.retrySentRequests = retrySentRequests;
      return this;
    }

    public Builder setNonRetriableExceptions(final Class[] nonRetriableExceptions) {
      this.nonRetriableExceptions = toSetOfIOExceptionClasses(nonRetriableExceptions);
      return this;
    }

    @Override
    public String toString() {
      return "Builder{" +
              "connectionsMaxCount=" + this.connectionsMaxCount +
              ", connectionsKeepAliveSeconds=" + this.connectionsKeepAliveSeconds +
              ", connectionsTtlSeconds=" + this.connectionsTtlSeconds +
              ", validateConnectionsAfterInactivityMs=" + this.validateConnectionsAfterInactivityMs +
              ", connectTimeoutMs=" + this.connectTimeoutMs +
              ", socketTimeoutMs=" + this.socketTimeoutMs +
              ", retriesMaxCount=" + this.retriesMaxCount +
              ", retriesMaxIntervalMs=" + this.retriesMaxIntervalMs +
              ", retriesMinIntervalMs=" + this.retriesMinIntervalMs +
              ", retrySentRequests=" + this.retrySentRequests +
              ", nonRetriableExceptions=" + this.nonRetriableExceptions +
              '}';
    }
  }

}

// © 2023-2024 SAP SE or an SAP affiliate company. All rights reserved.