software.amazon.glue.ExponentialHttpRequestRetryStrategy Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of glue-catalog-extensions-for-iceberg Show documentation
AWS Glue Catalog Extensions for Apache Iceberg
The newest version!
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *  http://aws.amazon.com/apache2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package software.amazon.glue;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
import java.time.Instant;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import javax.net.ssl.SSLException;
import org.apache.hc.client5.http.HttpRequestRetryStrategy;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.concurrent.CancellableDependency;
import org.apache.hc.core5.http.ConnectionClosedException;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpRequest;
import org.apache.hc.core5.http.HttpResponse;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.Method;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.TimeValue;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;

/**
 * Defines an exponential HTTP request retry strategy and provides the same characteristics as the
 * {@link org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy}, using the following list
 * of non-retriable I/O exception classes:
 *
 * 
 *   InterruptedIOException
 *   
UnknownHostException
 *   
ConnectException
 *   
ConnectionClosedException
 *   
NoRouteToHostException
 *   
SSLException
 * 
 *
 * The following retriable HTTP status codes are defined:
 *
 * 
 *   SC_TOO_MANY_REQUESTS (429)
 *   
SC_BAD_GATEWAY (502)
 *   
SC_SERVICE_UNAVAILABLE (503)
 *   
SC_GATEWAY_TIMEOUT (504)
 * 
 *
 * Most code and behavior is taken from {@link
 * org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy}, with minor modifications to
 * {@link #getRetryInterval(HttpResponse, int, HttpContext)} to achieve exponential backoff.
 */
class ExponentialHttpRequestRetryStrategy implements HttpRequestRetryStrategy {
  private final int maxRetries;
  private final Set> nonRetriableExceptions;
  private final Set retriableCodes;

  ExponentialHttpRequestRetryStrategy(int maximumRetries) {
    Preconditions.checkArgument(
        maximumRetries > 0, "Cannot set retries to %s, the value must be positive", maximumRetries);
    this.maxRetries = maximumRetries;
    this.retriableCodes =
        ImmutableSet.of(
            HttpStatus.SC_TOO_MANY_REQUESTS,
            HttpStatus.SC_SERVICE_UNAVAILABLE,
            HttpStatus.SC_BAD_GATEWAY,
            HttpStatus.SC_GATEWAY_TIMEOUT);
    this.nonRetriableExceptions =
        ImmutableSet.of(
            InterruptedIOException.class,
            UnknownHostException.class,
            ConnectException.class,
            ConnectionClosedException.class,
            NoRouteToHostException.class,
            SSLException.class);
  }

  @Override
  public boolean retryRequest(
      HttpRequest request, IOException exception, int execCount, HttpContext context) {
    if (execCount > maxRetries) {
      // Do not retry if over max retries
      return false;
    }

    if (nonRetriableExceptions.contains(exception.getClass())) {
      return false;
    } else {
      for (Class rejectException : nonRetriableExceptions) {
        if (rejectException.isInstance(exception)) {
          return false;
        }
      }
    }

    if (request instanceof CancellableDependency
        && ((CancellableDependency) request).isCancelled()) {
      return false;
    }

    // Retry if the request is considered idempotent
    return Method.isIdempotent(request.getMethod());
  }

  @Override
  public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) {
    return execCount <= maxRetries && retriableCodes.contains(response.getCode());
  }

  @Override
  public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) {
    // a server may send a 429 / 503 with a Retry-After header
    // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
    Header header = response.getFirstHeader(HttpHeaders.RETRY_AFTER);
    TimeValue retryAfter = null;
    if (header != null) {
      String value = header.getValue();
      try {
        retryAfter = TimeValue.ofSeconds(Long.parseLong(value));
      } catch (NumberFormatException ignore) {
        Instant retryAfterDate = DateUtils.parseStandardDate(value);
        if (retryAfterDate != null) {
          retryAfter =
              TimeValue.ofMilliseconds(retryAfterDate.toEpochMilli() - System.currentTimeMillis());
        }
      }

      if (TimeValue.isPositive(retryAfter)) {
        return retryAfter;
      }
    }

    int delayMillis = 1000 * (int) Math.min(Math.pow(2.0, (long) execCount - 1), 64.0);
    int jitter = ThreadLocalRandom.current().nextInt(Math.max(1, (int) (delayMillis * 0.1)));

    return TimeValue.ofMilliseconds(delayMillis + jitter);
  }
}