All Downloads are FREE. Search and download functionalities are using the official Maven repository.

net.snowflake.ingest.utils.HttpUtil Maven / Gradle / Ivy

There is a newer version: 3.0.0
Show newest version
/*
 * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
 */

package net.snowflake.ingest.utils;

import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;

import java.security.Security;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import net.snowflake.client.core.SFSessionProperty;
import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
import net.snowflake.client.jdbc.internal.apache.http.HttpRequest;
import net.snowflake.client.jdbc.internal.apache.http.HttpResponse;
import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException;
import net.snowflake.client.jdbc.internal.apache.http.auth.AuthScope;
import net.snowflake.client.jdbc.internal.apache.http.auth.Credentials;
import net.snowflake.client.jdbc.internal.apache.http.auth.UsernamePasswordCredentials;
import net.snowflake.client.jdbc.internal.apache.http.client.CredentialsProvider;
import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler;
import net.snowflake.client.jdbc.internal.apache.http.client.ServiceUnavailableRetryStrategy;
import net.snowflake.client.jdbc.internal.apache.http.client.config.RequestConfig;
import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext;
import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute;
import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier;
import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder;
import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner;
import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats;
import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext;
import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts;
import net.snowflake.ingest.streaming.internal.StreamingIngestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Created by hyu on 8/10/17. */
public class HttpUtil {
  public static final String USE_PROXY = "http.useProxy";
  public static final String PROXY_HOST = "http.proxyHost";
  public static final String PROXY_PORT = "http.proxyPort";
  public static final String NON_PROXY_HOSTS = "http.nonProxyHosts";
  public static final String HTTP_PROXY_USER = "http.proxyUser";
  public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
  private static final String SNOWFLAKE_DOMAIN_NAME = ".snowflakecomputing.com";

  private static final String PROXY_SCHEME = "http";
  private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
  private static final Duration TOTAL_RETRY_DURATION = Duration.of(120, ChronoUnit.SECONDS);
  private static final Duration RETRY_INTERVAL = Duration.of(3, ChronoUnit.SECONDS);

  /**
   * How many times to retry when an IO exception is thrown. Value here is chosen to match the total
   * value of {@link HttpUtil.TOTAL_RETRY_DURATION} when exponential backoff of up to 4 seconds per
   * retry is used.
   */
  private static final int MAX_RETRIES = 10;

  private static volatile CloseableHttpClient httpClient;

  private static PoolingHttpClientConnectionManager connectionManager;

  private static IdleConnectionMonitorThread idleConnectionMonitorThread;

  /**
   * This lock is to synchronize on idleConnectionMonitorThread to avoid setting starting a thread
   * which was already started. (To avoid {@link IllegalThreadStateException})
   */
  private static final ReentrantLock idleConnectionMonitorThreadLock = new ReentrantLock(true);

  private static final int DEFAULT_CONNECTION_TIMEOUT_MINUTES = 1;
  private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 1;

  /**
   * After how many seconds of inactivity should be idle connections evicted from the connection
   * pool.
   */
  private static final int DEFAULT_EVICT_IDLE_AFTER_SECONDS = 60;

  // Default is 2, but scaling it up to 100 to match with default_max_connections
  private static final int DEFAULT_MAX_CONNECTIONS_PER_ROUTE = 100;

  // 100 is close to max partition number we have seen for a kafka topic ingesting into snowflake.
  private static final int DEFAULT_MAX_CONNECTIONS = 100;

  // Interval in which we check if there are connections which needs to be closed.
  private static final long IDLE_HTTP_CONNECTION_MONITOR_THREAD_INTERVAL_MS =
      TimeUnit.SECONDS.toMillis(5);

  // Only connections that are currently owned, not checked out, are subject to idle timeouts.
  private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;

  /**
   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
   * @return Instance of CloseableHttpClient
   */
  public static CloseableHttpClient getHttpClient(String accountName) {
    if (httpClient == null) {
      synchronized (HttpUtil.class) {
        if (httpClient == null) {
          initHttpClient(accountName);
        }
      }
    }

    initIdleConnectionMonitoringThread();

    return httpClient;
  }

  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);

  private static void initHttpClient(String accountName) {

    Security.setProperty("ocsp.enable", "true");

    SSLContext sslContext = SSLContexts.createDefault();

    SSLConnectionSocketFactory f =
        new SSLConnectionSocketFactory(
            sslContext, new String[] {"TLSv1.2"}, null, new DefaultHostnameVerifier());
    // Set connectionTimeout which is the timeout until a connection with the server is established
    // Set connectionRequestTimeout which is the time to wait for getting a connection from the
    // connection pool
    // Set socketTimeout which is the max time gap between two consecutive data packets
    RequestConfig requestConfig =
        RequestConfig.custom()
            .setConnectTimeout(
                (int)
                    TimeUnit.MILLISECONDS.convert(
                        DEFAULT_CONNECTION_TIMEOUT_MINUTES, TimeUnit.MINUTES))
            .setConnectionRequestTimeout(
                (int)
                    TimeUnit.MILLISECONDS.convert(
                        DEFAULT_CONNECTION_TIMEOUT_MINUTES, TimeUnit.MINUTES))
            .setSocketTimeout(
                (int)
                    TimeUnit.MILLISECONDS.convert(
                        DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES, TimeUnit.MINUTES))
            .build();

    // Below pooling client connection manager uses time_to_live value as -1 which means it will not
    // refresh a persisted connection
    connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
    connectionManager.setMaxTotal(DEFAULT_MAX_CONNECTIONS);

    // Use an anonymous class to implement the interface ServiceUnavailableRetryStrategy() The max
    // retry time is 3. The interval time is backoff.
    HttpClientBuilder clientBuilder =
        HttpClientBuilder.create()
            .setConnectionManager(connectionManager)
            .evictIdleConnections(DEFAULT_EVICT_IDLE_AFTER_SECONDS, TimeUnit.SECONDS)
            .setSSLSocketFactory(f)
            .setServiceUnavailableRetryStrategy(getServiceUnavailableRetryStrategy())
            .setRetryHandler(getHttpRequestRetryHandler())
            .setDefaultRequestConfig(requestConfig);

    // proxy settings
    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
      if (System.getProperty(PROXY_PORT) == null) {
        throw new IllegalArgumentException(
            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
      }
      if (System.getProperty(PROXY_HOST) == null) {
        throw new IllegalArgumentException(
            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
      }
      String proxyHost = System.getProperty(PROXY_HOST);
      int proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT));
      HttpHost proxy = new HttpHost(proxyHost, proxyPort, PROXY_SCHEME);
      DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
      clientBuilder = clientBuilder.setRoutePlanner(routePlanner);

      // Check if proxy username and password are set
      final String proxyUser = System.getProperty(HTTP_PROXY_USER);
      final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
      if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
        Credentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
        AuthScope authScope = new AuthScope(proxyHost, proxyPort);
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(authScope, credentials);
        clientBuilder = clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
      }
    }

    httpClient = clientBuilder.build();
  }

  /** Starts a daemon thread to monitor idle connections in http connection manager. */
  private static void initIdleConnectionMonitoringThread() {
    idleConnectionMonitorThreadLock.lock();
    try {
      // Start a new thread only if connectionManager was init before and
      // daemon thread was not started before or was closed because of SimpleIngestManager close
      if (connectionManager != null
          && (idleConnectionMonitorThread == null || idleConnectionMonitorThread.isShutdown())) {
        // Monitors in a separate thread where it closes any idle connections
        // https://hc.apache.org/httpcomponents-client-4.5.x/current/tutorial/html/connmgmt.html
        idleConnectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
        idleConnectionMonitorThread.setDaemon(true);
        idleConnectionMonitorThread.start();
      }
    } catch (Exception e) {
      LOGGER.warn("Unable to start Daemon thread for Http Idle Connection Monitoring", e);
    } finally {
      idleConnectionMonitorThreadLock.unlock();
    }
  }

  private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrategy() {
    return new ServiceUnavailableRetryStrategy() {
      final int REQUEST_TIMEOUT = 408;
      final int TOO_MANY_REQUESTS = 429;
      final int SERVER_ERRORS = 500;

      @Override
      public boolean retryRequest(
          final HttpResponse response, final int executionCount, final HttpContext context) {
        Object firstFault = context.getAttribute(FIRST_FAULT_TIMESTAMP);
        long totalRetryDurationSoFarInSeconds = 0;
        if (firstFault == null) {
          context.setAttribute(FIRST_FAULT_TIMESTAMP, Instant.now());
        } else {
          Instant firstFaultInstant = (Instant) firstFault;
          Instant now = Instant.now();
          totalRetryDurationSoFarInSeconds = Duration.between(firstFaultInstant, now).getSeconds();

          if (totalRetryDurationSoFarInSeconds > TOTAL_RETRY_DURATION.getSeconds()) {
            LOGGER.info(
                String.format(
                    "Reached the max retry time of %d seconds, not retrying anymore",
                    TOTAL_RETRY_DURATION.getSeconds()));
            return false;
          }
        }

        int statusCode = response.getStatusLine().getStatusCode();
        boolean needNextRetry =
            (statusCode == REQUEST_TIMEOUT
                || statusCode == TOO_MANY_REQUESTS
                || statusCode >= SERVER_ERRORS);
        if (needNextRetry) {
          long interval = getRetryInterval();
          LOGGER.info(
              "In retryRequest for service unavailability with statusCode:{} and uri:{}",
              statusCode,
              getRequestUriFromContext(context));
          LOGGER.info(
              "Sleep time in millisecond: {}, retryCount: {}, total retry duration: {}s / {}s",
              interval,
              executionCount,
              totalRetryDurationSoFarInSeconds,
              TOTAL_RETRY_DURATION.getSeconds());
        }
        return needNextRetry;
      }

      @Override
      public long getRetryInterval() {
        return RETRY_INTERVAL.toMillis();
      }
    };
  }

  /**
   * Retry handler logic. Retry at most {@link HttpUtil.MAX_RETRIES} times if any of the following
   * exceptions is thrown by request execution:
   *
   * 
    *
  • No response from Service exception (NoHttpResponseException) *
  • javax.net.ssl.SSLException: Connection reset. *
* * @return retryHandler to add to http client. */ static HttpRequestRetryHandler getHttpRequestRetryHandler() { return (exception, executionCount, httpContext) -> { final String requestURI = getRequestUriFromContext(httpContext); if (executionCount > MAX_RETRIES) { LOGGER.info("Max retry exceeded for requestURI:{}", requestURI); return false; } if (exception instanceof NoHttpResponseException || exception instanceof javax.net.ssl.SSLException || exception instanceof java.net.SocketException || exception instanceof java.net.UnknownHostException || exception instanceof java.net.SocketTimeoutException) { LOGGER.info( "Retrying request which caused {} with " + "URI:{}, retryCount:{} and maxRetryCount:{}", exception.getClass().getName(), requestURI, executionCount, MAX_RETRIES); StreamingIngestUtils.sleepForRetry(executionCount); return true; } LOGGER.info("No retry for URI:{} with exception {}", requestURI, exception.toString()); return false; }; } private static String getRequestUriFromContext(final HttpContext httpContext) { HttpClientContext clientContext = HttpClientContext.adapt(httpContext); HttpRequest httpRequest = clientContext.getRequest(); return httpRequest.getRequestLine().getUri(); } /** * Helper method to decide whether to add any properties related to proxy server. These properties * are passed on to snowflake JDBC while calling put API. * * @return proxy parameters that could be used by JDBC */ public static Properties generateProxyPropertiesForJDBC() { Properties proxyProperties = new Properties(); if (Boolean.parseBoolean(System.getProperty(USE_PROXY))) { if (isNullOrEmpty(System.getProperty(PROXY_PORT))) { throw new IllegalArgumentException( "proxy port number is not provided, please assign proxy port to http.proxyPort option"); } if (isNullOrEmpty(System.getProperty(PROXY_HOST))) { throw new IllegalArgumentException( "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option"); } // Set proxy host and port proxyProperties.put(SFSessionProperty.USE_PROXY.getPropertyKey(), "true"); proxyProperties.put( SFSessionProperty.PROXY_HOST.getPropertyKey(), System.getProperty(PROXY_HOST)); proxyProperties.put( SFSessionProperty.PROXY_PORT.getPropertyKey(), System.getProperty(PROXY_PORT)); // Check if proxy username and password are set final String proxyUser = System.getProperty(HTTP_PROXY_USER); final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD); if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) { proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser); proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword); } // Check if http.nonProxyHosts was set final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS); if (!isNullOrEmpty(nonProxyHosts)) { proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts); } } return proxyProperties; } /** Thread to monitor expired and idle connection, if found clear it and return it back to pool */ private static class IdleConnectionMonitorThread extends Thread { private final PoolingHttpClientConnectionManager connectionManager; private volatile boolean shutdown; public IdleConnectionMonitorThread(PoolingHttpClientConnectionManager connectionManager) { super(); this.connectionManager = connectionManager; } @Override public void run() { try { LOGGER.debug("Starting Idle Connection Monitor Thread "); synchronized (this) { while (!shutdown) { wait(IDLE_HTTP_CONNECTION_MONITOR_THREAD_INTERVAL_MS); StringBuilder sb = new StringBuilder(); sb.append( createPoolStatsInfo("Total Pool Stats = ", connectionManager.getTotalStats())); Set routes = connectionManager.getRoutes(); if (routes != null) { for (HttpRoute route : routes) { sb.append(createPoolStatsForRoute(connectionManager, route)); } } LOGGER.debug("[IdleConnectionMonitorThread] Pool Stats:\n" + sb); // Close expired connections connectionManager.closeExpiredConnections(); // Optionally, close connections // that have been idle longer than 30 sec connectionManager.closeIdleConnections( DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS, TimeUnit.SECONDS); } } } catch (InterruptedException ex) { LOGGER.warn("Terminating Idle Connection Monitor Thread "); } } private void shutdown() { if (!shutdown) { LOGGER.debug("Shutdown Idle Connection Monitor Thread "); shutdown = true; synchronized (this) { notifyAll(); } } } private boolean isShutdown() { return this.shutdown; } } /** Shuts down the daemon thread. */ public static void shutdownHttpConnectionManagerDaemonThread() { idleConnectionMonitorThread.shutdown(); } /** Create Pool stats for a route */ private static String createPoolStatsForRoute( PoolingHttpClientConnectionManager connectionManager, HttpRoute route) { PoolStats routeStats = connectionManager.getStats(route); return createPoolStatsInfo( String.format("Pool Stats for route %s = ", route.getTargetHost().toURI()), routeStats); } /** Returns a string with a title and pool stats. */ private static String createPoolStatsInfo(String title, PoolStats poolStats) { if (poolStats != null) { return title + poolStats + "\n"; } return title; } /** * Changes the account name to the format accountName.snowflakecomputing.com then returns a * boolean to indicate if we should go through a proxy or not. */ public static Boolean shouldBypassProxy(String accountName) { String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME; return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost); } /** * The target hostname input is compared with the hosts in the '|' separated list provided by the * http.nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we * need to replace the '.' and '*' characters since those are special regex constructs that mean * 'any character,' and 'repeat 0 or more times.' */ private static Boolean isInNonProxyHosts(String targetHost) { String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*"); String[] nonProxyHostsArray = nonProxyHosts.split("\\|"); for (String i : nonProxyHostsArray) { if (Pattern.compile(i).matcher(targetHost).matches()) { return true; } } return false; } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy