All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.solr.client.solrj.impl.HttpClientUtil Maven / Gradle / Ivy

There is a newer version: 9.7.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.client.solrj.impl;

import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.ssl.SSLContexts;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Utility class for creating/configuring httpclient instances. 
 * 
 * This class can touch internal HttpClient details and is subject to change.
 * 
 * @lucene.experimental
 */
public class HttpClientUtil {
  
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
  
  public static final int DEFAULT_CONNECT_TIMEOUT = 60000;
  public static final int DEFAULT_SO_TIMEOUT = 600000;
  public static final int DEFAULT_MAXCONNECTIONSPERHOST = 100000;
  public static final int DEFAULT_MAXCONNECTIONS = 100000;
  
  private static final int VALIDATE_AFTER_INACTIVITY_DEFAULT = 3000;
  private static final int EVICT_IDLE_CONNECTIONS_DEFAULT = 50000;
  private static final String VALIDATE_AFTER_INACTIVITY = "validateAfterInactivity";
  private static final String EVICT_IDLE_CONNECTIONS = "evictIdleConnections";

  // Maximum connections allowed per host
  public static final String PROP_MAX_CONNECTIONS_PER_HOST = "maxConnectionsPerHost";
  // Maximum total connections allowed
  public static final String PROP_MAX_CONNECTIONS = "maxConnections";
  // Retry http requests on error
  public static final String PROP_USE_RETRY = "retry";
  // Allow compression (deflate,gzip) if server supports it
  public static final String PROP_ALLOW_COMPRESSION = "allowCompression";
  // Basic auth username 
  public static final String PROP_BASIC_AUTH_USER = "httpBasicAuthUser";
  // Basic auth password 
  public static final String PROP_BASIC_AUTH_PASS = "httpBasicAuthPassword";

  /**
   * System property consulted to determine if the default {@link SocketFactoryRegistryProvider} 
   * will require hostname validation of SSL Certificates.  The default behavior is to enforce 
   * peer name validation.
   * 

* This property will have no effect if {@link #setSocketFactoryRegistryProvider} is used to override * the default {@link SocketFactoryRegistryProvider} *

*/ public static final String SYS_PROP_CHECK_PEER_NAME = "solr.ssl.checkPeerName"; // * NOTE* The following params configure the default request config and this // is overridden by SolrJ clients. Use the setters on the SolrJ clients to // to configure these settings if that is the intent. // Follow redirects public static final String PROP_FOLLOW_REDIRECTS = "followRedirects"; // socket timeout measured in ms, closes a socket if read // takes longer than x ms to complete. throws // java.net.SocketTimeoutException: Read timed out exception public static final String PROP_SO_TIMEOUT = "socketTimeout"; // connection timeout measures in ms, closes a socket if connection // cannot be established within x ms. with a // java.net.SocketTimeoutException: Connection timed out public static final String PROP_CONNECTION_TIMEOUT = "connTimeout"; /** * A Java system property to select the {@linkplain HttpClientBuilderFactory} used for * configuring the {@linkplain HttpClientBuilder} instance by default. */ public static final String SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY = "solr.httpclient.builder.factory"; /** * A Java system property to select the {@linkplain SocketFactoryRegistryProvider} used for * configuring the Apache HTTP clients. */ public static final String SYS_PROP_SOCKET_FACTORY_REGISTRY_PROVIDER = "solr.httpclient.socketFactory.registry.provider"; static final DefaultHttpRequestRetryHandler NO_RETRY = new DefaultHttpRequestRetryHandler( 0, false); private static volatile SolrHttpClientBuilder httpClientBuilder; private static SolrHttpClientContextBuilder httpClientRequestContextBuilder = new SolrHttpClientContextBuilder(); private static volatile SocketFactoryRegistryProvider socketFactoryRegistryProvider; private static volatile String cookiePolicy; private static final List interceptors = new CopyOnWriteArrayList<>(); static { resetHttpClientBuilder(); // Configure the SocketFactoryRegistryProvider if user has specified the provider type. String socketFactoryRegistryProviderClassName = System.getProperty(SYS_PROP_SOCKET_FACTORY_REGISTRY_PROVIDER); if (socketFactoryRegistryProviderClassName != null) { log.debug("Using {}", socketFactoryRegistryProviderClassName); try { socketFactoryRegistryProvider = (SocketFactoryRegistryProvider)Class.forName(socketFactoryRegistryProviderClassName).getConstructor().newInstance(); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | InvocationTargetException | NoSuchMethodException e) { throw new RuntimeException("Unable to instantiate Solr SocketFactoryRegistryProvider", e); } } // Configure the HttpClientBuilder if user has specified the factory type. String factoryClassName = System.getProperty(SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY); if (factoryClassName != null) { log.debug ("Using {}", factoryClassName); try { HttpClientBuilderFactory factory = (HttpClientBuilderFactory)Class.forName(factoryClassName).newInstance(); httpClientBuilder = factory.getHttpClientBuilder(Optional.of(SolrHttpClientBuilder.create())); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new RuntimeException("Unable to instantiate Solr HttpClientBuilderFactory", e); } } } public static abstract class SocketFactoryRegistryProvider { /** Must be non-null */ public abstract Registry getSocketFactoryRegistry(); } private static class DynamicInterceptor implements HttpRequestInterceptor { @Override public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { // don't synchronize traversal - can lead to deadlock - CopyOnWriteArrayList is critical // we also do not want to have to acquire the mutex when the list is empty or put a global // mutex around the process calls interceptors.forEach(new Consumer() { @Override public void accept(HttpRequestInterceptor interceptor) { try { interceptor.process(request, context); } catch (Exception e) { log.error("", e); } } }); } } public static void setHttpClientBuilder(SolrHttpClientBuilder newHttpClientBuilder) { httpClientBuilder = newHttpClientBuilder; } public static void setHttpClientProvider(SolrHttpClientBuilder newHttpClientBuilder) { httpClientBuilder = newHttpClientBuilder; } /** * @see #SYS_PROP_CHECK_PEER_NAME */ public static void setSocketFactoryRegistryProvider(SocketFactoryRegistryProvider newRegistryProvider) { socketFactoryRegistryProvider = newRegistryProvider; } public static SolrHttpClientBuilder getHttpClientBuilder() { return httpClientBuilder; } /** * @see #SYS_PROP_CHECK_PEER_NAME */ public static SocketFactoryRegistryProvider getSocketFactoryRegistryProvider() { return socketFactoryRegistryProvider; } public static void resetHttpClientBuilder() { socketFactoryRegistryProvider = new DefaultSocketFactoryRegistryProvider(); httpClientBuilder = SolrHttpClientBuilder.create(); } private static final class DefaultSocketFactoryRegistryProvider extends SocketFactoryRegistryProvider { @Override public Registry getSocketFactoryRegistry() { // this mimics PoolingHttpClientConnectionManager's default behavior, // except that we explicitly use SSLConnectionSocketFactory.getSystemSocketFactory() // to pick up the system level default SSLContext (where javax.net.ssl.* properties // related to keystore & truststore are specified) RegistryBuilder builder = RegistryBuilder. create(); builder.register("http", PlainConnectionSocketFactory.getSocketFactory()); // logic to turn off peer host check SSLConnectionSocketFactory sslConnectionSocketFactory = null; boolean sslCheckPeerName = toBooleanDefaultIfNull( toBooleanObject(System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME)), true); if (sslCheckPeerName) { sslConnectionSocketFactory = SSLConnectionSocketFactory.getSystemSocketFactory(); } else { sslConnectionSocketFactory = new SSLConnectionSocketFactory(SSLContexts.createSystemDefault(), NoopHostnameVerifier.INSTANCE); log.debug("{} is false, hostname checks disabled.", HttpClientUtil.SYS_PROP_CHECK_PEER_NAME); } builder.register("https", sslConnectionSocketFactory); return builder.build(); } } /** * Creates new http client by using the provided configuration. * * @param params * http client configuration, if null a client with default * configuration (no additional configuration) is created. */ public static CloseableHttpClient createClient(SolrParams params) { return createClient(params, createPoolingConnectionManager()); } /** test usage subject to change @lucene.experimental */ static PoolingHttpClientConnectionManager createPoolingConnectionManager() { return new PoolingHttpClientConnectionManager(socketFactoryRegistryProvider.getSocketFactoryRegistry()); } public static CloseableHttpClient createClient(SolrParams params, PoolingHttpClientConnectionManager cm) { if (params == null) { params = new ModifiableSolrParams(); } return createClient(params, cm, false); } public static CloseableHttpClient createClient(final SolrParams params, PoolingHttpClientConnectionManager cm, boolean sharedConnectionManager, HttpRequestExecutor httpRequestExecutor) { final ModifiableSolrParams config = new ModifiableSolrParams(params); if (log.isDebugEnabled()) { log.debug("Creating new http client, config: {}", config); } cm.setMaxTotal(params.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000)); cm.setDefaultMaxPerRoute(params.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000)); cm.setValidateAfterInactivity(Integer.getInteger(VALIDATE_AFTER_INACTIVITY, VALIDATE_AFTER_INACTIVITY_DEFAULT)); HttpClientBuilder newHttpClientBuilder = HttpClientBuilder.create(); if (sharedConnectionManager) { newHttpClientBuilder.setConnectionManagerShared(true); } else { newHttpClientBuilder.setConnectionManagerShared(false); } ConnectionKeepAliveStrategy keepAliveStrat = new ConnectionKeepAliveStrategy() { @Override public long getKeepAliveDuration(HttpResponse response, HttpContext context) { // we only close connections based on idle time, not ttl expiration return -1; } }; if (httpClientBuilder.getAuthSchemeRegistryProvider() != null) { newHttpClientBuilder.setDefaultAuthSchemeRegistry(httpClientBuilder.getAuthSchemeRegistryProvider().getAuthSchemeRegistry()); } if (httpClientBuilder.getCookieSpecRegistryProvider() != null) { newHttpClientBuilder.setDefaultCookieSpecRegistry(httpClientBuilder.getCookieSpecRegistryProvider().getCookieSpecRegistry()); } if (httpClientBuilder.getCredentialsProviderProvider() != null) { newHttpClientBuilder.setDefaultCredentialsProvider(httpClientBuilder.getCredentialsProviderProvider().getCredentialsProvider()); } newHttpClientBuilder.addInterceptorLast(new DynamicInterceptor()); newHttpClientBuilder = newHttpClientBuilder.setKeepAliveStrategy(keepAliveStrat) .evictIdleConnections((long) Integer.getInteger(EVICT_IDLE_CONNECTIONS, EVICT_IDLE_CONNECTIONS_DEFAULT), TimeUnit.MILLISECONDS); if (httpRequestExecutor != null) { newHttpClientBuilder.setRequestExecutor(httpRequestExecutor); } HttpClientBuilder builder = setupBuilder(newHttpClientBuilder, params); HttpClient httpClient = builder.setConnectionManager(cm).build(); assert ObjectReleaseTracker.track(httpClient); return (CloseableHttpClient) httpClient; } /** * Creates new http client by using the provided configuration. * */ public static CloseableHttpClient createClient(final SolrParams params, PoolingHttpClientConnectionManager cm, boolean sharedConnectionManager) { return createClient(params, cm, sharedConnectionManager, null); } private static HttpClientBuilder setupBuilder(HttpClientBuilder builder, SolrParams config) { Builder requestConfigBuilder = RequestConfig.custom() .setRedirectsEnabled(config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false)).setDecompressionEnabled(false) .setConnectTimeout(config.getInt(HttpClientUtil.PROP_CONNECTION_TIMEOUT, DEFAULT_CONNECT_TIMEOUT)) .setSocketTimeout(config.getInt(HttpClientUtil.PROP_SO_TIMEOUT, DEFAULT_SO_TIMEOUT)); String cpolicy = cookiePolicy; if (cpolicy != null) { requestConfigBuilder.setCookieSpec(cpolicy); } RequestConfig requestConfig = requestConfigBuilder.build(); HttpClientBuilder retBuilder = builder.setDefaultRequestConfig(requestConfig); if (config.getBool(HttpClientUtil.PROP_USE_RETRY, true)) { retBuilder = retBuilder.setRetryHandler(new SolrHttpRequestRetryHandler(Integer.getInteger("solr.httpclient.retries", 3))); } else { retBuilder = retBuilder.setRetryHandler(NO_RETRY); } final String basicAuthUser = config.get(HttpClientUtil.PROP_BASIC_AUTH_USER); final String basicAuthPass = config.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); if (basicAuthUser != null && basicAuthPass != null) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass)); retBuilder.setDefaultCredentialsProvider(credsProvider); } if (config.getBool(HttpClientUtil.PROP_ALLOW_COMPRESSION, false)) { retBuilder.addInterceptorFirst(new UseCompressionRequestInterceptor()); retBuilder.addInterceptorFirst(new UseCompressionResponseInterceptor()); } else { retBuilder.disableContentCompression(); } return retBuilder; } public static void close(HttpClient httpClient) { org.apache.solr.common.util.IOUtils.closeQuietly((CloseableHttpClient) httpClient); assert ObjectReleaseTracker.release(httpClient); } public static void addRequestInterceptor(HttpRequestInterceptor interceptor) { interceptors.add(interceptor); } public static void removeRequestInterceptor(HttpRequestInterceptor interceptor) { interceptors.remove(interceptor); } public static void clearRequestInterceptors() { interceptors.clear(); } private static class UseCompressionRequestInterceptor implements HttpRequestInterceptor { @Override public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip, deflate"); } } } private static class UseCompressionResponseInterceptor implements HttpResponseInterceptor { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response .setEntity(new GzipDecompressingEntity(response.getEntity())); return; } if (codecs[i].getName().equalsIgnoreCase("deflate")) { response.setEntity(new DeflateDecompressingEntity(response .getEntity())); return; } } } } } protected static class GzipDecompressingEntity extends HttpEntityWrapper { private boolean gzipInputStreamCreated = false; private InputStream gzipInputStream = null; public GzipDecompressingEntity(final HttpEntity entity) { super(entity); } /** * Return a InputStream of uncompressed data. * If there is an issue with the compression of the data, a null InputStream will be returned, * and the underlying compressed InputStream will be closed. * * The same input stream will be returned if the underlying entity is not repeatable. * If the underlying entity is repeatable, then a new input stream will be created. */ @Override public InputStream getContent() throws IOException, IllegalStateException { if (!gzipInputStreamCreated || wrappedEntity.isRepeatable()) { gzipInputStreamCreated = true; InputStream wrappedContent = wrappedEntity.getContent(); if (wrappedContent != null) { try { gzipInputStream = new GZIPInputStream(wrappedContent); } catch (IOException ioException) { try { Utils.readFully(wrappedContent); } catch (IOException ignored) { } finally { IOUtils.closeQuietly(wrappedContent); } throw new IOException("Cannot open GZipInputStream for response", ioException); } } } return gzipInputStream; } @Override public long getContentLength() { return -1; } } private static class DeflateDecompressingEntity extends GzipDecompressingEntity { public DeflateDecompressingEntity(final HttpEntity entity) { super(entity); } @Override public InputStream getContent() throws IOException, IllegalStateException { // InflaterInputStream does not throw a ZipException in the constructor, // so it does not need the same checks as the GZIPInputStream. return new InflaterInputStream(wrappedEntity.getContent()); } } public static void setHttpClientRequestContextBuilder(SolrHttpClientContextBuilder httpClientContextBuilder) { httpClientRequestContextBuilder = httpClientContextBuilder; } /** * Create a HttpClientContext object and {@link HttpClientContext#setUserToken(Object)} * to an internal singleton. It allows to reuse underneath {@link HttpClient} * in connection pools if client authentication is enabled. */ public static HttpClientContext createNewHttpClientRequestContext() { HttpClientContext context = httpClientRequestContextBuilder.createContext(HttpSolrClient.cacheKey); return context; } public static Builder createDefaultRequestConfigBuilder() { String cpolicy = cookiePolicy; Builder builder = RequestConfig.custom(); builder.setSocketTimeout(DEFAULT_SO_TIMEOUT) .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT) .setRedirectsEnabled(false) .setDecompressionEnabled(false); // we do our own compression / decompression if (cpolicy != null) { builder.setCookieSpec(cpolicy); } return builder; } public static void setCookiePolicy(String policyName) { cookiePolicy = policyName; } /** * @lucene.internal */ static boolean toBooleanDefaultIfNull(Boolean bool, boolean valueIfNull) { if (bool == null) { return valueIfNull; } return bool.booleanValue() ? true : false; } /** * @lucene.internal */ static Boolean toBooleanObject(String str) { if ("true".equalsIgnoreCase(str)) { return Boolean.TRUE; } else if ("false".equalsIgnoreCase(str)) { return Boolean.FALSE; } // no match return null; } }




© 2015 - 2025 Weber Informatics LLC | Privacy Policy