org.apache.solr.client.solrj.impl.HttpClientUtil Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.ssl.SSLContexts;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utility class for creating/configuring httpclient instances.
*
* This class can touch internal HttpClient details and is subject to change.
*
* @lucene.experimental
* @deprecated Used to configure the Apache HTTP client. Please use The Http2 client
*/
@Deprecated(since = "9.0")
public class HttpClientUtil {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final int DEFAULT_CONNECT_TIMEOUT = 60000;
public static final int DEFAULT_SO_TIMEOUT = 600000;
public static final int DEFAULT_MAXCONNECTIONSPERHOST = 100000;
public static final int DEFAULT_MAXCONNECTIONS = 100000;
private static final int VALIDATE_AFTER_INACTIVITY_DEFAULT = 3000;
private static final int EVICT_IDLE_CONNECTIONS_DEFAULT = 50000;
private static final String VALIDATE_AFTER_INACTIVITY = "validateAfterInactivity";
private static final String EVICT_IDLE_CONNECTIONS = "evictIdleConnections";
// Maximum connections allowed per host
public static final String PROP_MAX_CONNECTIONS_PER_HOST = "maxConnectionsPerHost";
// Maximum total connections allowed
public static final String PROP_MAX_CONNECTIONS = "maxConnections";
// Retry http requests on error
public static final String PROP_USE_RETRY = "retry";
// Allow compression (deflate,gzip) if server supports it
public static final String PROP_ALLOW_COMPRESSION = "allowCompression";
// Basic auth username
public static final String PROP_BASIC_AUTH_USER = "httpBasicAuthUser";
// Basic auth password
public static final String PROP_BASIC_AUTH_PASS = "httpBasicAuthPassword";
/**
* System property consulted to determine if the default {@link SocketFactoryRegistryProvider}
* will require hostname validation of SSL Certificates. The default behavior is to enforce peer
* name validation.
*
*
This property will have no effect if {@link #setSocketFactoryRegistryProvider} is used to
* override the default {@link SocketFactoryRegistryProvider}
*/
public static final String SYS_PROP_CHECK_PEER_NAME = "solr.ssl.checkPeerName";
// * NOTE* The following params configure the default request config and this
// is overridden by SolrJ clients. Use the setters on the SolrJ clients to
// to configure these settings if that is the intent.
// Follow redirects
public static final String PROP_FOLLOW_REDIRECTS = "followRedirects";
// socket timeout measured in ms, closes a socket if read
// takes longer than x ms to complete. throws
// java.net.SocketTimeoutException: Read timed out exception
public static final String PROP_SO_TIMEOUT = "socketTimeout";
// connection timeout measures in ms, closes a socket if connection
// cannot be established within x ms. with a
// java.net.SocketTimeoutException: Connection timed out
public static final String PROP_CONNECTION_TIMEOUT = "connTimeout";
/**
* A Java system property to select the {@linkplain HttpClientBuilderFactory} used for configuring
* the {@linkplain HttpClientBuilder} instance by default.
*/
public static final String SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY =
"solr.httpclient.builder.factory";
/**
* A Java system property to select the {@linkplain SocketFactoryRegistryProvider} used for
* configuring the Apache HTTP clients.
*/
public static final String SYS_PROP_SOCKET_FACTORY_REGISTRY_PROVIDER =
"solr.httpclient.socketFactory.registry.provider";
static final DefaultHttpRequestRetryHandler NO_RETRY =
new DefaultHttpRequestRetryHandler(0, false);
private static volatile SolrHttpClientBuilder httpClientBuilder;
private static SolrHttpClientContextBuilder httpClientRequestContextBuilder =
new SolrHttpClientContextBuilder();
private static volatile SocketFactoryRegistryProvider socketFactoryRegistryProvider;
private static volatile String cookiePolicy;
private static final List interceptors = new CopyOnWriteArrayList<>();
static {
resetHttpClientBuilder();
// Configure the SocketFactoryRegistryProvider if user has specified the provider type.
String socketFactoryRegistryProviderClassName =
System.getProperty(SYS_PROP_SOCKET_FACTORY_REGISTRY_PROVIDER);
if (socketFactoryRegistryProviderClassName != null) {
log.debug("Using {}", socketFactoryRegistryProviderClassName);
try {
socketFactoryRegistryProvider =
Class.forName(socketFactoryRegistryProviderClassName)
.asSubclass(SocketFactoryRegistryProvider.class)
.getDeclaredConstructor()
.newInstance();
} catch (InstantiationException
| IllegalAccessException
| ClassNotFoundException
| InvocationTargetException
| NoSuchMethodException e) {
throw new RuntimeException("Unable to instantiate Solr SocketFactoryRegistryProvider", e);
}
}
// Configure the HttpClientBuilder if user has specified the factory type.
String factoryClassName = System.getProperty(SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY);
if (factoryClassName != null) {
log.debug("Using {}", factoryClassName);
try {
HttpClientBuilderFactory factory =
Class.forName(factoryClassName)
.asSubclass(HttpClientBuilderFactory.class)
.getDeclaredConstructor()
.newInstance();
httpClientBuilder = factory.getHttpClientBuilder(SolrHttpClientBuilder.create());
} catch (InstantiationException
| IllegalAccessException
| ClassNotFoundException
| InvocationTargetException
| NoSuchMethodException e) {
throw new RuntimeException("Unable to instantiate Solr HttpClientBuilderFactory", e);
}
}
}
public abstract static class SocketFactoryRegistryProvider {
/** Must be non-null */
public abstract Registry getSocketFactoryRegistry();
}
private static class DynamicInterceptor implements HttpRequestInterceptor {
@Override
public void process(HttpRequest request, HttpContext context)
throws HttpException, IOException {
// don't synchronize traversal - can lead to deadlock - CopyOnWriteArrayList is critical
// we also do not want to have to acquire the mutex when the list is empty or put a global
// mutex around the process calls
interceptors.forEach(
new Consumer() {
@Override
public void accept(HttpRequestInterceptor interceptor) {
try {
interceptor.process(request, context);
} catch (Exception e) {
log.error("", e);
}
}
});
}
}
public static void setHttpClientBuilder(SolrHttpClientBuilder newHttpClientBuilder) {
httpClientBuilder = newHttpClientBuilder;
}
public static void setHttpClientProvider(SolrHttpClientBuilder newHttpClientBuilder) {
httpClientBuilder = newHttpClientBuilder;
}
/**
* @see #SYS_PROP_CHECK_PEER_NAME
*/
public static void setSocketFactoryRegistryProvider(
SocketFactoryRegistryProvider newRegistryProvider) {
socketFactoryRegistryProvider = newRegistryProvider;
}
public static SolrHttpClientBuilder getHttpClientBuilder() {
return httpClientBuilder;
}
/**
* @see #SYS_PROP_CHECK_PEER_NAME
*/
public static SocketFactoryRegistryProvider getSocketFactoryRegistryProvider() {
return socketFactoryRegistryProvider;
}
public static void resetHttpClientBuilder() {
socketFactoryRegistryProvider = new DefaultSocketFactoryRegistryProvider();
httpClientBuilder = SolrHttpClientBuilder.create();
}
private static final class DefaultSocketFactoryRegistryProvider
extends SocketFactoryRegistryProvider {
@Override
public Registry getSocketFactoryRegistry() {
// this mimics PoolingHttpClientConnectionManager's default behavior,
// except that we explicitly use SSLConnectionSocketFactory.getSystemSocketFactory()
// to pick up the system level default SSLContext (where javax.net.ssl.* properties
// related to keystore & truststore are specified)
RegistryBuilder builder =
RegistryBuilder.create();
builder.register("http", PlainConnectionSocketFactory.getSocketFactory());
// logic to turn off peer host check
SSLConnectionSocketFactory sslConnectionSocketFactory = null;
boolean sslCheckPeerName =
toBooleanDefaultIfNull(
toBooleanObject(System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME)), true);
if (sslCheckPeerName) {
sslConnectionSocketFactory = SSLConnectionSocketFactory.getSystemSocketFactory();
} else {
sslConnectionSocketFactory =
new SSLConnectionSocketFactory(
SSLContexts.createSystemDefault(), NoopHostnameVerifier.INSTANCE);
log.debug(
"{} is false, hostname checks disabled.", HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
}
builder.register("https", sslConnectionSocketFactory);
return builder.build();
}
}
/**
* Creates new http client by using the provided configuration.
*
* @param params http client configuration, if null a client with default configuration (no
* additional configuration) is created.
*/
public static CloseableHttpClient createClient(SolrParams params) {
return createClient(params, createPoolingConnectionManager());
}
/** test usage subject to change @lucene.experimental */
static PoolingHttpClientConnectionManager createPoolingConnectionManager() {
return new PoolingHttpClientConnectionManager(
socketFactoryRegistryProvider.getSocketFactoryRegistry());
}
public static CloseableHttpClient createClient(
SolrParams params, PoolingHttpClientConnectionManager cm) {
if (params == null) {
params = new ModifiableSolrParams();
}
return createClient(params, cm, false);
}
public static CloseableHttpClient createClient(
final SolrParams params,
PoolingHttpClientConnectionManager cm,
boolean sharedConnectionManager,
HttpRequestExecutor httpRequestExecutor) {
final ModifiableSolrParams config = new ModifiableSolrParams(params);
if (log.isDebugEnabled()) {
log.debug("Creating new http client, config: {}", config);
}
cm.setMaxTotal(params.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000));
cm.setDefaultMaxPerRoute(params.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000));
cm.setValidateAfterInactivity(
Integer.getInteger(VALIDATE_AFTER_INACTIVITY, VALIDATE_AFTER_INACTIVITY_DEFAULT));
HttpClientBuilder newHttpClientBuilder = HttpClientBuilder.create();
if (sharedConnectionManager) {
newHttpClientBuilder.setConnectionManagerShared(true);
} else {
newHttpClientBuilder.setConnectionManagerShared(false);
}
ConnectionKeepAliveStrategy keepAliveStrat =
new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// we only close connections based on idle time, not ttl expiration
return -1;
}
};
if (httpClientBuilder.getAuthSchemeRegistryProvider() != null) {
newHttpClientBuilder.setDefaultAuthSchemeRegistry(
httpClientBuilder.getAuthSchemeRegistryProvider().getAuthSchemeRegistry());
}
if (httpClientBuilder.getCookieSpecRegistryProvider() != null) {
newHttpClientBuilder.setDefaultCookieSpecRegistry(
httpClientBuilder.getCookieSpecRegistryProvider().getCookieSpecRegistry());
}
if (httpClientBuilder.getCredentialsProviderProvider() != null) {
newHttpClientBuilder.setDefaultCredentialsProvider(
httpClientBuilder.getCredentialsProviderProvider().getCredentialsProvider());
}
newHttpClientBuilder.addInterceptorLast(new DynamicInterceptor());
newHttpClientBuilder =
newHttpClientBuilder
.setKeepAliveStrategy(keepAliveStrat)
.evictIdleConnections(
(long) Integer.getInteger(EVICT_IDLE_CONNECTIONS, EVICT_IDLE_CONNECTIONS_DEFAULT),
TimeUnit.MILLISECONDS);
if (httpRequestExecutor != null) {
newHttpClientBuilder.setRequestExecutor(httpRequestExecutor);
}
HttpClientBuilder builder = setupBuilder(newHttpClientBuilder, params);
HttpClient httpClient = builder.setConnectionManager(cm).build();
assert ObjectReleaseTracker.track(httpClient);
return (CloseableHttpClient) httpClient;
}
/** Creates new http client by using the provided configuration. */
public static CloseableHttpClient createClient(
final SolrParams params,
PoolingHttpClientConnectionManager cm,
boolean sharedConnectionManager) {
return createClient(params, cm, sharedConnectionManager, null);
}
private static HttpClientBuilder setupBuilder(HttpClientBuilder builder, SolrParams config) {
Builder requestConfigBuilder =
RequestConfig.custom()
.setRedirectsEnabled(config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false))
.setDecompressionEnabled(false)
.setConnectTimeout(
config.getInt(HttpClientUtil.PROP_CONNECTION_TIMEOUT, DEFAULT_CONNECT_TIMEOUT))
.setSocketTimeout(config.getInt(HttpClientUtil.PROP_SO_TIMEOUT, DEFAULT_SO_TIMEOUT));
String cpolicy = cookiePolicy;
if (cpolicy != null) {
requestConfigBuilder.setCookieSpec(cpolicy);
}
RequestConfig requestConfig = requestConfigBuilder.build();
HttpClientBuilder retBuilder = builder.setDefaultRequestConfig(requestConfig);
if (config.getBool(HttpClientUtil.PROP_USE_RETRY, true)) {
retBuilder =
retBuilder.setRetryHandler(
new SolrHttpRequestRetryHandler(Integer.getInteger("solr.httpclient.retries", 3)));
} else {
retBuilder = retBuilder.setRetryHandler(NO_RETRY);
}
final String basicAuthUser = config.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
final String basicAuthPass = config.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
if (basicAuthUser != null && basicAuthPass != null) {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
retBuilder.setDefaultCredentialsProvider(credsProvider);
}
if (config.getBool(HttpClientUtil.PROP_ALLOW_COMPRESSION, false)) {
retBuilder.addInterceptorFirst(new UseCompressionRequestInterceptor());
retBuilder.addInterceptorFirst(new UseCompressionResponseInterceptor());
} else {
retBuilder.disableContentCompression();
}
retBuilder.useSystemProperties();
return retBuilder;
}
public static void close(HttpClient httpClient) {
org.apache.solr.common.util.IOUtils.closeQuietly((CloseableHttpClient) httpClient);
assert ObjectReleaseTracker.release(httpClient);
}
public static void addRequestInterceptor(HttpRequestInterceptor interceptor) {
interceptors.add(interceptor);
}
public static void removeRequestInterceptor(HttpRequestInterceptor interceptor) {
interceptors.remove(interceptor);
}
public static void clearRequestInterceptors() {
interceptors.clear();
}
private static class UseCompressionRequestInterceptor implements HttpRequestInterceptor {
@Override
public void process(HttpRequest request, HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip, deflate");
}
}
}
private static class UseCompressionResponseInterceptor implements HttpResponseInterceptor {
@Override
public void process(final HttpResponse response, final HttpContext context)
throws HttpException, IOException {
HttpEntity entity = response.getEntity();
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (int i = 0; i < codecs.length; i++) {
if (codecs[i].getName().equalsIgnoreCase("gzip")) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
if (codecs[i].getName().equalsIgnoreCase("deflate")) {
response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
protected static class GzipDecompressingEntity extends HttpEntityWrapper {
private boolean gzipInputStreamCreated = false;
private InputStream gzipInputStream = null;
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
}
/**
* Return a InputStream of uncompressed data. If there is an issue with the compression of the
* data, a null InputStream will be returned, and the underlying compressed InputStream will be
* closed.
*
* The same input stream will be returned if the underlying entity is not repeatable. If the
* underlying entity is repeatable, then a new input stream will be created.
*/
@Override
public InputStream getContent() throws IOException, IllegalStateException {
if (!gzipInputStreamCreated || wrappedEntity.isRepeatable()) {
gzipInputStreamCreated = true;
InputStream wrappedContent = wrappedEntity.getContent();
if (wrappedContent != null) {
try {
gzipInputStream = new GZIPInputStream(wrappedContent);
} catch (IOException ioException) {
try (wrappedContent) {
Utils.readFully(wrappedContent);
} catch (IOException ignored) {
}
throw new IOException("Cannot open GZipInputStream for response", ioException);
}
}
}
return gzipInputStream;
}
@Override
public long getContentLength() {
return -1;
}
}
private static class DeflateDecompressingEntity extends GzipDecompressingEntity {
public DeflateDecompressingEntity(final HttpEntity entity) {
super(entity);
}
@Override
public InputStream getContent() throws IOException, IllegalStateException {
// InflaterInputStream does not throw a ZipException in the constructor,
// so it does not need the same checks as the GZIPInputStream.
return new InflaterInputStream(wrappedEntity.getContent());
}
}
public static void setHttpClientRequestContextBuilder(
SolrHttpClientContextBuilder httpClientContextBuilder) {
httpClientRequestContextBuilder = httpClientContextBuilder;
}
/**
* Create a HttpClientContext object and {@link HttpClientContext#setUserToken(Object)} to an
* internal singleton. It allows to reuse underneath {@link HttpClient} in connection pools if
* client authentication is enabled.
*/
public static HttpClientContext createNewHttpClientRequestContext() {
HttpClientContext context =
httpClientRequestContextBuilder.createContext(HttpSolrClient.cacheKey);
return context;
}
public static Builder createDefaultRequestConfigBuilder() {
String cpolicy = cookiePolicy;
Builder builder = RequestConfig.custom();
builder
.setSocketTimeout(DEFAULT_SO_TIMEOUT)
.setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
.setRedirectsEnabled(false)
.setDecompressionEnabled(false); // we do our own compression / decompression
if (cpolicy != null) {
builder.setCookieSpec(cpolicy);
}
return builder;
}
public static void setCookiePolicy(String policyName) {
cookiePolicy = policyName;
}
/**
* @lucene.internal
*/
static boolean toBooleanDefaultIfNull(Boolean bool, boolean valueIfNull) {
if (bool == null) {
return valueIfNull;
}
return bool.booleanValue() ? true : false;
}
/**
* @lucene.internal
*/
static Boolean toBooleanObject(String str) {
if ("true".equalsIgnoreCase(str)) {
return Boolean.TRUE;
} else if ("false".equalsIgnoreCase(str)) {
return Boolean.FALSE;
}
// no match
return null;
}
}