Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import static org.apache.solr.common.util.Utils.getObjectByPath;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
import java.net.ConnectException;
import java.net.CookieStore;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Phaser;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.V2RequestSupport;
import org.apache.solr.client.solrj.embedded.SSLConfig;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteExecutionException;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.impl.HttpListenerFactory.RequestResponseListener;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.util.AsyncListener;
import org.apache.solr.client.solrj.util.Cancellable;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.common.util.Utils;
import org.eclipse.jetty.client.HttpClient;
import org.eclipse.jetty.client.HttpClientTransport;
import org.eclipse.jetty.client.HttpProxy;
import org.eclipse.jetty.client.Origin.Address;
import org.eclipse.jetty.client.Origin.Protocol;
import org.eclipse.jetty.client.ProtocolHandlers;
import org.eclipse.jetty.client.ProxyConfiguration;
import org.eclipse.jetty.client.Socks4Proxy;
import org.eclipse.jetty.client.api.AuthenticationStore;
import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.api.Result;
import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
import org.eclipse.jetty.client.util.FormRequestContent;
import org.eclipse.jetty.client.util.InputStreamRequestContent;
import org.eclipse.jetty.client.util.InputStreamResponseListener;
import org.eclipse.jetty.client.util.MultiPartRequestContent;
import org.eclipse.jetty.client.util.OutputStreamRequestContent;
import org.eclipse.jetty.client.util.StringRequestContent;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.HttpFields;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.HttpStatus;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.http2.client.HTTP2Client;
import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
import org.eclipse.jetty.io.ClientConnector;
import org.eclipse.jetty.util.BlockingArrayQueue;
import org.eclipse.jetty.util.Fields;
import org.eclipse.jetty.util.HttpCookieStore;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
/**
* Difference between this {@link Http2SolrClient} and {@link HttpSolrClient}:
*
*
*
{@link Http2SolrClient} sends requests in HTTP/2
*
{@link Http2SolrClient} can point to multiple urls
*
{@link Http2SolrClient} does not expose its internal httpClient like {@link
* HttpSolrClient#getHttpClient()}, sharing connection pools should be done by {@link
* Http2SolrClient.Builder#withHttpClient(Http2SolrClient)}
*
*
* @lucene.experimental
*/
public class Http2SolrClient extends SolrClient {
public static final String REQ_PRINCIPAL_KEY = "solr-req-principal";
private static volatile SSLConfig defaultSSLConfig;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
private static final Charset FALLBACK_CHARSET = StandardCharsets.UTF_8;
private static final String DEFAULT_PATH = "/select";
private static final List errPath = Arrays.asList("metadata", "error-class");
private final HttpClient httpClient;
private Set urlParamNames;
private final long idleTimeoutMillis;
private final long requestTimeoutMillis;
// updating parser instance needs to go via the setter to ensure update of defaultParserMimeTypes
private ResponseParser parser = new BinaryResponseParser();
private Set defaultParserMimeTypes;
protected RequestWriter requestWriter = new BinaryRequestWriter();
private List listenerFactory = new ArrayList<>();
private final AsyncTracker asyncTracker = new AsyncTracker();
/** The URL of the Solr server. */
private final String serverBaseUrl;
private final boolean closeClient;
private ExecutorService executor;
private boolean shutdownExecutor;
final String basicAuthAuthorizationStr;
private AuthenticationStoreHolder authenticationStore;
protected Http2SolrClient(String serverBaseUrl, Builder builder) {
if (serverBaseUrl != null) {
if (!serverBaseUrl.equals("/") && serverBaseUrl.endsWith("/")) {
serverBaseUrl = serverBaseUrl.substring(0, serverBaseUrl.length() - 1);
}
if (serverBaseUrl.startsWith("//")) {
serverBaseUrl = serverBaseUrl.substring(1, serverBaseUrl.length());
}
this.serverBaseUrl = serverBaseUrl;
} else {
this.serverBaseUrl = null;
}
this.idleTimeoutMillis = builder.idleTimeoutMillis;
if (builder.httpClient != null) {
this.httpClient = builder.httpClient;
this.closeClient = false;
} else {
this.httpClient = createHttpClient(builder);
this.closeClient = true;
}
this.basicAuthAuthorizationStr = builder.basicAuthAuthorizationStr;
if (builder.requestWriter != null) {
this.requestWriter = builder.requestWriter;
}
if (builder.responseParser != null) {
this.parser = builder.responseParser;
}
updateDefaultMimeTypeForParser();
if (builder.requestTimeoutMillis != null) {
this.requestTimeoutMillis = builder.requestTimeoutMillis;
} else {
this.requestTimeoutMillis = -1;
}
this.httpClient.setFollowRedirects(Boolean.TRUE.equals(builder.followRedirects));
if (builder.urlParamNames != null) {
this.urlParamNames = builder.urlParamNames;
} else {
this.urlParamNames = Set.of();
}
assert ObjectReleaseTracker.track(this);
}
public void addListenerFactory(HttpListenerFactory factory) {
this.listenerFactory.add(factory);
}
// internal usage only
HttpClient getHttpClient() {
return httpClient;
}
// internal usage only
ProtocolHandlers getProtocolHandlers() {
return httpClient.getProtocolHandlers();
}
private HttpClient createHttpClient(Builder builder) {
HttpClient httpClient;
executor = builder.executor;
if (executor == null) {
BlockingArrayQueue queue = new BlockingArrayQueue<>(256, 256);
this.executor =
new ExecutorUtil.MDCAwareThreadPoolExecutor(
32, 256, 60, TimeUnit.SECONDS, queue, new SolrNamedThreadFactory("h2sc"));
shutdownExecutor = true;
} else {
shutdownExecutor = false;
}
SslContextFactory.Client sslContextFactory;
if (builder.sslConfig == null) {
sslContextFactory = getDefaultSslContextFactory();
} else {
sslContextFactory = builder.sslConfig.createClientContextFactory();
}
ClientConnector clientConnector = new ClientConnector();
clientConnector.setReuseAddress(true);
clientConnector.setSslContextFactory(sslContextFactory);
clientConnector.setSelectors(2);
HttpClientTransport transport;
if (builder.useHttp1_1) {
if (log.isDebugEnabled()) {
log.debug("Create Http2SolrClient with HTTP/1.1 transport");
}
transport = new HttpClientTransportOverHTTP(clientConnector);
httpClient = new HttpClient(transport);
if (builder.maxConnectionsPerHost != null) {
httpClient.setMaxConnectionsPerDestination(builder.maxConnectionsPerHost);
}
} else {
if (log.isDebugEnabled()) {
log.debug("Create Http2SolrClient with HTTP/2 transport");
}
HTTP2Client http2client = new HTTP2Client(clientConnector);
transport = new HttpClientTransportOverHTTP2(http2client);
httpClient = new HttpClient(transport);
httpClient.setMaxConnectionsPerDestination(4);
}
httpClient.setExecutor(this.executor);
httpClient.setStrictEventOrdering(false);
httpClient.setConnectBlocking(true);
httpClient.setFollowRedirects(false);
httpClient.setMaxRequestsQueuedPerDestination(
asyncTracker.getMaxRequestsQueuedPerDestination());
httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, AGENT));
httpClient.setIdleTimeout(idleTimeoutMillis);
if (builder.cookieStore != null) {
httpClient.setCookieStore(builder.cookieStore);
}
this.authenticationStore = new AuthenticationStoreHolder();
httpClient.setAuthenticationStore(this.authenticationStore);
httpClient.setConnectTimeout(builder.connectionTimeoutMillis);
setupProxy(builder, httpClient);
try {
httpClient.start();
} catch (Exception e) {
close(); // make sure we clean up
throw new RuntimeException(e);
}
return httpClient;
}
private void setupProxy(Builder builder, HttpClient httpClient) {
if (builder.proxyHost == null) {
return;
}
Address address = new Address(builder.proxyHost, builder.proxyPort);
final ProxyConfiguration.Proxy proxy;
if (builder.proxyIsSocks4) {
proxy = new Socks4Proxy(address, builder.proxyIsSecure);
} else {
final Protocol protocol;
if (builder.useHttp1_1) {
protocol = HttpClientTransportOverHTTP.HTTP11;
} else {
// see HttpClientTransportOverHTTP2#newOrigin
String protocolName = builder.proxyIsSecure ? "h2" : "h2c";
protocol = new Protocol(List.of(protocolName), false);
}
proxy = new HttpProxy(address, builder.proxyIsSecure, protocol);
}
httpClient.getProxyConfiguration().addProxy(proxy);
}
@Override
public void close() {
// we wait for async requests, so far devs don't want to give sugar for this
asyncTracker.waitForComplete();
try {
if (closeClient) {
httpClient.stop();
httpClient.destroy();
}
} catch (Exception e) {
throw new RuntimeException("Exception on closing client", e);
} finally {
if (shutdownExecutor) {
ExecutorUtil.shutdownAndAwaitTermination(executor);
}
}
assert ObjectReleaseTracker.release(this);
}
public void setAuthenticationStore(AuthenticationStore authenticationStore) {
this.authenticationStore.updateAuthenticationStore(authenticationStore);
}
public boolean isV2ApiRequest(final SolrRequest> request) {
return request instanceof V2Request || request.getPath().contains("/____v2");
}
public long getIdleTimeout() {
return idleTimeoutMillis;
}
public static class OutStream implements Closeable {
private final String origCollection;
private final ModifiableSolrParams origParams;
private final OutputStreamRequestContent content;
private final InputStreamResponseListener responseListener;
private final boolean isXml;
public OutStream(
String origCollection,
ModifiableSolrParams origParams,
OutputStreamRequestContent content,
InputStreamResponseListener responseListener,
boolean isXml) {
this.origCollection = origCollection;
this.origParams = origParams;
this.content = content;
this.responseListener = responseListener;
this.isXml = isXml;
}
boolean belongToThisStream(SolrRequest> solrRequest, String collection) {
ModifiableSolrParams solrParams = new ModifiableSolrParams(solrRequest.getParams());
return origParams.toNamedList().equals(solrParams.toNamedList())
&& Objects.equals(origCollection, collection);
}
public void write(byte b[]) throws IOException {
this.content.getOutputStream().write(b);
}
public void flush() throws IOException {
this.content.getOutputStream().flush();
}
@Override
public void close() throws IOException {
if (isXml) {
write("".getBytes(FALLBACK_CHARSET));
}
this.content.getOutputStream().close();
}
// TODO this class should be hidden
public InputStreamResponseListener getResponseListener() {
return responseListener;
}
}
public OutStream initOutStream(String baseUrl, UpdateRequest updateRequest, String collection)
throws IOException {
String contentType = requestWriter.getUpdateContentType();
final ModifiableSolrParams origParams = new ModifiableSolrParams(updateRequest.getParams());
// The parser 'wt=' and 'version=' params are used instead of the
// original params
ModifiableSolrParams requestParams = new ModifiableSolrParams(origParams);
requestParams.set(CommonParams.WT, parser.getWriterType());
requestParams.set(CommonParams.VERSION, parser.getVersion());
String basePath = baseUrl;
if (collection != null) basePath += "/" + collection;
if (!basePath.endsWith("/")) basePath += "/";
OutputStreamRequestContent content = new OutputStreamRequestContent(contentType);
Request postRequest =
httpClient
.newRequest(basePath + "update" + requestParams.toQueryString())
.method(HttpMethod.POST)
.body(content);
decorateRequest(postRequest, updateRequest, false);
InputStreamResponseListener responseListener = new InputStreamReleaseTrackingResponseListener();
postRequest.send(responseListener);
boolean isXml = ClientUtils.TEXT_XML.equals(requestWriter.getUpdateContentType());
OutStream outStream = new OutStream(collection, origParams, content, responseListener, isXml);
if (isXml) {
outStream.write("".getBytes(FALLBACK_CHARSET));
}
return outStream;
}
public void send(OutStream outStream, SolrRequest> req, String collection) throws IOException {
assert outStream.belongToThisStream(req, collection);
this.requestWriter.write(req, outStream.content.getOutputStream());
if (outStream.isXml) {
// check for commit or optimize
SolrParams params = req.getParams();
if (params != null) {
String fmt = null;
if (params.getBool(UpdateParams.OPTIMIZE, false)) {
fmt = "";
} else if (params.getBool(UpdateParams.COMMIT, false)) {
fmt = "";
}
if (fmt != null) {
byte[] content =
String.format(
Locale.ROOT, fmt, params.getBool(UpdateParams.WAIT_SEARCHER, false) + "")
.getBytes(FALLBACK_CHARSET);
outStream.write(content);
}
}
}
outStream.flush();
}
@SuppressWarnings("StaticAssignmentOfThrowable")
private static final Exception CANCELLED_EXCEPTION = new Exception();
private static final Cancellable FAILED_MAKING_REQUEST_CANCELLABLE = () -> {};
public Cancellable asyncRequest(
SolrRequest> solrReq, String collection, AsyncListener> asyncListener) {
MDCCopyHelper mdcCopyHelper = new MDCCopyHelper();
SolrRequest> solrRequest = unwrapV2Request(solrReq);
Request req;
try {
String url = getRequestPath(solrRequest, collection);
InputStreamResponseListener listener =
new InputStreamReleaseTrackingResponseListener() {
@Override
public void onHeaders(Response response) {
super.onHeaders(response);
executor.execute(
() -> {
InputStream is = getInputStream();
try {
NamedList