org.parosproxy.paros.network.HttpSender Maven / Gradle / Ivy
Show all versions of zap Show documentation
/*
*
* Paros and its related class files.
*
* Paros is an HTTP/HTTPS proxy for assessing web application security.
* Copyright (C) 2003-2004 Chinotec Technologies Company
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the Clarified Artistic License
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Clarified Artistic License for more details.
*
* You should have received a copy of the Clarified Artistic License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
// ZAP: 2011/09/19 Added debugging
// ZAP: 2012/04/23 Removed unnecessary cast.
// ZAP: 2012/05/08 Use custom http client on "Connection: Upgrade" in executeMethod().
// Retrieve upgraded socket and save for later use in send() method.
// ZAP: 2012/08/07 Issue 342 Support the HttpSenderListener
// ZAP: 2012/12/27 Do not read request body on Server-Sent Event streams.
// ZAP: 2013/01/03 Resolved Checkstyle issues: removed throws HttpException
// declaration where IOException already appears,
// introduced two helper methods for notifying listeners.
// ZAP: 2013/01/19 Issue 459: Active scanner locking
// ZAP: 2013/01/23 Clean up of exception handling/logging.
// ZAP: 2013/01/30 Issue 478: Allow to choose to send ZAP's managed cookies on
// a single Cookie request header and set it as the default
// ZAP: 2013/07/10 Issue 720: Cannot send non standard http methods
// ZAP: 2013/07/14 Issue 729: Update NTLM authentication code
// ZAP: 2013/07/25 Added support for sending the message from the perspective of a User
// ZAP: 2013/08/31 Reauthentication when sending a message from the perspective of a User
// ZAP: 2013/09/07 Switched to using HttpState for requesting User for cookie management
// ZAP: 2013/09/26 Issue 716: ZAP flags its own HTTP responses
// ZAP: 2013/09/26 Issue 656: Content-length: 0 in GET requests
// ZAP: 2013/09/29 Deprecating configuring HTTP Authentication through Options
// ZAP: 2013/11/16 Issue 837: Update, always, the HTTP request sent/forward by ZAP's proxy
// ZAP: 2013/12/11 Corrected log.info calls to use debug
// ZAP: 2014/03/04 Issue 1043: Custom active scan dialog
// ZAP: 2014/03/23 Issue 412: Enable unsafe SSL/TLS renegotiation option not saved
// ZAP: 2014/03/23 Issue 416: Normalise how multiple related options are managed throughout ZAP
// and enhance the usability of some options
// ZAP: 2014/03/29 Issue 1132: HttpSender ignores the "Send single cookie request header" option
// ZAP: 2014/08/14 Issue 1291: 407 Proxy Authentication Required while active scanning
// ZAP: 2014/10/25 Issue 1062: Added a getter for the HttpClient.
// ZAP: 2014/10/28 Issue 1390: Force https on cfu call
// ZAP: 2014/11/25 Issue 1411: Changed getUser() visibility
// ZAP: 2014/12/11 Added JavaDoc to constructor and removed the instance variable allowState.
// ZAP: 2015/04/09 Allow to specify the maximum number of retries on I/O error.
// ZAP: 2015/04/09 Allow to specify the maximum number of redirects.
// ZAP: 2015/04/09 Allow to specify if circular redirects are allowed.
// ZAP: 2015/06/12 Issue 1459: Add an HTTP sender listener script
// ZAP: 2016/05/24 Issue 2463: Websocket not proxied when outgoing proxy is set
// ZAP: 2016/05/27 Issue 2484: Circular Redirects
// ZAP: 2016/06/08 Set User-Agent header defined in options as default for (internal) CONNECT
// requests
// ZAP: 2016/06/10 Allow to validate the URI of the redirections before being followed
// ZAP: 2016/08/04 Added removeListener(..)
// ZAP: 2016/12/07 Add initiator constant for AJAX spider requests
// ZAP: 2016/12/12 Add initiator constant for Forced Browse requests
// ZAP: 2017/03/27 Introduce HttpRequestConfig.
// ZAP: 2017/06/12 Allow to ignore listeners.
// ZAP: 2017/06/19 Allow to send a request with custom socket timeout.
// ZAP: 2017/11/20 Add initiator constant for Token Generator requests.
// ZAP: 2017/11/27 Use custom CookieSpec (ZapCookieSpec).
// ZAP: 2017/12/20 Apply socket connect timeout (Issue 4171).
// ZAP: 2018/02/06 Make the lower case changes locale independent (Issue 4327).
// ZAP: 2018/02/19 Added WEB_SOCKET_INITIATOR.
// ZAP: 2018/02/23 Issue 1161: Allow to override the global session tracking setting
// Fix Session Tracking button sync
// ZAP: 2018/08/03 Added AUTHENTICATION_HELPER_INITIATOR.
// ZAP: 2018/09/17 Set the user to messages created for redirections (Issue 2531).
// ZAP: 2018/10/12 Deprecate getClient(), it exposes implementation details.
// ZAP: 2019/03/24 Removed commented and unused sendAndReceive method.
// ZAP: 2019/06/01 Normalise line endings.
// ZAP: 2019/06/05 Normalise format/style.
// ZAP: 2019/08/19 Reinstate proxy auth credentials when HTTP state is changed.
// ZAP: 2019/09/17 Use remove() instead of set(null) on IN_LISTENER.
// ZAP: 2019/09/25 Add option to disable cookies
package org.parosproxy.paros.network;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpHost;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodDirector;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.InvalidRedirectLocationException;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NTCredentials;
import org.apache.commons.httpclient.ProxyHost;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.auth.AuthPolicy;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.log4j.Logger;
import org.zaproxy.zap.ZapGetMethod;
import org.zaproxy.zap.ZapHttpConnectionManager;
import org.zaproxy.zap.network.HttpRedirectionValidator;
import org.zaproxy.zap.network.HttpRequestConfig;
import org.zaproxy.zap.network.HttpSenderListener;
import org.zaproxy.zap.network.ZapCookieSpec;
import org.zaproxy.zap.network.ZapNTLMScheme;
import org.zaproxy.zap.users.User;
public class HttpSender {
public static final int PROXY_INITIATOR = 1;
public static final int ACTIVE_SCANNER_INITIATOR = 2;
public static final int SPIDER_INITIATOR = 3;
public static final int FUZZER_INITIATOR = 4;
public static final int AUTHENTICATION_INITIATOR = 5;
public static final int MANUAL_REQUEST_INITIATOR = 6;
public static final int CHECK_FOR_UPDATES_INITIATOR = 7;
public static final int BEAN_SHELL_INITIATOR = 8;
public static final int ACCESS_CONTROL_SCANNER_INITIATOR = 9;
public static final int AJAX_SPIDER_INITIATOR = 10;
public static final int FORCED_BROWSE_INITIATOR = 11;
public static final int TOKEN_GENERATOR_INITIATOR = 12;
public static final int WEB_SOCKET_INITIATOR = 13;
public static final int AUTHENTICATION_HELPER_INITIATOR = 14;
private static Logger log = Logger.getLogger(HttpSender.class);
private static ProtocolSocketFactory sslFactory = null;
private static Protocol protocol = null;
private static List listeners = new ArrayList<>();
private static Comparator listenersComparator = null;;
private User user = null;
static {
try {
protocol = Protocol.getProtocol("https");
sslFactory = protocol.getSocketFactory();
} catch (Exception e) {
}
// avoid init again if already initialized
if (sslFactory == null || !(sslFactory instanceof SSLConnector)) {
Protocol.registerProtocol(
"https",
new Protocol("https", (ProtocolSocketFactory) new SSLConnector(true), 443));
}
AuthPolicy.registerAuthScheme(AuthPolicy.NTLM, ZapNTLMScheme.class);
CookiePolicy.registerCookieSpec(CookiePolicy.DEFAULT, ZapCookieSpec.class);
CookiePolicy.registerCookieSpec(CookiePolicy.BROWSER_COMPATIBILITY, ZapCookieSpec.class);
}
private static HttpMethodHelper helper = new HttpMethodHelper();
private static String userAgent = "";
private static final ThreadLocal IN_LISTENER = new ThreadLocal();
private HttpClient client = null;
private HttpClient clientViaProxy = null;
private ConnectionParam param = null;
private MultiThreadedHttpConnectionManager httpConnManager = null;
private MultiThreadedHttpConnectionManager httpConnManagerProxy = null;
private boolean followRedirect = false;
private boolean useCookies;
private boolean useGlobalState;
private int initiator = -1;
/*
* public HttpSender(ConnectionParam connectionParam, boolean allowState) { this
* (connectionParam, allowState, -1); }
*/
/**
* Constructs an {@code HttpSender}.
*
* If {@code useGlobalState} is {@code true} the HttpSender will use the HTTP state given by
* {@code ConnectionParam#getHttpState()} iff {@code ConnectionParam#isHttpStateEnabled()}
* returns {@code true} otherwise it doesn't have any state (i.e. cookies are disabled). If
* {@code useGlobalState} is {@code false} it uses a non shared HTTP state. The actual state
* used is overridden, per message, when {@code HttpMessage#getRequestingUser()} returns non
* {@code null}.
*
*
The {@code initiator} is used to indicate the component that is sending the messages when
* the {@code HttpSenderListener}s are notified of messages sent and received.
*
* @param connectionParam the parameters used to setup the connections to target hosts
* @param useGlobalState {@code true} if the messages sent/received should use the global HTTP
* state, {@code false} if should use a non shared HTTP state
* @param initiator the ID of the initiator of the HTTP messages sent
* @see ConnectionParam#getHttpState()
* @see HttpSenderListener
* @see HttpMessage#getRequestingUser()
*/
public HttpSender(ConnectionParam connectionParam, boolean useGlobalState, int initiator) {
this.param = connectionParam;
this.initiator = initiator;
client = createHttpClient();
clientViaProxy = createHttpClientViaProxy();
setAllowCircularRedirects(true);
// Set how cookie headers are sent no matter of the "allowState", in case a state is forced
// by
// other extensions (e.g. Authentication)
final boolean singleCookieRequestHeader = param.isSingleCookieRequestHeader();
client.getParams()
.setBooleanParameter(
HttpMethodParams.SINGLE_COOKIE_HEADER, singleCookieRequestHeader);
clientViaProxy
.getParams()
.setBooleanParameter(
HttpMethodParams.SINGLE_COOKIE_HEADER, singleCookieRequestHeader);
String defaultUserAgent = param.getDefaultUserAgent();
client.getParams()
.setParameter(
HttpMethodDirector.PARAM_DEFAULT_USER_AGENT_CONNECT_REQUESTS,
defaultUserAgent);
clientViaProxy
.getParams()
.setParameter(
HttpMethodDirector.PARAM_DEFAULT_USER_AGENT_CONNECT_REQUESTS,
defaultUserAgent);
setUseGlobalState(useGlobalState);
setUseCookies(true);
}
private void setClientsCookiePolicy(String policy) {
client.getParams().setCookiePolicy(policy);
clientViaProxy.getParams().setCookiePolicy(policy);
}
public static SSLConnector getSSLConnector() {
return (SSLConnector) protocol.getSocketFactory();
}
private void checkState() {
if (!useCookies) {
resetState();
setClientsCookiePolicy(CookiePolicy.IGNORE_COOKIES);
} else if (useGlobalState) {
if (param.isHttpStateEnabled()) {
client.setState(param.getHttpState());
clientViaProxy.setState(param.getHttpState());
setProxyAuth(clientViaProxy);
setClientsCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
} else {
setClientsCookiePolicy(CookiePolicy.IGNORE_COOKIES);
}
} else {
resetState();
setClientsCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
}
}
private void resetState() {
HttpState state = new HttpState();
HttpState proxyState = new HttpState();
client.setState(state);
clientViaProxy.setState(proxyState);
setProxyAuth(clientViaProxy);
}
/**
* Sets whether or not the global state should be used.
*
*
Refer to {@link #HttpSender(ConnectionParam, boolean, int)} for details on how the global
* state is used.
*
* @param enableGlobalState {@code true} if the global state should be used, {@code false}
* otherwise.
* @since 2.8.0
*/
public void setUseGlobalState(boolean enableGlobalState) {
this.useGlobalState = enableGlobalState;
checkState();
}
/**
* Sets whether or not the requests sent should keep track of cookies.
*
* @param shouldUseCookies {@code true} if cookies should be used, {@code false} otherwise.
* @since 2.9.0
*/
public void setUseCookies(boolean shouldUseCookies) {
this.useCookies = shouldUseCookies;
checkState();
}
private HttpClient createHttpClient() {
httpConnManager = new MultiThreadedHttpConnectionManager();
setCommonManagerParams(httpConnManager);
return new HttpClient(httpConnManager);
}
private HttpClient createHttpClientViaProxy() {
if (!param.isUseProxyChain()) {
return createHttpClient();
}
httpConnManagerProxy = new MultiThreadedHttpConnectionManager();
setCommonManagerParams(httpConnManagerProxy);
HttpClient clientProxy = new HttpClient(httpConnManagerProxy);
clientProxy
.getHostConfiguration()
.setProxy(param.getProxyChainName(), param.getProxyChainPort());
setProxyAuth(clientProxy);
return clientProxy;
}
private void setProxyAuth(HttpClient client) {
setProxyAuth(client.getState());
}
private void setProxyAuth(HttpState state) {
if (param.isUseProxyChain() && param.isUseProxyChainAuth()) {
String realm = param.getProxyChainRealm();
state.setProxyCredentials(
new AuthScope(
param.getProxyChainName(),
param.getProxyChainPort(),
realm.isEmpty() ? AuthScope.ANY_REALM : realm),
new NTCredentials(
param.getProxyChainUserName(),
param.getProxyChainPassword(),
"",
realm));
}
}
public int executeMethod(HttpMethod method, HttpState state) throws IOException {
int responseCode = -1;
String hostName;
hostName = method.getURI().getHost();
method.setDoAuthentication(true);
HostConfiguration hc = null;
HttpClient requestClient;
if (isConnectionUpgrade(method)) {
requestClient = new HttpClient(new ZapHttpConnectionManager());
if (param.isUseProxy(hostName)) {
requestClient
.getHostConfiguration()
.setProxy(param.getProxyChainName(), param.getProxyChainPort());
setProxyAuth(requestClient);
}
} else if (param.isUseProxy(hostName)) {
requestClient = clientViaProxy;
} else {
requestClient = client;
}
if (this.initiator == CHECK_FOR_UPDATES_INITIATOR) {
// Use the 'strict' SSLConnector, i.e. one that performs all the usual cert checks
// The 'standard' one 'trusts' everything
// This is to ensure that all 'check-for update' calls are made to the expected https
// urls
// without this is would be possible to intercept and change the response which could
// result
// in the user downloading and installing a malicious add-on
hc =
new HostConfiguration() {
@Override
public synchronized void setHost(URI uri) {
try {
setHost(new HttpHost(uri.getHost(), uri.getPort(), getProtocol()));
} catch (URIException e) {
throw new IllegalArgumentException(e.toString());
}
};
};
hc.setHost(
hostName,
method.getURI().getPort(),
new Protocol("https", (ProtocolSocketFactory) new SSLConnector(false), 443));
if (param.isUseProxy(hostName)) {
hc.setProxyHost(
new ProxyHost(param.getProxyChainName(), param.getProxyChainPort()));
setProxyAuth(requestClient);
}
}
// ZAP: Check if a custom state is being used
if (state != null) {
// Make sure cookies are enabled
method.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
setProxyAuth(state);
}
responseCode = requestClient.executeMethod(hc, method, state);
return responseCode;
}
/**
* Tells whether or not the given {@code method} has a {@code Connection} request header with
* {@code Upgrade} value.
*
* @param method the method that will be checked
* @return {@code true} if the {@code method} has a connection upgrade, {@code false} otherwise
*/
private static boolean isConnectionUpgrade(HttpMethod method) {
Header connectionHeader = method.getRequestHeader("connection");
if (connectionHeader == null) {
return false;
}
return connectionHeader.getValue().toLowerCase(Locale.ROOT).contains("upgrade");
}
public void shutdown() {
if (httpConnManager != null) {
httpConnManager.shutdown();
}
if (httpConnManagerProxy != null) {
httpConnManagerProxy.shutdown();
}
}
public void sendAndReceive(HttpMessage msg) throws IOException {
sendAndReceive(msg, followRedirect);
}
/**
* Send and receive a HttpMessage.
*
* @param msg
* @param isFollowRedirect
* @throws HttpException
* @throws IOException
* @see #sendAndReceive(HttpMessage, HttpRequestConfig)
*/
public void sendAndReceive(HttpMessage msg, boolean isFollowRedirect) throws IOException {
log.debug(
"sendAndReceive "
+ msg.getRequestHeader().getMethod()
+ " "
+ msg.getRequestHeader().getURI()
+ " start");
msg.setTimeSentMillis(System.currentTimeMillis());
try {
notifyRequestListeners(msg);
if (!isFollowRedirect
|| !(msg.getRequestHeader().getMethod().equalsIgnoreCase(HttpRequestHeader.POST)
|| msg.getRequestHeader()
.getMethod()
.equalsIgnoreCase(HttpRequestHeader.PUT))) {
// ZAP: Reauthentication when sending a message from the perspective of a User
sendAuthenticated(msg, isFollowRedirect);
return;
}
// ZAP: Reauthentication when sending a message from the perspective of a User
sendAuthenticated(msg, false);
HttpMessage temp = msg.cloneAll();
temp.setRequestingUser(getUser(msg));
// POST/PUT method cannot be redirected by library. Need to follow by code
// loop 1 time only because httpclient can handle redirect itself after first GET.
for (int i = 0;
i < 1
&& (HttpStatusCode.isRedirection(
temp.getResponseHeader().getStatusCode())
&& temp.getResponseHeader().getStatusCode()
!= HttpStatusCode.NOT_MODIFIED);
i++) {
String location = temp.getResponseHeader().getHeader(HttpHeader.LOCATION);
URI baseUri = temp.getRequestHeader().getURI();
URI newLocation = new URI(baseUri, location, false);
temp.getRequestHeader().setURI(newLocation);
temp.getRequestHeader().setMethod(HttpRequestHeader.GET);
temp.getRequestHeader().setHeader(HttpHeader.CONTENT_LENGTH, null);
// ZAP: Reauthentication when sending a message from the perspective of a User
sendAuthenticated(temp, true);
}
msg.setResponseHeader(temp.getResponseHeader());
msg.setResponseBody(temp.getResponseBody());
} finally {
msg.setTimeElapsedMillis((int) (System.currentTimeMillis() - msg.getTimeSentMillis()));
log.debug(
"sendAndReceive "
+ msg.getRequestHeader().getMethod()
+ " "
+ msg.getRequestHeader().getURI()
+ " took "
+ msg.getTimeElapsedMillis());
notifyResponseListeners(msg);
}
}
private void notifyRequestListeners(HttpMessage msg) {
if (IN_LISTENER.get() != null) {
// This is a request from one of the listeners - prevent infinite recursion
return;
}
try {
IN_LISTENER.set(true);
for (HttpSenderListener listener : listeners) {
try {
listener.onHttpRequestSend(msg, initiator, this);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
} finally {
IN_LISTENER.remove();
}
}
private void notifyResponseListeners(HttpMessage msg) {
if (IN_LISTENER.get() != null) {
// This is a request from one of the listeners - prevent infinite recursion
return;
}
try {
IN_LISTENER.set(true);
for (HttpSenderListener listener : listeners) {
try {
listener.onHttpResponseReceive(msg, initiator, this);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
} finally {
IN_LISTENER.remove();
}
}
public User getUser(HttpMessage msg) {
if (this.user != null) {
// If its set for the sender it overrides the message
return user;
}
return msg.getRequestingUser();
}
// ZAP: Make sure a message that needs to be authenticated is authenticated
private void sendAuthenticated(HttpMessage msg, boolean isFollowRedirect) throws IOException {
sendAuthenticated(msg, isFollowRedirect, null);
}
private void sendAuthenticated(
HttpMessage msg, boolean isFollowRedirect, HttpMethodParams params) throws IOException {
// Modify the request message if a 'Requesting User' has been set
User forceUser = this.getUser(msg);
if (initiator != AUTHENTICATION_INITIATOR && forceUser != null)
forceUser.processMessageToMatchUser(msg);
log.debug("Sending message to: " + msg.getRequestHeader().getURI().toString());
// Send the message
send(msg, isFollowRedirect, params);
// If there's a 'Requesting User', make sure the response corresponds to an authenticated
// session and, if not, attempt a reauthentication and try again
if (initiator != AUTHENTICATION_INITIATOR
&& forceUser != null
&& !msg.getRequestHeader().isImage()
&& !forceUser.isAuthenticated(msg)) {
log.debug(
"First try to send authenticated message failed for "
+ msg.getRequestHeader().getURI()
+ ". Authenticating and trying again...");
forceUser.queueAuthentication(msg);
forceUser.processMessageToMatchUser(msg);
send(msg, isFollowRedirect, params);
} else log.debug("SUCCESSFUL");
}
private void send(HttpMessage msg, boolean isFollowRedirect, HttpMethodParams params)
throws IOException {
HttpMethod method = null;
HttpResponseHeader resHeader = null;
try {
method = runMethod(msg, isFollowRedirect, params);
// successfully executed;
resHeader = HttpMethodHelper.getHttpResponseHeader(method);
resHeader.setHeader(
HttpHeader.TRANSFER_ENCODING,
null); // replaceAll("Transfer-Encoding: chunked\r\n",
// "");
msg.setResponseHeader(resHeader);
msg.getResponseBody().setCharset(resHeader.getCharset());
msg.getResponseBody().setLength(0);
// ZAP: Do not read response body for Server-Sent Events stream
// ZAP: Moreover do not set content length to zero
if (!msg.isEventStream()) {
msg.getResponseBody().append(method.getResponseBody());
}
msg.setResponseFromTargetHost(true);
// ZAP: set method to retrieve upgraded channel later
if (method instanceof ZapGetMethod) {
msg.setUserObject(method);
}
} finally {
if (method != null) {
method.releaseConnection();
}
}
}
private HttpMethod runMethod(HttpMessage msg, boolean isFollowRedirect, HttpMethodParams params)
throws IOException {
HttpMethod method = null;
// no more retry
modifyUserAgent(msg);
method = helper.createRequestMethod(msg.getRequestHeader(), msg.getRequestBody(), params);
if (!(method instanceof EntityEnclosingMethod) || method instanceof ZapGetMethod) {
// cant do this for EntityEnclosingMethod methods - it will fail
method.setFollowRedirects(isFollowRedirect);
}
// ZAP: Use custom HttpState if needed
User forceUser = this.getUser(msg);
if (forceUser != null) {
this.executeMethod(method, forceUser.getCorrespondingHttpState());
} else {
this.executeMethod(method, null);
}
HttpMethodHelper.updateHttpRequestHeaderSent(msg.getRequestHeader(), method);
return method;
}
public void setFollowRedirect(boolean followRedirect) {
this.followRedirect = followRedirect;
}
private void modifyUserAgent(HttpMessage msg) {
try {
// no modification to user agent if empty
if (userAgent.equals("") || msg.getRequestHeader().isEmpty()) {
return;
}
// append new user agent to existing user agent
String currentUserAgent = msg.getRequestHeader().getHeader(HttpHeader.USER_AGENT);
if (currentUserAgent == null) {
currentUserAgent = "";
}
if (currentUserAgent.indexOf(userAgent) >= 0) {
// user agent already in place, exit
return;
}
String delimiter = "";
if (!currentUserAgent.equals("") && !currentUserAgent.endsWith(" ")) {
delimiter = " ";
}
currentUserAgent = currentUserAgent + delimiter + userAgent;
msg.getRequestHeader().setHeader(HttpHeader.USER_AGENT, currentUserAgent);
} catch (Exception e) {
}
}
/** @return Returns the userAgent. */
public static String getUserAgent() {
return userAgent;
}
/** @param userAgent The userAgent to set. */
public static void setUserAgent(String userAgent) {
HttpSender.userAgent = userAgent;
}
private void setCommonManagerParams(MultiThreadedHttpConnectionManager mgr) {
int timeout = (int) TimeUnit.SECONDS.toMillis(this.param.getTimeoutInSecs());
mgr.getParams().setSoTimeout(timeout);
mgr.getParams().setConnectionTimeout(timeout);
mgr.getParams().setStaleCheckingEnabled(true);
// Set to arbitrary large values to prevent locking
mgr.getParams().setDefaultMaxConnectionsPerHost(10000);
mgr.getParams().setMaxTotalConnections(200000);
// to use for HttpClient 3.0.1
// mgr.getParams().setDefaultMaxConnectionsPerHost((Constant.MAX_HOST_CONNECTION > 5) ? 15 :
// 3*Constant.MAX_HOST_CONNECTION);
// mgr.getParams().setMaxTotalConnections(mgr.getParams().getDefaultMaxConnectionsPerHost()*10);
// mgr.getParams().setConnectionTimeout(60000); // use default
}
/*
* Send and receive a HttpMessage.
*
* @param msg
*
* @param isFollowRedirect
*
* @throws HttpException
*
* @throws IOException
*/
/*
* private void sendAndReceive(HttpMessage msg, boolean isFollowRedirect, HttpOutputStream pipe,
* byte[] buf) throws HttpException, IOException { log.debug("sendAndReceive " +
* msg.getRequestHeader().getMethod() + " " + msg.getRequestHeader().getURI() + " start");
* msg.setTimeSentMillis(System.currentTimeMillis());
*
* try { if (!isFollowRedirect || !
* (msg.getRequestHeader().getMethod().equalsIgnoreCase(HttpRequestHeader.POST) ||
* msg.getRequestHeader().getMethod().equalsIgnoreCase(HttpRequestHeader.PUT)) ) { send(msg,
* isFollowRedirect, pipe, buf); return; } else { send(msg, false, pipe, buf); }
*
* HttpMessage temp = msg.cloneAll(); // POST/PUT method cannot be redirected by library. Need
* to follow by code
*
* // loop 1 time only because httpclient can handle redirect itself after first GET. for (int
* i=0; i<1 && (HttpStatusCode.isRedirection(temp.getResponseHeader().getStatusCode()) &&
* temp.getResponseHeader().getStatusCode() != HttpStatusCode.NOT_MODIFIED); i++) { String
* location = temp.getResponseHeader().getHeader(HttpHeader.LOCATION); URI baseUri =
* temp.getRequestHeader().getURI(); URI newLocation = new URI(baseUri, location, false);
* temp.getRequestHeader().setURI(newLocation);
*
* temp.getRequestHeader().setMethod(HttpRequestHeader.GET);
* temp.getRequestHeader().setContentLength(0); send(temp, true, pipe, buf); }
*
* msg.setResponseHeader(temp.getResponseHeader()); msg.setResponseBody(temp.getResponseBody());
*
* } finally { msg.setTimeElapsedMillis((int)
* (System.currentTimeMillis()-msg.getTimeSentMillis())); log.debug("sendAndReceive " +
* msg.getRequestHeader().getMethod() + " " + msg.getRequestHeader().getURI() + " took " +
* msg.getTimeElapsedMillis()); } }
*/
/*
* Do not use this unless sure what is doing. This method works but proxy may skip the pipe
* without properly handle the filter.
*
* @param msg
*
* @param isFollowRedirect
*
* @param pipe
*
* @param buf
*
* @throws HttpException
*
* @throws IOException
*/
/*
* private void send(HttpMessage msg, boolean isFollowRedirect, HttpOutputStream pipe, byte[]
* buf) throws HttpException, IOException { HttpMethod method = null; HttpResponseHeader
* resHeader = null;
*
* try { method = runMethod(msg, isFollowRedirect); // successfully executed; resHeader =
* HttpMethodHelper.getHttpResponseHeader(method);
* resHeader.setHeader(HttpHeader.TRANSFER_ENCODING, null); //
* replaceAll("Transfer-Encoding: chunked\r\n", ""); msg.setResponseHeader(resHeader);
* msg.getResponseBody().setCharset(resHeader.getCharset()); msg.getResponseBody().setLength(0);
*
* // process response for each listener
*
* pipe.write(msg.getResponseHeader()); pipe.flush();
*
* if (msg.getResponseHeader().getContentLength() >= 0 &&
* msg.getResponseHeader().getContentLength() < 20480) { // save time expanding buffer in
* HttpBody if (msg.getResponseHeader().getContentLength() > 0) {
* msg.getResponseBody().setBody(method.getResponseBody()); pipe.write(msg.getResponseBody());
* pipe.flush();
*
* } } else { //byte[] buf = new byte[4096]; InputStream in = method.getResponseBodyAsStream();
*
* int len = 0; while (in != null && (len = in.read(buf)) > 0) { pipe.write(buf, 0, len);
* pipe.flush();
*
* msg.getResponseBody().append(buf, len); } } } finally { if (method != null) {
* method.releaseConnection(); } } }
*/
public static void addListener(HttpSenderListener listener) {
listeners.add(listener);
Collections.sort(listeners, getListenersComparator());
}
public static void removeListener(HttpSenderListener listener) {
listeners.remove(listener);
}
private static Comparator getListenersComparator() {
if (listenersComparator == null) {
createListenersComparator();
}
return listenersComparator;
}
private static synchronized void createListenersComparator() {
if (listenersComparator == null) {
listenersComparator =
new Comparator() {
@Override
public int compare(HttpSenderListener o1, HttpSenderListener o2) {
int order1 = o1.getListenerOrder();
int order2 = o2.getListenerOrder();
if (order1 < order2) {
return -1;
} else if (order1 > order2) {
return 1;
}
return 0;
}
};
}
}
/**
* Set the user to scan as. If null then the current session will be used.
*
* @param user
*/
public void setUser(User user) {
this.user = user;
}
/**
* @return the HTTP client implementation.
* @deprecated (2.8.0) Do not use, this exposes implementation details which might change
* without warning. It will be removed in a following version.
*/
@Deprecated
public HttpClient getClient() {
return this.client;
}
/**
* Sets whether or not the authentication headers ("Authorization" and "Proxy-Authorization")
* already present in the request should be removed if received an authentication challenge
* (status codes 401 and 407).
*
* If {@code true} new authentication headers will be generated and the old ones removed
* otherwise the authentication headers already present in the request will be used to
* authenticate.
*
*
Default is {@code false}, i.e. use the headers already present in the request header.
*
*
Processes that reuse messages previously sent should consider setting this to {@code
* true}, otherwise new authentication challenges might fail.
*
* @param removeHeaders {@code true} if the the authentication headers already present should be
* removed when challenged, {@code false} otherwise
*/
public void setRemoveUserDefinedAuthHeaders(boolean removeHeaders) {
client.getParams()
.setBooleanParameter(
HttpMethodDirector.PARAM_REMOVE_USER_DEFINED_AUTH_HEADERS, removeHeaders);
clientViaProxy
.getParams()
.setBooleanParameter(
HttpMethodDirector.PARAM_REMOVE_USER_DEFINED_AUTH_HEADERS, removeHeaders);
}
/**
* Sets the maximum number of retries of an unsuccessful request caused by I/O errors.
*
*
The default number of retries is 3.
*
* @param retries the number of retries
* @throws IllegalArgumentException if {@code retries} is negative.
* @since 2.4.0
*/
public void setMaxRetriesOnIOError(int retries) {
if (retries < 0) {
throw new IllegalArgumentException(
"Parameter retries must be greater or equal to zero.");
}
HttpMethodRetryHandler retryHandler = new DefaultHttpMethodRetryHandler(retries, false);
client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
clientViaProxy.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
}
/**
* Sets the maximum number of redirects that will be followed before failing with an exception.
*
*
The default maximum number of redirects is 100.
*
* @param maxRedirects the maximum number of redirects
* @throws IllegalArgumentException if {@code maxRedirects} is negative.
* @since 2.4.0
*/
public void setMaxRedirects(int maxRedirects) {
if (maxRedirects < 0) {
throw new IllegalArgumentException(
"Parameter maxRedirects must be greater or equal to zero.");
}
client.getParams().setIntParameter(HttpClientParams.MAX_REDIRECTS, maxRedirects);
clientViaProxy.getParams().setIntParameter(HttpClientParams.MAX_REDIRECTS, maxRedirects);
}
/**
* Sets whether or not circular redirects are allowed.
*
*
Circular redirects happen when a request redirects to itself, or when a same request was
* already accessed in a chain of redirects.
*
*
Since 2.5.0, the default is to allow circular redirects.
*
* @param allow {@code true} if circular redirects should be allowed, {@code false} otherwise
* @since 2.4.0
*/
public void setAllowCircularRedirects(boolean allow) {
client.getParams().setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, allow);
clientViaProxy
.getParams()
.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, allow);
}
/**
* Sends the request of given HTTP {@code message} with the given configurations.
*
* @param message the message that will be sent
* @param requestConfig the request configurations.
* @throws IllegalArgumentException if any of the parameters is {@code null}
* @throws IOException if an error occurred while sending the message or following the
* redirections
* @since 2.6.0
* @see #sendAndReceive(HttpMessage, boolean)
*/
public void sendAndReceive(HttpMessage message, HttpRequestConfig requestConfig)
throws IOException {
if (message == null) {
throw new IllegalArgumentException("Parameter message must not be null.");
}
if (requestConfig == null) {
throw new IllegalArgumentException("Parameter requestConfig must not be null.");
}
sendAndReceiveImpl(message, requestConfig);
if (requestConfig.isFollowRedirects()) {
followRedirections(message, requestConfig);
}
}
/**
* Helper method that sends the request of the given HTTP {@code message} with the given
* configurations.
*
*
No redirections are followed (see {@link #followRedirections(HttpMessage,
* HttpRequestConfig)}).
*
* @param message the message that will be sent.
* @param requestConfig the request configurations.
* @throws IOException if an error occurred while sending the message or following the
* redirections.
*/
private void sendAndReceiveImpl(HttpMessage message, HttpRequestConfig requestConfig)
throws IOException {
if (log.isDebugEnabled()) {
log.debug(
"Sending "
+ message.getRequestHeader().getMethod()
+ " "
+ message.getRequestHeader().getURI());
}
message.setTimeSentMillis(System.currentTimeMillis());
try {
if (requestConfig.isNotifyListeners()) {
notifyRequestListeners(message);
}
HttpMethodParams params = null;
if (requestConfig.getSoTimeout() != HttpRequestConfig.NO_VALUE_SET) {
params = new HttpMethodParams();
params.setSoTimeout(requestConfig.getSoTimeout());
}
sendAuthenticated(message, false, params);
} finally {
message.setTimeElapsedMillis(
(int) (System.currentTimeMillis() - message.getTimeSentMillis()));
if (log.isDebugEnabled()) {
log.debug(
"Received response after "
+ message.getTimeElapsedMillis()
+ "ms for "
+ message.getRequestHeader().getMethod()
+ " "
+ message.getRequestHeader().getURI());
}
if (requestConfig.isNotifyListeners()) {
notifyResponseListeners(message);
}
}
}
/**
* Follows redirections using the response of the given {@code message}. The {@code validator}
* in the given request configuration will be called for each redirection received. After the
* call to this method the given {@code message} will have the contents of the last response
* received (possibly the response of a redirection).
*
*
The validator is notified of each message sent and received (first message and
* redirections followed, if any).
*
* @param message the message that will be sent, must not be {@code null}
* @param requestConfig the request configuration that contains the validator responsible for
* validation of redirections, must not be {@code null}.
* @throws IOException if an error occurred while sending the message or following the
* redirections
* @see #isRedirectionNeeded(int)
*/
private void followRedirections(HttpMessage message, HttpRequestConfig requestConfig)
throws IOException {
HttpRedirectionValidator validator = requestConfig.getRedirectionValidator();
validator.notifyMessageReceived(message);
User requestingUser = getUser(message);
HttpMessage redirectMessage = message;
int maxRedirections =
client.getParams().getIntParameter(HttpClientParams.MAX_REDIRECTS, 100);
for (int i = 0;
i < maxRedirections
&& isRedirectionNeeded(redirectMessage.getResponseHeader().getStatusCode());
i++) {
URI newLocation = extractRedirectLocation(redirectMessage);
if (newLocation == null || !validator.isValid(newLocation)) {
return;
}
redirectMessage = redirectMessage.cloneAll();
redirectMessage.setRequestingUser(requestingUser);
redirectMessage.getRequestHeader().setURI(newLocation);
if (isRequestRewriteNeeded(redirectMessage)) {
redirectMessage.getRequestHeader().setMethod(HttpRequestHeader.GET);
redirectMessage.getRequestHeader().setHeader(HttpHeader.CONTENT_TYPE, null);
redirectMessage.getRequestHeader().setHeader(HttpHeader.CONTENT_LENGTH, null);
redirectMessage.setRequestBody("");
}
sendAndReceiveImpl(redirectMessage, requestConfig);
validator.notifyMessageReceived(redirectMessage);
// Update the response of the (original) message
message.setResponseHeader(redirectMessage.getResponseHeader());
message.setResponseBody(redirectMessage.getResponseBody());
}
}
/**
* Tells whether or not a redirection is needed based on the given status code.
*
*
A redirection is needed if the status code is 301, 302, 303, 307 or 308.
*
* @param statusCode the status code that will be checked
* @return {@code true} if a redirection is needed, {@code false} otherwise
* @see #isRequestRewriteNeeded(HttpMessage)
*/
private static boolean isRedirectionNeeded(int statusCode) {
switch (statusCode) {
case 301:
case 302:
case 303:
case 307:
case 308:
return true;
default:
return false;
}
}
/**
* Tells whether or not the (original) request of the redirection, should be rewritten.
*
*
For status codes 301 and 302 the request should be changed from POST to GET when following
* redirections, for status code 303 it should be changed to GET for all methods except GET/HEAD
* (mimicking the behaviour of browsers, which per RFC 7231, Section 6.4 is now OK).
*
* @param message the message with the redirection.
* @return {@code true} if the request should be rewritten, {@code false} otherwise
* @see #isRedirectionNeeded(int)
*/
private static boolean isRequestRewriteNeeded(HttpMessage message) {
int statusCode = message.getResponseHeader().getStatusCode();
String method = message.getRequestHeader().getMethod();
if (statusCode == 301 || statusCode == 302) {
return HttpRequestHeader.POST.equalsIgnoreCase(method);
}
return statusCode == 303
&& !(HttpRequestHeader.GET.equalsIgnoreCase(method)
|| HttpRequestHeader.HEAD.equalsIgnoreCase(method));
}
/**
* Extracts a {@code URI} from the {@code Location} header of the given HTTP {@code message}.
*
*
If there's no {@code Location} header this method returns {@code null}.
*
* @param message the HTTP message that will processed
* @return the {@code URI} created from the value of the {@code Location} header, might be
* {@code null}
* @throws InvalidRedirectLocationException if the value of {@code Location} header is not a
* valid {@code URI}
*/
private static URI extractRedirectLocation(HttpMessage message)
throws InvalidRedirectLocationException {
String location = message.getResponseHeader().getHeader(HttpHeader.LOCATION);
if (location == null) {
if (log.isDebugEnabled()) {
log.debug("No Location header found: " + message.getResponseHeader());
}
return null;
}
try {
return new URI(message.getRequestHeader().getURI(), location, true);
} catch (URIException ex) {
throw new InvalidRedirectLocationException(
"Invalid redirect location: " + location, location, ex);
}
}
}