org.littleshoot.proxy.impl.ProxyUtils Maven / Gradle / Ivy
Show all versions of littleproxy Show documentation
package org.littleshoot.proxy.impl;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.DefaultHttpResponse;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaderValues;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMessage;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpObject;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpVersion;
import io.netty.handler.codec.http.LastHttpContent;
import io.netty.util.AsciiString;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
/**
* Utilities for the proxy.
*/
public class ProxyUtils {
/**
* Hop-by-hop headers that should be removed when proxying, as defined by the
* (HTTP 1.1 spec, section 13.5.1).
* Transfer-Encoding is NOT included in this list, since LittleProxy
* does not typically modify the transfer encoding. See also {@link #shouldRemoveHopByHopHeader(String)}.
*
* Header names are stored as lowercase to make case-insensitive comparisons easier.
*/
@SuppressWarnings("deprecation") // Don't remove header names from this set until they're removed from Netty, just in case someone's still using them.
private static final Set SHOULD_NOT_PROXY_HOP_BY_HOP_HEADERS = ImmutableSet.of(
HttpHeaderNames.CONNECTION.toString(),
HttpHeaderNames.KEEP_ALIVE.toString(),
HttpHeaderNames.PROXY_AUTHENTICATE.toString(),
HttpHeaderNames.PROXY_AUTHORIZATION.toString(),
HttpHeaderNames.TE.toString(),
HttpHeaderNames.TRAILER.toString(),
/* Note: Not removing Transfer-Encoding since LittleProxy does not normally re-chunk content.
HttpHeaderNames.TRANSFER_ENCODING.toString(), */
HttpHeaderNames.UPGRADE.toString()
);
private static final Logger LOG = LoggerFactory.getLogger(ProxyUtils.class);
/**
* Splits comma-separated header values (such as Connection) into their individual tokens.
*/
private static final Splitter COMMA_SEPARATED_HEADER_VALUE_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
// Schemes are case-insensitive:
// https://tools.ietf.org/html/rfc3986#section-3.1
private static final Pattern HTTP_PREFIX = Pattern.compile("^(http|ws)s?://.*",
Pattern.CASE_INSENSITIVE);
/**
* Strips the host from a URI string. This will turn "https://host.com/path"
* into "/path".
*
* @param uri
* The URI to transform.
* @return A string with the URI stripped.
*/
public static String stripHost(final String uri) {
if (!HTTP_PREFIX.matcher(uri).matches()) {
// It's likely a URI path, not the full URI (i.e. the host is
// already stripped).
return uri;
}
final String noHttpUri = StringUtils.substringAfter(uri, "://");
final int slashIndex = noHttpUri.indexOf("/");
if (slashIndex == -1) {
return "/";
}
return noHttpUri.substring(slashIndex);
}
/**
* If an HttpObject implements the market interface LastHttpContent, it
* represents the last chunk of a transfer.
*
* @see io.netty.handler.codec.http.LastHttpContent
*/
public static boolean isLastChunk(final HttpObject httpObject) {
return httpObject instanceof LastHttpContent;
}
/**
* If an HttpObject is not the last chunk, then that means there are other
* chunks that will follow.
*
* @see io.netty.handler.codec.http.FullHttpMessage
*/
public static boolean isChunked(final HttpObject httpObject) {
return !isLastChunk(httpObject);
}
/**
* Parses the host and port an HTTP request is being sent to.
*
* @param httpRequest
* The request.
* @return The host and port string.
*/
public static String parseHostAndPort(final HttpRequest httpRequest) {
return parseHostAndPort(httpRequest.uri());
}
/**
* Parses the host and port an HTTP request is being sent to.
*
* @param uri
* The URI.
* @return The host and port string.
*/
public static String parseHostAndPort(final String uri) {
final String tempUri;
if (!HTTP_PREFIX.matcher(uri).matches()) {
// Browsers particularly seem to send requests in this form when
// they use CONNECT.
tempUri = uri;
} else {
// We can't just take a substring from a hard-coded index because it
// could be either http or https.
tempUri = StringUtils.substringAfter(uri, "://");
}
final String hostAndPort;
if (tempUri.contains("/")) {
hostAndPort = tempUri.substring(0, tempUri.indexOf("/"));
} else {
hostAndPort = tempUri;
}
return hostAndPort;
}
/**
* Make a copy of the response including all mutable fields.
*
* @param original
* The original response to copy from.
* @return The copy with all mutable fields from the original.
*/
public static HttpResponse copyMutableResponseFields(
final HttpResponse original) {
HttpResponse copy;
if (original instanceof DefaultFullHttpResponse) {
ByteBuf content = ((DefaultFullHttpResponse) original).content();
copy = new DefaultFullHttpResponse(original.protocolVersion(),
original.status(), content);
} else {
copy = new DefaultHttpResponse(original.protocolVersion(),
original.status());
}
final Collection headerNames = original.headers().names();
for (final String name : headerNames) {
final List values = original.headers().getAll(name);
copy.headers().set(name, values);
}
return copy;
}
/**
* Adds the Via header to specify that the message has passed through the proxy. The specified alias will be
* appended to the Via header line. The alias may be the hostname of the machine proxying the request, or a
* pseudonym. From RFC 7230, section 5.7.1:
*
The received-by portion of the field value is normally the host and
optional port number of a recipient server or client that
subsequently forwarded the message. However, if the real host is
considered to be sensitive information, a sender MAY replace it with
a pseudonym.
*
*
*
* @param httpMessage HTTP message to add the Via header to
* @param alias the alias to provide in the Via header for this proxy
*/
public static void addVia(HttpMessage httpMessage, String alias) {
String newViaHeader = String.valueOf(httpMessage.protocolVersion().majorVersion()) +
'.' +
httpMessage.protocolVersion().minorVersion() +
' ' +
alias;
final List vias;
if (httpMessage.headers().contains(HttpHeaderNames.VIA)) {
List existingViaHeaders = httpMessage.headers().getAll(HttpHeaderNames.VIA);
vias = new ArrayList<>(existingViaHeaders);
vias.add(newViaHeader);
} else {
vias = Collections.singletonList(newViaHeader);
}
httpMessage.headers().set(HttpHeaderNames.VIA, vias);
}
/**
* Returns true if the specified string is either "true" or
* "on" ignoring case.
*
* @param val
* The string in question.
* @return true if the specified string is either "true" or
* "on" ignoring case, otherwise false.
*/
public static boolean isTrue(final String val) {
return checkTrueOrFalse(val, "true", "on");
}
/**
* Returns true if the specified string is either "false" or
* "off" ignoring case.
*
* @param val
* The string in question.
* @return true if the specified string is either "false" or
* "off" ignoring case, otherwise false.
*/
public static boolean isFalse(final String val) {
return checkTrueOrFalse(val, "false", "off");
}
public static boolean extractBooleanDefaultFalse(final Properties props,
final String key) {
final String throttle = props.getProperty(key);
if (StringUtils.isNotBlank(throttle)) {
return "true".equalsIgnoreCase(throttle.trim());
}
return false;
}
public static int extractInt(final Properties props, final String key) {
return extractInt(props, key, -1);
}
public static int extractInt(final Properties props, final String key, int defaultValue) {
final String readThrottleString = props.getProperty(key);
if (StringUtils.isNotBlank(readThrottleString) && NumberUtils.isCreatable(readThrottleString)) {
return Integer.parseInt(readThrottleString);
}
return defaultValue;
}
public static boolean isCONNECT(HttpObject httpObject) {
return httpObject instanceof HttpRequest && HttpMethod.CONNECT.equals(((HttpRequest) httpObject).method());
}
/**
* Returns true if the specified HttpRequest is a HEAD request.
*
* @param httpRequest http request
* @return true if request is a HEAD, otherwise false
*/
public static boolean isHEAD(HttpRequest httpRequest) {
return httpRequest != null && HttpMethod.HEAD.equals(httpRequest.method());
}
private static boolean checkTrueOrFalse(final String val,
final String str1, final String str2) {
final String str = val.trim();
return StringUtils.isNotBlank(str)
&& (str.equalsIgnoreCase(str1) || str.equalsIgnoreCase(str2));
}
/**
* Returns true if the HTTP message cannot contain an entity body, according to the HTTP spec. This code is taken directly
* from {@link io.netty.handler.codec.http.HttpObjectDecoder#isContentAlwaysEmpty(HttpMessage)}.
*
* @param msg HTTP message
* @return true if the HTTP message is always empty, false if the message may have entity content.
*/
public static boolean isContentAlwaysEmpty(HttpMessage msg) {
if (msg instanceof HttpResponse) {
HttpResponse res = (HttpResponse) msg;
int code = res.status().code();
// Correctly handle return codes of 1xx.
//
// See:
// - https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html Section 4.4
// - https://github.com/netty/netty/issues/222
if (code >= 100 && code < 200) {
// According to RFC 7231, section 6.1, 1xx responses have no content (https://tools.ietf.org/html/rfc7231#section-6.2):
// 1xx responses are terminated by the first empty line after
// the status-line (the empty line signaling the end of the header
// section).
// Hixie 76 websocket handshake responses contain a 16-byte body, so their content is not empty; but Hixie 76
// was a draft specification that was superceded by RFC 6455. Since it is rarely used and doesn't conform to
// RFC 7231, we do not support or make special allowance for Hixie 76 responses.
return true;
}
switch (code) {
case 204: case 205: case 304:
return true;
}
}
return false;
}
/**
* Returns true if the HTTP response from the server is expected to indicate its own message length/end-of-message. Returns false
* if the server is expected to indicate the end of the HTTP entity by closing the connection.
*
* This method is based on the allowed message length indicators in the HTTP specification, section 4.4:
*
4.4 Message Length
The transfer-length of a message is the length of the message-body as it appears in the message; that is, after any transfer-codings have been applied. When a message-body is included with a message, the transfer-length of that body is determined by one of the following (in order of precedence):
1.Any response message which "MUST NOT" include a message-body (such as the 1xx, 204, and 304 responses and any response to a HEAD request) is always terminated by the first empty line after the header fields, regardless of the entity-header fields present in the message.
2.If a Transfer-Encoding header field (section 14.41) is present and has any value other than "identity", then the transfer-length is defined by use of the "chunked" transfer-coding (section 3.6), unless the message is terminated by closing the connection.
3.If a Content-Length header field (section 14.13) is present, its decimal value in OCTETs represents both the entity-length and the transfer-length. The Content-Length header field MUST NOT be sent if these two lengths are different (i.e., if a Transfer-Encoding
header field is present). If a message is received with both a Transfer-Encoding header field and a Content-Length header field, the latter MUST be ignored.
[LP note: multipart/byteranges support has been removed from the HTTP 1.1 spec by RFC 7230, section A.2. Since it is seldom used, LittleProxy does not check for it.]
5.By the server closing the connection. (Closing the connection cannot be used to indicate the end of a request body, since that would leave no possibility for the server to send back a response.)
*
*
* The rules for Transfer-Encoding are clarified in RFC 7230, section 3.3.1 and 3.3.3 (3):
*
If any transfer coding other than
chunked is applied to a response payload body, the sender MUST either
apply chunked as the final transfer coding or terminate the message
by closing the connection.
*
*
*
* @param response the HTTP response object
* @return true if the message will indicate its own message length, or false if the server is expected to indicate the message length by closing the connection
*/
public static boolean isResponseSelfTerminating(HttpResponse response) {
if (isContentAlwaysEmpty(response)) {
return true;
}
// if there is a Transfer-Encoding value, determine whether the final encoding is "chunked", which makes the message self-terminating
List allTransferEncodingHeaders = getAllCommaSeparatedHeaderValues(HttpHeaderNames.TRANSFER_ENCODING, response);
if (!allTransferEncodingHeaders.isEmpty()) {
String finalEncoding = allTransferEncodingHeaders.get(allTransferEncodingHeaders.size() - 1);
// per #3 above: "If a message is received with both a Transfer-Encoding header field and a Content-Length header field, the latter MUST be ignored."
// since the Transfer-Encoding field is present, the message is self-terminating if and only if the final Transfer-Encoding value is "chunked"
return HttpHeaderValues.CHUNKED.toString().equals(finalEncoding);
}
String contentLengthHeader = response.headers().get(HttpHeaderNames.CONTENT_LENGTH);
return contentLengthHeader != null && !contentLengthHeader.isEmpty();
// not checking for multipart/byteranges, since it is seldom used and its use as a message length indicator was removed in RFC 7230
// none of the other message length indicators are present, so the only way the server can indicate the end
// of this message is to close the connection
}
/**
* Retrieves all comma-separated values for headers with the specified name on the HttpMessage. Any whitespace (spaces
* or tabs) surrounding the values will be removed. Empty values (e.g. two consecutive commas, or a value followed
* by a comma and no other value) will be removed; they will not appear as empty elements in the returned list.
* If the message contains repeated headers, their values will be added to the returned list in the order in which
* the headers appear. For example, if a message has headers like:
*
* Transfer-Encoding: gzip,deflate
* Transfer-Encoding: chunked
*
* This method will return a list of three values: "gzip", "deflate", "chunked".
*
* Placing values on multiple header lines is allowed under certain circumstances
* in RFC 2616 section 4.2, and in RFC 7230 section 3.2.2 quoted here:
*
A sender MUST NOT generate multiple header fields with the same field
name in a message unless either the entire field value for that
header field is defined as a comma-separated list [i.e., #(values)]
or the header field is a well-known exception (as noted below).
A recipient MAY combine multiple header fields with the same field
name into one "field-name: field-value" pair, without changing the
semantics of the message, by appending each subsequent field value to
the combined field value in order, separated by a comma. The order
in which header fields with the same field name are received is
therefore significant to the interpretation of the combined field
value; a proxy MUST NOT change the order of these field values when
forwarding a message.
*
* @param headerName the name of the header for which values will be retrieved
* @param httpMessage the HTTP message whose header values will be retrieved
* @return a list of single header values, or an empty list if the header was not present in the message or contained no values
*/
public static List getAllCommaSeparatedHeaderValues(AsciiString headerName, HttpMessage httpMessage) {
List allHeaders = httpMessage.headers().getAll(headerName);
if (allHeaders.isEmpty()) {
return Collections.emptyList();
}
ImmutableList.Builder headerValues = ImmutableList.builder();
for (String header : allHeaders) {
List commaSeparatedValues = splitCommaSeparatedHeaderValues(header);
headerValues.addAll(commaSeparatedValues);
}
return headerValues.build();
}
/**
* Duplicates the status line and headers of an HttpResponse object. Does not duplicate any content associated with that response.
*
* @param originalResponse HttpResponse to be duplicated
* @return a new HttpResponse with the same status line and headers
*/
public static HttpResponse duplicateHttpResponse(HttpResponse originalResponse) {
DefaultHttpResponse newResponse = new DefaultHttpResponse(originalResponse.protocolVersion(), originalResponse.status());
newResponse.headers().add(originalResponse.headers());
return newResponse;
}
/**
* Attempts to resolve the local machine's hostname.
*
* @return the local machine's hostname, or null if a hostname cannot be determined
*/
public static String getHostName() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (IOException | RuntimeException e) {
LOG.debug("Ignored exception", e);
} // An exception here must not stop the proxy. Android could throw a
// runtime exception, since it not allows network access in the main
// process.
LOG.info("Could not lookup localhost");
return null;
}
/**
* Determines if the specified header should be removed from the proxied response because it is a hop-by-hop header, as defined by the
* HTTP 1.1 spec in section 13.5.1. The comparison is case-insensitive, so "Connection" will be treated the same as "connection" or "CONNECTION".
* From section 13.5.1 :
*
* The following HTTP/1.1 headers are hop-by-hop headers:
* - Connection
* - Keep-Alive
* - Proxy-Authenticate
* - Proxy-Authorization
* - TE
* - Trailers [LittleProxy note: actual header name is Trailer]
* - Transfer-Encoding [LittleProxy note: this header is not normally removed when proxying, since the proxy does not re-chunk
* responses. The exception is when an HttpObjectAggregator is enabled, which aggregates chunked content and removes
* the 'Transfer-Encoding: chunked' header itself.]
* - Upgrade
*
* All other headers defined by HTTP/1.1 are end-to-end headers.
*
*
* @param headerName the header name
* @return true if this header is a hop-by-hop header and should be removed when proxying, otherwise false
*/
public static boolean shouldRemoveHopByHopHeader(String headerName) {
return SHOULD_NOT_PROXY_HOP_BY_HOP_HEADERS.contains(headerName);
}
/**
* Removes all headers that should not be forwarded. See RFC 2616 13.5.1
* End-to-end and Hop-by-hop Headers.
*
* @param headers
* The headers to modify
*/
public static void stripHopByHopHeaders(HttpHeaders headers) {
// Not explicitly documented, but remove is case-insensitive as HTTP header handling function should be
SHOULD_NOT_PROXY_HOP_BY_HOP_HEADERS.forEach(headers::remove);
}
/**
* Splits comma-separated header values into tokens. For example, if the value of the Connection header is "Transfer-Encoding, close",
* this method will return "Transfer-Encoding" and "close". This method strips trims any optional whitespace from
* the tokens. Unlike {@link #getAllCommaSeparatedHeaderValues(AsciiString, HttpMessage)}, this method only operates on
* a single header value, rather than all instances of the header in a message.
*
* @param headerValue the un-tokenized header value (must not be null)
* @return all tokens within the header value, or an empty list if there are no values
*/
public static List splitCommaSeparatedHeaderValues(String headerValue) {
return ImmutableList.copyOf(COMMA_SEPARATED_HEADER_VALUE_SPLITTER.split(headerValue));
}
/**
* Creates a new {@link FullHttpResponse} with the specified String as the body contents (encoded using UTF-8).
*
* @param httpVersion HTTP version of the response
* @param status HTTP status code
* @param body body to include in the FullHttpResponse; will be UTF-8 encoded
* @return new http response object
*/
public static FullHttpResponse createFullHttpResponse(HttpVersion httpVersion,
HttpResponseStatus status,
String body) {
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
ByteBuf content = Unpooled.copiedBuffer(bytes);
return createFullHttpResponse(httpVersion, status, "text/html; charset=utf-8", content, bytes.length);
}
/**
* Creates a new {@link FullHttpResponse} with no content
*
* @param httpVersion HTTP version of the response
* @param status HTTP status code
* @return new http response object
*/
public static FullHttpResponse createFullHttpResponse(HttpVersion httpVersion,
HttpResponseStatus status) {
return createFullHttpResponse(httpVersion, status, null, null, 0);
}
/**
* Creates a new {@link FullHttpResponse} with the specified body.
*
* @param httpVersion HTTP version of the response
* @param status HTTP status code
* @param contentType the Content-Type of the body
* @param body body to include in the FullHttpResponse; if null
* @param contentLength number of bytes to send in the Content-Length header; should equal the number of bytes in the ByteBuf
* @return new http response object
*/
public static FullHttpResponse createFullHttpResponse(HttpVersion httpVersion,
HttpResponseStatus status,
String contentType,
ByteBuf body,
int contentLength) {
DefaultFullHttpResponse response;
if (body != null) {
response = new DefaultFullHttpResponse(httpVersion, status, body);
response.headers().set(HttpHeaderNames.CONTENT_LENGTH, contentLength);
response.headers().set(HttpHeaderNames.CONTENT_TYPE, contentType);
} else {
response = new DefaultFullHttpResponse(httpVersion, status);
}
return response;
}
/**
* Given an HttpHeaders instance, removes 'sdch' from the 'Accept-Encoding'
* header list (if it exists) and returns the modified instance.
*
* Removes all occurrences of 'sdch' from the 'Accept-Encoding' header.
* @param headers The headers to modify.
*/
public static void removeSdchEncoding(HttpHeaders headers) {
List encodings = headers.getAll(HttpHeaderNames.ACCEPT_ENCODING);
headers.remove(HttpHeaderNames.ACCEPT_ENCODING);
for (String encoding : encodings) {
if (encoding != null) {
// The former regex should remove occurrences of 'sdch' while the
// latter regex should take care of the dangling comma case when
// 'sdch' was the first element in the list and there are other
// encodings.
encoding = encoding.replaceAll(",? *(sdch|SDCH)", "").replaceFirst("^ *, *", "");
if (StringUtils.isNotBlank(encoding)) {
headers.add(HttpHeaderNames.ACCEPT_ENCODING, encoding);
}
}
}
}
/**
* Tests whether the given response indicates that the connection is switching
* to the WebSocket protocol.
*
* @param response the response to check.
* @return true if switching to the WebSocket protocol; false otherwise;
*/
public static boolean isSwitchingToWebSocketProtocol(HttpResponse response) {
return (response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS)
&& response.headers().contains(HttpHeaderNames.CONNECTION, HttpHeaderNames.UPGRADE, true)
&& response.headers().contains(HttpHeaderNames.UPGRADE, "websocket", true);
}
/**
* Tests whether the given request indicates that the connection is switching
* to the WebSocket protocol.
*
* @param request the request to check.
* @return true if switching to the WebSocket protocol; false otherwise;
*/
public static boolean isSwitchingToWebSocketProtocol(HttpRequest request) {
return request.headers().contains(HttpHeaderNames.CONNECTION, HttpHeaderNames.UPGRADE, true)
&& request.headers().contains(HttpHeaderNames.UPGRADE, "websocket", true);
}
}