org.apache.brooklyn.util.http.HttpTool Maven / Gradle / Ivy
Show all versions of brooklyn-utils-common Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.brooklyn.util.http;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSession;
import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.crypto.SslTrustUtils;
import org.apache.brooklyn.util.exceptions.Exceptions;
import org.apache.brooklyn.util.net.URLParamEncoder;
import org.apache.brooklyn.util.stream.Streams;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.Duration;
import org.apache.brooklyn.util.time.Time;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeSocketFactory;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Throwables;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
/**
* A utility tool for HTTP operations.
*/
public class HttpTool {
private static final Logger LOG = LoggerFactory.getLogger(HttpTool.class);
static final ExecutorService executor = Executors.newCachedThreadPool();
/**
* Connects to the given url and returns the connection.
* Caller should {@code connection.getInputStream().close()} the result of this
* (especially if they are making heavy use of this method).
*/
public static URLConnection connectToUrl(String u) throws Exception {
final URL url = new URL(u);
final AtomicReference exception = new AtomicReference();
// sometimes openConnection hangs, so run in background
Future f = executor.submit(new Callable() {
@Override
public URLConnection call() {
try {
HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
URLConnection connection = url.openConnection();
TrustingSslSocketFactory.configure(connection);
connection.connect();
connection.getContentLength(); // Make sure the connection is made.
return connection;
} catch (Exception e) {
exception.set(e);
LOG.debug("Error connecting to url "+url+" (propagating): "+e, e);
}
return null;
}
});
try {
URLConnection result = null;
try {
result = f.get(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw e;
} catch (Exception e) {
LOG.debug("Error connecting to url "+url+", probably timed out (rethrowing): "+e);
throw new IllegalStateException("Connect to URL not complete within 60 seconds, for url "+url+": "+e);
}
if (exception.get() != null) {
LOG.debug("Error connecting to url "+url+", thread caller of "+exception, new Throwable("source of rethrown error "+exception));
throw exception.get();
} else {
return result;
}
} finally {
f.cancel(true);
}
}
public static int getHttpStatusCode(String url) throws Exception {
URLConnection connection = connectToUrl(url);
long startTime = System.currentTimeMillis();
int status = ((HttpURLConnection) connection).getResponseCode();
// read fully if possible, then close everything, trying to prevent cached threads at server
consumeAndCloseQuietly((HttpURLConnection) connection);
if (LOG.isDebugEnabled())
LOG.debug("connection to {} ({}ms) gives {}", new Object[] { url, (System.currentTimeMillis()-startTime), status });
return status;
}
public static String getContent(String url) {
try {
return Streams.readFullyStringAndClose(SslTrustUtils.trustAll(new URL(url).openConnection()).getInputStream());
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
public static String getErrorContent(String url) {
try {
HttpURLConnection connection = (HttpURLConnection) connectToUrl(url);
long startTime = System.currentTimeMillis();
String err;
int status;
try {
InputStream errStream = connection.getErrorStream();
err = Streams.readFullyStringAndClose(errStream);
status = connection.getResponseCode();
} finally {
closeQuietly(connection);
}
if (LOG.isDebugEnabled())
LOG.debug("read of err {} ({}ms) complete; http code {}", new Object[] { url, Time.makeTimeStringRounded(System.currentTimeMillis() - startTime), status});
return err;
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
/**
* Consumes the input stream entirely and then cleanly closes the connection.
* Ignores all exceptions completely, not even logging them!
*
* Consuming the stream fully is useful for preventing idle TCP connections.
* @see Persistent Connections
*/
public static void consumeAndCloseQuietly(HttpURLConnection connection) {
try { Streams.readFully(connection.getInputStream()); } catch (Exception e) {}
closeQuietly(connection);
}
/**
* Closes all streams of the connection, and disconnects it. Ignores all exceptions completely,
* not even logging them!
*/
public static void closeQuietly(HttpURLConnection connection) {
try { connection.disconnect(); } catch (Exception e) {}
try { connection.getInputStream().close(); } catch (Exception e) {}
try { connection.getOutputStream().close(); } catch (Exception e) {}
try { connection.getErrorStream().close(); } catch (Exception e) {}
}
/** Apache HTTP commons utility for trusting all.
*
* For generic java HTTP usage, see {@link SslTrustUtils#trustAll(java.net.URLConnection)}
* and static constants in the same class. */
public static class TrustAllStrategy implements TrustStrategy {
@Override
public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
return true;
}
}
public static HttpClientBuilder httpClientBuilder() {
return new HttpClientBuilder();
}
// TODO deprecate this and use the new Apache Commons HttpClientBuilder instead
@SuppressWarnings("deprecation")
public static class HttpClientBuilder {
private ClientConnectionManager clientConnectionManager;
private HttpParams httpParams;
private URI uri;
private Integer port;
private Credentials credentials;
private boolean laxRedirect;
private Boolean https;
private SchemeSocketFactory socketFactory;
private ConnectionReuseStrategy reuseStrategy;
private boolean trustAll;
private boolean trustSelfSigned;
public static HttpClientBuilder fromBuilder(HttpClientBuilder other) {
HttpClientBuilder result = httpClientBuilder();
result.clientConnectionManager = other.clientConnectionManager;
result.httpParams = other.httpParams;
result.uri = other.uri;
result.port = other.port;
result.credentials = other.credentials;
result.laxRedirect = other.laxRedirect;
result.https = other.https;
result.socketFactory = other.socketFactory;
result.reuseStrategy = other.reuseStrategy;
result.trustAll = other.trustAll;
result.trustSelfSigned = other.trustSelfSigned;
return result;
}
public HttpClientBuilder clientConnectionManager(ClientConnectionManager val) {
this.clientConnectionManager = checkNotNull(val, "clientConnectionManager");
return this;
}
public HttpClientBuilder httpParams(HttpParams val) {
checkState(httpParams == null, "Must not call httpParams multiple times, or after other methods like connectionTimeout");
this.httpParams = checkNotNull(val, "httpParams");
return this;
}
public HttpClientBuilder connectionTimeout(Duration val) {
if (httpParams == null) httpParams = new BasicHttpParams();
long millis = checkNotNull(val, "connectionTimeout").toMilliseconds();
if (millis > Integer.MAX_VALUE) throw new IllegalStateException("HttpClient only accepts upto max-int millis for connectionTimeout, but given "+val);
HttpConnectionParams.setConnectionTimeout(httpParams, (int) millis);
return this;
}
public HttpClientBuilder socketTimeout(Duration val) {
if (httpParams == null) httpParams = new BasicHttpParams();
long millis = checkNotNull(val, "socketTimeout").toMilliseconds();
if (millis > Integer.MAX_VALUE) throw new IllegalStateException("HttpClient only accepts upto max-int millis for socketTimeout, but given "+val);
HttpConnectionParams.setSoTimeout(httpParams, (int) millis);
return this;
}
public HttpClientBuilder reuseStrategy(ConnectionReuseStrategy val) {
this.reuseStrategy = checkNotNull(val, "reuseStrategy");
return this;
}
public HttpClientBuilder uri(String val) {
return uri(URI.create(checkNotNull(val, "uri")));
}
public HttpClientBuilder uri(URI val) {
this.uri = checkNotNull(val, "uri");
if (https == null) https = ("https".equalsIgnoreCase(uri.getScheme()));
return this;
}
public HttpClientBuilder port(int val) {
this.port = val;
return this;
}
public HttpClientBuilder credentials(Credentials val) {
this.credentials = checkNotNull(val, "credentials");
return this;
}
public HttpClientBuilder credential(Optional val) {
if (val.isPresent()) credentials = val.get();
return this;
}
/** similar to curl --post301 -L` */
public HttpClientBuilder laxRedirect(boolean val) {
this.laxRedirect = val;
return this;
}
public HttpClientBuilder https(boolean val) {
this.https = val;
return this;
}
public HttpClientBuilder socketFactory(SchemeSocketFactory val) {
this.socketFactory = checkNotNull(val, "socketFactory");
return this;
}
public HttpClientBuilder trustAll() {
return trustAll(true);
}
public HttpClientBuilder trustSelfSigned() {
return trustSelfSigned(true);
}
public HttpClientBuilder trustAll(boolean val) {
trustAll = val;
return this;
}
public HttpClientBuilder trustSelfSigned(boolean val) {
this.trustSelfSigned = val;
return this;
}
public HttpClient build() {
final DefaultHttpClient httpClient = new DefaultHttpClient(clientConnectionManager);
httpClient.setParams(httpParams);
// support redirects for POST (similar to `curl --post301 -L`)
// http://stackoverflow.com/questions/3658721/httpclient-4-error-302-how-to-redirect
if (laxRedirect) {
httpClient.setRedirectStrategy(new LaxRedirectStrategy());
}
if (reuseStrategy != null) {
httpClient.setReuseStrategy(reuseStrategy);
}
if (https == Boolean.TRUE || (uri!=null && uri.toString().startsWith("https:"))) {
try {
if (port == null) {
port = (uri != null && uri.getPort() >= 0) ? uri.getPort() : 443;
}
if (socketFactory == null) {
if (trustAll) {
TrustStrategy trustStrategy = new TrustAllStrategy();
X509HostnameVerifier hostnameVerifier = SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
socketFactory = new SSLSocketFactory(trustStrategy, hostnameVerifier);
} else if (trustSelfSigned) {
TrustStrategy trustStrategy = new TrustSelfSignedStrategy();
X509HostnameVerifier hostnameVerifier = SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
socketFactory = new SSLSocketFactory(trustStrategy, hostnameVerifier);
} else {
// Using default https scheme: based on default java truststore, which is pretty strict!
}
}
if (socketFactory != null) {
Scheme sch = new Scheme("https", port, socketFactory);
httpClient.getConnectionManager().getSchemeRegistry().register(sch);
}
} catch (Exception e) {
LOG.warn("Error setting trust for uri {}", uri);
throw Exceptions.propagate(e);
}
}
// Set credentials
if (uri != null && credentials != null) {
String hostname = uri.getHost();
int port = uri.getPort();
httpClient.getCredentialsProvider().setCredentials(new AuthScope(hostname, port), credentials);
}
if (uri==null && credentials!=null) {
LOG.warn("credentials have no effect in builder unless URI for host is specified");
}
return httpClient;
}
}
public static class HttpRequestBuilder {
private Class requestClass;
private Map headers;
private URI uri;
private HttpEntity body;
public HttpRequestBuilder(Class requestClass) {
this.requestClass = requestClass;
this.headers = MutableMap.of();
}
public HttpRequestBuilder uri(URI uri) {
this.uri = uri;
return this;
}
public HttpRequestBuilder headers(Map headers) {
if (headers != null) {
this.headers.putAll(headers);
}
return this;
}
public HttpRequestBuilder headers(Multimap headers) {
if (headers != null) {
for (Map.Entry entry : headers.entries()) {
this.headers.put(entry.getKey(), entry.getValue());
}
}
return this;
}
public HttpRequestBuilder body(byte[] body) {
if (body != null) {
this.body = new ByteArrayEntity(body);
}
return this;
}
public HttpRequestBuilder body(String body) {
if (body != null) {
this.body(body.getBytes(Charset.forName("UTF-8")));
}
return this;
}
public HttpRequestBuilder body(Map body) {
if (body != null) {
Collection httpParams = new ArrayList(body.size());
for (Entry param : body.entrySet()) {
httpParams.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
this.body = new UrlEncodedFormEntity(httpParams);
}
return this;
}
public R build() {
try {
R request = this.requestClass.newInstance();
request.setURI(this.uri);
for (Map.Entry entry : this.headers.entrySet()) {
request.addHeader(entry.getKey(), entry.getValue());
}
if (this.body != null) {
if (request instanceof HttpPost) {
((HttpPost) request).setEntity(this.body);
} else if (request instanceof HttpPut) {
((HttpPut) request).setEntity(this.body);
} else {
throw new Exception(this.requestClass.getSimpleName() + " does not support a request body");
}
}
return request;
} catch (Exception e) {
LOG.warn("Cannot create the HTTP request for uri {}", this.uri);
throw Exceptions.propagate(e);
}
}
}
public static class HttpGetBuilder extends HttpRequestBuilder {
public HttpGetBuilder(URI uri) {
super(HttpGet.class);
this.uri(uri);
}
}
public static class HttpHeadBuilder extends HttpRequestBuilder {
public HttpHeadBuilder(URI uri) {
super(HttpHead.class);
this.uri(uri);
}
}
public static class HttpDeleteBuilder extends HttpRequestBuilder {
public HttpDeleteBuilder(URI uri) {
super(HttpDelete.class);
this.uri(uri);
}
}
public static class HttpPostBuilder extends HttpRequestBuilder {
public HttpPostBuilder(URI uri) {
super(HttpPost.class);
this.uri(uri);
}
}
public static class HttpPutBuilder extends HttpRequestBuilder {
public HttpPutBuilder(URI uri) {
super(HttpPut.class);
this.uri(uri);
}
}
public static HttpToolResponse httpGet(HttpClient httpClient, URI uri, Map headers) {
HttpGet req = new HttpGetBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpGet(HttpClient httpClient, URI uri, Multimap headers) {
HttpGet req = new HttpGetBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpPost(HttpClient httpClient, URI uri, Multimap headers, byte[] body) {
HttpPost req = new HttpPostBuilder(uri).headers(headers).body(body).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpPost(HttpClient httpClient, URI uri, Map headers, byte[] body) {
HttpPost req = new HttpPostBuilder(uri).headers(headers).body(body).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpPut(HttpClient httpClient, URI uri, Multimap headers, byte[] body) {
HttpPut req = new HttpPutBuilder(uri).headers(headers).body(body).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpPut(HttpClient httpClient, URI uri, Map headers, byte[] body) {
HttpPut req = new HttpPutBuilder(uri).headers(headers).body(body).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpPost(HttpClient httpClient, URI uri, Map headers, Map params) {
HttpPost req = new HttpPostBuilder(uri).body(params).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpDelete(HttpClient httpClient, URI uri, Multimap headers) {
HttpDelete req = new HttpDeleteBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpDelete(HttpClient httpClient, URI uri, Map headers) {
HttpDelete req = new HttpDeleteBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpHead(HttpClient httpClient, URI uri, Multimap headers) {
HttpHead req = new HttpHeadBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse httpHead(HttpClient httpClient, URI uri, Map headers) {
HttpHead req = new HttpHeadBuilder(uri).headers(headers).build();
return execAndConsume(httpClient, req);
}
public static HttpToolResponse execAndConsume(HttpClient httpClient, HttpUriRequest req) {
long startTime = System.currentTimeMillis();
try {
HttpResponse httpResponse = httpClient.execute(req);
try {
return new HttpToolResponse(httpResponse, startTime);
} finally {
EntityUtils.consume(httpResponse.getEntity());
}
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
public static boolean isStatusCodeHealthy(int code) { return (code>=200 && code<=299); }
public static String toBasicAuthorizationValue(UsernamePasswordCredentials credentials) {
return "Basic "+Base64.encodeBase64String( (credentials.getUserName()+":"+credentials.getPassword()).getBytes() );
}
public static String encodeUrlParams(Map data) {
if (data==null) return "";
Iterable args = Iterables.transform(data.entrySet(),
new Function,String>() {
@Override public String apply(Map.Entry entry) {
Object k = entry.getKey();
Object v = entry.getValue();
return URLParamEncoder.encode(Strings.toString(k)) + (v != null ? "=" + URLParamEncoder.encode(Strings.toString(v)) : "");
}
});
return Joiner.on("&").join(args);
}
}