com.metaeffekt.artifact.analysis.utils.HTTPClientDownloader Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.utils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/**
* Use this class to download files with (or without) a proxy via HTTP or HTTPS.
*/
@Deprecated // use WebAccess
public class HTTPClientDownloader {
private final Logger LOG = LoggerFactory.getLogger(getClass());
private final HttpHost proxy;
private final CredentialsProvider credentialsProvider;
/**
* Initialize with setting up a proxy (host, port, scheme, username, password).
* Username and password are optional, if the proxy requires authentication it will come back and
* ask for it, otherwise they will remain unused.
*
* @param scheme The proxy scheme
* @param host The proxy host
* @param port The proxy port
* @param username Proxy user (if authentication is required)
* @param password Proxy password (if authentication is required)
*/
public HTTPClientDownloader(String scheme, String host, int port, String username, String password) {
this.proxy = new HttpHost(host, port, scheme);
this.credentialsProvider = new BasicCredentialsProvider();
this.credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
}
/**
* Initialize without setting up a proxy.
*/
public HTTPClientDownloader() {
this.proxy = null;
this.credentialsProvider = null;
}
/**
* @return A closeable HTTP Client with the given credentials.
*/
private CloseableHttpClient getHttpClient() {
HttpClientBuilder httpClient = HttpClients.custom();
httpClient.setDefaultCredentialsProvider(credentialsProvider);
return httpClient.build();
}
/**
* Download a file from a given URL to a destination file. If specified in the constructor, it will use a proxy to
* download the file.
*
* @param url The url to download the file from.
* @param dstFile The destination file where the downloaded content should go to.
* @param attempts The amount of times to retry fetching the response if failed.
*/
public void download(URL url, File dstFile, int attempts) {
attempts = Math.max(1, attempts);
RuntimeException caughtException = null;
for (int i = 0; i < attempts; i++) {
try {
download(url, dstFile);
return;
} catch (RuntimeException e) {
LOG.warn("Attempt {}/{}: Retrying fetching [{}]: {}", i + 1, attempts, url, e.getMessage());
caughtException = e;
}
}
LOG.error("Attempt {}/{}: Unable to fetch [{}]: {}", attempts, attempts, url, caughtException.getMessage());
throw caughtException;
}
/**
* Download a file from a given URL to a destination file. If specified in the constructor, it will use a proxy to
* download the file.
*
* @param url The url to download the file from.
* @param dstFile The destination file where the downloaded content should go to.
*/
public void download(URL url, File dstFile) {
try {
try (CloseableHttpClient httpclient = getHttpClient()) {
// create a target from the given URL
HttpHost target = new HttpHost(url.getHost(), protocolToPort(url.getProtocol()), url.getProtocol());
// check if a proxy is required. If not, the default RequestConfig can be used
RequestConfig config;
if (proxy != null)
config = RequestConfig.custom()
.setProxy(proxy)
.build();
else
config = RequestConfig.custom()
.build();
// create the request and pass the RequestConfig
HttpGet request = new HttpGet(url.getPath());
request.setConfig(config);
if (proxy == null)
LOG.info("Executing request [{}] to [{}]", request.getRequestLine(), target);
else
LOG.info("Executing request [{}] to [{}] via proxy [{}]", request.getRequestLine(), target, this.proxy);
// execute the request and fetch the response
CloseableHttpResponse response = httpclient.execute(target, request);
HttpEntity entity = response.getEntity();
if (entity != null) {
if (dstFile == null) {
LOG.error("Download destination file for request [{}] is [null]", url);
throw new RuntimeException("Unable to download file from [" + url + "] since no local destination file was specified.");
}
if (!dstFile.getAbsoluteFile().getParentFile().exists() && !dstFile.getAbsoluteFile().getParentFile().mkdirs()) {
LOG.error("Unable to create parent directory to file [{}]", dstFile.getAbsoluteFile());
throw new RuntimeException("Unable to download file from [" + url + "] to [" + dstFile.getAbsolutePath() + "] since the parent directory does not exist and could not be created.");
}
try (FileOutputStream outstream = new FileOutputStream(dstFile)) {
entity.writeTo(outstream);
}
} else {
LOG.error("Response from [{}] is [null]", url);
throw new RuntimeException("Response from [" + url + "] is [null].");
}
// check if file was downloaded successfully or if it only seemed to have worked
if (!dstFile.exists()) {
LOG.warn("File was downloaded but destination file [{}] does not exist", dstFile.getAbsolutePath());
LOG.error("Unable to download file from [{}] to [{}]", url, dstFile);
throw new RuntimeException("Unable to download file from [" + url + "] to [" + dstFile.getAbsolutePath() + "]");
}
}
} catch (IOException e) {
LOG.error("An error occurred while getting the response from [{}]: {}", url, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
/**
* Returns the plain text response from a given URL.
*
* @param url The URL to fetch the response from.
* @param attempts The amount of times to retry fetching the response if failed.
* @return The plain text response from the URL.
*/
public List getResponseStringFromUrl(URL url, int attempts) {
attempts = Math.max(1, attempts);
RuntimeException caughtException = null;
for (int i = 0; i < attempts; i++) {
try {
return getResponseStringFromUrl(url);
} catch (RuntimeException e) {
LOG.warn("Attempt {}/{}: Retrying fetching [{}]: {}", i + 1, attempts, url, e.getMessage());
caughtException = e;
}
}
LOG.error("Attempt {}/{}: Unable to fetch [{}]: {}", attempts, attempts, url, caughtException.getMessage());
throw caughtException;
}
/**
* Returns the plain text response from a given URL.
*
* @param url The URL to fetch the response from.
* @return The plain text response from the URL.
*/
public List getResponseStringFromUrl(URL url) {
try {
try (CloseableHttpClient httpclient = getHttpClient()) {
// create a target from the given URL
HttpHost target = new HttpHost(url.getHost(), protocolToPort(url.getProtocol()), url.getProtocol());
// check if a proxy is required. If not, the default RequestConfig can be used
RequestConfig config;
if (proxy != null)
config = RequestConfig.custom()
.setProxy(proxy)
.build();
else
config = RequestConfig.custom()
.build();
// create the request and pass the RequestConfig
HttpGet request = new HttpGet(url.getPath());
request.setConfig(config);
if (proxy == null)
LOG.info("Executing request [{}] to [{}]", request.getRequestLine(), target);
else
LOG.info("Executing request [{}] to [{}] via proxy [{}]", request.getRequestLine(), target, this.proxy);
// execute the request and fetch the response
CloseableHttpResponse response = httpclient.execute(target, request);
HttpEntity entity = response.getEntity();
InputStream contentInputStream = entity.getContent();
// read the temp downloaded file to a list
List result = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(contentInputStream));
String line;
while ((line = reader.readLine()) != null) {
result.add(line);
}
// warn if response contains the string 'Access Denied', since this is most likely unwanted
if (result.stream().anyMatch(res -> res.contains("Access Denied")))
LOG.warn("Response from url [{}] contains string 'Access Denied'", url);
return result;
}
} catch (Exception e) {
LOG.error("An error occurred while getting the response from [{}]: {}", url, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
/**
* @param url The URL to fetch the file size from.
* @return The file size of a file specified by a URL.
*/
public long getFileSizeFromUrl(URL url) {
try {
try (CloseableHttpClient httpclient = getHttpClient()) {
// create a target from the given URL
HttpHost target = new HttpHost(url.getHost(), protocolToPort(url.getProtocol()), url.getProtocol());
// check if a proxy is required. If not, the default RequestConfig can be used
RequestConfig config;
if (proxy != null)
config = RequestConfig.custom()
.setProxy(proxy)
.build();
else
config = RequestConfig.custom()
.build();
// create the request and pass the RequestConfig
HttpGet request = new HttpGet(url.getPath());
request.setConfig(config);
if (proxy == null)
LOG.info("Executing request [{}] to [{}]", request.getRequestLine(), target);
else
LOG.info("Executing request [{}] to [{}] via proxy [{}]", request.getRequestLine(), target, this.proxy);
// execute the request and get the content length from the response
CloseableHttpResponse response = httpclient.execute(target, request);
if (response.getLastHeader("Content-Length") == null) {
LOG.warn("Response Header did not contain [Content-Length], getting result content and extracting byte length");
// execute the request and fetch the response
HttpEntity entity = response.getEntity();
InputStream contentInputStream = entity.getContent();
// read the temp downloaded file to a list
List result = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(contentInputStream));
String line;
while ((line = reader.readLine()) != null) {
result.add(line);
}
// warn if response contains the string 'Access Denied', since this is most likely unwanted
if (result.stream().anyMatch(res -> res.contains("Access Denied")))
LOG.warn("Response from url [{}] contains string 'Access Denied'. This might or might not be a bad sign.", url);
return result.stream().map(String::getBytes).mapToInt(v -> v.length).sum();
}
String size = response.getLastHeader("Content-Length").getValue();
return Long.parseLong(size);
}
} catch (IOException e) {
LOG.error("An error occurred while getting the response from [{}]: {}", url, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
public final static int DEFAULT_PORT = 80;
public final static int FTP_PORT = 20;
public final static int HTTPS_PORT = 443;
public final static int HTTP_PORT = 80;
/**
* Converts a protocol to its according port.
* Such as: http, https
*
* @param protocol The protocol to translate to a port
* @return The matching port
*/
private int protocolToPort(String protocol) {
switch (protocol.toLowerCase()) {
case "ftp":
return FTP_PORT;
case "https":
return HTTPS_PORT;
case "http":
return HTTP_PORT;
default:
LOG.warn("Unknown protocol [{}], returning default port [{}]", protocol, DEFAULT_PORT);
return DEFAULT_PORT;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy