org.metaeffekt.artifact.resolver.download.WebAccess Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-resolver Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.metaeffekt.artifact.resolver.download;

import lombok.Getter;
import lombok.NonNull;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.metaeffekt.core.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Optional;
import java.util.function.Consumer;

public class WebAccess {

    private final static Logger LOG = LoggerFactory.getLogger(WebAccess.class);

    private final HttpHost proxy;

    private final CredentialsProvider credentialsProvider;

    /**
     * Parameter for construction and holder of central configuration data
     */
    @Getter
    private final ProxyConfig proxyConfig;

    /**
     * Set up a proxy in between the host to access (host, port, scheme, username, password).

     * Username and password are optional, only used if authentication is requested by the proxy.
     *
     * @param proxyConfig The proxy configuration.
     */
    public WebAccess(ProxyConfig proxyConfig) {
        LOG.debug("Setting up proxy credentials for downloader.");

        if (proxyConfig.isValid()) {
            LOG.info("Using proxy: {}://{}:{}.", proxyConfig.getScheme(), proxyConfig.getHost(), proxyConfig.getPort());

            // remember proxyConfig
            this.proxyConfig = proxyConfig;
            this.proxy = new HttpHost(proxyConfig.getHost(), proxyConfig.getPort(), proxyConfig.getScheme());

            if (!StringUtils.isBlank(proxyConfig.getUsername())) {
                LOG.info("Using proxy user: [{}].", proxyConfig.getUsername());
                this.credentialsProvider = new BasicCredentialsProvider();
                this.credentialsProvider.setCredentials(AuthScope.ANY,
                        new UsernamePasswordCredentials(proxyConfig.getUsername(), proxyConfig.getPassword()));
            } else {
                this.credentialsProvider = null;
            }
        } else {
            LOG.info("No valid proxy configured. Operating without proxy.");
            this.proxyConfig = null;
            this.proxy = null;
            this.credentialsProvider = null;
        }
    }

    public RequestConfig getProxyRequestConfig() {
        if (proxy != null) {
            return RequestConfig.custom().setProxy(proxy).build();
        } else {
            return RequestConfig.custom().build();
        }
    }

    public static class WebSession implements Closeable {

        private final CloseableHttpClient httpClient;

        private final WebAccess webAccess;

        public WebSession(WebAccess webAccess) {
            this.webAccess = webAccess;
            this.httpClient = createHttpClient();
        }

        @Override
        public void close() throws IOException {
            httpClient.close();
        }

        private CloseableHttpClient createHttpClient() {
            final HttpClientBuilder httpClient = HttpClients.custom();
            final int timeOutInSeconds = 10;
            final RequestConfig requestConfig = RequestConfig.custom()
                    // max time to establish a connection with remote host/server
                    .setConnectTimeout(timeOutInSeconds * 1000)
                    // time to wait for getting a connection from the connection manager/pool
                    .setConnectionRequestTimeout(timeOutInSeconds * 1000)
                    // max time gap between two consecutive data packets while transferring data from server to client
                    .setSocketTimeout(timeOutInSeconds * 1000)
                    // disable content compression. somehow this is not enough so i disable it again for httpClient.
                    .setContentCompressionEnabled(false)
                    .build();
            httpClient.setDefaultRequestConfig(requestConfig);
            // disable automagic decompression. enabling this will cause unexpected decompression and fail checksums.
            httpClient.disableContentCompression();

            if (webAccess.credentialsProvider != null) {
                httpClient.setDefaultCredentialsProvider(webAccess.credentialsProvider);
            }

            return httpClient.build();
        }

        @NonNull
        public Optional downloadToUtf8String(String uri, Consumer errorConsumer) {
            final HttpGet get = new HttpGet(uri);

            get.setConfig(this.webAccess.getProxyRequestConfig());

            final HttpClientContext context = new HttpClientContext();
            try (final CloseableHttpResponse response = httpClient.execute(get, context)) {
                if (response.getStatusLine().getStatusCode() == 200) {
                    try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
                        IOUtils.copyLarge(response.getEntity().getContent(), bos);
                        return Optional.of(new String(bos.toByteArray(), StandardCharsets.UTF_8));
                    }
                } else {
                    LOG.info("Cannot download URI [{}]. Site returned: {}", uri, response.getStatusLine());
                    errorConsumer.accept(response);
                    return Optional.empty();
                }
            } catch(Exception e) {
                LOG.info("Cannot download uri [{}]. Request timed out.", uri, e);
                return Optional.empty();
            }
        }

        public Optional downloadFile(String uri, File file, Consumer errorConsumer) {
            final HttpGet get = new HttpGet(uri);

            get.setConfig(this.webAccess.getProxyRequestConfig());

            final HttpClientContext context = new HttpClientContext();
            try (final CloseableHttpResponse response = httpClient.execute(get, context)) {
                if (response.getStatusLine().getStatusCode() == 200) {
                    long downloadStartTime = System.nanoTime();

                    FileUtils.copyInputStreamToFile(response.getEntity().getContent(), file);

                    long downloadEndTime = System.nanoTime();
                    double totalTimeSeconds = (downloadEndTime - downloadStartTime) / 1e9;
                    long fileSizeKB = Files.size(file.toPath()) / (1024);
                    LOG.debug("Finished download stats: uri [{}] to file [{}] of size [{}]KB in [{}]s at approx [{}]KB/s.",
                            uri,
                            file,
                            fileSizeKB,
                            String.format("%.3f", totalTimeSeconds),
                            String.format("%.2f", (((double) fileSizeKB) / totalTimeSeconds))
                    );

                    return Optional.of(file);
                } else {
                    LOG.info("Cannot download URI [{}]. Site returned: {}", uri, response.getStatusLine());
                    errorConsumer.accept(response);
                    return Optional.empty();
                }
            } catch(Exception e) {
                LOG.info("Cannot download uri [{}]. Request timed out.", uri, e);
                return Optional.empty();
            }
        }
    }

    public WebSession createSession() {
        return new WebSession(this);
    }
}