pipiz.downloader.HttpClientDownloader Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of Buggy Show documentation

Web crawler - easy for use

The newest version!

package pipiz.downloader;

import lombok.extern.slf4j.Slf4j;
import org.apache.http.Header;
import org.apache.http.client.fluent.Request;
import pipiz.entity.HttpResponse;
import pipiz.util.IoUtils;

import java.net.URI;

import static pipiz.util.UrlUtils.convertToUri;

@Slf4j
public class HttpClientDownloader implements IDownloader {

    @Override
    public HttpResponse download(String s) throws Exception {
        URI uri = convertToUri(s);
        if (uri == null) {
            return null;
        }

        try {
            org.apache.http.HttpResponse response = Request.Get(uri)
                .connectTimeout(30000).socketTimeout(30000)
                .execute()
                .returnResponse();

            String responseContent = IoUtils.readString(response.getEntity().getContent());
            int statusCode = response.getStatusLine().getStatusCode();
            Header[] headers = response.getAllHeaders();

            return new HttpResponse(responseContent, headers, statusCode);
        } catch (Exception e) {
            log.error("", e);
            throw e;
        }
    }
}