us.codecraft.webmagic.SimpleHttpClient Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of webmagic-extension Show documentation

The newest version!

package us.codecraft.webmagic;

import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.model.PageMapper;
import us.codecraft.webmagic.proxy.ProxyProvider;

/**
 * @author [email protected]
 *         Date: 2017/5/27
 * @since 0.7.0
 */
public class SimpleHttpClient {

    private final HttpClientDownloader httpClientDownloader;

    private final Site site;

    public SimpleHttpClient() {
        this(Site.me());
    }

    public SimpleHttpClient(Site site) {
        this.site = site;
        this.httpClientDownloader = new HttpClientDownloader();
    }

    public void setProxyProvider(ProxyProvider proxyProvider){
        this.httpClientDownloader.setProxyProvider(proxyProvider);
    }

    public  T get(String url, Class clazz) {
        return get(new Request(url), clazz);
    }

    public  T get(Request request, Class clazz) {
        Page page = httpClientDownloader.download(request, site.toTask());
        if (!page.isDownloadSuccess()) {
            return null;
        }
        return new PageMapper(clazz).get(page);
    }

    public Page get(String url) {
        return httpClientDownloader.download(new Request(url), site.toTask());
    }

    public Page get(Request request) {
        return httpClientDownloader.download(request, site.toTask());
    }

}