mtons.spider.config.Config Maven / Gradle / Ivy

Go to download
package mtons.spider.config;

import mtons.spider.http.Request;
import mtons.spider.http.Style;

import java.util.ArrayList;
import java.util.List;

/**
 * 站点配置信息, 每个Spider对应一个Config
 *
 * Created by langhsu on 16/6/6.
 */
public class Config {
    /**
     * 网站域名
     */
    private String domain; // 域名

    /**
     * 最大遍历深度, 默认为0 即不限制
     */
    private int degree = 0;

    /**
     * 是否开启Gzip压缩
     */
    private boolean useGzip;

    /**
     * 任务线程数
     */
    private int threads = 2; // 线程数

    /**
     * 连接超时时间, 单位为毫秒
     */
    private int timeout = 10 * 1000;

    /**
     * 每次请求的间隔时间
     */
    private int interval = 2 * 1000;

    /**
     * 重试次数
     */
    private int maxRetries = 5; // 请求失败, 重试次数

    /**
     * 页面请求
     */
    private List requests = new ArrayList<>();

    public String getDomain() {
        return domain;
    }

    public void setDomain(String domain) {
        this.domain = domain;
    }

    public List getRequests() {
        return requests;
    }

    public void setRequests(List requests) {
        this.requests = requests;
    }

    public Config addRequest(Request request) {
        requests.add(request);
        return this;
    }

    public int getDegree() {
        return degree;
    }

    public void setDegree(int degree) {
        this.degree = degree;
    }

    public boolean isUseGzip() {
        return useGzip;
    }

    public void setUseGzip(boolean useGzip) {
        this.useGzip = useGzip;
    }

    public int getThreads() {
        return threads;
    }

    public void setThreads(int threads) {
        this.threads = threads;
    }

    public int getTimeout() {
        return timeout;
    }

    public void setTimeout(int timeout) {
        this.timeout = timeout;
    }

    public int getInterval() {
        return interval;
    }

    public void setInterval(int interval) {
        this.interval = interval;
    }

    public int getMaxRetries() {
        return maxRetries;
    }

    public void setMaxRetries(int maxRetries) {
        this.maxRetries = maxRetries;
    }

    public Config addRequest(Style pageType, String url) {
        requests.add(new Request(url, pageType));
        return this;
    }
}