us.codecraft.webmagic.recover.RecoverSample Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of webmagic-samples Show documentation

There is a newer version: 1.0.2

package us.codecraft.webmagic.recover;


import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.samples.SinaBlogProcessor;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;

/**
 * @author [email protected] 

 */
public class RecoverSample {

    public static void main(String[] args) {
        String storage = "queue";
        String duplicate = "duplicate";
        Spider spider = new Spider(new SinaBlogProcessor());
        DuplicateRemover remover = new DuplicateStorageRemover(duplicate);
        spider.setScheduler(new MmapQueueScheduler(remover, storage));
        spider.addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
                .run();
    }
}