pipiz.scheduler.InMemoryScheduler Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of Buggy Show documentation

Web crawler - easy for use

The newest version!

package pipiz.scheduler;

import lombok.extern.slf4j.Slf4j;

import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;

@Slf4j
public class InMemoryScheduler implements IScheduler {
    private Queue urls = new ConcurrentLinkedQueue<>();
    private List processed = new LinkedList<>();
    private long interval;

    public synchronized void put(String url) {
        if (processed.contains(url) || urls.contains(url)) {
            return;
        }
        urls.add(url);
    }

    @Override
    public synchronized void done(String url) {
        if (!processed.contains(url)) {
            processed.add(url);
        }
    }

    @Override
    public void setInterval(long millis) {
        if (millis > 0) {
            this.interval = millis;
        }
    }

    public String get() {
        String url;
        int retry = 10;
        while ((url = urls.poll()) == null && retry > 0) {
            sleep(6000);
            log.debug("wait {} millis to retry get url, remaining {} times retry", 6000, retry);
            retry--;
        }
        sleep(interval);
        return url;
    }

    private void sleep(long millis) {
        if (millis > 0) {
            try {
                Thread.sleep(millis);
            } catch (InterruptedException e) {
                log.error("", e);
            }
        }
    }
}