cn.wanghaomiao.seimi.def.DefaultRedisQueue Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of SeimiCrawler Show documentation
一个支持分布式的可以高效开发且可以高效运行的爬虫框架。设计思想上融合了spring与scrapy的优点。
There is a newer version: 2.1.4
package cn.wanghaomiao.seimi.def;

import cn.wanghaomiao.seimi.annotation.Queue;
import cn.wanghaomiao.seimi.core.SeimiQueue;
import cn.wanghaomiao.seimi.struct.Request;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;

import java.util.List;

/**
 * @author 汪浩淼 [[email protected]]
 * @since 2015/8/21.
 */
@Queue
public class DefaultRedisQueue implements SeimiQueue {
    /**
     * host,port,password,queueName需要开发者自行在配置文件中注入，如：
     
         
         
         classpath:config/seimi.properties
         
         
     
     再配一个properties文件
     redis.host=192.168.1.11
     redis.port=6379
     redis.password=
     */
    @Value("${redis.host:127.0.0.1}")
    private String host;
    @Value("${redis.port:6379}")
    private int port;
    @Value("${redis.password:}")
    private String password;
    private String quueName = "SEIMI_CRAWLER_QUEUE_";
    private JedisPool wpool = null;
    private Logger logger = LoggerFactory.getLogger(getClass());
    public void refresh(){
        if (wpool!=null){
            this.wpool.destroy();
            this.wpool = null;
        }
    }
    public JedisPool getWritePool() {
        if (wpool == null) {
            JedisPoolConfig config = new JedisPoolConfig();
            config.setMaxTotal(500);
            config.setMaxIdle(200);
            config.setMinIdle(100);
            config.setMaxWaitMillis(1000 * 100);
            logger.info("create redisPool host={},port={}",this.host,this.port);
            if (StringUtils.isNotBlank(password)){
                wpool = new JedisPool(config, this.host, this.port, 0, password);
            }else {
                wpool = new JedisPool(config, this.host, this.port, 0);
            }
        }
        return wpool;
    }

    public Jedis getWClient() {
        return getWritePool().getResource();
    }
    @Override
    public Request bPop(String crawlerName) {
        Jedis jedis = null;
        Request request = null;
        try {
            jedis = getWClient();
            List res = jedis.brpop(0,quueName+crawlerName);
            request = JSON.parseObject(res.get(1),Request.class);
        }catch (Exception e){
            logger.warn(e.getMessage());
            refresh();
        }finally {
            if (jedis!=null){
                jedis.close();
            }
        }
        return request;
    }

    @Override
    public boolean push(Request req) {
        Jedis jedis = null;
        boolean res = false;
        try {
            jedis = getWClient();
            res = jedis.lpush(quueName+req.getCrawlerName(),JSON.toJSONString(req))>0;
        }catch (Exception e){
            logger.warn(e.getMessage());
            refresh();
        }finally {
            if (jedis!=null){
                jedis.close();
            }
        }
        return res;
    }

    @Override
    public int len(String crawlerName) {
        long len = 0;
        Jedis jedis = null;
        try {
            jedis = getWClient();
            len = jedis.llen(quueName+crawlerName);
        }catch (Exception e){
            logger.warn(e.getMessage());
            refresh();
        }finally {
            if (jedis!=null){
                jedis.close();
            }
        }
        return (int) len;
    }

    public String getHost() {
        return host;
    }

    public void setHost(String host) {
        this.host = host;
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public String getPassword() {
        return password;
    }

    public void setPassword(String password) {
        this.password = password;
    }

    public String getQuueName() {
        return quueName;
    }

    public void setQuueName(String quueName) {
        this.quueName = quueName;
    }
}