me.zhyd.hunter.config.platform.BasePlatform Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of blog-hunter Show documentation

博客猎手，基于webMagic的博客爬取工具，支持慕课、csdn、iteye、cnblogs、掘金和V2EX等各大主流博客平台。博客千万篇，版权第一条。狩猎不规范，亲人两行泪。

The newest version!

package me.zhyd.hunter.config.platform;

import com.alibaba.fastjson.JSONObject;
import me.zhyd.hunter.config.HunterConfig;
import me.zhyd.hunter.config.HunterConfigTemplate;
import me.zhyd.hunter.util.PlatformUtil;

import java.util.Collections;
import java.util.Map;
import java.util.Set;

/**
 * @author yadong.zhang (yadong.zhang0415(a)gmail.com)
 * @version 1.0
 * @since 1.8
 */
public abstract class BasePlatform implements InnerPlatform {
    String platform;

    public BasePlatform(String platform) {
        this.platform = platform;
    }

    protected final HunterConfig get(String url) {

        String host = PlatformUtil.getHost(url);
        String domain = PlatformUtil.getDomain(url);

        String platformConfig = HunterConfigTemplate.getConfig(platform);
        JSONObject platformObj = JSONObject.parseObject(platformConfig);
        String br = "\r\n", header = null;
        Set> entries = platformObj.entrySet();
        for (Map.Entry entry : entries) {
            if ("header".equals(entry.getKey())) {
                header = "Host=" + host + br + "Referer=" + domain;
                entry.setValue(header);
            } else if ("entryUrls".equals(entry.getKey())) {
                entry.setValue(Collections.singletonList(url));
            } else {
                if (platform.equals(Platform.ITEYE.getPlatform()) && "domain".equals(entry.getKey())) {
                    entry.setValue(host);
                }
            }
        }
        HunterConfig config = JSONObject.toJavaObject(platformObj, HunterConfig.class);
        config.setSingle(true);
        return config;
    }
}