com.saucesubfresh.starter.crawler.pipeline.AbstractFormatPipeline Maven / Gradle / Ivy

Go to download
package com.saucesubfresh.starter.crawler.pipeline;

import com.saucesubfresh.starter.crawler.domain.SpiderRequest;
import com.saucesubfresh.starter.crawler.domain.SpiderResponse;
import com.saucesubfresh.starter.crawler.generator.KeyGenerator;
import org.apache.commons.lang3.StringUtils;

import java.util.*;

/**
 * 抽象数据格式化类，子类实现包括
 *
 * 数据格式化策略（当勾选的字段数量 == 全部字段数量，则进行转置，否则不进行）
 *
 * @author lijunping on 2022/4/26
 */
public abstract class AbstractFormatPipeline implements FormatPipeline {

    @Override
    public void process(SpiderRequest request, SpiderResponse response) {
        List> formatResult = doFormat(request, response);
        response.setFormatResult(formatResult);
    }

    protected List> formatObject(Map fields){
        List> formatResult = new ArrayList<>();
        formatResult.add(fields);
        return formatResult;
    }

    @SuppressWarnings("unchecked")
    protected List> formatList(Map fields){
        List> formatResult = new ArrayList<>();
        Map> dataTmp = new HashMap<>();
        fields.forEach((key, value)-> dataTmp.put(key, (List) value));
        int maxSize = dataTmp.values().stream().map(List::size).max(Comparator.comparing(Integer::intValue)).orElse(0);

        for (int i = 0; i  rowData = new HashMap<>();
            int finalI = i;
            dataTmp.forEach((key, value)-> rowData.put(key, finalI >= value.size() ? null : value.get(finalI)));
            formatResult.add(rowData);
        }
        return formatResult;
    }

    protected abstract List> doFormat(SpiderRequest request, SpiderResponse response);
}