com.qiniu.datasource.BaiduBosContainer Maven / Gradle / Ivy

Go to download
package com.qiniu.datasource;

import com.baidubce.auth.DefaultBceCredentials;
import com.baidubce.services.bos.BosClient;
import com.baidubce.services.bos.BosClientConfiguration;
import com.baidubce.services.bos.model.BosObjectSummary;
import com.qiniu.common.SuitsException;
import com.qiniu.convert.Converter;
import com.qiniu.convert.JsonObjectPair;
import com.qiniu.convert.StringBuilderPair;
import com.qiniu.convert.StringMapPair;
import com.qiniu.interfaces.IStorageLister;
import com.qiniu.interfaces.IResultOutput;
import com.qiniu.interfaces.IStringFormat;
import com.qiniu.interfaces.ITypeConvert;
import com.qiniu.persistence.FileSaveMapper;
import com.qiniu.util.CloudApiUtils;
import com.qiniu.util.ConvertingUtils;

import java.io.IOException;
import java.util.List;
import java.util.Map;

public class BaiduBosContainer extends CloudStorageContainer> {

//    private String accessKeyId;
//    private String accessKeySecret;
    private BosClientConfiguration configuration;

    public BaiduBosContainer(String accessKeyId, String accessKeySecret, BosClientConfiguration configuration, String endpoint,
                             String bucket, Map> prefixesMap, List antiPrefixes,
                             boolean prefixLeft, boolean prefixRight, Map indexMap, List fields,
                             int unitLen, int threads) throws IOException {
        super(bucket, prefixesMap, antiPrefixes, prefixLeft, prefixRight, indexMap, fields, unitLen, threads);
//        this.accessKeyId = accessKeyId;
//        this.accessKeySecret = accessKeySecret;
        this.configuration = configuration;
        this.configuration.setCredentials(new DefaultBceCredentials(accessKeyId, accessKeySecret));
        this.configuration.setEndpoint(endpoint);
        BosClient bosClient = new BosClient(this.configuration);
        try {
            BaiduLister baiduLister = new BaiduLister(bosClient, bucket, null, null, null, 1);
            baiduLister.close();
            baiduLister = null;
        } catch (SuitsException e) {
            bosClient.shutdown();
            throw e;
        }
        BosObjectSummary test = new BosObjectSummary();
        test.setKey("test");
        ConvertingUtils.toPair(test, indexMap, new StringMapPair());
    }

    @Override
    public String getSourceName() {
        return "baidu";
    }

    @Override
    protected ITypeConvert> getNewConverter() {
        return new Converter>() {
            @Override
            public Map convertToV(BosObjectSummary line) throws IOException {
                return ConvertingUtils.toPair(line, indexMap, new StringMapPair());
            }
        };
    }

    @Override
    protected ITypeConvert getNewStringConverter() {
        IStringFormat stringFormatter;
        if ("json".equals(saveFormat)) {
            stringFormatter = line -> ConvertingUtils.toPair(line, fields, new JsonObjectPair()).toString();
        } else if ("yaml".equals(saveFormat)) {
            stringFormatter = line -> ConvertingUtils.toStringWithIndent(line, fields);
        } else {
            stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(saveSeparator));
        }
        return new Converter() {
            @Override
            public String convertToV(BosObjectSummary line) throws IOException {
                return stringFormatter.toFormatString(line);
            }
        };
    }

    @Override
    protected IResultOutput getNewResultSaver(String order) throws IOException {
        return order != null ? new FileSaveMapper(savePath, getSourceName(), order) : new FileSaveMapper(savePath);
    }

    @Override
    protected IStorageLister getLister(String prefix, String marker, String start, String end, int unitLen) throws SuitsException {
        if (marker == null || "".equals(marker)) marker = CloudApiUtils.getAliOssMarker(start);
        BosClient bosClient = new BosClient(this.configuration);
        try {
            return new BaiduLister(new BosClient(configuration), bucket, prefix, marker, end, unitLen);
        } catch (SuitsException e) {
            bosClient.shutdown();
            throw e;
        }
    }
}