com.aliyun.datahub.client.impl.batch.BatchDeserializer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of aliyun-sdk-datahub Show documentation
The newest version!
package com.aliyun.datahub.client.impl.batch;

import com.aliyun.datahub.client.exception.DatahubClientException;
import com.aliyun.datahub.client.impl.batch.header.BatchHeader;
import com.aliyun.datahub.client.impl.compress.Compressor;
import com.aliyun.datahub.client.impl.compress.CompressorFactory;
import com.aliyun.datahub.client.impl.schemaregistry.SchemaRegistryClient;
import com.aliyun.datahub.client.model.CompressType;
import com.aliyun.datahub.client.model.RecordEntry;
import com.aliyun.datahub.client.model.RecordRespMeta;
import com.aliyun.datahub.client.model.RecordSchema;
import com.aliyun.datahub.client.util.CrcUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;

public abstract class BatchDeserializer {
    private final static Logger LOGGER = LoggerFactory.getLogger(BatchDeserializer.class);

    protected String projectName;
    protected String topicName;
    protected String shardId;
    protected SchemaRegistryClient schemaRegistry;

    public BatchDeserializer() {
    }

    protected InputStream decompress(byte[] bytes, BatchHeader header) throws IOException {
        CompressType compressType = header.getCompressType();
        Compressor compressor = CompressorFactory.getCompressor(compressType);
        if (compressor == null) {
            return new ByteArrayInputStream(bytes, header.getDataOffset(), bytes.length - header.getDataOffset());
        }
        byte[] rawBuf = compressor.decompress(bytes, header.getDataOffset(), bytes.length - header.getDataOffset(), header.getRawDataSize());
        return new ByteArrayInputStream(rawBuf);
    }

    protected RecordSchema getSchema(int schemaVersion) {
        return schemaVersion != -1 ? schemaRegistry.getSchema(projectName, topicName, schemaVersion) : null;
    }

    public String getProjectName() {
        return projectName;
    }

    public BatchDeserializer setProjectName(String projectName) {
        this.projectName = projectName;
        return this;
    }

    public String getTopicName() {
        return topicName;
    }

    public BatchDeserializer setTopicName(String topicName) {
        this.topicName = topicName;
        return this;
    }

    public String getShardId() {
        return shardId;
    }

    public BatchDeserializer setShardId(String shardId) {
        this.shardId = shardId;
        return this;
    }

    public SchemaRegistryClient getSchemaRegistry() {
        return schemaRegistry;
    }

    public BatchDeserializer setSchemaRegistry(SchemaRegistryClient schemaRegistry) {
        this.schemaRegistry = schemaRegistry;
        return this;
    }

    public List deserialize(byte[] bytes, RecordRespMeta meta) {
        BatchHeader header = BatchHeader.parseHeader(bytes);
        checkBuffer(bytes, header);
        try (InputStream inputStream = decompress(bytes, header)) {
            List recordEntryList = deserializeRecord(inputStream, header);
            for (RecordEntry entry : recordEntryList) {
                entry.setShardId(shardId);
                entry.setCursor(meta.getCursor());
                entry.setSequence(meta.getSequence());
                entry.setSerial(meta.getSerial());
                entry.setSystemTime(meta.getSystemTime());
                entry.setLatestSequence(meta.getLatestSequence());
                entry.setLatestTime(meta.getLatestTime());
            }
            return recordEntryList;
        } catch (Exception e) {
            e.printStackTrace();
            LOGGER.error("Deserialize record failed, {}/{}", projectName, topicName, e);
            throw new DatahubClientException(e.getMessage());
        }
    }

    private void checkBuffer(byte[] bytes, BatchHeader header) {
        if (bytes.length != header.getLength()) {
            throw new DatahubClientException("Check payload length fail, need: " + header.getLength() + ", real: " + bytes.length);
        }

        if (header.getCrc32() != 0) {
            int computeCrc = CrcUtils.getCrc32(bytes, header.getHeaderSize(), header.getLength() - header.getHeaderSize());
            if (header.getCrc32() != computeCrc) {
                throw new DatahubClientException("Check crc fail. expect:" + header.getCrc32() + ", real:" + computeCrc);
            }
        }
    }

    protected abstract List deserializeRecord(InputStream inputStream, BatchHeader header);
}