net.snowflake.ingest.streaming.internal.ChunkMetadata Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of snowflake-ingest-sdk Show documentation
Snowflake Ingest SDK
There is a newer version: 3.0.1
/*
 * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
 */

package net.snowflake.ingest.streaming.internal;

import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
import net.snowflake.ingest.utils.Utils;

/** Metadata for a chunk that sends to Snowflake as part of the register blob request */
class ChunkMetadata {
  private final String dbName;
  private final String schemaName;
  private final String tableName;
  private Long chunkStartOffset;
  private final Integer chunkLength;
  private final Integer uncompressedChunkLength;
  private final List channels;
  private final String chunkMD5;
  private final EpInfo epInfo;
  private final Long encryptionKeyId;
  private final Long firstInsertTimeInMs;
  private final Long lastInsertTimeInMs;

  static Builder builder() {
    return new Builder();
  }

  /** Builder class to build a ChunkMetadata */
  static class Builder {
    private String dbName;
    private String schemaName;
    private String tableName;
    private Long chunkStartOffset;
    private Integer chunkLength; // compressedChunkLength

    private Integer uncompressedChunkLength;

    private List channels;
    private String chunkMD5;
    private EpInfo epInfo;
    private Long encryptionKeyId;
    private Long firstInsertTimeInMs;
    private Long lastInsertTimeInMs;

    Builder setOwningTableFromChannelContext(ChannelFlushContext channelFlushContext) {
      this.dbName = channelFlushContext.getDbName();
      this.schemaName = channelFlushContext.getSchemaName();
      this.tableName = channelFlushContext.getTableName();
      return this;
    }

    Builder setEpInfo(EpInfo epInfo) {
      this.epInfo = epInfo;
      return this;
    }

    Builder setChunkStartOffset(Long startOffset) {
      this.chunkStartOffset = startOffset;
      return this;
    }

    Builder setChunkLength(Integer chunkLength) {
      this.chunkLength = chunkLength;
      return this;
    }

    /**
     * Currently we send estimated uncompressed size that is close to the actual parquet data size
     * and mostly about user data but parquet encoding overhead may be slightly different.
     */
    public Builder setUncompressedChunkLength(Integer uncompressedChunkLength) {
      this.uncompressedChunkLength = uncompressedChunkLength;
      return this;
    }

    Builder setChannelList(List channels) {
      this.channels = channels;
      return this;
    }

    Builder setChunkMD5(String chunkMD5) {
      this.chunkMD5 = chunkMD5;
      return this;
    }

    Builder setEncryptionKeyId(Long encryptionKeyId) {
      this.encryptionKeyId = encryptionKeyId;
      return this;
    }

    Builder setFirstInsertTimeInMs(Long firstInsertTimeInMs) {
      this.firstInsertTimeInMs = firstInsertTimeInMs;
      return this;
    }

    Builder setLastInsertTimeInMs(Long lastInsertTimeInMs) {
      this.lastInsertTimeInMs = lastInsertTimeInMs;
      return this;
    }

    ChunkMetadata build() {
      return new ChunkMetadata(this);
    }
  }

  private ChunkMetadata(Builder builder) {
    Utils.assertStringNotNullOrEmpty("chunk database name", builder.dbName);
    Utils.assertStringNotNullOrEmpty("chunk schema name", builder.schemaName);
    Utils.assertStringNotNullOrEmpty("chunk table name", builder.tableName);
    Utils.assertNotNull("chunk start offset", builder.chunkStartOffset);
    Utils.assertNotNull("chunk length", builder.chunkLength);
    Utils.assertNotNull("chunk channels", builder.channels);
    Utils.assertNotNull("chunk MD5", builder.chunkMD5);
    Utils.assertNotNull("chunk ep info", builder.epInfo);
    Utils.assertNotNull("encryption key id", builder.encryptionKeyId);
    Utils.assertNotNull("chunk first insert time in ms", builder.firstInsertTimeInMs);
    Utils.assertNotNull("chunk last insert time in ms", builder.lastInsertTimeInMs);

    this.dbName = builder.dbName;
    this.schemaName = builder.schemaName;
    this.tableName = builder.tableName;
    this.chunkStartOffset = builder.chunkStartOffset;
    this.chunkLength = builder.chunkLength;
    this.uncompressedChunkLength = builder.uncompressedChunkLength;
    this.channels = builder.channels;
    this.chunkMD5 = builder.chunkMD5;
    this.epInfo = builder.epInfo;
    this.encryptionKeyId = builder.encryptionKeyId;
    this.firstInsertTimeInMs = builder.firstInsertTimeInMs;
    this.lastInsertTimeInMs = builder.lastInsertTimeInMs;
  }

  /**
   * Advance the start offset of a chunk
   *
   * @param offset
   */
  void advanceStartOffset(int offset) {
    this.chunkStartOffset += offset;
  }

  @JsonProperty("database")
  String getDBName() {
    return this.dbName;
  }

  @JsonProperty("schema")
  String getSchemaName() {
    return this.schemaName;
  }

  @JsonProperty("table")
  String getTableName() {
    return this.tableName;
  }

  @JsonProperty("chunk_start_offset")
  Long getChunkStartOffset() {
    return this.chunkStartOffset;
  }

  @JsonProperty("chunk_length")
  Integer getChunkLength() {
    return chunkLength;
  }

  @JsonProperty("chunk_length_uncompressed")
  public Integer getUncompressedChunkLength() {
    return uncompressedChunkLength;
  }

  @JsonProperty("channels")
  List getChannels() {
    return this.channels;
  }

  @JsonProperty("chunk_md5")
  public String getChunkMD5() {
    return this.chunkMD5;
  }

  @JsonProperty("eps")
  EpInfo getEpInfo() {
    return this.epInfo;
  }

  @JsonProperty("encryption_key_id")
  Long getEncryptionKeyId() {
    return this.encryptionKeyId;
  }

  @JsonProperty("first_insert_time_in_ms")
  Long getFirstInsertTimeInMs() {
    return this.firstInsertTimeInMs;
  }

  @JsonProperty("last_insert_time_in_ms")
  Long getLastInsertTimeInMs() {
    return this.lastInsertTimeInMs;
  }
}