a.clickzetta-java.1.3.15.source-code.file_meta_data.proto Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of clickzetta-java Show documentation

The java SDK for clickzetta's Lakehouse

There is a newer version: 2.0.0

syntax = "proto3";

option java_multiple_files = true;

option java_outer_classname = "FileMetaDataProto";

import "bucket_info.proto";
import "file_format_type.proto";
import "statistics.proto";
import "virtual_value_info.proto";

package cz.proto;
option java_package = "cz.proto";

enum FileType {
  UNKNOWN_FILE = 0;
  DATA_FILE = 1;
  DELTA_FILE = 2;
}

message BlockInfo {
  uint64 offset = 1;
  uint64 length = 2;
  // other block meta of file
}

message DataLayout {
  // footer offset range
  BlockInfo footer = 1;
  // block (parquet row group or orc stripe) offset range
  repeated BlockInfo blocks = 2;
  // bucket:[blocks]
  // For a partitioned table, there might be multiple buckets
  // with same bucket_id, but fall into different partitions.
  repeated BucketInfo buckets = 3;
  // A set of values of virtual columns in the file.
  repeated VirtualValueInfo value_info = 4;
  // Number of values of virtual columns in the original file data.
  optional uint32 original_virtual_value_count = 8;
  // Number of rows to limit a parquet page or orc row group.
  optional int32 row_index_stride = 5;
  // Number of rows per block (parquet row group or orc stripe)
  repeated int64 block_row_counts = 6;
  // other organization info of file

  // Indicates whether the file is sealed or not.
  optional bool data_sealed = 9;
}

message ExtendedMetaData {
  map key_value = 1;
}

message FileMetaData {
  optional uint32 data_source_id = 20;
  string file_path = 1;
  FileFormatType file_format = 2;
  FileType file_type = 3;
  int64 file_slice_id = 4;
  optional int64 file_slice_version = 5;
  int32 compaction_level = 6;

  optional uint32 cluster_spec_id = 7;          // TableMeta.cluster_info_spec.spec_id
  optional uint32 sort_order_spec_id = 8;       // TableMeta.sort_order_spec.spec_id
  optional uint32 primary_key_spec_id = 9;      // TableMeta.primary_key_spec.spec_id

  DataLayout layout = 10;
  ExtendedMetaData extended_metadata = 11;

  // file level statistics
  StatsData stats = 16;
}