• Home
• org.alluxio
• alluxio-shaded-hadoop3-client
• 305
• source code
• block_worker.proto

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

grpc.block_worker.proto Maven / Gradle / Ivy

syntax = "proto2";

option java_multiple_files = true;
option java_package = "alluxio.grpc";
option java_outer_classname = "BlockWorkerProto";

package alluxio.grpc.block;

import "proto/dataserver/protocol.proto";
import "grpc/common.proto";
import "grpc/file_system_master.proto";

// The block worker service
service BlockWorker {
  /**
   * Returns the status of the file or directory.
   */
  rpc GetStatus (grpc.file.GetStatusPRequest) returns (grpc.file.GetStatusPResponse);
  /**
   * If the path points to a file, the method returns a singleton with its file information.
   * If the path points to a directory, the method returns a list with file information for the
   * directory contents.
   */
  rpc ListStatus(grpc.file.ListStatusPRequest) returns (stream grpc.file.ListStatusPResponse);


  rpc ReadBlock (stream ReadRequest) returns (stream ReadResponse);
  rpc WriteBlock (stream WriteRequest) returns (stream WriteResponse);

  rpc AsyncCache (AsyncCacheRequest) returns (AsyncCacheResponse);
  rpc Cache (CacheRequest) returns (CacheResponse);
  rpc Load(LoadRequest)returns (LoadResponse);
  rpc LoadFile(LoadFileRequest)returns (LoadFileResponse);
  rpc Copy(CopyRequest)returns (CopyResponse);
  rpc RemoveBlock (RemoveBlockRequest) returns (RemoveBlockResponse);
  rpc MoveBlock (MoveBlockRequest) returns (MoveBlockResponse);

  rpc Move(MoveRequest) returns (MoveResponse);

  // TODO(lu) Move to metrics worker
  rpc ClearMetrics (ClearMetricsRequest) returns (ClearMetricsResponse);

  rpc FreeWorker (FreeWorkerRequest) returns (FreeWorkerResponse);

  /**
   * Creates a file.
   */
  rpc CreateFile(grpc.file.CreateFilePRequest) returns (grpc.file.CreateFilePResponse);

  /**
   * Marks a file as completed.
   */
  rpc CompleteFile(grpc.file.CompleteFilePRequest) returns (grpc.file.CompleteFilePResponse);

  /**
   * Deletes a file or a directory and returns whether the remove operation succeeded.
   */
  rpc Remove(grpc.file.DeletePRequest) returns (grpc.file.DeletePResponse);

  /**
   * Renames a file or a directory.
   */
  rpc Rename(grpc.file.RenamePRequest) returns (grpc.file.RenamePResponse);

  /**
   * Creates a directory.
   */
  rpc CreateDirectory(grpc.file.CreateDirectoryPRequest) returns (grpc.file.CreateDirectoryPResponse);

  /**
   * Checks the existence of a file or directory.
   */
  rpc Exists(grpc.file.ExistsPRequest) returns (grpc.file.ExistsPResponse);

  /**
   * Sets file or directory attributes.
   */
  rpc SetAttribute(grpc.file.SetAttributePRequest) returns (grpc.file.SetAttributePResponse);

  /**
   * Caches data from UFS.
   */
  rpc CacheData(CacheDataRequest) returns (CacheDataResponse);
}

message UfsStatus {
  optional string name = 1;
  optional bool is_directory = 2;
  optional int64 last_modified_time_ms = 3;
  optional string owner = 4;
  optional string group = 5;
  optional int32 mode = 6;
  map xattr = 7;
  optional string ufs_full_path = 8;

  optional UfsFileStatus ufs_file_status = 100;
}

message UfsFileStatus {
  optional string content_hash = 1;
  optional int64 content_length = 2;
  optional int64 block_size = 3;
}

// The check request
message CheckRequest {}

// The check response
message CheckResponse {}

// The data chunk.
// next available id: 2
message Chunk {
  optional bytes data = 1;
}

// The read/write request type. It can either be an Alluxio block operation or a UFS file operation.
// next available id: 3
enum RequestType {
  ALLUXIO_BLOCK = 0;
  UFS_FILE = 1;
}

// The read request.
// next available id: 9
message ReadRequest {
  optional int64 block_id = 1;
  optional int64 offset = 2;
  optional int64 length = 3;
  // Whether the block should be promoted before reading
  optional bool promote = 4;
  optional int64 chunk_size = 5;

  // This is only set for UFS block read.
  optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 6;

  // Read receipt
  optional int64 offset_received = 7;

  // Is position read to a small buffer
  optional bool position_short = 8;
}

// The read response.
// next available id: 2
message ReadResponse {
  optional Chunk chunk = 1;
}

// The write request command.
// next available id: 11
message WriteRequestCommand {
  optional RequestType type = 1;
  // The block ID or UFS file ID.
  optional int64 id = 2;
  optional int64 offset = 3;
  // This is only applicable for block write.
  optional int32 tier = 4;
  optional bool flush = 5;
  // Cancel, close and error will be handled by standard gRPC stream APIs.
  optional alluxio.proto.dataserver.CreateUfsFileOptions create_ufs_file_options = 6;
  optional string medium_type = 8;
  optional bool pin_on_create = 9;
  optional int64 space_to_reserve = 10;
}

// The write request.
// next available id: 3
message WriteRequest {
  oneof value {
    WriteRequestCommand command = 1;
    Chunk chunk = 2;
  }
}

// The write response.
// next available id: 3
message WriteResponse {
  optional int64 offset = 1;
  // Errors will be handled by standard gRPC stream APIs.
  optional string contentHash = 2;
}

// Request for caching a block asynchronously
// Deprecated and will be removed in v3.0
message AsyncCacheRequest {
  optional int64 block_id = 1;
  // TODO(calvin): source host and port should be replace with WorkerNetAddress
  optional string source_host = 2;
  optional int32 source_port = 3;
  optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4;
  optional int64 length = 5;
}

// Request for caching a block synchronously/asynchronously
// next available id: 7
message CacheRequest {
  optional int64 block_id = 1;
  // TODO(calvin): source host and port should be replace with WorkerNetAddress
  optional string source_host = 2;
  optional int32 source_port = 3;
  optional alluxio.proto.dataserver.OpenUfsBlockOptions open_ufs_block_options = 4;
  optional int64 length = 5;
  optional bool async = 6;
}

message CacheDataRequest {
  optional string ufs_path = 1;
  optional int64 pos = 2;
  optional int64 length = 3;
  optional bool async = 4;
}

message CacheDataResponse {}

// Request for load a block into alluxio
// next available id: 3
message LoadRequest {
  repeated Block blocks = 1;
  required UfsReadOptions options = 2;
}

message UfsReadOptions{
  required string tag = 1;
  // is position short or not, used for HDFS performance optimization.
  // When the client buffer size is large ( > 2MB) and reads are guaranteed to be somewhat
  // sequential, the `pread` API to HDFS is not as efficient as simple `read`.
  // We introduce a heuristic to choose which API to use.
  required bool position_short = 2;
  optional int64 bandwidth = 3;
  optional string user = 4;
}

message Block{
  optional int64 block_id = 1;
    // The block length.
  required int64 length = 2;
  optional string ufs_path = 3;
  // The offset of the block in within ufs the file.
  optional int64 offset_in_file = 4;
  optional int64 mountId = 5;
  optional UfsStatus ufs_status = 6;
}

message LoadResponse {
  required TaskStatus status = 1;
  repeated BlockStatus block_status = 2;
}

enum TaskStatus {
  SUCCESS = 0;
  FAILURE = 1;
  PARTIAL_FAILURE = 2;
}

message LoadFileRequest {
  repeated LoadSubTask subtasks = 1;
  required UfsReadOptions options = 2;
  optional bool skip_if_exists = 3;
}

// A subtask of a load file request. either a load data or load metadata.
message LoadSubTask {
  oneof task {
    LoadDataSubTask load_data_subtask = 1;
    LoadMetadataSubTask load_metadata_subtask = 2;
  }
}

message LoadDataSubTask {
  required int64 length = 1;
  optional string ufs_path = 2;
  optional int64 offset_in_file = 3;
  optional UfsStatus ufs_status = 4;
  optional WorkerNetAddress main_worker = 5;
}

message LoadMetadataSubTask {
  optional UfsStatus ufs_status = 1;
}

message File{
  optional string alluxio_path = 1;
  optional string ufs_path = 2;
  // file length
  optional int64 length = 3;
  optional int64 mountId = 4;
}

message LoadFileResponse {
  required TaskStatus status = 1;
  repeated LoadFailure failures = 2;
  optional int32 num_skipped = 3;
  optional int64 bytes_skipped = 4;
}

message FreeWorkerRequest{}

message FreeWorkerResponse{}

message BlockStatus {
  required Block block = 1;
  // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code].
  required int32 code = 2;
  // A developer-facing error message
  optional string message = 3;
  optional bool retryable = 4;
}

// load subtask failure, either metadata or load block we only need to input one of them
message LoadFailure {
  optional LoadSubTask subtask = 1;
  // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code].
  optional int32 code = 2;
  // A developer-facing error message
  optional string message = 3;
  optional bool retryable = 4;
}

// Response for an async cache request
message AsyncCacheResponse {}

// Response for an async cache request
message CacheResponse {}

message CopyRequest {
  repeated Route routes = 1;
  optional UfsReadOptions ufs_read_options = 2;
  required WriteOptions write_options= 3;
}

message Route {
  required string src = 1;
  required string dst = 2;
  optional int64 length = 3;
}

message CopyResponse {
  required TaskStatus status = 1;
  repeated RouteFailure failures = 2;
}

message WriteOptions {
  optional bool overwrite = 1;
  optional grpc.file.WritePType write_type = 2 [default = CACHE_THROUGH];
  optional bool check_content = 3;
}

message RouteFailure {
  required Route route = 1;
  // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code].
  required int32 code = 2;
  // A developer-facing error message
  optional string message = 3;
  optional bool retryable = 4;
  optional bool is_skip = 5;
}

// next available id: 2
message RemoveBlockRequest {
  optional int64 block_id = 1;
}

message RemoveBlockResponse {}

message MoveBlockRequest {
  optional int64 block_id = 1;
  optional string medium_type = 2;
}

message MoveBlockResponse {}

message ClearMetricsRequest {}

message ClearMetricsResponse {}

message MoveRequest {
  repeated Route routes = 1;
  optional UfsReadOptions ufs_read_options = 2;
  required WriteOptions write_options= 3;
}

message MoveResponse {
  required TaskStatus status = 1;
  repeated RouteFailure failures = 2;
}