google.cloud.dataplex.v1.processing.proto Maven / Gradle / Ivy

Go to download
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.dataplex.v1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";

option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
option java_multiple_files = true;
option java_outer_classname = "ProcessingProto";
option java_package = "com.google.cloud.dataplex.v1";

// DataScan scheduling and trigger settings.
message Trigger {
  // The scan runs once via `RunDataScan` API.
  message OnDemand {}

  // The scan is scheduled to run periodically.
  message Schedule {
    // Required. [Cron](https://en.wikipedia.org/wiki/Cron) schedule for running
    // scans periodically.
    //
    // To explicitly set a timezone in the cron tab, apply a prefix in the
    // cron tab: **"CRON_TZ=${IANA_TIME_ZONE}"** or **"TZ=${IANA_TIME_ZONE}"**.
    // The **${IANA_TIME_ZONE}** may only be a valid string from IANA time zone
    // database
    // ([wikipedia](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List)).
    // For example, `CRON_TZ=America/New_York 1 * * * *`, or
    // `TZ=America/New_York 1 * * * *`.
    //
    // This field is required for Schedule scans.
    string cron = 1 [(google.api.field_behavior) = REQUIRED];
  }

  // DataScan scheduling and trigger settings.
  //
  // If not specified, the default is `onDemand`.
  oneof mode {
    // The scan runs once via `RunDataScan` API.
    OnDemand on_demand = 100;

    // The scan is scheduled to run periodically.
    Schedule schedule = 101;
  }
}

// The data source for DataScan.
message DataSource {
  // The source is required and immutable. Once it is set, it cannot be change
  // to others.
  oneof source {
    // Immutable. The Dataplex entity that represents the data source (e.g.
    // BigQuery table) for DataScan, of the form:
    // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/entities/{entity_id}`.
    string entity = 100 [
      (google.api.field_behavior) = IMMUTABLE,
      (google.api.resource_reference) = {
        type: "dataplex.googleapis.com/Entity"
      }
    ];

    // Immutable. The service-qualified full resource name of the cloud resource
    // for a DataScan job to scan against. The field could be: BigQuery table of
    // type "TABLE" for DataProfileScan/DataQualityScan Format:
    // //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID
    string resource = 101 [(google.api.field_behavior) = IMMUTABLE];
  }
}

// The data scanned during processing (e.g. in incremental DataScan)
message ScannedData {
  // A data range denoted by a pair of start/end values of a field.
  message IncrementalField {
    // The field that contains values which monotonically increases over time
    // (e.g. a timestamp column).
    string field = 1;

    // Value that marks the start of the range.
    string start = 2;

    // Value that marks the end of the range.
    string end = 3;
  }

  // The range of scanned data
  oneof data_range {
    // The range denoted by values of an incremental field
    IncrementalField incremental_field = 1;
  }
}