google.cloud.dataplex.v1.logs.proto Maven / Gradle / Ivy
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.dataplex.v1;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
option go_package = "cloud.google.com/go/dataplex/apiv1/dataplexpb;dataplexpb";
option java_multiple_files = true;
option java_outer_classname = "LogsProto";
option java_package = "com.google.cloud.dataplex.v1";
// The payload associated with Discovery data processing.
message DiscoveryEvent {
// The type of the event.
enum EventType {
// An unspecified event type.
EVENT_TYPE_UNSPECIFIED = 0;
// An event representing discovery configuration in effect.
CONFIG = 1;
// An event representing a metadata entity being created.
ENTITY_CREATED = 2;
// An event representing a metadata entity being updated.
ENTITY_UPDATED = 3;
// An event representing a metadata entity being deleted.
ENTITY_DELETED = 4;
// An event representing a partition being created.
PARTITION_CREATED = 5;
// An event representing a partition being updated.
PARTITION_UPDATED = 6;
// An event representing a partition being deleted.
PARTITION_DELETED = 7;
}
// The type of the entity.
enum EntityType {
// An unspecified event type.
ENTITY_TYPE_UNSPECIFIED = 0;
// Entities representing structured data.
TABLE = 1;
// Entities representing unstructured data.
FILESET = 2;
}
// Details about configuration events.
message ConfigDetails {
// A list of discovery configuration parameters in effect.
// The keys are the field paths within DiscoverySpec.
// Eg. includePatterns, excludePatterns, csvOptions.disableTypeInference,
// etc.
map parameters = 1;
}
// Details about the entity.
message EntityDetails {
// The name of the entity resource.
// The name is the fully-qualified resource name.
string entity = 1;
// The type of the entity resource.
EntityType type = 2;
}
// Details about the partition.
message PartitionDetails {
// The name to the partition resource.
// The name is the fully-qualified resource name.
string partition = 1;
// The name to the containing entity resource.
// The name is the fully-qualified resource name.
string entity = 2;
// The type of the containing entity resource.
EntityType type = 3;
// The locations of the data items (e.g., a Cloud Storage objects) sampled
// for metadata inference.
repeated string sampled_data_locations = 4;
}
// Details about the action.
message ActionDetails {
// The type of action.
// Eg. IncompatibleDataSchema, InvalidDataFormat
string type = 1;
}
// The log message.
string message = 1;
// The id of the associated lake.
string lake_id = 2;
// The id of the associated zone.
string zone_id = 3;
// The id of the associated asset.
string asset_id = 4;
// The data location associated with the event.
string data_location = 5;
// The type of the event being logged.
EventType type = 10;
// Additional details about the event.
oneof details {
// Details about discovery configuration in effect.
ConfigDetails config = 20;
// Details about the entity associated with the event.
EntityDetails entity = 21;
// Details about the partition associated with the event.
PartitionDetails partition = 22;
// Details about the action associated with the event.
ActionDetails action = 23;
}
}
// The payload associated with Job logs that contains events describing jobs
// that have run within a Lake.
message JobEvent {
// The type of the job.
enum Type {
// Unspecified job type.
TYPE_UNSPECIFIED = 0;
// Spark jobs.
SPARK = 1;
// Notebook jobs.
NOTEBOOK = 2;
}
// The completion status of the job.
enum State {
// Unspecified job state.
STATE_UNSPECIFIED = 0;
// Job successfully completed.
SUCCEEDED = 1;
// Job was unsuccessful.
FAILED = 2;
// Job was cancelled by the user.
CANCELLED = 3;
// Job was cancelled or aborted via the service executing the job.
ABORTED = 4;
}
// The service used to execute the job.
enum Service {
// Unspecified service.
SERVICE_UNSPECIFIED = 0;
// Cloud Dataproc.
DATAPROC = 1;
}
// Job Execution trigger.
enum ExecutionTrigger {
// The job execution trigger is unspecified.
EXECUTION_TRIGGER_UNSPECIFIED = 0;
// The job was triggered by Dataplex based on trigger spec from task
// definition.
TASK_CONFIG = 1;
// The job was triggered by the explicit call of Task API.
RUN_REQUEST = 2;
}
// The log message.
string message = 1;
// The unique id identifying the job.
string job_id = 2;
// The time when the job started running.
google.protobuf.Timestamp start_time = 3;
// The time when the job ended running.
google.protobuf.Timestamp end_time = 4;
// The job state on completion.
State state = 5;
// The number of retries.
int32 retries = 6;
// The type of the job.
Type type = 7;
// The service used to execute the job.
Service service = 8;
// The reference to the job within the service.
string service_job = 9;
// Job execution trigger.
ExecutionTrigger execution_trigger = 11;
}
// These messages contain information about sessions within an environment.
// The monitored resource is 'Environment'.
message SessionEvent {
// The type of the event.
enum EventType {
// An unspecified event type.
EVENT_TYPE_UNSPECIFIED = 0;
// Event when the session is assigned to a user.
START = 1;
// Event for stop of a session.
STOP = 2;
// Query events in the session.
QUERY = 3;
// Event for creation of a cluster. It is not yet assigned to a user.
// This comes before START in the sequence
CREATE = 4;
}
// Execution details of the query.
message QueryDetail {
// Query Execution engine.
enum Engine {
// An unspecified Engine type.
ENGINE_UNSPECIFIED = 0;
// Spark-sql engine is specified in Query.
SPARK_SQL = 1;
// BigQuery engine is specified in Query.
BIGQUERY = 2;
}
// The unique Query id identifying the query.
string query_id = 1;
// The query text executed.
string query_text = 2;
// Query Execution engine.
Engine engine = 3;
// Time taken for execution of the query.
google.protobuf.Duration duration = 4;
// The size of results the query produced.
int64 result_size_bytes = 5;
// The data processed by the query.
int64 data_processed_bytes = 6;
}
// The log message.
string message = 1;
// The information about the user that created the session. It will be the
// email address of the user.
string user_id = 2;
// Unique identifier for the session.
string session_id = 3;
// The type of the event.
EventType type = 4;
// Additional information about the Query metadata.
oneof detail {
// The execution details of the query.
QueryDetail query = 5;
}
// The status of the event.
bool event_succeeded = 6;
// If the session is associated with an environment with fast startup enabled,
// and was created before being assigned to a user.
bool fast_startup_enabled = 7;
// The idle duration of a warm pooled session before it is assigned to user.
google.protobuf.Duration unassigned_duration = 8;
}
// Payload associated with Governance related log events.
message GovernanceEvent {
// Information about Entity resource that the log event is associated with.
message Entity {
// Type of entity.
enum EntityType {
// An unspecified Entity type.
ENTITY_TYPE_UNSPECIFIED = 0;
// Table entity type.
TABLE = 1;
// Fileset entity type.
FILESET = 2;
}
// The Entity resource the log event is associated with.
// Format:
// `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/entities/{entity_id}`
string entity = 1 [(google.api.resource_reference) = {
type: "dataplex.googleapis.com/Entity"
}];
// Type of entity.
EntityType entity_type = 2;
}
// Type of governance log event.
enum EventType {
// An unspecified event type.
EVENT_TYPE_UNSPECIFIED = 0;
// Resource IAM policy update event.
RESOURCE_IAM_POLICY_UPDATE = 1;
// BigQuery table create event.
BIGQUERY_TABLE_CREATE = 2;
// BigQuery table update event.
BIGQUERY_TABLE_UPDATE = 3;
// BigQuery table delete event.
BIGQUERY_TABLE_DELETE = 4;
// BigQuery connection create event.
BIGQUERY_CONNECTION_CREATE = 5;
// BigQuery connection update event.
BIGQUERY_CONNECTION_UPDATE = 6;
// BigQuery connection delete event.
BIGQUERY_CONNECTION_DELETE = 7;
// BigQuery taxonomy created.
BIGQUERY_TAXONOMY_CREATE = 10;
// BigQuery policy tag created.
BIGQUERY_POLICY_TAG_CREATE = 11;
// BigQuery policy tag deleted.
BIGQUERY_POLICY_TAG_DELETE = 12;
// BigQuery set iam policy for policy tag.
BIGQUERY_POLICY_TAG_SET_IAM_POLICY = 13;
// Access policy update event.
ACCESS_POLICY_UPDATE = 14;
// Number of resources matched with particular Query.
GOVERNANCE_RULE_MATCHED_RESOURCES = 15;
// Rule processing exceeds the allowed limit.
GOVERNANCE_RULE_SEARCH_LIMIT_EXCEEDS = 16;
// Rule processing errors.
GOVERNANCE_RULE_ERRORS = 17;
// Governance rule processing Event.
GOVERNANCE_RULE_PROCESSING = 18;
}
// The log message.
string message = 1;
// The type of the event.
EventType event_type = 2;
// Entity resource information if the log event is associated with a
// specific entity.
optional Entity entity = 3;
}
// These messages contain information about the execution of a datascan.
// The monitored resource is 'DataScan'
// Next ID: 13
message DataScanEvent {
// The type of the data scan.
enum ScanType {
// An unspecified data scan type.
SCAN_TYPE_UNSPECIFIED = 0;
// Data scan for data profile.
DATA_PROFILE = 1;
// Data scan for data quality.
DATA_QUALITY = 2;
}
// The job state of the data scan.
enum State {
// Unspecified job state.
STATE_UNSPECIFIED = 0;
// Data scan job started.
STARTED = 1;
// Data scan job successfully completed.
SUCCEEDED = 2;
// Data scan job was unsuccessful.
FAILED = 3;
// Data scan job was cancelled.
CANCELLED = 4;
// Data scan job was createed.
CREATED = 5;
}
// The trigger type for the data scan.
enum Trigger {
// An unspecified trigger type.
TRIGGER_UNSPECIFIED = 0;
// Data scan triggers on demand.
ON_DEMAND = 1;
// Data scan triggers as per schedule.
SCHEDULE = 2;
}
// The scope of job for the data scan.
enum Scope {
// An unspecified scope type.
SCOPE_UNSPECIFIED = 0;
// Data scan runs on all of the data.
FULL = 1;
// Data scan runs on incremental data.
INCREMENTAL = 2;
}
// Data profile result for data scan job.
message DataProfileResult {
// The count of rows processed in the data scan job.
int64 row_count = 1;
}
// Data quality result for data scan job.
message DataQualityResult {
// The count of rows processed in the data scan job.
int64 row_count = 1;
// Whether the data quality result was `pass` or not.
bool passed = 2;
// The result of each dimension for data quality result.
// The key of the map is the name of the dimension.
// The value is the bool value depicting whether the dimension result was
// `pass` or not.
map dimension_passed = 3;
// The table-level data quality score for the data scan job.
//
// The data quality score ranges between [0, 100] (up to two decimal
// points).
float score = 4;
// The score of each dimension for data quality result.
// The key of the map is the name of the dimension.
// The value is the data quality score for the dimension.
//
// The score ranges between [0, 100] (up to two decimal
// points).
map dimension_score = 5;
// The score of each column scanned in the data scan job.
// The key of the map is the name of the column.
// The value is the data quality score for the column.
//
// The score ranges between [0, 100] (up to two decimal
// points).
map column_score = 6;
}
// Applied configs for data profile type data scan job.
message DataProfileAppliedConfigs {
// The percentage of the records selected from the dataset for DataScan.
//
// * Value ranges between 0.0 and 100.0.
// * Value 0.0 or 100.0 imply that sampling was not applied.
float sampling_percent = 1;
// Boolean indicating whether a row filter was applied in the DataScan job.
bool row_filter_applied = 2;
// Boolean indicating whether a column filter was applied in the DataScan
// job.
bool column_filter_applied = 3;
}
// Applied configs for data quality type data scan job.
message DataQualityAppliedConfigs {
// The percentage of the records selected from the dataset for DataScan.
//
// * Value ranges between 0.0 and 100.0.
// * Value 0.0 or 100.0 imply that sampling was not applied.
float sampling_percent = 1;
// Boolean indicating whether a row filter was applied in the DataScan job.
bool row_filter_applied = 2;
}
// Post scan actions result for data scan job.
message PostScanActionsResult {
// The result of BigQuery export post scan action.
message BigQueryExportResult {
// Execution state for the exporting.
enum State {
// The exporting state is unspecified.
STATE_UNSPECIFIED = 0;
// The exporting completed successfully.
SUCCEEDED = 1;
// The exporting is no longer running due to an error.
FAILED = 2;
// The exporting is skipped due to no valid scan result to export
// (usually caused by scan failed).
SKIPPED = 3;
}
// Execution state for the BigQuery exporting.
State state = 1;
// Additional information about the BigQuery exporting.
string message = 2;
}
// The result of BigQuery export post scan action.
BigQueryExportResult bigquery_export_result = 1;
}
// The data source of the data scan
string data_source = 1;
// The identifier of the specific data scan job this log entry is for.
string job_id = 2;
// The time when the data scan job was created.
google.protobuf.Timestamp create_time = 12;
// The time when the data scan job started to run.
google.protobuf.Timestamp start_time = 3;
// The time when the data scan job finished.
google.protobuf.Timestamp end_time = 4;
// The type of the data scan.
ScanType type = 5;
// The status of the data scan job.
State state = 6;
// The message describing the data scan job event.
string message = 7;
// A version identifier of the spec which was used to execute this job.
string spec_version = 8;
// The trigger type of the data scan job.
Trigger trigger = 9;
// The scope of the data scan (e.g. full, incremental).
Scope scope = 10;
// The result of the data scan job.
oneof result {
// Data profile result for data profile type data scan.
DataProfileResult data_profile = 101;
// Data quality result for data quality type data scan.
DataQualityResult data_quality = 102;
}
// The applied configs in the data scan job.
oneof appliedConfigs {
// Applied configs for data profile type data scan.
DataProfileAppliedConfigs data_profile_configs = 201;
// Applied configs for data quality type data scan.
DataQualityAppliedConfigs data_quality_configs = 202;
}
// The result of post scan actions.
PostScanActionsResult post_scan_actions_result = 11;
}
// Information about the result of a data quality rule for data quality scan.
// The monitored resource is 'DataScan'.
message DataQualityScanRuleResult {
// The type of the data quality rule.
enum RuleType {
// An unspecified rule type.
RULE_TYPE_UNSPECIFIED = 0;
// See
// [DataQualityRule.NonNullExpectation][google.cloud.dataplex.v1.DataQualityRule.NonNullExpectation].
NON_NULL_EXPECTATION = 1;
// See
// [DataQualityRule.RangeExpectation][google.cloud.dataplex.v1.DataQualityRule.RangeExpectation].
RANGE_EXPECTATION = 2;
// See
// [DataQualityRule.RegexExpectation][google.cloud.dataplex.v1.DataQualityRule.RegexExpectation].
REGEX_EXPECTATION = 3;
// See
// [DataQualityRule.RowConditionExpectation][google.cloud.dataplex.v1.DataQualityRule.RowConditionExpectation].
ROW_CONDITION_EXPECTATION = 4;
// See
// [DataQualityRule.SetExpectation][google.cloud.dataplex.v1.DataQualityRule.SetExpectation].
SET_EXPECTATION = 5;
// See
// [DataQualityRule.StatisticRangeExpectation][google.cloud.dataplex.v1.DataQualityRule.StatisticRangeExpectation].
STATISTIC_RANGE_EXPECTATION = 6;
// See
// [DataQualityRule.TableConditionExpectation][google.cloud.dataplex.v1.DataQualityRule.TableConditionExpectation].
TABLE_CONDITION_EXPECTATION = 7;
// See
// [DataQualityRule.UniquenessExpectation][google.cloud.dataplex.v1.DataQualityRule.UniquenessExpectation].
UNIQUENESS_EXPECTATION = 8;
// See
// [DataQualityRule.SqlAssertion][google.cloud.dataplex.v1.DataQualityRule.SqlAssertion].
SQL_ASSERTION = 9;
}
// The evaluation type of the data quality rule.
enum EvaluationType {
// An unspecified evaluation type.
EVALUATION_TYPE_UNSPECIFIED = 0;
// The rule evaluation is done at per row level.
PER_ROW = 1;
// The rule evaluation is done for an aggregate of rows.
AGGREGATE = 2;
}
// Whether the data quality rule passed or failed.
enum Result {
// An unspecified result.
RESULT_UNSPECIFIED = 0;
// The data quality rule passed.
PASSED = 1;
// The data quality rule failed.
FAILED = 2;
}
// Identifier of the specific data scan job this log entry is for.
string job_id = 1;
// The data source of the data scan (e.g. BigQuery table name).
string data_source = 2;
// The column which this rule is evaluated against.
string column = 3;
// The name of the data quality rule.
string rule_name = 4;
// The type of the data quality rule.
RuleType rule_type = 5;
// The evaluation type of the data quality rule.
EvaluationType evalution_type = 6;
// The dimension of the data quality rule.
string rule_dimension = 7;
// The passing threshold ([0.0, 100.0]) of the data quality rule.
double threshold_percent = 8;
// The result of the data quality rule.
Result result = 9;
// The number of rows evaluated against the data quality rule.
// This field is only valid for rules of PER_ROW evaluation type.
int64 evaluated_row_count = 10;
// The number of rows which passed a rule evaluation.
// This field is only valid for rules of PER_ROW evaluation type.
int64 passed_row_count = 11;
// The number of rows with null values in the specified column.
int64 null_row_count = 12;
// The number of rows returned by the SQL statement in a SQL assertion rule.
// This field is only valid for SQL assertion rules.
int64 assertion_row_count = 13;
}