json.schema.metadataIngestion.databaseServiceQueryLineagePipeline.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of openmetadata-spec Show documentation
There is a newer version: 1.6.0-rc1
{
  "$id": "https://open-metadata.org/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "DatabaseServiceQueryLineagePipeline",
  "description": "DatabaseService Query Lineage Pipeline Configuration.",
  "type": "object",
  "definitions": {
    "databaseLineageConfigType": {
      "description": "Database Source Config Usage Pipeline type",
      "type": "string",
      "enum": ["DatabaseLineage"],
      "default": "DatabaseLineage"
    }
  },
  "properties": {
    "type": {
      "description": "Pipeline type",
      "$ref": "#/definitions/databaseLineageConfigType",
      "default": "DatabaseLineage"
    },
    "queryLogDuration": {
      "description": "Configuration to tune how far we want to look back in query logs to process lineage data.",
      "type": "integer",
      "default": 1,
      "title": "Query Log Duration"
    },
    "queryLogFilePath": {
      "description": "Configuration to set the file path for query logs",
      "type": "string",
      "title": "Query Log File Path"
    },
    "resultLimit": {
      "description": "Configuration to set the limit for query logs",
      "type": "integer",
      "default": 1000,
      "title": "Result Limit"
    },
    "parsingTimeoutLimit": {
      "description": "Configuration to set the timeout for parsing the query in seconds.",
      "type": "integer",
      "default": 300,
      "title": "Parsing Timeout Limit"
    },
    "filterCondition": {
      "description": "Configuration the condition to filter the query history.",
      "type": "string",
      "title": "Filter Condition"
    },
    "schemaFilterPattern": {
      "description": "Regex to only fetch tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Schema Filter Pattern"
    },
    "tableFilterPattern": {
      "description": "Regex exclude tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Table Filter Pattern"
    },
    "databaseFilterPattern": {
      "description": "Regex to only fetch databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Database Filter Pattern"
    },
    "overrideViewLineage":{
      "title": "Override View Lineage",
      "description": "Set the 'Override View Lineage' toggle to control whether to override the existing view lineage.",
      "type": "boolean",
      "default": false
    },
    "processViewLineage": {
      "title": "Process View Lineage",
      "description": "Set the 'Process View Lineage' toggle to control whether to process view lineage.",
      "type": "boolean",
      "default": true
    },
    "processQueryLineage": {
      "title": "Process Query Lineage",
      "description": "Set the 'Process Query Lineage' toggle to control whether to process query lineage.",
      "type": "boolean",
      "default": true
    },
    "processStoredProcedureLineage": {
      "title": "Process Stored Procedure Lineage",
      "description": "Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored procedure lineage.",
      "type": "boolean",
      "default": true
    },
    "threads": {
      "description": "Number of Threads to use in order to parallelize lineage ingestion.",
      "type": "integer",
      "default": 1,
      "title": "Number of Threads",
      "minimum": 1
    }
  },
  "additionalProperties": false
}