json.schema.metadataIngestion.databaseServiceProfilerPipeline.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of openmetadata-spec Show documentation
There is a newer version: 1.6.0-rc1
{
  "$id": "https://open-metadata.org/schema/metadataIngestion/databaseServiceProfilerPipeline.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "DatabaseServiceProfilerPipeline",
  "description": "DatabaseService Profiler Pipeline Configuration.",
  "type": "object",
  "definitions": {
    "profilerConfigType": {
      "description": "Profiler Source Config Pipeline type",
      "type": "string",
      "enum": ["Profiler"],
      "default": "Profiler"
    }
  },
  "properties": {
    "type": {
      "description": "Pipeline type",
      "$ref": "#/definitions/profilerConfigType",
      "default": "Profiler"
    },
    "classificationFilterPattern": {
      "description": "Regex to only compute metrics for table that matches the given tag, tiers, gloassary pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Classification Filter Pattern"
    },
    "schemaFilterPattern": {
      "description": "Regex to only fetch tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Schema Filter Pattern"
    },
    "tableFilterPattern": {
      "description": "Regex exclude tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Table Filter Pattern"
    },
    "databaseFilterPattern": {
      "description": "Regex to only fetch databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Database Filter Pattern"
    },
    "includeViews": {
      "description": "Optional configuration to turn off fetching metadata for views.",
      "type": "boolean",
      "default": false,
      "title": "Include Views"
    },
    "useFqnForFiltering": {
      "description": "Regex will be applied on fully qualified name (e.g service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name)",
      "type": "boolean",
      "default": false,
      "title": "Use FQN For Filtering"
    },
    "computeMetrics": {
      "description": "Option to turn on/off computing profiler metrics.",
      "type": "boolean",
      "default": true,
      "title": "Compute Metrics"
    },
    "computeTableMetrics": {
      "description": "Option to turn on/off table metric computation. If enabled, profiler will compute table level metrics.",
      "type": "boolean",
      "default": true,
      "title": "Compute Table Metrics"
    },
    "computeColumnMetrics": {
      "description": "Option to turn on/off column metric computation. If enabled, profiler will compute column level metrics.",
      "type": "boolean",
      "default": true,
      "title": "Compute Column Metrics"
    },
    "useStatistics": {
      "description": "Use system tables to extract metrics. Metrics that cannot be gathered from system tables will use the default methods. Using system tables can be faster but requires gathering statistics before running (for example using the ANALYZE procedure). More information can be found in the documentation: https://docs.openmetadata.org/latest/profler",
      "type": "boolean",
      "default": false,
      "title": "Use Gathered Statistics"
    },
    "profileSampleType": {
      "$ref": "../entity/data/table.json#/definitions/profileSampleType",
      "title": "Profile Sample Type"
    },
    "profileSample": {
      "description": "Percentage of data or no. of rows used to compute the profiler metrics and run data quality tests",
      "type": "number",
      "default": null,
      "title": "Profile Sample"
    },
    "samplingMethodType": {
      "$ref": "../entity/data/table.json#/definitions/samplingMethodType",
      "title": "Sampling Method Type"
    },
    "threadCount": {
      "description": "Number of threads to use during metric computations",
      "type": "number",
      "default": 5,
      "title": "Thread Count"
    },
    "timeoutSeconds": {
      "description": "Profiler Timeout in Seconds",
      "type": "integer",
      "default": 43200,
      "title": "Timeout (in sec.)"
    }
  },
  "additionalProperties": false
}