json.schema.metadataIngestion.pipelineServiceMetadataPipeline.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of openmetadata-spec Show documentation
There is a newer version: 1.6.0-rc1
{
  "$id": "https://open-metadata.org/schema/metadataIngestion/pipelineServiceMetadataPipeline.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "PipelineServiceMetadataPipeline",
  "description": "PipelineService Metadata Pipeline Configuration.",
  "type": "object",
  "definitions": {
    "pipelineMetadataConfigType": {
      "description": "Pipeline Source Config Metadata Pipeline type",
      "type": "string",
      "enum": ["PipelineMetadata"],
      "default": "PipelineMetadata"
    }
  },
  "properties": {
    "type": {
      "description": "Pipeline type",
      "$ref": "#/definitions/pipelineMetadataConfigType",
      "default": "PipelineMetadata"
    },
    "includeLineage": {
      "description": "Optional configuration to turn off fetching lineage from pipelines.",
      "type": "boolean",
      "default": true,
      "title": "Include Lineage"
    },
    "includeOwners": {
      "description": "Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.",
      "type": "boolean",
      "default": true,
      "title": "Include Owners"
    },
    "pipelineFilterPattern": {
      "description": "Regex exclude pipelines.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern",
      "title": "Pipeline Filter Pattern"
    },
    "lineageInformation": {
      "description": "Details required to generate Lineage",
      "type": "object",
      "title": "Lineage Information",
      "properties": {
        "dbServiceNames": {
          "title": "Database Service Names List",
          "description": "List of Database Service Names for creation of lineage",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "storageServiceNames": {
          "title": "Storage Service Names List",
          "description": "List of Storage Service Names for creation of lineage",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "overrideLineage":{
      "title": "Override Lineage",
      "description": "Set the 'Override Lineage' toggle to control whether to override the existing lineage.",
      "type": "boolean",
      "default": false
    },
    "markDeletedPipelines": {
      "description": "Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, etc., with that Pipeline will be deleted",
      "type": "boolean",
      "default": true,
      "title": "Mark Deleted Pipeline"
    },
    "includeTags": {
      "description": "Optional configuration to toggle the tags ingestion.",
      "type": "boolean",
      "default": true,
      "title": "Include Tags"
    },
    "includeUnDeployedPipelines": {
      "description": "Optional configuration to toggle whether the un-deployed pipelines should be ingested or not. If set to false, only deployed pipelines will be ingested.",
      "type": "boolean",
      "default": true,
      "title": "Include UnDeployed Pipelines"
    },
    "overrideMetadata":{
      "title": "Override Metadata",
      "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
      "type": "boolean",
      "default": false
    }
  },
  "additionalProperties": false
}