json.schema.metadataIngestion.storageServiceMetadataPipeline.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of openmetadata-spec Show documentation
There is a newer version: 1.6.0-rc1
{
  "$id": "https://open-metadata.org/schema/metadataIngestion/storageServiceMetadataPipeline.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "StorageServiceMetadataPipeline",
  "description": "StorageService Metadata Pipeline Configuration.",
  "type": "object",
  "definitions": {
    "storageMetadataConfigType": {
      "description": "Object Store Source Config Metadata Pipeline type",
      "type": "string",
      "enum": ["StorageMetadata"],
      "default": "StorageMetadata"
    },
    "noMetadataConfigurationSource": {
      "title": "No Global Manifest",
      "description": "No manifest file available. Ingestion would look for bucket-level metadata file instead",
      "type": "object",
      "additionalProperties": false
    }
  },
  "properties": {
    "type": {
      "description": "Pipeline type",
      "$ref": "#/definitions/storageMetadataConfigType",
      "default": "StorageMetadata"
    },
    "containerFilterPattern": {
      "description": "Regex to only fetch containers that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern"
    },
    "storageMetadataConfigSource": {
      "mask": true,
      "title": "Storage Metadata Configuration Source",
      "oneOf": [
        {
          "$ref": "#/definitions/noMetadataConfigurationSource"
        },
        {
          "$ref": "./storage/storageMetadataLocalConfig.json"
        },
        {
          "$ref": "./storage/storageMetadataHttpConfig.json"
        },
        {
          "$ref": "./storage/storageMetadataS3Config.json"
        },
        {
          "$ref": "./storage/storageMetadataADLSConfig.json"
        },
        {
          "$ref": "./storage/storageMetadataGCSConfig.json"
        }
      ]
    },
    "markDeletedContainers": {
      "description": "Optional configuration to soft delete containers in OpenMetadata if the source containers are deleted. Also, if the topic is deleted, all the associated entities with that containers will be deleted",
      "type": "boolean",
      "default": true,
      "title": "Mark Deleted Containers"
    },
    "overrideMetadata":{
      "title": "Override Metadata",
      "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
      "type": "boolean",
      "default": false
    },
    "includeTags": {
      "description": "Optional configuration to toggle the tags ingestion.",
      "type": "boolean",
      "default": false,
      "title": "Include Tags"
    }
  },
  "additionalProperties": false
}