json.schema.metadataIngestion.databaseServiceMetadataPipeline.json Maven / Gradle / Ivy

Go to download
{
  "$id": "https://open-metadata.org//schema/entity/services/ingestionPipelines/databaseServiceMetadataPipeline.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "DatabaseServiceMetadataPipeline",
  "description": "DatabaseService Metadata Pipeline Configuration.",
  "definitions": {
    "databaseMetadataConfigType": {
      "description": "Database Source Config Metadata Pipeline type",
      "type": "string",
      "enum": ["DatabaseMetadata"],
      "default": "DatabaseMetadata"
    },
    "dbtBucketDetails": {
      "title": "DBT Bucket Details",
      "description": "Details of the bucket where the dbt files are stored",
      "type": "object",
      "properties": {
        "dbtBucketName": {
          "title": "DBT Bucket Name",
          "description": "Name of the bucket where the dbt files are stored",
          "type": "string"
        },
        "dbtObjectPrefix": {
          "title": "DBT Object Prefix",
          "description": "Path of the folder where the dbt files are stored",
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "dbtCloudConfig": {
      "title": "DBT Cloud Config Source",
      "description": "DBT Catalog and Manifest HTTP path configuration.",
      "type": "object",
      "properties": {
        "dbtCloudAuthToken": {
          "title": "DBT Cloud Authentication Token",
          "description": "DBT cloud account authentication token",
          "type": "string",
          "format": "password"
        },
        "dbtCloudAccountId": {
          "title": "DBT Cloud Account Id",
          "description": "DBT cloud account Id",
          "type": "string"
        }
      },
      "additionalProperties": false,
      "required": ["dbtCloudAuthToken", "dbtCloudAccountId"]
    },
    "dbtLocalConfig": {
      "title": "DBT Local Config Source",
      "description": "DBT Catalog and Manifest file path config.",
      "type": "object",
      "properties": {
        "dbtCatalogFilePath": {
          "title": "DBT Catalog File Path",
          "description": "DBT catalog file path to extract dbt models with their column schemas.",
          "type": "string"
        },
        "dbtManifestFilePath": {
          "title": "DBT Manifest File Path",
          "description": "DBT manifest file path to extract dbt models and associate with tables.",
          "type": "string"
        }
      },
      "additionalProperties": false,
      "required": ["dbtCatalogFilePath", "dbtManifestFilePath"]
    },
    "dbtHttpConfig": {
      "title": "DBT HTTP Config Source",
      "description": "DBT Catalog and Manifest HTTP path configuration.",
      "type": "object",
      "properties": {
        "dbtCatalogHttpPath": {
          "title": "DBT Catalog HTTP File Path",
          "description": "DBT catalog http file path to extract dbt models with their column schemas.",
          "type": "string"
        },
        "dbtManifestHttpPath": {
          "title": "DBT Manifest HTTP File Path",
          "description": "DBT manifest http file path to extract dbt models and associate with tables.",
          "type": "string"
        }
      },
      "additionalProperties": false,
      "required": ["dbtCatalogHttpPath", "dbtManifestHttpPath"]
    },
    "dbtS3Config": {
      "title": "DBT S3 Config Source",
      "description": "DBT Catalog and Manifest files in S3 bucket. We will search for catalog.json and manifest.json.",
      "properties": {
        "dbtSecurityConfig": {
          "title": "DBT S3 Security Config",
          "$ref": "../security/credentials/awsCredentials.json"
        },
        "dbtPrefixConfig": {
          "title": "DBT Prefix Config",
          "$ref": "#/definitions/dbtBucketDetails"
        }
      }
    },
    "dbtGCSConfig": {
      "title": "DBT GCS Config Source",
      "description": "DBT Catalog and Manifest files in GCS storage. We will search for catalog.json and manifest.json.",
      "properties": {
        "dbtSecurityConfig": {
          "title": "DBT GCS Security Config",
          "$ref": "../security/credentials/gcsCredentials.json"
        },
        "dbtPrefixConfig": {
          "title": "DBT Prefix Config",
          "$ref": "#/definitions/dbtBucketDetails"
        }
      }
    }
  },
  "properties": {
    "type": {
      "description": "Pipeline type",
      "$ref": "#/definitions/databaseMetadataConfigType",
      "default": "DatabaseMetadata"
    },
    "markDeletedTables": {
      "description": "Optional configuration to soft delete tables in OpenMetadata if the source tables are deleted.",
      "type": "boolean",
      "default": true
    },
    "includeTables": {
      "description": "Optional configuration to turn off fetching metadata for tables.",
      "type": "boolean",
      "default": true
    },
    "includeViews": {
      "description": "Optional configuration to turn off fetching metadata for views.",
      "type": "boolean",
      "default": true
    },
    "includeTags": {
      "description": "Optional configuration to turn off fetching metadata for tags.",
      "type": "boolean",
      "default": true
    },
    "schemaFilterPattern": {
      "description": "Regex to only fetch tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern"
    },
    "tableFilterPattern": {
      "description": "Regex exclude tables or databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern"
    },
    "databaseFilterPattern": {
      "description": "Regex to only fetch databases that matches the pattern.",
      "$ref": "../type/filterPattern.json#/definitions/filterPattern"
    },
    "dbtConfigSource": {
      "title": "DBT Configuration Source",
      "description": "Available sources to fetch DBT catalog and manifest files.",
      "oneOf": [
        {
          "$ref": "#/definitions/dbtCloudConfig"
        },
        {
          "$ref": "#/definitions/dbtLocalConfig"
        },
        {
          "$ref": "#/definitions/dbtHttpConfig"
        },
        {
          "$ref": "#/definitions/dbtS3Config"
        },
        {
          "$ref": "#/definitions/dbtGCSConfig"
        }
      ]
    }
  },
  "additionalProperties": false
}