et-import.2.0.1.source-code.package.pkl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of asset-import Show documentation
Show all versions of asset-import Show documentation
Atlan custom package for importing asset metadata
/* SPDX-License-Identifier: Apache-2.0
Copyright 2024 Atlan Pte. Ltd. */
amends "modulepath:/Config.pkl"
import "pkl:semver"
import "modulepath:/BuildInfo.pkl"
packageId = "@csa/asset-import"
packageName = "Asset Import"
version = semver.Version(BuildInfo.VERSION)
description = "Import assets from a CSV file."
iconUrl = "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg"
docsUrl = "https://solutions.atlan.com/asset-import/"
implementationLanguage = "Kotlin"
containerImage = "ghcr.io/atlanhq/\(name):\(BuildInfo.VERSION)"
containerImagePullPolicy = if (BuildInfo.VERSION.endsWith("-SNAPSHOT")) "Always" else "IfNotPresent"
containerCommand {
"/dumb-init"
"--"
"java"
"com.atlan.pkg.aim.Importer"
}
outputs {
files {
["debug-logs"] = "/tmp/debug.log"
}
}
keywords {
"kotlin"
"utility"
}
preview = true
uiConfig {
tasks {
["Source"] {
description = "Inputs"
inputs {
["import_type"] = new Radio {
title = "Import metadata from"
helpText = "Select how you want to provide the file(s) containing metadata to be imported, produced by one of the Asset Export packages."
possibleValues {
["DIRECT"] = "Direct file upload(s)"
["CLOUD"] = "Object storage"
}
default = "DIRECT"
}
["cloud_source"] = new CredentialInput {
title = "Import metadata from"
required = true
credType = "csa-connectors-objectstore"
helpText = "Enter details for the object store from which to retrieve the file(s) containing metadata to be imported."
}
}
}
["Assets"] {
description = "Assets to import"
inputs {
["assets_file"] = new FileUploader {
title = "Assets file"
fileTypes {
"text/csv"
}
required = false
helpText = "Select the file containing assets to import, produced by one of the Asset Export packages."
placeholderText = "Select assets CSV file"
}
["assets_prefix"] = new TextInput {
title = "Prefix (path)"
helpText = "Enter the directory (path) within the object store from which to retrieve the file containing asset metadata."
placeholderText = "path/to/file"
width = 4
}
["assets_key"] = new TextInput {
title = "Object key (filename)"
helpText = "Enter the object key (filename), including its extension, within the object store and prefix."
placeholderText = "assets.csv"
width = 4
}
["assets_upsert_semantic"] = new Radio {
title = "Input handling"
required = false
possibleValues {
["upsert"] = "Create full"
["partial"] = "Create partial"
["update"] = "Update only"
}
default = "update"
helpText = "Whether to allow the creation of new assets from the input CSV (full or partial assets), or ensure assets are only updated if they already exist in Atlan."
}
["assets_config"] = new Radio {
title = "Options"
required = true
possibleValues {
["default"] = "Default"
["advanced"] = "Advanced"
}
default = "default"
helpText = "Options to optimize how assets are imported."
}
["assets_attr_to_overwrite"] = new DropDown {
title = "Remove attributes, if empty"
required = false
possibleValues {
["certificateStatus"] = "Certificate"
["announcementType"] = "Announcement"
["displayName"] = "Display name"
["description"] = "Description (system)"
["userDescription"] = "Description (user)"
["ownerUsers"] = "Owners (users)"
["ownerGroups"] = "Owners (groups)"
["assignedTerms"] = "Assigned terms"
["readme"] = "README"
}
helpText = "Select attributes you want to clear (remove) from assets if their value is blank in the provided file."
multiSelect = true
width = 8
}
["assets_fail_on_errors"] = new BooleanInput {
title = "Fail on errors"
required = false
helpText = "Whether an invalid value in a field should cause the import to fail (Yes) or log a warning, skip that value, and proceed (No)."
}
["assets_case_sensitive"] = new BooleanInput {
title = "Case-sensitive match for updates"
required = false
helpText = "Whether to use case-sensitive matching when running in update-only mode (Yes) or try case-insensitive matching (No)."
}
["assets_table_view_agnostic"] = new BooleanInput {
title = "Table/view agnostic?"
required = false
helpText = "Whether to treat tables, views and materialized views as interchangeable (Yes) or strictly adhere to specified types in the input (No)."
}
["assets_field_separator"] = new TextInput {
title = "Field separator"
required = false
helpText = "Character used to separate fields in the input file (for example, ',' or ';')."
placeholderText = ","
width = 4
}
["assets_batch_size"] = new NumericInput {
title = "Batch size"
required = false
helpText = "Maximum number of rows to process at a time (per API request)."
placeholderValue = 20
width = 4
}
["track_batches"] = new BooleanInput {
title = "Track asset details"
required = false
hide = true
defaultSelection = true
helpText = "Whether to track details about every asset across batches (Yes) or only counts (No)."
}
}
}
["Glossaries"] {
description = "Glossaries to import"
inputs {
["glossaries_file"] = new FileUploader {
title = "Glossaries file"
fileTypes { "text/csv" }
required = false
helpText = "Select the file containing glossaries, categories and terms to import, produced by one of the Asset Export packages."
placeholderText = "Select glossaries CSV file"
}
["glossaries_prefix"] = new TextInput {
title = "Prefix (path)"
helpText = "Enter the directory (path) within the object store from which to retrieve the file containing glossaries, categories and terms."
placeholderText = "path/to/file"
width = 4
}
["glossaries_key"] = new TextInput {
title = "Object key (filename)"
helpText = "Enter the object key (filename), including its extension, within the object store and prefix."
placeholderText = "glossaries.csv"
width = 4
}
["glossaries_upsert_semantic"] = new Radio {
title = "Input handling"
required = false
possibleValues {
["upsert"] = "Create and update"
["update"] = "Update only"
}
default = "upsert"
helpText = "Whether to allow the creation of new glossaries, categories and terms from the input CSV, or ensure these are only updated if they already exist in Atlan."
}
["glossaries_config"] = new Radio {
title = "Options"
required = true
possibleValues {
["default"] = "Default"
["advanced"] = "Advanced"
}
default = "default"
helpText = "Options to optimize how assets are imported."
}
["glossaries_attr_to_overwrite"] = new DropDown {
title = "Remove attributes, if empty"
required = false
possibleValues {
["certificateStatus"] = "Certificate"
["announcementType"] = "Announcement"
["displayName"] = "Display name"
["description"] = "Description (system)"
["userDescription"] = "Description (user)"
["ownerUsers"] = "Owners (users)"
["ownerGroups"] = "Owners (groups)"
["assignedTerms"] = "Assigned terms"
["readme"] = "README"
}
helpText = "Select attributes you want to clear (remove) from glossaries, categories and terms if their value is blank in the provided file."
multiSelect = true
width = 8
}
["glossaries_fail_on_errors"] = new BooleanInput {
title = "Fail on errors"
required = false
helpText = "Whether an invalid value in a field should cause the import to fail (Yes) or log a warning, skip that value, and proceed (No)."
}
["glossaries_field_separator"] = new TextInput {
title = "Field separator"
required = false
helpText = "Character used to separate fields in the input file (for example, ',' or ';')."
placeholderText = ","
width = 4
}
["glossaries_batch_size"] = new NumericInput {
title = "Batch size"
required = false
helpText = "Maximum number of rows to process at a time (per API request)."
placeholderValue = 20
width = 4
}
}
}
["Data products"] {
description = "Data products to import"
inputs {
["data_products_file"] = new FileUploader {
title = "Data products file"
fileTypes { "text/csv" }
required = false
helpText = "Select the file containing domains and data products to import, produced by one of the Asset Export packages."
placeholderText = "Select data products CSV file"
}
["data_products_prefix"] = new TextInput {
title = "Prefix (path)"
helpText = "Enter the directory (path) within the object store from which to retrieve the file containing domains and data products."
placeholderText = "path/to/file"
width = 4
}
["data_products_key"] = new TextInput {
title = "Object key (filename)"
helpText = "Enter the object key (filename), including its extension, within the object store and prefix."
placeholderText = "data_products.csv"
width = 4
}
["data_products_upsert_semantic"] = new Radio {
title = "Input handling"
required = false
possibleValues {
["upsert"] = "Create and update"
["update"] = "Update only"
}
default = "upsert"
helpText = "Whether to allow the creation of new domains and data products from the input CSV, or ensure these are only updated if they already exist in Atlan."
}
["data_products_config"] = new Radio {
title = "Options"
required = true
possibleValues {
["default"] = "Default"
["advanced"] = "Advanced"
}
default = "default"
helpText = "Options to optimize how domains and data products are imported."
}
["data_products_attr_to_overwrite"] = new DropDown {
title = "Remove attributes, if empty"
required = false
possibleValues {
["certificateStatus"] = "Certificate"
["announcementType"] = "Announcement"
["displayName"] = "Display name"
["description"] = "Description (system)"
["userDescription"] = "Description (user)"
["ownerUsers"] = "Owners (users)"
["ownerGroups"] = "Owners (groups)"
["assignedTerms"] = "Assigned terms"
["readme"] = "README"
}
helpText = "Select attributes you want to clear (remove) from domains and data products if their value is blank in the provided file."
multiSelect = true
width = 8
}
["data_products_fail_on_errors"] = new BooleanInput {
title = "Fail on errors"
required = false
helpText = "Whether an invalid value in a field should cause the import to fail (Yes) or log a warning, skip that value, and proceed (No)."
}
["data_products_field_separator"] = new TextInput {
title = "Field separator"
required = false
helpText = "Character used to separate fields in the input file (for example, ',' or ';')."
placeholderText = ","
width = 4
}
["data_products_batch_size"] = new NumericInput {
title = "Batch size"
required = false
helpText = "Maximum number of rows to process at a time (per API request)."
placeholderValue = 20
width = 4
}
}
}
}
rules {
new UIRule {
whenInputs { ["import_type"] = "DIRECT" }
required { "assets_file" "glossaries_file" "data_products_file" }
}
new UIRule {
whenInputs { ["import_type"] = "CLOUD" }
required { "cloud_source" "assets_prefix" "assets_key" "glossaries_prefix" "glossaries_key" "data_products_prefix" "data_products_key" }
}
new UIRule {
whenInputs { ["assets_config"] = "advanced" }
required {
"assets_attr_to_overwrite"
"assets_fail_on_errors"
"assets_case_sensitive"
"assets_table_view_agnostic"
"assets_field_separator"
"assets_batch_size"
}
}
new UIRule {
whenInputs { ["glossaries_config"] = "advanced" }
required {
"glossaries_attr_to_overwrite"
"glossaries_fail_on_errors"
"glossaries_field_separator"
"glossaries_batch_size"
}
}
new UIRule {
whenInputs { ["data_products_config"] = "advanced" }
required {
"data_products_attr_to_overwrite"
"data_products_fail_on_errors"
"data_products_field_separator"
"data_products_batch_size"
}
}
}
}