Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.atlan.samples.loaders.models.LineageDetails Maven / Gradle / Ivy
// Generated by delombok at Thu Sep 07 11:44:18 UTC 2023
/* SPDX-License-Identifier: Apache-2.0 */
/* Copyright 2023 Atlan Pte. Ltd. */
package com.atlan.samples.loaders.models;
import com.atlan.Atlan;
import com.atlan.exception.AtlanException;
import com.atlan.model.assets.ICatalog;
import com.atlan.model.assets.LineageProcess;
import com.atlan.model.enums.AtlanAnnouncementType;
import com.atlan.model.enums.AtlanConnectorType;
import com.atlan.model.enums.CertificateStatus;
import com.atlan.util.AssetBatch;
import java.util.*;
import java.util.regex.Pattern;
/**
* Utility class for capturing the full details provided about a column.
*/
public class LineageDetails extends AssetDetails {
@java.lang.SuppressWarnings("all")
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(LineageDetails.class);
public static final String COL_S_CONNECTOR = "CONNECTOR (S)";
public static final String COL_S_CONNECTION = "CONNECTION (S)";
public static final String COL_S_ASSET_TYPE = "SOURCE ASSET TYPE";
public static final String COL_S_ASSET = "SOURCE ASSET";
public static final String COL_ORCHESTRATOR = "ORCHESTRATOR";
public static final String COL_PROCESS_ID = "PROCESS ID";
public static final String COL_PROCESS_TYPE = "PROCESS TYPE";
public static final String COL_T_ASSET = "TARGET ASSET";
public static final String COL_T_ASSET_TYPE = "TARGET ASSET TYPE";
public static final String COL_T_CONNECTOR = "CONNECTOR (T)";
public static final String COL_T_CONNECTION = "CONNECTION (T)";
public static final String COL_SQL_CODE = "SQL / CODE";
public static final String COL_PROCESS_URL = "PROCESS URL";
private static final Pattern CONNECTION_QN_PREFIX = Pattern.compile("default/[a-z0-9]+/[0-9]{10}/.*");
private static final List REQUIRED = List.of(COL_S_ASSET_TYPE, COL_S_ASSET, COL_ORCHESTRATOR, COL_PROCESS_TYPE, COL_PROCESS_ID, COL_T_ASSET, COL_T_ASSET_TYPE);
private AssetHeader sourceAsset;
private String processConnectionQualifiedName;
private String processId;
private AssetHeader targetAsset;
private String sqlCode;
private String processUrl;
/**
* {@inheritDoc}
*/
@Override
public String getIdentity() {
return processConnectionQualifiedName + "/" + processId;
}
/**
* Retrieve the orchestrator details from the provided row.
*
* @param row of data
* @return the connection details for the orchestrator for that row
*/
public static ConnectionDetails getOrchestratorFromRow(Map row) {
if (getMissingFields(row, List.of(COL_ORCHESTRATOR, COL_PROCESS_TYPE)).isEmpty()) {
String processType = row.get(COL_PROCESS_TYPE);
AtlanConnectorType connectorType = AtlanConnectorType.fromValue(processType);
if (connectorType != null) {
return ConnectionDetails.getHeader(row.get(COL_ORCHESTRATOR), connectorType);
}
}
return null;
}
/**
* Retrieve the source connection details from the provided row.
*
* @param row of data
* @return the connection details for the source asset on that row
*/
public static ConnectionDetails getSourceConnectionFromRow(Map row) {
if (getMissingFields(row, List.of(COL_S_CONNECTOR, COL_S_CONNECTION)).isEmpty()) {
return ConnectionDetails.getHeader(row.get(COL_S_CONNECTION), AtlanConnectorType.fromValue(row.get(COL_S_CONNECTOR).toLowerCase(Locale.ROOT)));
}
return null;
}
/**
* Retrieve the target connection details from the provided row.
*
* @param row of data
* @return the connection details for the target asset on that row
*/
public static ConnectionDetails getTargetConnectionFromRow(Map row) {
if (getMissingFields(row, List.of(COL_T_CONNECTOR, COL_T_CONNECTION)).isEmpty()) {
return ConnectionDetails.getHeader(row.get(COL_T_CONNECTION), AtlanConnectorType.fromValue(row.get(COL_T_CONNECTOR).toLowerCase(Locale.ROOT)));
}
return null;
}
/**
* Construct the qualifiedName of a connection that will be used for lineage processes.
*
* @param connectionCache cache of connections
* @param row of data
* @return the qualifiedName for the connection for lineage processes (if it exists), or null if it does not yet exist
*/
static String getProcessConnectionQN(Map connectionCache, Map row) {
ConnectionDetails connection = getOrchestratorFromRow(row);
return connectionCache.getOrDefault(connection, null);
}
/**
* Build up details about the lineage on the provided row.
*
* @param connectionCache a cache of connections that have first been resolved across the spreadsheet
* @param row a row of data from the spreadsheet, as a map from column name to value
* @param delim delimiter used in cells that can contain multiple values
* @return the lineage details for that row
*/
public static LineageDetails getFromRow(Map connectionCache, Map row, String delim) {
LineageDetailsBuilder builder = getFromRow(LineageDetails.builder(), row, delim);
if (getMissingFields(row, REQUIRED).isEmpty()) {
String sourceAsset = row.get(COL_S_ASSET);
String sourceType = row.get(COL_S_ASSET_TYPE);
String targetAsset = row.get(COL_T_ASSET);
String targetType = row.get(COL_T_ASSET_TYPE);
// Check if we were given fully-qualified names for the source and/or target assets
// (if so, use them and ignore the connection details)
String sourceAssetQN = null;
String targetAssetQN = null;
if (CONNECTION_QN_PREFIX.matcher(sourceAsset).matches()) {
sourceAssetQN = sourceAsset;
}
if (CONNECTION_QN_PREFIX.matcher(targetAsset).matches()) {
targetAssetQN = targetAsset;
}
// If either of the provided names are not fully-qualified, look up the connections
// to prepend them
if (sourceAssetQN == null) {
String sourceConnectionQN = ConnectionDetails.getQualifiedName(connectionCache, row, COL_S_CONNECTOR, COL_S_CONNECTION);
if (sourceConnectionQN != null) {
sourceAssetQN = sourceConnectionQN + "/" + sourceAsset;
}
}
if (targetAssetQN == null) {
String targetConnectionQN = ConnectionDetails.getQualifiedName(connectionCache, row, COL_T_CONNECTOR, COL_T_CONNECTION);
if (targetConnectionQN != null) {
targetAssetQN = targetConnectionQN + "/" + targetAsset;
}
}
// Then build up the process qualifiedName
String processId = row.get(COL_PROCESS_ID);
String processConnectionQN = getProcessConnectionQN(connectionCache, row);
AssetHeader source = AssetHeader.of(sourceType, sourceAssetQN);
AssetHeader target = AssetHeader.of(targetType, targetAssetQN);
if (source != null && target != null && processConnectionQN != null) {
return builder.sourceAsset(source).targetAsset(target).processConnectionQualifiedName(processConnectionQN).processId(processId).sqlCode(row.get(COL_SQL_CODE)).processUrl(row.get(COL_PROCESS_URL)).stub(false).build();
}
}
return null;
}
/**
* Create lineage processes in bulk, if they do not exist, or update them if they do (idempotent).
*
* @param processes the map of processes to ensure exist, keyed by processQualifiedName
* @param batchSize maximum number of processes to create per batch
*/
public static void upsert(Map> processes, int batchSize) {
AssetBatch batch = new AssetBatch(Atlan.getDefaultClient(), LineageProcess.TYPE_NAME, batchSize);
Map> toTag = new HashMap<>();
long totalResults = processes.size();
long localCount = 0;
try {
for (Map.Entry> entry : processes.entrySet()) {
Set assets = entry.getValue();
Set inputs = new HashSet<>();
Set outputs = new HashSet<>();
Set atlanTagNames = new HashSet<>();
String description = null;
CertificateStatus certificate = null;
String certificateMessage = null;
AtlanAnnouncementType announcementType = null;
String announcementTitle = null;
String announcementMessage = null;
Set ownerUsers = new HashSet<>();
Set ownerGroups = new HashSet<>();
String sqlCode = null;
String processUrl = null;
String processId = null;
String processConnectionQN = null;
for (LineageDetails details : assets) {
processId = details.getProcessId();
processConnectionQN = details.getProcessConnectionQualifiedName();
AssetHeader source = details.getSourceAsset();
AssetHeader target = details.getTargetAsset();
ICatalog input = ICatalog.getLineageReference(source.getTypeName(), source.getQualifiedName());
ICatalog output = ICatalog.getLineageReference(target.getTypeName(), target.getQualifiedName());
inputs.add(input);
outputs.add(output);
if (details.getDescription() != null && !details.getDescription().isEmpty()) {
description = details.getDescription();
}
if (details.getCertificate() != null) {
certificate = details.getCertificate();
}
if (details.getCertificateStatusMessage() != null && !details.getCertificateStatusMessage().isEmpty()) {
certificateMessage = details.getCertificateStatusMessage();
}
if (details.getAnnouncementType() != null) {
announcementType = details.getAnnouncementType();
}
if (details.getAnnouncementTitle() != null && !details.getAnnouncementTitle().isEmpty()) {
announcementTitle = details.getAnnouncementTitle();
}
if (details.getAnnouncementMessage() != null && !details.getAnnouncementMessage().isEmpty()) {
announcementMessage = details.getAnnouncementMessage();
}
if (details.getOwnerUsers() != null) {
ownerUsers.addAll(details.getOwnerUsers());
}
if (details.getOwnerGroups() != null) {
ownerGroups.addAll(details.getOwnerGroups());
}
if (details.getSqlCode() != null && !details.getSqlCode().isEmpty()) {
sqlCode = details.getSqlCode();
}
if (details.getProcessUrl() != null && !details.getProcessUrl().isEmpty()) {
processUrl = details.getProcessUrl();
}
if (details.getAtlanTags() != null) {
atlanTagNames.addAll(details.getAtlanTags());
}
}
if (processConnectionQN != null) {
LineageProcess.LineageProcessBuilder builder = LineageProcess.creator(processId, processConnectionQN, processId, new ArrayList<>(inputs), new ArrayList<>(outputs), null).description(description).certificateStatus(certificate).certificateStatusMessage(certificateMessage).announcementType(announcementType).announcementTitle(announcementTitle).announcementMessage(announcementMessage).ownerUsers(ownerUsers).ownerGroups(ownerGroups).sql(sqlCode).code(sqlCode).sourceURL(processUrl);
LineageProcess process = builder.build();
if (!atlanTagNames.isEmpty()) {
toTag.put(process.getQualifiedName(), new ArrayList<>(atlanTagNames));
}
localCount++;
if (batch.add(process) != null) {
log.info(" ... processed {}/{} ({}%)", localCount, totalResults, Math.round(((double) localCount / totalResults) * 100));
}
}
}
// And don't forget to flush out any that remain
if (batch.flush() != null) {
log.info(" ... processed {}/{} ({}%)", localCount, totalResults, Math.round(((double) localCount / totalResults) * 100));
}
} catch (AtlanException e) {
log.error("Unable to bulk-upsert lineage processes.", e);
}
// Atlan tags must be added in a second pass, after the asset exists
appendAtlanTags(toTag, LineageProcess.TYPE_NAME);
}
public static class AssetHeader {
private String typeName;
private String qualifiedName;
public static AssetHeader of(String typeName, String qualifiedName) {
if (typeName != null && !typeName.isEmpty() && qualifiedName != null && !qualifiedName.isEmpty()) {
return AssetHeader.builder().typeName(typeName).qualifiedName(qualifiedName).build();
}
return null;
}
@java.lang.SuppressWarnings("all")
AssetHeader(final String typeName, final String qualifiedName) {
this.typeName = typeName;
this.qualifiedName = qualifiedName;
}
@java.lang.SuppressWarnings("all")
public static class AssetHeaderBuilder {
@java.lang.SuppressWarnings("all")
private String typeName;
@java.lang.SuppressWarnings("all")
private String qualifiedName;
@java.lang.SuppressWarnings("all")
AssetHeaderBuilder() {
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public LineageDetails.AssetHeader.AssetHeaderBuilder typeName(final String typeName) {
this.typeName = typeName;
return this;
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public LineageDetails.AssetHeader.AssetHeaderBuilder qualifiedName(final String qualifiedName) {
this.qualifiedName = qualifiedName;
return this;
}
@java.lang.SuppressWarnings("all")
public LineageDetails.AssetHeader build() {
return new LineageDetails.AssetHeader(this.typeName, this.qualifiedName);
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public java.lang.String toString() {
return "LineageDetails.AssetHeader.AssetHeaderBuilder(typeName=" + this.typeName + ", qualifiedName=" + this.qualifiedName + ")";
}
}
@java.lang.SuppressWarnings("all")
public static LineageDetails.AssetHeader.AssetHeaderBuilder builder() {
return new LineageDetails.AssetHeader.AssetHeaderBuilder();
}
@java.lang.SuppressWarnings("all")
public String getTypeName() {
return this.typeName;
}
@java.lang.SuppressWarnings("all")
public String getQualifiedName() {
return this.qualifiedName;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public boolean equals(final java.lang.Object o) {
if (o == this) return true;
if (!(o instanceof LineageDetails.AssetHeader)) return false;
final LineageDetails.AssetHeader other = (LineageDetails.AssetHeader) o;
if (!other.canEqual((java.lang.Object) this)) return false;
final java.lang.Object this$typeName = this.getTypeName();
final java.lang.Object other$typeName = other.getTypeName();
if (this$typeName == null ? other$typeName != null : !this$typeName.equals(other$typeName)) return false;
final java.lang.Object this$qualifiedName = this.getQualifiedName();
final java.lang.Object other$qualifiedName = other.getQualifiedName();
if (this$qualifiedName == null ? other$qualifiedName != null : !this$qualifiedName.equals(other$qualifiedName)) return false;
return true;
}
@java.lang.SuppressWarnings("all")
protected boolean canEqual(final java.lang.Object other) {
return other instanceof LineageDetails.AssetHeader;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public int hashCode() {
final int PRIME = 59;
int result = 1;
final java.lang.Object $typeName = this.getTypeName();
result = result * PRIME + ($typeName == null ? 43 : $typeName.hashCode());
final java.lang.Object $qualifiedName = this.getQualifiedName();
result = result * PRIME + ($qualifiedName == null ? 43 : $qualifiedName.hashCode());
return result;
}
}
@java.lang.SuppressWarnings("all")
public static abstract class LineageDetailsBuilder> extends AssetDetails.AssetDetailsBuilder {
@java.lang.SuppressWarnings("all")
private AssetHeader sourceAsset;
@java.lang.SuppressWarnings("all")
private String processConnectionQualifiedName;
@java.lang.SuppressWarnings("all")
private String processId;
@java.lang.SuppressWarnings("all")
private AssetHeader targetAsset;
@java.lang.SuppressWarnings("all")
private String sqlCode;
@java.lang.SuppressWarnings("all")
private String processUrl;
@java.lang.Override
@java.lang.SuppressWarnings("all")
protected abstract B self();
@java.lang.Override
@java.lang.SuppressWarnings("all")
public abstract C build();
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B sourceAsset(final AssetHeader sourceAsset) {
this.sourceAsset = sourceAsset;
return self();
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B processConnectionQualifiedName(final String processConnectionQualifiedName) {
this.processConnectionQualifiedName = processConnectionQualifiedName;
return self();
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B processId(final String processId) {
this.processId = processId;
return self();
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B targetAsset(final AssetHeader targetAsset) {
this.targetAsset = targetAsset;
return self();
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B sqlCode(final String sqlCode) {
this.sqlCode = sqlCode;
return self();
}
/**
* @return {@code this}.
*/
@java.lang.SuppressWarnings("all")
public B processUrl(final String processUrl) {
this.processUrl = processUrl;
return self();
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public java.lang.String toString() {
return "LineageDetails.LineageDetailsBuilder(super=" + super.toString() + ", sourceAsset=" + this.sourceAsset + ", processConnectionQualifiedName=" + this.processConnectionQualifiedName + ", processId=" + this.processId + ", targetAsset=" + this.targetAsset + ", sqlCode=" + this.sqlCode + ", processUrl=" + this.processUrl + ")";
}
}
@java.lang.SuppressWarnings("all")
private static final class LineageDetailsBuilderImpl extends LineageDetails.LineageDetailsBuilder {
@java.lang.SuppressWarnings("all")
private LineageDetailsBuilderImpl() {
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
protected LineageDetails.LineageDetailsBuilderImpl self() {
return this;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public LineageDetails build() {
return new LineageDetails(this);
}
}
@java.lang.SuppressWarnings("all")
protected LineageDetails(final LineageDetails.LineageDetailsBuilder b) {
super(b);
this.sourceAsset = b.sourceAsset;
this.processConnectionQualifiedName = b.processConnectionQualifiedName;
this.processId = b.processId;
this.targetAsset = b.targetAsset;
this.sqlCode = b.sqlCode;
this.processUrl = b.processUrl;
}
@java.lang.SuppressWarnings("all")
public static LineageDetails.LineageDetailsBuilder builder() {
return new LineageDetails.LineageDetailsBuilderImpl();
}
@java.lang.SuppressWarnings("all")
public AssetHeader getSourceAsset() {
return this.sourceAsset;
}
@java.lang.SuppressWarnings("all")
public String getProcessConnectionQualifiedName() {
return this.processConnectionQualifiedName;
}
@java.lang.SuppressWarnings("all")
public String getProcessId() {
return this.processId;
}
@java.lang.SuppressWarnings("all")
public AssetHeader getTargetAsset() {
return this.targetAsset;
}
@java.lang.SuppressWarnings("all")
public String getSqlCode() {
return this.sqlCode;
}
@java.lang.SuppressWarnings("all")
public String getProcessUrl() {
return this.processUrl;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public boolean equals(final java.lang.Object o) {
if (o == this) return true;
if (!(o instanceof LineageDetails)) return false;
final LineageDetails other = (LineageDetails) o;
if (!other.canEqual((java.lang.Object) this)) return false;
final java.lang.Object this$sourceAsset = this.getSourceAsset();
final java.lang.Object other$sourceAsset = other.getSourceAsset();
if (this$sourceAsset == null ? other$sourceAsset != null : !this$sourceAsset.equals(other$sourceAsset)) return false;
final java.lang.Object this$processConnectionQualifiedName = this.getProcessConnectionQualifiedName();
final java.lang.Object other$processConnectionQualifiedName = other.getProcessConnectionQualifiedName();
if (this$processConnectionQualifiedName == null ? other$processConnectionQualifiedName != null : !this$processConnectionQualifiedName.equals(other$processConnectionQualifiedName)) return false;
final java.lang.Object this$processId = this.getProcessId();
final java.lang.Object other$processId = other.getProcessId();
if (this$processId == null ? other$processId != null : !this$processId.equals(other$processId)) return false;
final java.lang.Object this$targetAsset = this.getTargetAsset();
final java.lang.Object other$targetAsset = other.getTargetAsset();
if (this$targetAsset == null ? other$targetAsset != null : !this$targetAsset.equals(other$targetAsset)) return false;
final java.lang.Object this$sqlCode = this.getSqlCode();
final java.lang.Object other$sqlCode = other.getSqlCode();
if (this$sqlCode == null ? other$sqlCode != null : !this$sqlCode.equals(other$sqlCode)) return false;
final java.lang.Object this$processUrl = this.getProcessUrl();
final java.lang.Object other$processUrl = other.getProcessUrl();
if (this$processUrl == null ? other$processUrl != null : !this$processUrl.equals(other$processUrl)) return false;
return true;
}
@java.lang.SuppressWarnings("all")
protected boolean canEqual(final java.lang.Object other) {
return other instanceof LineageDetails;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public int hashCode() {
final int PRIME = 59;
int result = 1;
final java.lang.Object $sourceAsset = this.getSourceAsset();
result = result * PRIME + ($sourceAsset == null ? 43 : $sourceAsset.hashCode());
final java.lang.Object $processConnectionQualifiedName = this.getProcessConnectionQualifiedName();
result = result * PRIME + ($processConnectionQualifiedName == null ? 43 : $processConnectionQualifiedName.hashCode());
final java.lang.Object $processId = this.getProcessId();
result = result * PRIME + ($processId == null ? 43 : $processId.hashCode());
final java.lang.Object $targetAsset = this.getTargetAsset();
result = result * PRIME + ($targetAsset == null ? 43 : $targetAsset.hashCode());
final java.lang.Object $sqlCode = this.getSqlCode();
result = result * PRIME + ($sqlCode == null ? 43 : $sqlCode.hashCode());
final java.lang.Object $processUrl = this.getProcessUrl();
result = result * PRIME + ($processUrl == null ? 43 : $processUrl.hashCode());
return result;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
public java.lang.String toString() {
return "LineageDetails(super=" + super.toString() + ", sourceAsset=" + this.getSourceAsset() + ", processConnectionQualifiedName=" + this.getProcessConnectionQualifiedName() + ", processId=" + this.getProcessId() + ", targetAsset=" + this.getTargetAsset() + ", sqlCode=" + this.getSqlCode() + ", processUrl=" + this.getProcessUrl() + ")";
}
}
