datahub.spark.conf.SparkLineageConf Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of acryl-spark-lineage Show documentation

Library to push data lineage from spark to datahub

There is a newer version: 0.2.17

package datahub.spark.conf;

import com.typesafe.config.Config;
import io.datahubproject.openlineage.config.DatahubOpenlineageConfig;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;

@Getter
@Builder
@Setter
public class SparkLineageConf {
  final DatahubOpenlineageConfig openLineageConf;
  @Builder.Default final boolean coalesceEnabled = true;
  @Builder.Default final boolean emitCoalescePeriodically = false;
  final SparkAppContext sparkAppContext;
  final DatahubEmitterConfig datahubEmitterConfig;
  @Builder.Default final List tags = new LinkedList<>();

  @Builder.Default final List domains = new LinkedList<>();

  public static SparkLineageConf toSparkLineageConf(
      Config sparkConfig,
      SparkAppContext sparkAppContext,
      DatahubEmitterConfig datahubEmitterConfig) {
    SparkLineageConfBuilder builder = SparkLineageConf.builder();
    DatahubOpenlineageConfig datahubOpenlineageConfig =
        SparkConfigParser.sparkConfigToDatahubOpenlineageConf(sparkConfig, sparkAppContext);
    builder.openLineageConf(datahubOpenlineageConfig);
    builder.coalesceEnabled(SparkConfigParser.isCoalesceEnabled(sparkConfig));
    if (SparkConfigParser.getTags(sparkConfig) != null) {
      builder.tags(Arrays.asList(Objects.requireNonNull(SparkConfigParser.getTags(sparkConfig))));
    }

    if (SparkConfigParser.getDomains(sparkConfig) != null) {
      builder.domains(
          Arrays.asList(Objects.requireNonNull(SparkConfigParser.getDomains(sparkConfig))));
    }

    builder.emitCoalescePeriodically(SparkConfigParser.isEmitCoalescePeriodically(sparkConfig));
    if (sparkAppContext != null) {
      builder.sparkAppContext(sparkAppContext);
    }

    if (datahubEmitterConfig != null) {
      builder.datahubEmitterConfig = datahubEmitterConfig;
    }
    return builder.build();
  }
}