bio.ferlab.datalake.spark3.transformation.HashTransformation.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of datalake-spark3_2.12 Show documentation

Library built on top of Apache Spark to speed-up data lakes development..

There is a newer version: 14.8.0

package bio.ferlab.datalake.spark3.transformation

import org.apache.spark.sql.{Column, DataFrame}
import org.apache.spark.sql.functions.lit
import org.apache.spark.sql.types.StringType

sealed trait HashTransformation[A] extends Transformation {self =>
  val columns: A
  val nullValues: Column = lit(null).cast(StringType)
}

object HashTransformation {
  trait SimpleHashTransformation extends HashTransformation[Seq[String]]
  trait DynamicHashTransformation extends HashTransformation[DataFrame => Seq[String]]
}