bio.ferlab.datalake.spark3.transformation.When.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of datalake-spark3_2.12 Show documentation

Library built on top of Apache Spark to speed-up data lakes development..

There is a newer version: 14.8.0

package bio.ferlab.datalake.spark3.transformation

import org.apache.spark.sql.{Column, DataFrame}
import org.apache.spark.sql.functions._

case class When(column: String, conditions: List[(Column, Any)], otherwise: Any) extends Transformation {
  override def transform: DataFrame => DataFrame = { df =>

    //step 1 initial
    val initialWhen = when(conditions.head._1, conditions.head._2)

    //step 2 add all the when() together
    val whens: Column =
      conditions.tail.foldLeft(initialWhen) {
        case (previousWhen, (currentCondition, currentValue)) =>
          previousWhen.when(currentCondition, currentValue)
      }

    //step 3 add otherwise() to all the whens()
    val whensWithOtherwise = whens.otherwise(otherwise)

      //step 4 create a column with the when statement
    df.withColumn(column, whensWithOtherwise)
  }
}