All Downloads are FREE. Search and download functionalities are using the official Maven repository.

bio.ferlab.datalake.spark3.transformation.Coalesce.scala Maven / Gradle / Ivy

There is a newer version: 14.8.0
Show newest version
package bio.ferlab.datalake.spark3.transformation
import org.apache.spark.sql.functions.coalesce
import org.apache.spark.sql.{Column, DataFrame}

case class Coalesce(values: Seq[Column], columns: String*) extends Transformation {
  /**
   * Main method of the trait.
   * It defines the logic to transform the input dataframe.
   *
   * @return a transformed dataframe
   */
  override def transform: DataFrame => DataFrame = { df =>
    columns.foldLeft(df){ case (d, c) => d.withColumn(c, coalesce(values:_*))}
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy