bio.ferlab.datalake.spark3.transformation.InputFileName.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of datalake-spark3_2.12 Show documentation

Library built on top of Apache Spark to speed-up data lakes development..

There is a newer version: 14.8.0

package bio.ferlab.datalake.spark3.transformation

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions._

case class InputFileName(columnName: String, regex: Option[String] = None) extends Transformation {
  override def transform: DataFrame => DataFrame = { df =>
    regex.fold(
      df.withColumn(columnName, input_file_name())
    )(rg =>
      df.withColumn(columnName, regexp_extract(input_file_name(), rg, 1))
    )
  }
}