tech.ytsaurus.spyt.fs.YtClientConfigurationConverter.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of spark-yt-file-system_2.12 Show documentation

Spark over YTsaurus

The newest version!

package tech.ytsaurus.spyt.fs

import org.apache.hadoop.conf.Configuration
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.internal.SQLConf
import tech.ytsaurus.spyt.fs.conf._
import tech.ytsaurus.spyt.wrapper.client.YtClientConfiguration

object YtClientConfigurationConverter {
  def ytClientConfiguration(spark: SparkSession): YtClientConfiguration = {
    ytClientConfiguration(spark.sparkContext.hadoopConfiguration)
  }

  def ytClientConfiguration(conf: Configuration): YtClientConfiguration = {
    YtClientConfiguration(conf.getYtConf(_))
  }

  def ytClientConfiguration(conf: SparkConf): YtClientConfiguration = {
    ytClientConfiguration(hadoopConf(conf.getAll))
  }

  def ytClientConfiguration(sqlConf: SQLConf): YtClientConfiguration = {
    ytClientConfiguration(hadoopConf(sqlConf.getAllConfs.toArray))
  }

  private def hadoopConf(conf: Array[(String, String)]): Configuration = {
    val hadoopConf = new Configuration()
    for ((key, value) <- conf if key.startsWith("spark.hadoop.")) {
      hadoopConf.set(key.substring("spark.hadoop.".length), value)
    }
    hadoopConf
  }
}