• Home
• tech.ytsaurus
• spark-yt-data-source-base_2.12
• 2.3.0
• source code
• SparkYtWriteConfiguration.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

tech.ytsaurus.spyt.format.conf.SparkYtWriteConfiguration.scala Maven / Gradle / Ivy

package tech.ytsaurus.spyt.format.conf

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.internal.SQLConf
import tech.ytsaurus.spyt.fs.conf._

import scala.concurrent.duration.Duration

case class SparkYtWriteConfiguration(miniBatchSize: Int,
                                     batchSize: Int,
                                     dynBatchSize: Int,
                                     timeout: Duration,
                                     typeV3Format: Boolean)

object SparkYtWriteConfiguration {

  import SparkYtConfiguration._

  def apply(sqlc: SQLContext): SparkYtWriteConfiguration = SparkYtWriteConfiguration(
    sqlc.ytConf(Write.MiniBatchSize),
    sqlc.ytConf(Write.BatchSize),
    sqlc.ytConf(Write.DynBatchSize),
    sqlc.ytConf(Write.Timeout),
    sqlc.ytConf(Write.TypeV3)
  )

  def apply(sqlc: SQLConf): SparkYtWriteConfiguration = SparkYtWriteConfiguration(
    sqlc.ytConf(Write.MiniBatchSize),
    sqlc.ytConf(Write.BatchSize),
    sqlc.ytConf(Write.DynBatchSize),
    sqlc.ytConf(Write.Timeout),
    sqlc.ytConf(Write.TypeV3)
  )
}