com.alpine.plugin.test.utils.TestSparkContexts.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of plugin-test Show documentation

plugin-test

There is a newer version: 1.6-beta-2

package com.alpine.plugin.test.utils

import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkContext
import org.apache.spark.sql.{SQLContext, SparkSession}

object TestSparkContexts {

  lazy val sparkSession: SparkSession = {
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("akka").setLevel(Level.WARN)
    SparkSession.builder().appName("udf testings")
        .master("local")
        .config("spark.ui.enabled", value = false)
        .getOrCreate()
  }

  lazy val sc: SparkContext = sparkSession.sparkContext

  @deprecated("use spark session directly")
  lazy val sqlContext: SQLContext =sparkSession.sqlContext

}