jove.spark.SparkKernel.scala Maven / Gradle / Ivy

Go to download
package jove
package spark

import java.io.File
import jove.scala.ScalaModuleID
import jove.kernel.helpers.{ BuildInfo => MainBuildInfo }
import jove.spark.helpers.BuildInfo
import org.apache.spark.repl.JoveSparkIMain

import scalaz.{-\/, \/-}
import jove.sbt.{MavenRepository, ModuleID, DefaultMavenRepository, Resolver}
import jove.scala.imainlike.IMainKernel

object SparkKernel extends IMainKernel {
  // FIXME Move to config file
  val scalaVersion = Some(MainBuildInfo.scalaVersion)
  // Forcing the scala libs version because of binary incompatibilities between them and spark-repl
  // in case of non exact match
  val forceScalaVersion = true
  val resolvers = Seq(
    Resolver.sonatypeRepo("snapshots"),
    Resolver.defaultLocal,
    DefaultMavenRepository,
    Resolver.sonatypeRepo("releases"),
    MavenRepository("Apache Spark 1.3 Snapshot 1", "https://repository.apache.org/content/repositories/orgapachespark-1068")
  )
  val dependencies = {
    var l = List(
      \/-(ModuleID("org.scala-lang", "scala-library", MainBuildInfo.scalaVersion, Some("compile"))),
      -\/(ScalaModuleID("sh.jove", "jove-bridge", "0.1.0-SNAPSHOT")),
      -\/(ScalaModuleID("org.apache.spark", "spark-repl", jove.spark.helpers.BuildInfo.sparkVersion))
    )

    if (MainBuildInfo.scalaVersion startsWith "2.11.")
      l ::= -\/(ScalaModuleID("com.lihaoyi", "ammonite-repl", MainBuildInfo.ammoniteVersion))

    l
  }

  override def rootClassLoader: ClassLoader = classOf[org.apache.spark.SparkContext].getClassLoader

  def imain(classPath: Seq[File], classLoader: ClassLoader, error: String => Unit) =
    JoveSparkIMain.withSparkContext(classLoader, Some(classPath.map(_.getAbsolutePath) mkString File.pathSeparator), error)
}