• Home
• com.twitter
• scalding-spark_2.12
• 0.18.0-RC1
• source code
• SparkMode.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.twitter.scalding.spark_backend.SparkMode.scala Maven / Gradle / Ivy

package com.twitter.scalding.spark_backend

import com.twitter.scalding.{ Config, Mode }
import com.twitter.scalding.typed.{ Resolver, TypedSource, TypedSink }
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import scala.concurrent.{ Future, ExecutionContext }

case class SparkMode(session: SparkSession, sources: Resolver[TypedSource, SparkSource], sink: Resolver[TypedSink, SparkSink]) extends Mode {
  def newWriter(): SparkWriter =
    new SparkWriter(this)
}

object SparkMode {
  def empty(session: SparkSession): SparkMode =
    SparkMode(session, Resolver.empty, Resolver.empty)
}

trait SparkSource[+A] {
  def read(session: SparkSession, config: Config)(implicit ec: ExecutionContext): Future[RDD[_ <: A]]
}

trait SparkSink[-A] {
  def write(session: SparkSession, config: Config, rdd: RDD[_ <: A])(implicit ec: ExecutionContext): Future[Unit]
}