spark.jobserver.SqlTestJob.scala Maven / Gradle / Ivy
package spark.jobserver
import com.typesafe.config.Config
import org.apache.spark.sql.SQLContext
/**
* A test job that accepts a SQLContext, as opposed to the regular SparkContext.
* Just initializes some dummy data into a table.
*/
object SqlLoaderJob extends SparkSqlJob {
case class Address(firstName: String, lastName: String, street: String, city: String)
val addresses = Seq(
Address("Bob", "Charles", "101 A St.", "San Jose"),
Address("Sandy", "Charles", "10200 Ranch Rd.", "Purple City"),
Address("Randy", "Charles", "101 A St.", "San Jose")
)
def validate(sql: SQLContext, config: Config): SparkJobValidation = SparkJobValid
def runJob(sql: SQLContext, config: Config): Any = {
import sql.implicits._
val addrRdd = sql.sparkContext.parallelize(addresses)
addrRdd.toDF().registerTempTable("addresses")
addrRdd.count()
}
}
/**
* This job simply runs the SQL in the config.
*/
object SqlTestJob extends SparkSqlJob {
def validate(sql: SQLContext, config: Config): SparkJobValidation = SparkJobValid
def runJob(sql: SQLContext, config: Config): Any = {
sql.sql(config.getString("sql")).collect()
}
}