• Home
• io.hydrosphere
• mist-lib_2.12
• 1.1.3
• source code
• DatasetEncoding.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

mist.api.encoding.spark.DatasetEncoding.scala Maven / Gradle / Ivy

package mist.api.encoding.spark

import mist.api.data._
import mist.api.encoding.JsEncoder
import org.apache.spark.sql.Dataset

trait DatasetEncoding {

  implicit val datasetEncoder: JsEncoder[Dataset[_]] = JsEncoder(ds => {
    val rowEncoder = new SchemedRowEncoder(ds.schema)

    val rdd = ds.queryExecution.toRdd.mapPartitions(iter => {
      new Iterator[JsData] {
        override def hasNext: Boolean = iter.hasNext
        override def next(): JsData = {
          val row = iter.next()
          rowEncoder.encode(row)
        }
      }
    })

    JsList(rdd.collect().toSeq)
  })
}

object DatasetEncoding extends DatasetEncoding