• Home
• io.hydrosphere
• mist-lib_2.12
• 1.1.3
• source code
• DataFrameEncoding.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

mist.api.encoding.spark.DataFrameEncoding.scala Maven / Gradle / Ivy

package mist.api.encoding.spark

import mist.api.data.{JsData, JsList}
import mist.api.encoding.JsEncoder
import org.apache.spark.sql.DataFrame

trait DataFrameEncoding {

  implicit val dataFrameEncoder: JsEncoder[DataFrame] = JsEncoder(df => {
    val rowEncoder = new SchemedRowEncoder(df.schema)
    val x = df.queryExecution.toRdd.mapPartitions(iter => {
      new Iterator[JsData] {
        override def hasNext: Boolean = iter.hasNext
        override def next(): JsData = {
          val row = iter.next()
          rowEncoder.encode(row)
        }
      }
    });
    JsList(x.collect().toSeq)
  })

}

object DataFrameEncoding extends DataFrameEncoding