• Home
• tech.mlsql
• mlsql-ets-3.0_2.12
• 2.1.0
• source code
• StreamJobs.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

tech.mlsql.plugins.ets.StreamJobs.scala Maven / Gradle / Ivy

package tech.mlsql.plugins.ets

import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.expressions.UserDefinedFunction
import streaming.dsl.mmlib.SQLAlg
import streaming.dsl.mmlib.algs.param.WowParams

/**
 * 14/12/2020 WilliamZhu([email protected])
 */
class StreamJobs (override val uid: String) extends SQLAlg with WowParams {
  def this() = this(Identifiable.randomUID("tech.mlsql.plugins.ets.StreamJobs"))

  override def train(df: DataFrame, path: String, params: Map[String, String]): DataFrame = {
    val names = df.sparkSession.streams.active.map(item=>item.name)
    import df.sparkSession.implicits._
    df.sparkSession.createDataset[String](names).toDF("values")
  }

  override def batchPredict(df: DataFrame, path: String, params: Map[String, String]): DataFrame = train(df, path, params)

  override def load(sparkSession: SparkSession, path: String, params: Map[String, String]): Any = ???

  override def predict(sparkSession: SparkSession, _model: Any, name: String, params: Map[String, String]): UserDefinedFunction = ???
}