icu.wuhufly.dws.recent_state08.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of shtd-bd Show documentation

bigdata source code for shtd

The newest version!

package icu.wuhufly.dws

import icu.wuhufly.utils.{CreateUtils, WriteUtils}
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrame, SparkSession}

object recent_state08 {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = CreateUtils.getSpark()
    val sc: SparkContext = spark.sparkContext
    import spark.implicits._
    import org.apache.spark.sql.functions._

    spark.sql("use dwd")
    val df: DataFrame = spark.sql(
      """
        |select machine_id, record_state, change_start_time, change_end_time
        |from (
        |select machine_id, record_state, change_start_time, change_end_time, rn,
        |  max(rn) over(partition by machine_id) as max
        |from (
        |select ChangeMachineID as machine_id, ChangeRecordState as record_state, ChangeStartTime as change_start_time,
        |  ChangeEndTime as change_end_time,
        |  row_number() over(partition by ChangeMachineID order by ChangeStartTime desc) as rn
        |  from fact_change_record
        |) t1
        |) t1
        | where (rn = 2 and max > 1)
        |  or (max = 1 and rn = 1)
        |""".stripMargin)

    WriteUtils.writeToCK(
      "recent_state", df, "machine_id"
    )

    sc.stop()
  }
}