org.apache.spark.dataflint.listener.model.scala Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of spark_2.13 Show documentation
Open Source Data-Application Performance Monitoring for Apache Spark
The newest version!
package org.apache.spark.dataflint.listener

import com.fasterxml.jackson.annotation.JsonIgnore
import org.apache.spark.scheduler.SparkListenerEvent
import org.apache.spark.util.kvstore.KVIndex

case class DatabricksAdditionalExecutionInfo(
                       executionId: Long,
                       version: Long,
                       nodeIdToRddScopeId: Map[Long, String]) {
}

case class DatabricksAdditionalExecutionEvent(databricksAdditionalExecutionInfo: DatabricksAdditionalExecutionInfo) extends SparkListenerEvent

class DatabricksAdditionalExecutionWrapper(val info: DatabricksAdditionalExecutionInfo) {
  @KVIndex
  @JsonIgnore
  def id: Long = info.executionId
}

case class IcebergCommitMetrics(
                                 durationMS: Long,
                                 attempts: Long,
                                 addedDataFiles: Long,
                                 removedDataFiles: Long,
                                 totalDataFiles: Long,
                                 addedDeleteFiles: Long,
                                 addedEqualityDeleteFiles: Long,
                                 addedPositionalDeleteFiles: Long,
                                 removedDeleteFiles: Long,
                                 removedEqualityDeleteFiles: Long,
                                 removedPositionalDeleteFiles: Long,
                                 totalDeleteFiles: Long,
                                 addedRecords: Long,
                                 removedRecords: Long,
                                 totalRecords: Long,
                                 addedFilesSizeInBytes: Long,
                                 removedFilesSizeInBytes: Long,
                                 totalFilesSizeInBytes: Long,
                                 addedPositionalDeletes: Long,
                                 removedPositionalDeletes: Long,
                                 totalPositionalDeletes: Long,
                                 addedEqualityDeletes: Long,
                                 removedEqualityDeletes: Long,
                                 totalEqualityDeletes: Long
                               )

case class IcebergCommitInfo(executionId: Int, tableName: String, commitId: Long, operation: String, metrics: IcebergCommitMetrics )

case class IcebergCommitEvent(icebergCommit: IcebergCommitInfo) extends SparkListenerEvent

class IcebergCommitWrapper(val info: IcebergCommitInfo) {

  @JsonIgnore @KVIndex
  def id: String = info.executionId.toString
}