org.apache.spark.streaming.ui.BatchPage.scala Maven / Gradle / Ivy

Go to download
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.streaming.ui

import javax.servlet.http.HttpServletRequest

import scala.xml._

import org.apache.commons.lang3.StringEscapeUtils

import org.apache.spark.streaming.Time
import org.apache.spark.streaming.ui.StreamingJobProgressListener.SparkJobId
import org.apache.spark.ui.{UIUtils => SparkUIUtils, WebUIPage}
import org.apache.spark.ui.jobs.UIData.JobUIData

private[ui] case class SparkJobIdWithUIData(sparkJobId: SparkJobId, jobUIData: Option[JobUIData])

private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") {
  private val streamingListener = parent.listener
  private val sparkListener = parent.ssc.sc.jobProgressListener

  private def columns: Seq[Node] = {
    Output Op Id
      Description
      Output Op Duration
      Status
      Job Id
      Job Duration
      Stages: Succeeded/Total
      Tasks (for all stages): Succeeded/Total
      Error
  }

  private def generateJobRow(
      outputOpData: OutputOperationUIData,
      outputOpDescription: Seq[Node],
      formattedOutputOpDuration: String,
      numSparkJobRowsInOutputOp: Int,
      isFirstRow: Boolean,
      sparkJob: SparkJobIdWithUIData): Seq[Node] = {
    if (sparkJob.jobUIData.isDefined) {
      generateNormalJobRow(outputOpData, outputOpDescription, formattedOutputOpDuration,
        numSparkJobRowsInOutputOp, isFirstRow, sparkJob.jobUIData.get)
    } else {
      generateDroppedJobRow(outputOpData, outputOpDescription, formattedOutputOpDuration,
        numSparkJobRowsInOutputOp, isFirstRow, sparkJob.sparkJobId)
    }
  }

  private def generateOutputOpRowWithoutSparkJobs(
    outputOpData: OutputOperationUIData,
    outputOpDescription: Seq[Node],
    formattedOutputOpDuration: String): Seq[Node] = {
    
      {outputOpData.id.toString}
      {outputOpDescription}
      {formattedOutputOpDuration}
      {outputOpStatusCell(outputOpData, rowspan = 1)}
      
      -
      
      -
      
      -
      
      -
      
      -
    
  }

  /**
   * Generate a row for a Spark Job. Because duplicated output op infos needs to be collapsed into
   * one cell, we use "rowspan" for the first row of an output op.
   */
  private def generateNormalJobRow(
      outputOpData: OutputOperationUIData,
      outputOpDescription: Seq[Node],
      formattedOutputOpDuration: String,
      numSparkJobRowsInOutputOp: Int,
      isFirstRow: Boolean,
      sparkJob: JobUIData): Seq[Node] = {
    val duration: Option[Long] = {
      sparkJob.submissionTime.map { start =>
        val end = sparkJob.completionTime.getOrElse(System.currentTimeMillis())
        end - start
      }
    }
    val lastFailureReason =
      sparkJob.stageIds.sorted.reverse.flatMap(sparkListener.stageIdToInfo.get).
      dropWhile(_.failureReason == None).take(1). // get the first info that contains failure
      flatMap(info => info.failureReason).headOption.getOrElse("")
    val formattedDuration = duration.map(d => SparkUIUtils.formatDuration(d)).getOrElse("-")
    val detailUrl = s"${SparkUIUtils.prependBaseUri(parent.basePath)}/jobs/job?id=${sparkJob.jobId}"

    // In the first row, output op id and its information needs to be shown. In other rows, these
    // cells will be taken up due to "rowspan".
    // scalastyle:off
    val prefixCells =
      if (isFirstRow) {
        {outputOpData.id.toString}
        
          {outputOpDescription}
        
        {formattedOutputOpDuration} ++
        {outputOpStatusCell(outputOpData, numSparkJobRowsInOutputOp)}
      } else {
        Nil
      }
    // scalastyle:on

    
      {prefixCells}
      
        
          {sparkJob.jobId}{sparkJob.jobGroup.map(id => s"($id)").getOrElse("")}
        
      
      
        {formattedDuration}
      
      
        {sparkJob.completedStageIndices.size}/{sparkJob.stageIds.size - sparkJob.numSkippedStages}
        {if (sparkJob.numFailedStages > 0) s"(${sparkJob.numFailedStages} failed)"}
        {if (sparkJob.numSkippedStages > 0) s"(${sparkJob.numSkippedStages} skipped)"}
      
      
        {
          SparkUIUtils.makeProgressBar(
            started = sparkJob.numActiveTasks,
            completed = sparkJob.numCompletedTasks,
            failed = sparkJob.numFailedTasks,
            skipped = sparkJob.numSkippedTasks,
            total = sparkJob.numTasks - sparkJob.numSkippedTasks)
        }
      
      {UIUtils.failureReasonCell(lastFailureReason)}
    
  }

  /**
   * If a job is dropped by sparkListener due to exceeding the limitation, we only show the job id
   * with "-" cells.
   */
  private def generateDroppedJobRow(
      outputOpData: OutputOperationUIData,
      outputOpDescription: Seq[Node],
      formattedOutputOpDuration: String,
      numSparkJobRowsInOutputOp: Int,
      isFirstRow: Boolean,
      jobId: Int): Seq[Node] = {
    // In the first row, output op id and its information needs to be shown. In other rows, these
    // cells will be taken up due to "rowspan".
    // scalastyle:off
    val prefixCells =
      if (isFirstRow) {
        {outputOpData.id.toString}
          {outputOpDescription}
          {formattedOutputOpDuration} ++
          {outputOpStatusCell(outputOpData, numSparkJobRowsInOutputOp)}
      } else {
        Nil
      }
    // scalastyle:on

    
      {prefixCells}
      
        {if (jobId >= 0) jobId.toString else "-"}
      
      
      -
      
      -
      
      -
      
      -
    
  }

  private def generateOutputOpIdRow(
      outputOpData: OutputOperationUIData,
      sparkJobs: Seq[SparkJobIdWithUIData]): Seq[Node] = {
    val formattedOutputOpDuration =
      if (outputOpData.duration.isEmpty) {
        "-"
      } else {
        SparkUIUtils.formatDuration(outputOpData.duration.get)
      }

    val description = generateOutputOpDescription(outputOpData)

    if (sparkJobs.isEmpty) {
      generateOutputOpRowWithoutSparkJobs(outputOpData, description, formattedOutputOpDuration)
    } else {
      val firstRow =
        generateJobRow(
          outputOpData,
          description,
          formattedOutputOpDuration,
          sparkJobs.size,
          true,
          sparkJobs.head)
      val tailRows =
        sparkJobs.tail.map { sparkJob =>
          generateJobRow(
            outputOpData,
            description,
            formattedOutputOpDuration,
            sparkJobs.size,
            false,
            sparkJob)
        }
      (firstRow ++ tailRows).flatten
    }
  }

  private def generateOutputOpDescription(outputOp: OutputOperationUIData): Seq[Node] = {
    
      {outputOp.name}
      
          +details
      
      
        {outputOp.description}
      
    
  }

  private def getJobData(sparkJobId: SparkJobId): Option[JobUIData] = {
    sparkListener.activeJobs.get(sparkJobId).orElse {
      sparkListener.completedJobs.find(_.jobId == sparkJobId).orElse {
        sparkListener.failedJobs.find(_.jobId == sparkJobId)
      }
    }
  }

  private def generateOutputOperationStatusForUI(failure: String): String = {
    if (failure.startsWith("org.apache.spark.SparkException")) {
      "Failed due to Spark job error\n" + failure
    } else {
      var nextLineIndex = failure.indexOf("\n")
      if (nextLineIndex < 0) {
        nextLineIndex = failure.length
      }
      val firstLine = failure.substring(0, nextLineIndex)
      s"Failed due to error: $firstLine\n$failure"
    }
  }

  /**
   * Generate the job table for the batch.
   */
  private def generateJobTable(batchUIData: BatchUIData): Seq[Node] = {
    val outputOpIdToSparkJobIds = batchUIData.outputOpIdSparkJobIdPairs.groupBy(_.outputOpId).
      map { case (outputOpId, outputOpIdAndSparkJobIds) =>
        // sort SparkJobIds for each OutputOpId
        (outputOpId, outputOpIdAndSparkJobIds.map(_.sparkJobId).toSeq.sorted)
      }

    val outputOps: Seq[(OutputOperationUIData, Seq[SparkJobId])] =
      batchUIData.outputOperations.map { case (outputOpId, outputOperation) =>
        val sparkJobIds = outputOpIdToSparkJobIds.getOrElse(outputOpId, Seq.empty)
        (outputOperation, sparkJobIds)
      }.toSeq.sortBy(_._1.id)
    sparkListener.synchronized {
      val outputOpWithJobs = outputOps.map { case (outputOpData, sparkJobIds) =>
          (outputOpData,
            sparkJobIds.map(sparkJobId => SparkJobIdWithUIData(sparkJobId, getJobData(sparkJobId))))
        }

      
          {columns}
        
          {
            outputOpWithJobs.map { case (outputOpData, sparkJobIds) =>
              generateOutputOpIdRow(outputOpData, sparkJobIds)
            }
          }
        
        
        
      
    }
  }

  def render(request: HttpServletRequest): Seq[Node] = streamingListener.synchronized {
    val batchTime = Option(request.getParameter("id")).map(id => Time(id.toLong)).getOrElse {
      throw new IllegalArgumentException(s"Missing id parameter")
    }
    val formattedBatchTime =
      UIUtils.formatBatchTime(batchTime.milliseconds, streamingListener.batchDuration)

    val batchUIData = streamingListener.getBatchUIData(batchTime).getOrElse {
      throw new IllegalArgumentException(s"Batch $formattedBatchTime does not exist")
    }

    val formattedSchedulingDelay =
      batchUIData.schedulingDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
    val formattedProcessingTime =
      batchUIData.processingDelay.map(SparkUIUtils.formatDuration).getOrElse("-")
    val formattedTotalDelay = batchUIData.totalDelay.map(SparkUIUtils.formatDuration).getOrElse("-")

    val inputMetadatas = batchUIData.streamIdToInputInfo.values.flatMap { inputInfo =>
      inputInfo.metadataDescription.map(desc => inputInfo.inputStreamId -> desc)
    }.toSeq
    val summary: NodeSeq =
      
        
          
            Batch Duration: 
            {SparkUIUtils.formatDuration(streamingListener.batchDuration)}
          
          
            Input data size: 
            {batchUIData.numRecords} records
          
          
            Scheduling delay: 
            {formattedSchedulingDelay}
          
          
            Processing time: 
            {formattedProcessingTime}
          
          
            Total delay: 
            {formattedTotalDelay}
          
          {
            if (inputMetadatas.nonEmpty) {
              
                Input Metadata:{generateInputMetadataTable(inputMetadatas)}
              
            }
          }
        
      

    val content = summary ++ generateJobTable(batchUIData)

    SparkUIUtils.headerSparkPage(s"Details of batch at $formattedBatchTime", content, parent)
  }

  def generateInputMetadataTable(inputMetadatas: Seq[(Int, String)]): Seq[Node] = {
    
        {inputMetadatas.flatMap(generateInputMetadataRow)}
      
      
        
          Input
          Metadata
        
      
      
    
  }

  def generateInputMetadataRow(inputMetadata: (Int, String)): Seq[Node] = {
    val streamId = inputMetadata._1

    
      {streamingListener.streamName(streamId).getOrElse(s"Stream-$streamId")}
      {metadataDescriptionToHTML(inputMetadata._2)}
    
  }

  private def metadataDescriptionToHTML(metadataDescription: String): Seq[Node] = {
    // tab to 4 spaces and "\n" to "
"
    Unparsed(StringEscapeUtils.escapeHtml4(metadataDescription).
      replaceAllLiterally("\t", "    ").replaceAllLiterally("\n", "
"))
  }

  private def outputOpStatusCell(outputOp: OutputOperationUIData, rowspan: Int): Seq[Node] = {
    outputOp.failureReason match {
      case Some(failureReason) =>
        val failureReasonForUI = UIUtils.createOutputOperationFailureForUI(failureReason)
        UIUtils.failureReasonCell(
          failureReasonForUI, rowspan, includeFirstLineInExpandDetails = false)
      case None =>
        if (outputOp.endTime.isEmpty) {
          -
        } else {
          Succeeded
        }
    }
  }
}