All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.spark.ui.jobs.StageTable.scala Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.ui.jobs

import java.util.Date

import scala.xml.Node
import scala.collection.mutable.HashSet

import org.apache.spark.scheduler.{SchedulingMode, StageInfo, TaskInfo}
import org.apache.spark.ui.UIUtils
import org.apache.spark.util.Utils


/** Page showing list of all ongoing and recently finished stages */
private[spark] class StageTable(val stages: Seq[StageInfo], val parent: JobProgressUI) {

  val listener = parent.listener
  val dateFmt = parent.dateFmt
  val isFairScheduler = listener.sc.getSchedulingMode == SchedulingMode.FAIR

  def toNodeSeq(): Seq[Node] = {
    listener.synchronized {
      stageTable(stageRow, stages)
    }
  }

  /** Special table which merges two header cells. */
  private def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]): Seq[Node] = {
    
        {if (isFairScheduler) {} else {}}
        
        {rows.map(r => makeRow(r))}
      
Stage IdPool NameDescription Submitted Duration Tasks: Succeeded/Total Shuffle Read Shuffle Write
} private def makeProgressBar(started: Int, completed: Int, failed: String, total: Int): Seq[Node] = { val completeWidth = "width: %s%%".format((completed.toDouble/total)*100) val startWidth = "width: %s%%".format((started.toDouble/total)*100)
{completed}/{total} {failed}
} private def stageRow(s: StageInfo): Seq[Node] = { val submissionTime = s.submissionTime match { case Some(t) => dateFmt.format(new Date(t)) case None => "Unknown" } val shuffleReadSortable = listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L) val shuffleRead = shuffleReadSortable match { case 0 => "" case b => Utils.bytesToString(b) } val shuffleWriteSortable = listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L) val shuffleWrite = shuffleWriteSortable match { case 0 => "" case b => Utils.bytesToString(b) } val startedTasks = listener.stageIdToTasksActive.getOrElse(s.stageId, HashSet[TaskInfo]()).size val completedTasks = listener.stageIdToTasksComplete.getOrElse(s.stageId, 0) val failedTasks = listener.stageIdToTasksFailed.getOrElse(s.stageId, 0) match { case f if f > 0 => "(%s failed)".format(f) case _ => "" } val totalTasks = s.numTasks val poolName = listener.stageIdToPool.get(s.stageId) val nameLink = {s.name} val description = listener.stageIdToDescription.get(s.stageId) .map(d =>
{d}
{nameLink}
).getOrElse(nameLink) val finishTime = s.completionTime.getOrElse(System.currentTimeMillis()) val duration = s.submissionTime.map(t => finishTime - t) {s.stageId} {if (isFairScheduler) { {poolName.get}} } {description} {submissionTime} {duration.map(d => parent.formatDuration(d)).getOrElse("Unknown")} {makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)} {shuffleRead} {shuffleWrite} } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy