org.apache.spark.ui.jobs.StageTable.scala Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import java.util.Date
import javax.servlet.http.HttpServletRequest
import scala.collection.JavaConverters._
import scala.xml._
import org.apache.commons.lang3.StringEscapeUtils
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1
import org.apache.spark.ui._
import org.apache.spark.util.Utils
private[ui] class StageTableBase(
store: AppStatusStore,
request: HttpServletRequest,
stages: Seq[v1.StageData],
tableHeaderID: String,
stageTag: String,
basePath: String,
subPath: String,
isFairScheduler: Boolean,
killEnabled: Boolean,
isFailedStage: Boolean) {
// stripXSS is called to remove suspicious characters used in XSS attacks
val allParameters = request.getParameterMap.asScala.toMap.map { case (k, v) =>
UIUtils.stripXSS(k) -> v.map(UIUtils.stripXSS).toSeq
}
val parameterOtherTable = allParameters.filterNot(_._1.startsWith(stageTag))
.map(para => para._1 + "=" + para._2(0))
val parameterStagePage = UIUtils.stripXSS(request.getParameter(stageTag + ".page"))
val parameterStageSortColumn = UIUtils.stripXSS(request.getParameter(stageTag + ".sort"))
val parameterStageSortDesc = UIUtils.stripXSS(request.getParameter(stageTag + ".desc"))
val parameterStagePageSize = UIUtils.stripXSS(request.getParameter(stageTag + ".pageSize"))
val parameterStagePrevPageSize =
UIUtils.stripXSS(request.getParameter(stageTag + ".prevPageSize"))
val stagePage = Option(parameterStagePage).map(_.toInt).getOrElse(1)
val stageSortColumn = Option(parameterStageSortColumn).map { sortColumn =>
UIUtils.decodeURLParameter(sortColumn)
}.getOrElse("Stage Id")
val stageSortDesc = Option(parameterStageSortDesc).map(_.toBoolean).getOrElse(
// New stages should be shown above old jobs by default.
stageSortColumn == "Stage Id"
)
val stagePageSize = Option(parameterStagePageSize).map(_.toInt).getOrElse(100)
val stagePrevPageSize = Option(parameterStagePrevPageSize).map(_.toInt)
.getOrElse(stagePageSize)
val page: Int = {
// If the user has changed to a larger page size, then go to page 1 in order to avoid
// IndexOutOfBoundsException.
if (stagePageSize <= stagePrevPageSize) {
stagePage
} else {
1
}
}
val currentTime = System.currentTimeMillis()
val toNodeSeq = try {
new StagePagedTable(
store,
stages,
tableHeaderID,
stageTag,
basePath,
subPath,
isFairScheduler,
killEnabled,
currentTime,
stagePageSize,
stageSortColumn,
stageSortDesc,
isFailedStage,
parameterOtherTable,
request
).table(page)
} catch {
case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
Error while rendering stage table:
{Utils.exceptionString(e)}
}
}
private[ui] class StageTableRowData(
val stage: v1.StageData,
val option: Option[v1.StageData],
val stageId: Int,
val attemptId: Int,
val schedulingPool: String,
val descriptionOption: Option[String],
val submissionTime: Date,
val formattedSubmissionTime: String,
val duration: Long,
val formattedDuration: String,
val inputRead: Long,
val inputReadWithUnit: String,
val outputWrite: Long,
val outputWriteWithUnit: String,
val shuffleRead: Long,
val shuffleReadWithUnit: String,
val shuffleWrite: Long,
val shuffleWriteWithUnit: String)
private[ui] class MissingStageTableRowData(
stageInfo: v1.StageData,
stageId: Int,
attemptId: Int) extends StageTableRowData(
stageInfo, None, stageId, attemptId, "", None, new Date(0), "", -1, "", 0, "", 0, "", 0, "", 0,
"")
/** Page showing list of all ongoing and recently finished stages */
private[ui] class StagePagedTable(
store: AppStatusStore,
stages: Seq[v1.StageData],
tableHeaderId: String,
stageTag: String,
basePath: String,
subPath: String,
isFairScheduler: Boolean,
killEnabled: Boolean,
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean,
isFailedStage: Boolean,
parameterOtherTable: Iterable[String],
request: HttpServletRequest) extends PagedTable[StageTableRowData] {
override def tableId: String = stageTag + "-table"
override def tableCssClass: String =
"table table-bordered table-condensed table-striped " +
"table-head-clickable table-cell-width-limited"
override def pageSizeFormField: String = stageTag + ".pageSize"
override def prevPageSizeFormField: String = stageTag + ".prevPageSize"
override def pageNumberFormField: String = stageTag + ".page"
val parameterPath = UIUtils.prependBaseUri(request, basePath) + s"/$subPath/?" +
parameterOtherTable.mkString("&")
override val dataSource = new StageDataSource(
store,
stages,
currentTime,
pageSize,
sortColumn,
desc
)
override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")
parameterPath +
s"&$pageNumberFormField=$page" +
s"&$stageTag.sort=$encodedSortColumn" +
s"&$stageTag.desc=$desc" +
s"&$pageSizeFormField=$pageSize" +
s"#$tableHeaderId"
}
override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")
s"$parameterPath&$stageTag.sort=$encodedSortColumn&$stageTag.desc=$desc#$tableHeaderId"
}
override def headers: Seq[Node] = {
// stageHeadersAndCssClasses has three parts: header title, tooltip information, and sortable.
// The tooltip information could be None, which indicates it does not have a tooltip.
// Otherwise, it has two parts: tooltip text, and position (true for left, false for default).
val stageHeadersAndCssClasses: Seq[(String, Option[(String, Boolean)], Boolean)] =
Seq(("Stage Id", None, true)) ++
{if (isFairScheduler) {Seq(("Pool Name", None, true))} else Seq.empty} ++
Seq(
("Description", None, true), ("Submitted", None, true), ("Duration", None, true),
("Tasks: Succeeded/Total", None, false),
("Input", Some((ToolTips.INPUT, false)), true),
("Output", Some((ToolTips.OUTPUT, false)), true),
("Shuffle Read", Some((ToolTips.SHUFFLE_READ, false)), true),
("Shuffle Write", Some((ToolTips.SHUFFLE_WRITE, true)), true)
) ++
{if (isFailedStage) {Seq(("Failure Reason", None, false))} else Seq.empty}
if (!stageHeadersAndCssClasses.filter(_._3).map(_._1).contains(sortColumn)) {
throw new IllegalArgumentException(s"Unknown column: $sortColumn")
}
val headerRow: Seq[Node] = {
stageHeadersAndCssClasses.map { case (header, tooltip, sortable) =>
val headerSpan = tooltip.map { case (title, left) =>
if (left) {
/* Place the shuffle write tooltip on the left (rather than the default position
of on top) because the shuffle write column is the last column on the right side and
the tooltip is wider than the column, so it doesn't fit on top. */
{header}
} else {
{header}
}
}.getOrElse(
{header}
)
if (header == sortColumn) {
val headerLink = Unparsed(
parameterPath +
s"&$stageTag.sort=${URLEncoder.encode(header, "UTF-8")}" +
s"&$stageTag.desc=${!desc}" +
s"&$stageTag.pageSize=$pageSize") +
s"#$tableHeaderId"
val arrow = if (desc) "▾" else "▴" // UP or DOWN
{headerSpan}
{Unparsed(arrow)}
} else {
if (sortable) {
val headerLink = Unparsed(
parameterPath +
s"&$stageTag.sort=${URLEncoder.encode(header, "UTF-8")}" +
s"&$stageTag.pageSize=$pageSize") +
s"#$tableHeaderId"
{headerSpan}
} else {
{headerSpan}
}
}
}
}
{headerRow}
}
override def row(data: StageTableRowData): Seq[Node] = {
{rowContent(data)}
}
private def rowContent(data: StageTableRowData): Seq[Node] = {
data.option match {
case None => missingStageRow(data.stageId)
case Some(stageData) =>
val info = data.stage
{if (data.attemptId > 0) {
{data.stageId} (retry {data.attemptId})
} else {
{data.stageId}
}} ++
{if (isFairScheduler) {
{data.schedulingPool}
} else {
Seq.empty
}} ++
{makeDescription(info, data.descriptionOption)}
{data.formattedSubmissionTime}
{data.formattedDuration}
{UIUtils.makeProgressBar(started = stageData.numActiveTasks,
completed = stageData.numCompleteTasks, failed = stageData.numFailedTasks,
skipped = 0, reasonToNumKilled = stageData.killedTasksSummary, total = info.numTasks)}
{data.inputReadWithUnit}
{data.outputWriteWithUnit}
{data.shuffleReadWithUnit}
{data.shuffleWriteWithUnit} ++
{
if (isFailedStage) {
failureReasonHtml(info)
} else {
Seq.empty
}
}
}
}
private def failureReasonHtml(s: v1.StageData): Seq[Node] = {
val failureReason = s.failureReason.getOrElse("")
val isMultiline = failureReason.indexOf('\n') >= 0
// Display the first line by default
val failureReasonSummary = StringEscapeUtils.escapeHtml4(
if (isMultiline) {
failureReason.substring(0, failureReason.indexOf('\n'))
} else {
failureReason
})
val details = if (isMultiline) {
// scalastyle:off
++
{failureReason}
// scalastyle:on
} else {
""
}
{failureReasonSummary}{details}
}
private def makeDescription(s: v1.StageData, descriptionOption: Option[String]): Seq[Node] = {
val basePathUri = UIUtils.prependBaseUri(request, basePath)
val killLink = if (killEnabled) {
val confirm =
s"if (window.confirm('Are you sure you want to kill stage ${s.stageId} ?')) " +
"{ this.parentNode.submit(); return true; } else { return false; }"
// SPARK-6846 this should be POST-only but YARN AM won't proxy POST
/*
val killLinkUri = s"$basePathUri/stages/stage/kill/"
*/
val killLinkUri = s"$basePathUri/stages/stage/kill/?id=${s.stageId}"
(kill)
} else {
Seq.empty
}
val nameLinkUri = s"$basePathUri/stages/stage/?id=${s.stageId}&attempt=${s.attemptId}"
val nameLink = {s.name}
val cachedRddInfos = store.rddList().filter { rdd => s.rddIds.contains(rdd.id) }
val details = if (s.details != null && s.details.nonEmpty) {
++
{if (cachedRddInfos.nonEmpty) {
Text("RDD: ") ++
cachedRddInfos.map { i =>
{i.name}
}
}}
{s.details}
}
val stageDesc = descriptionOption.map(UIUtils.makeDescription(_, basePathUri))
{stageDesc.getOrElse("")} {killLink} {nameLink} {details}
}
protected def missingStageRow(stageId: Int): Seq[Node] = {
{stageId} ++
{if (isFairScheduler) {- } else Seq.empty} ++
No data available for this stage ++ // Description