org.apache.spark.deploy.history.HistoryPage.scala Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.deploy.history
import javax.servlet.http.HttpServletRequest
import scala.xml.Node
import org.apache.spark.ui.{WebUIPage, UIUtils}
private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {
private val pageSize = 20
private val plusOrMinus = 2
def render(request: HttpServletRequest): Seq[Node] = {
val requestedPage = Option(request.getParameter("page")).getOrElse("1").toInt
val requestedFirst = (requestedPage - 1) * pageSize
val requestedIncomplete =
Option(request.getParameter("showIncomplete")).getOrElse("false").toBoolean
val allApps = parent.getApplicationList()
.filter(_.attempts.head.completed != requestedIncomplete)
val allAppsSize = allApps.size
val actualFirst = if (requestedFirst < allAppsSize) requestedFirst else 0
val appsToShow = allApps.slice(actualFirst, actualFirst + pageSize)
val actualPage = (actualFirst / pageSize) + 1
val last = Math.min(actualFirst + pageSize, allAppsSize) - 1
val pageCount = allAppsSize / pageSize + (if (allAppsSize % pageSize > 0) 1 else 0)
val secondPageFromLeft = 2
val secondPageFromRight = pageCount - 1
val hasMultipleAttempts = appsToShow.exists(_.attempts.size > 1)
val appTable =
if (hasMultipleAttempts) {
// Sorting is disable here as table sort on rowspan has issues.
// ref. SPARK-10172
UIUtils.listingTable(appWithAttemptHeader, appWithAttemptRow,
appsToShow, sortable = false)
} else {
UIUtils.listingTable(appHeader, appRow, appsToShow)
}
val providerConfig = parent.getProviderConfig()
val content =
{providerConfig.map { case (k, v) => - {k}: {v}
}}
{
// This displays the indices of pages that are within `plusOrMinus` pages of
// the current page. Regardless of where the current page is, this also links
// to the first and last page. If the current page +/- `plusOrMinus` is greater
// than the 2nd page from the first page or less than the 2nd page from the last
// page, `...` will be displayed.
if (allAppsSize > 0) {
val leftSideIndices =
rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete)
val rightSideIndices =
rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount,
requestedIncomplete)
Showing {actualFirst + 1}-{last + 1} of {allAppsSize}
{if (requestedIncomplete) "(Incomplete applications)"}
{
if (actualPage > 1) {
<
1
}
}
{if (actualPage - plusOrMinus > secondPageFromLeft) " ... "}
{leftSideIndices}
{actualPage}
{rightSideIndices}
{if (actualPage + plusOrMinus < secondPageFromRight) " ... "}
{
if (actualPage < pageCount) {
{pageCount}
>
}
}
++
appTable
} else if (requestedIncomplete) {
No incomplete applications found!
} else {
No completed applications found!
++
Did you specify the correct logging directory?
Please verify your setting of
spark.history.fs.logDirectory and whether you have the permissions to
access it.
It is also possible that your application did not run to
completion or did not stop the SparkContext.
}
}
{
if (requestedIncomplete) {
"Back to completed applications"
} else {
"Show incomplete applications"
}
}
UIUtils.basicSparkPage(content, "History Server")
}
private val appHeader = Seq(
"App ID",
"App Name",
"Started",
"Completed",
"Duration",
"Spark User",
"Last Updated")
private val appWithAttemptHeader = Seq(
"App ID",
"App Name",
"Attempt ID",
"Started",
"Completed",
"Duration",
"Spark User",
"Last Updated")
private def rangeIndices(
range: Seq[Int],
condition: Int => Boolean,
showIncomplete: Boolean): Seq[Node] = {
range.filter(condition).map(nextPage =>
{nextPage} )
}
private def attemptRow(
renderAttemptIdColumn: Boolean,
info: ApplicationHistoryInfo,
attempt: ApplicationAttemptInfo,
isFirst: Boolean): Seq[Node] = {
val uiAddress = HistoryServer.getAttemptURI(info.id, attempt.attemptId)
val startTime = UIUtils.formatDate(attempt.startTime)
val endTime = if (attempt.endTime > 0) UIUtils.formatDate(attempt.endTime) else "-"
val duration =
if (attempt.endTime > 0) {
UIUtils.formatDuration(attempt.endTime - attempt.startTime)
} else {
"-"
}
val lastUpdated = UIUtils.formatDate(attempt.lastUpdated)
{
if (isFirst) {
if (info.attempts.size > 1 || renderAttemptIdColumn) {
{info.id}
{info.name}
} else {
{info.id}
{info.name}
}
} else {
Nil
}
}
{
if (renderAttemptIdColumn) {
if (info.attempts.size > 1 && attempt.attemptId.isDefined) {
{attempt.attemptId.get}
} else {
}
} else {
Nil
}
}
{startTime}
{endTime}
{duration}
{attempt.sparkUser}
{lastUpdated}
}
private def appRow(info: ApplicationHistoryInfo): Seq[Node] = {
attemptRow(false, info, info.attempts.head, true)
}
private def appWithAttemptRow(info: ApplicationHistoryInfo): Seq[Node] = {
attemptRow(true, info, info.attempts.head, true) ++
info.attempts.drop(1).flatMap(attemptRow(true, info, _, false))
}
private def makePageLink(linkPage: Int, showIncomplete: Boolean): String = {
"/?" + Array(
"page=" + linkPage,
"showIncomplete=" + showIncomplete
).mkString("&")
}
}