org.apache.spark.ui.jobs.ExecutorTable.scala Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.ui.jobs
import scala.xml.{Node, Unparsed}
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1.StageData
import org.apache.spark.ui.{ToolTips, UIUtils}
import org.apache.spark.util.Utils
/** Stage summary grouped by executors. */
private[ui] class ExecutorTable(stage: StageData, store: AppStatusStore) {
import ApiHelper._
def toNodeSeq: Seq[Node] = {
Executor ID
Address
Task Time
Total Tasks
Failed Tasks
Killed Tasks
Succeeded Tasks
{if (hasInput(stage)) {
Input Size / Records
}}
{if (hasOutput(stage)) {
Output Size / Records
}}
{if (hasShuffleRead(stage)) {
Shuffle Read Size / Records
}}
{if (hasShuffleWrite(stage)) {
Shuffle Write Size / Records
}}
{if (hasBytesSpilled(stage)) {
Shuffle Spill (Memory)
Shuffle Spill (Disk)
}}
Blacklisted
{createExecutorTable(stage)}
}
private def createExecutorTable(stage: StageData) : Seq[Node] = {
val executorSummary = store.executorSummary(stage.stageId, stage.attemptId)
executorSummary.toSeq.sortBy(_._1).map { case (k, v) =>
val executor = store.asOption(store.executorSummary(k))
{k}
{
executor.map(_.executorLogs).getOrElse(Map.empty).map {
case (logName, logUrl) =>
}
}
{executor.map { e => e.hostPort }.getOrElse("CANNOT FIND ADDRESS")}
{UIUtils.formatDuration(v.taskTime)}
{v.failedTasks + v.succeededTasks + v.killedTasks}
{v.failedTasks}
{v.killedTasks}
{v.succeededTasks}
{if (hasInput(stage)) {
{s"${Utils.bytesToString(v.inputBytes)} / ${v.inputRecords}"}
}}
{if (hasOutput(stage)) {
{s"${Utils.bytesToString(v.outputBytes)} / ${v.outputRecords}"}
}}
{if (hasShuffleRead(stage)) {
{s"${Utils.bytesToString(v.shuffleRead)} / ${v.shuffleReadRecords}"}
}}
{if (hasShuffleWrite(stage)) {
{s"${Utils.bytesToString(v.shuffleWrite)} / ${v.shuffleWriteRecords}"}
}}
{if (hasBytesSpilled(stage)) {
{Utils.bytesToString(v.memoryBytesSpilled)}
{Utils.bytesToString(v.diskBytesSpilled)}
}}
{
if (executor.map(_.isBlacklisted).getOrElse(false)) {
for application
} else if (v.isBlacklistedForStage) {
for stage
} else {
false
}
}
}
}
}