org.apache.spark.ui.jobs.ExecutorTable.scala Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.ui.jobs

import scala.xml.{Node, Unparsed}

import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1.StageData
import org.apache.spark.ui.{ToolTips, UIUtils}
import org.apache.spark.util.Utils

/** Stage summary grouped by executors. */
private[ui] class ExecutorTable(stage: StageData, store: AppStatusStore) {

  import ApiHelper._

  def toNodeSeq: Seq[Node] = {
    
        {if (hasInput(stage)) {
          
        }}
        {if (hasOutput(stage)) {
          
        }}
        {if (hasShuffleRead(stage)) {
          
        }}
        {if (hasShuffleWrite(stage)) {
          
        }}
        {if (hasBytesSpilled(stage)) {
          
        }}
        
        {createExecutorTable(stage)}
      
      
        Executor ID
        Address
        Task Time
        Total Tasks
        Failed Tasks
        Killed Tasks
        Succeeded Tasks 
            Input Size / Records
          
            Output Size / Records
          
            
            Shuffle Read Size / Records
          
            
            Shuffle Write Size / Records
          Shuffle Spill (Memory)
          Shuffle Spill (Disk) 
          
          Blacklisted
          
        
      
      
    
    
  }

  private def createExecutorTable(stage: StageData) : Seq[Node] = {
    val executorSummary = store.executorSummary(stage.stageId, stage.attemptId)

    executorSummary.toSeq.sortBy(_._1).map { case (k, v) =>
      val executor = store.asOption(store.executorSummary(k))
      
        
          {k}
          
          {
            executor.map(_.executorLogs).getOrElse(Map.empty).map {
              case (logName, logUrl) => {logName}
            }
          }
          
        
        {executor.map { e => e.hostPort }.getOrElse("CANNOT FIND ADDRESS")}
        {UIUtils.formatDuration(v.taskTime)}
        {v.failedTasks + v.succeededTasks + v.killedTasks}
        {v.failedTasks}
        {v.killedTasks}
        {v.succeededTasks}
        {if (hasInput(stage)) {
          
            {s"${Utils.bytesToString(v.inputBytes)} / ${v.inputRecords}"}
          
        }}
        {if (hasOutput(stage)) {
          
            {s"${Utils.bytesToString(v.outputBytes)} / ${v.outputRecords}"}
          
        }}
        {if (hasShuffleRead(stage)) {
          
            {s"${Utils.bytesToString(v.shuffleRead)} / ${v.shuffleReadRecords}"}
          
        }}
        {if (hasShuffleWrite(stage)) {
          
            {s"${Utils.bytesToString(v.shuffleWrite)} / ${v.shuffleWriteRecords}"}
          
        }}
        {if (hasBytesSpilled(stage)) {
          
            {Utils.bytesToString(v.memoryBytesSpilled)}
          
          
            {Utils.bytesToString(v.diskBytesSpilled)}
          
        }}
        {
          if (executor.map(_.isBlacklisted).getOrElse(false)) {
            for application
          } else if (v.isBlacklistedForStage) {
            for stage
          } else {
            false
          }
        }
      
    }
  }

}