org.apache.spark.ui.exec.ExecutorsPage.scala Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of spark-core_2.11 Show documentation
There is a newer version: 2.4.8
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.ui.exec

import java.net.URLEncoder
import javax.servlet.http.HttpServletRequest

import scala.xml.Node

import org.apache.spark.status.api.v1.ExecutorSummary
import org.apache.spark.ui.{ToolTips, UIUtils, WebUIPage}
import org.apache.spark.util.Utils

// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
private[ui] case class ExecutorSummaryInfo(
    id: String,
    hostPort: String,
    rddBlocks: Int,
    memoryUsed: Long,
    diskUsed: Long,
    activeTasks: Int,
    failedTasks: Int,
    completedTasks: Int,
    totalTasks: Int,
    totalDuration: Long,
    totalInputBytes: Long,
    totalShuffleRead: Long,
    totalShuffleWrite: Long,
    maxMemory: Long,
    executorLogs: Map[String, String])


private[ui] class ExecutorsPage(
    parent: ExecutorsTab,
    threadDumpEnabled: Boolean)
  extends WebUIPage("") {
  private val listener = parent.listener

  def render(request: HttpServletRequest): Seq[Node] = {
    val storageStatusList = listener.storageStatusList
    val maxMem = storageStatusList.map(_.maxMem).sum
    val memUsed = storageStatusList.map(_.memUsed).sum
    val diskUsed = storageStatusList.map(_.diskUsed).sum
    val execInfo = for (statusId <- 0 until storageStatusList.size) yield
      ExecutorsPage.getExecInfo(listener, statusId)
    val execInfoSorted = execInfo.sortBy(_.id)
    val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty

    val execTable =
      
          {if (logsExist)  else Seq.empty}
          {if (threadDumpEnabled)  else Seq.empty}
        
          {execInfoSorted.map(execRow(_, logsExist))}
        
        
          Executor ID
          Address
          RDD Blocks
          Storage Memory
          Disk Used
          Active Tasks
          Failed Tasks
          Complete Tasks
          Total Tasks
          Task Time
          Input
          Shuffle Read
          
            
            
              Shuffle Write
            
          Logs Thread Dump
        
      

    val content =
      
        
          
            Memory:
              {Utils.bytesToString(memUsed)} Used
              ({Utils.bytesToString(maxMem)} Total) 
            Disk: {Utils.bytesToString(diskUsed)} Used 
          
        
      
      
        
          {execTable}
        
      ;

    UIUtils.headerSparkPage("Executors (" + execInfo.size + ")", content, parent)
  }

  /** Render an HTML row representing an executor */
  private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = {
    val maximumMemory = info.maxMemory
    val memoryUsed = info.memoryUsed
    val diskUsed = info.diskUsed
    
      {info.id}
      {info.hostPort}
      {info.rddBlocks}
      
        {Utils.bytesToString(memoryUsed)} /
        {Utils.bytesToString(maximumMemory)}
      
      
        {Utils.bytesToString(diskUsed)}
      
      {info.activeTasks}
      {info.failedTasks}
      {info.completedTasks}
      {info.totalTasks}
      
        {Utils.msDurationToString(info.totalDuration)}
      
      
        {Utils.bytesToString(info.totalInputBytes)}
      
      
        {Utils.bytesToString(info.totalShuffleRead)}
      
      
        {Utils.bytesToString(info.totalShuffleWrite)}
      
      {
        if (logsExist) {
          
            {
              info.executorLogs.map { case (logName, logUrl) =>
                
                  
                    {logName}
                  
                
              }
            }
          
        }
      }
      {
        if (threadDumpEnabled) {
          val encodedId = URLEncoder.encode(info.id, "UTF-8")
          
            Thread Dump
          
        } else {
          Seq.empty
        }
      }
    
  }

}

private[spark] object ExecutorsPage {
  /** Represent an executor's info as a map given a storage status index */
  def getExecInfo(listener: ExecutorsListener, statusId: Int): ExecutorSummary = {
    val status = listener.storageStatusList(statusId)
    val execId = status.blockManagerId.executorId
    val hostPort = status.blockManagerId.hostPort
    val rddBlocks = status.numBlocks
    val memUsed = status.memUsed
    val maxMem = status.maxMem
    val diskUsed = status.diskUsed
    val activeTasks = listener.executorToTasksActive.getOrElse(execId, 0)
    val failedTasks = listener.executorToTasksFailed.getOrElse(execId, 0)
    val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0)
    val totalTasks = activeTasks + failedTasks + completedTasks
    val totalDuration = listener.executorToDuration.getOrElse(execId, 0L)
    val totalInputBytes = listener.executorToInputBytes.getOrElse(execId, 0L)
    val totalShuffleRead = listener.executorToShuffleRead.getOrElse(execId, 0L)
    val totalShuffleWrite = listener.executorToShuffleWrite.getOrElse(execId, 0L)
    val executorLogs = listener.executorToLogUrls.getOrElse(execId, Map.empty)

    new ExecutorSummary(
      execId,
      hostPort,
      rddBlocks,
      memUsed,
      diskUsed,
      activeTasks,
      failedTasks,
      completedTasks,
      totalTasks,
      totalDuration,
      totalInputBytes,
      totalShuffleRead,
      totalShuffleWrite,
      maxMem,
      executorLogs
    )
  }
}