org.apache.spark.deploy.worker.ui.WorkerPage.scala Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of spark-core_2.11
There is a newer version: 2.4.8
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.deploy.worker.ui

import javax.servlet.http.HttpServletRequest

import scala.xml.Node

import org.json4s.JValue

import org.apache.spark.deploy.{ExecutorState, JsonProtocol}
import org.apache.spark.deploy.DeployMessages.{RequestWorkerState, WorkerStateResponse}
import org.apache.spark.deploy.master.DriverState
import org.apache.spark.deploy.worker.{DriverRunner, ExecutorRunner}
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils

private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {
  private val workerEndpoint = parent.worker.self

  override def renderJson(request: HttpServletRequest): JValue = {
    val workerState = workerEndpoint.askSync[WorkerStateResponse](RequestWorkerState)
    JsonProtocol.writeWorkerState(workerState)
  }

  def render(request: HttpServletRequest): Seq[Node] = {
    val workerState = workerEndpoint.askSync[WorkerStateResponse](RequestWorkerState)

    val executorHeaders = Seq("ExecutorID", "Cores", "State", "Memory", "Job Details", "Logs")
    val runningExecutors = workerState.executors
    val runningExecutorTable =
      UIUtils.listingTable(executorHeaders, executorRow, runningExecutors)
    val finishedExecutors = workerState.finishedExecutors
    val finishedExecutorTable =
      UIUtils.listingTable(executorHeaders, executorRow, finishedExecutors)

    val driverHeaders = Seq("DriverID", "Main Class", "State", "Cores", "Memory", "Logs", "Notes")
    val runningDrivers = workerState.drivers.sortBy(_.driverId).reverse
    val runningDriverTable = UIUtils.listingTable[DriverRunner](driverHeaders,
      driverRow(workerState.workerId, _), runningDrivers)
    val finishedDrivers = workerState.finishedDrivers.sortBy(_.driverId).reverse
    val finishedDriverTable = UIUtils.listingTable[DriverRunner](driverHeaders,
      driverRow(workerState.workerId, _), finishedDrivers)

    // For now we only show driver information if the user has submitted drivers to the cluster.
    // This is until we integrate the notion of drivers and applications in the UI.

    val content =
       
        
          
            ID: {workerState.workerId}
            
              Master URL: {workerState.masterUrl}
            
            Cores: {workerState.cores} ({workerState.coresUsed} Used)
            Memory: {Utils.megabytesToString(workerState.memory)}
              ({Utils.megabytesToString(workerState.memoryUsed)} Used)
          
          Back to Master
        
      
       
        
          
            
              
              Running Executors ({runningExecutors.size})
            
          
          
            {runningExecutorTable}
          
          {
            if (runningDrivers.nonEmpty) {
              
                
                  
                  Running Drivers ({runningDrivers.size})
                
               ++
              
                {runningDriverTable}
              
            }
          }
          {
            if (finishedExecutors.nonEmpty) {
              
                
                  
                  Finished Executors ({finishedExecutors.size})
                
               ++
              
                {finishedExecutorTable}
              
            }
          }
          {
            if (finishedDrivers.nonEmpty) {
              
                
                  
                  Finished Drivers ({finishedDrivers.size})
                
               ++
              
                {finishedDriverTable}
              
            }
          }
        
      ;
    UIUtils.basicSparkPage(request, content, "Spark Worker at %s:%s".format(
      workerState.host, workerState.port))
  }

  def executorRow(executor: ExecutorRunner): Seq[Node] = {
    val workerUrlRef = UIUtils.makeHref(parent.worker.reverseProxy, executor.workerId,
      parent.webUrl)
    val appUrlRef = UIUtils.makeHref(parent.worker.reverseProxy, executor.appId,
      executor.appDesc.appUiUrl)

    
      {executor.execId}
      {executor.cores}
      {executor.state}
      
        {Utils.megabytesToString(executor.memory)}
      
      
        
          ID: {executor.appId}
          Name:
          {
            if ({executor.state == ExecutorState.RUNNING} && executor.appDesc.appUiUrl.nonEmpty) {
               {executor.appDesc.name}
            } else {
              {executor.appDesc.name}
            }
          }
          
          User: {executor.appDesc.user}
        
      
      
        stdout
        stderr
      
    

  }

  def driverRow(workerId: String, driver: DriverRunner): Seq[Node] = {
    val workerUrlRef = UIUtils.makeHref(parent.worker.reverseProxy, workerId, parent.webUrl)
    
      {driver.driverId}
      {driver.driverDesc.command.arguments(2)}
      {driver.finalState.getOrElse(DriverState.RUNNING)}
      
        {driver.driverDesc.cores.toString}
      
      
        {Utils.megabytesToString(driver.driverDesc.mem)}
      
      
        stdout
        stderr
      
      
        {driver.finalException.getOrElse("")}
      
    
  }
}