org.apache.spark.deploy.master.ui.MasterPage.scala Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of spark-core_2.12
There is a newer version: 3.5.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.deploy.master.ui

import javax.servlet.http.HttpServletRequest

import scala.xml.Node

import org.json4s.JValue

import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, MasterStateResponse, RequestKillDriver, RequestMasterState}
import org.apache.spark.deploy.JsonProtocol
import org.apache.spark.deploy.master._
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils

private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
  private val master = parent.masterEndpointRef

  def getMasterState: MasterStateResponse = {
    master.askSync[MasterStateResponse](RequestMasterState)
  }

  override def renderJson(request: HttpServletRequest): JValue = {
    JsonProtocol.writeMasterState(getMasterState)
  }

  def handleAppKillRequest(request: HttpServletRequest): Unit = {
    handleKillRequest(request, id => {
      parent.master.idToApp.get(id).foreach { app =>
        parent.master.removeApplication(app, ApplicationState.KILLED)
      }
    })
  }

  def handleDriverKillRequest(request: HttpServletRequest): Unit = {
    handleKillRequest(request, id => {
      master.ask[KillDriverResponse](RequestKillDriver(id))
    })
  }

  private def handleKillRequest(request: HttpServletRequest, action: String => Unit): Unit = {
    if (parent.killEnabled &&
        parent.master.securityMgr.checkModifyPermissions(request.getRemoteUser)) {
      // stripXSS is called first to remove suspicious characters used in XSS attacks
      val killFlag =
        Option(UIUtils.stripXSS(request.getParameter("terminate"))).getOrElse("false").toBoolean
      val id = Option(UIUtils.stripXSS(request.getParameter("id")))
      if (id.isDefined && killFlag) {
        action(id.get)
      }

      Thread.sleep(100)
    }
  }

  /** Index view listing applications and executors */
  def render(request: HttpServletRequest): Seq[Node] = {
    val state = getMasterState

    val workerHeaders = Seq("Worker Id", "Address", "State", "Cores", "Memory")
    val workers = state.workers.sortBy(_.id)
    val aliveWorkers = state.workers.filter(_.state == WorkerState.ALIVE)
    val workerTable = UIUtils.listingTable(workerHeaders, workerRow, workers)

    val appHeaders = Seq("Application ID", "Name", "Cores", "Memory per Executor", "Submitted Time",
      "User", "State", "Duration")
    val activeApps = state.activeApps.sortBy(_.startTime).reverse
    val activeAppsTable = UIUtils.listingTable(appHeaders, appRow, activeApps)
    val completedApps = state.completedApps.sortBy(_.endTime).reverse
    val completedAppsTable = UIUtils.listingTable(appHeaders, appRow, completedApps)

    val driverHeaders = Seq("Submission ID", "Submitted Time", "Worker", "State", "Cores",
      "Memory", "Main Class")
    val activeDrivers = state.activeDrivers.sortBy(_.startTime).reverse
    val activeDriversTable = UIUtils.listingTable(driverHeaders, driverRow, activeDrivers)
    val completedDrivers = state.completedDrivers.sortBy(_.startTime).reverse
    val completedDriversTable = UIUtils.listingTable(driverHeaders, driverRow, completedDrivers)

    // For now we only show driver information if the user has submitted drivers to the cluster.
    // This is until we integrate the notion of drivers and applications in the UI.
    def hasDrivers: Boolean = activeDrivers.length > 0 || completedDrivers.length > 0

    val content =
        
          
            
              URL: {state.uri}
              {
                state.restUri.map { uri =>
                  
                    REST URL: {uri}
                     (cluster mode)
                  
                }.getOrElse { Seq.empty }
              }
              Alive Workers: {aliveWorkers.length}
              Cores in use: {aliveWorkers.map(_.cores).sum} Total,
                {aliveWorkers.map(_.coresUsed).sum} Used
              Memory in use:
                {Utils.megabytesToString(aliveWorkers.map(_.memory).sum)} Total,
                {Utils.megabytesToString(aliveWorkers.map(_.memoryUsed).sum)} Used
              Applications:
                {state.activeApps.length} Running,
                {state.completedApps.length} Completed 
              Drivers:
                {state.activeDrivers.length} Running,
                {state.completedDrivers.length} Completed 
              Status: {state.status}
            
          
        

        
          
            
              
                
                Workers ({workers.length})
              
            
            
              {workerTable}
            
          
        

        
          
            
              
                
                Running Applications ({activeApps.length})
              
            
            
              {activeAppsTable}
            
          
        

        
          {if (hasDrivers) {
             
               
                 
                   
                     
                     Running Drivers ({activeDrivers.length})
                   
                 
                 
                   {activeDriversTable}
                 
               
             
           }
          }
        

        
          
            
              
                
                Completed Applications ({completedApps.length})
              
            
            
              {completedAppsTable}
            
          
        

        
          {
            if (hasDrivers) {
              
                
                  
                    
                      
                      Completed Drivers ({completedDrivers.length})
                    
                  
                  
                    {completedDriversTable}
                  
                
              
            }
          }
        ;

    UIUtils.basicSparkPage(request, content, "Spark Master at " + state.uri)
  }

  private def workerRow(worker: WorkerInfo): Seq[Node] = {
    
      
        {
          if (worker.isAlive()) {
            
              {worker.id}
            
          } else {
            worker.id
          }
        }
      
      {worker.host}:{worker.port}
      {worker.state}
      {worker.cores} ({worker.coresUsed} Used)
      
        {Utils.megabytesToString(worker.memory)}
        ({Utils.megabytesToString(worker.memoryUsed)} Used)
      
    
  }

  private def appRow(app: ApplicationInfo): Seq[Node] = {
    val killLink = if (parent.killEnabled &&
      (app.state == ApplicationState.RUNNING || app.state == ApplicationState.WAITING)) {
      val confirm =
        s"if (window.confirm('Are you sure you want to kill application ${app.id} ?')) " +
          "{ this.parentNode.submit(); return true; } else { return false; }"
      
        
        
        (kill)
      
    }
    
      
        {app.id}
        {killLink}
      
      
        {
          if (app.isFinished) {
            app.desc.name
          } else {
            {app.desc.name}
          }
        }
      
      
        {app.coresGranted}
      
      
        {Utils.megabytesToString(app.desc.memoryPerExecutorMB)}
      
      {UIUtils.formatDate(app.submitDate)}
      {app.desc.user}
      {app.state.toString}
      {UIUtils.formatDuration(app.duration)}
    
  }

  private def driverRow(driver: DriverInfo): Seq[Node] = {
    val killLink = if (parent.killEnabled &&
      (driver.state == DriverState.RUNNING ||
        driver.state == DriverState.SUBMITTED ||
        driver.state == DriverState.RELAUNCHING)) {
      val confirm =
        s"if (window.confirm('Are you sure you want to kill driver ${driver.id} ?')) " +
          "{ this.parentNode.submit(); return true; } else { return false; }"
      
        
        
        (kill)
      
    }
    
      {driver.id} {killLink}
      {UIUtils.formatDate(driver.submitDate)}
      {driver.worker.map(w =>
        if (w.isAlive()) {
          
            {w.id.toString}
          
        } else {
          w.id.toString
        }).getOrElse("None")}
      
      {driver.state}
      
        {driver.desc.cores}
      
      
        {Utils.megabytesToString(driver.desc.mem.toLong)}
      
      {driver.desc.command.arguments(2)}
    
  }
}