org.apache.hadoop.mapred.ClusterStatus Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-core Show documentation
Hadoop is the distributed computing framework of Apache; hadoop-core contains the filesystem, job tracker and map/reduce modules
The newest version!
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;


import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.TaskType;

/**
 * Status information on the current state of the Map-Reduce cluster.
 * 
 * ClusterStatus provides clients with information such as:
 * 

 *   
 *   Size of the cluster. 
 *   
 *   
 *   Name of the trackers. 
 *   
 *   
 *   Task capacity of the cluster. 
 *   
 *   
 *   The number of currently running map & reduce tasks.
 *   
 *   
 *   State of the JobTracker.
 *   
 * 

 * 
 * Clients can query for the latest ClusterStatus, via 
 * {@link JobClient#getClusterStatus()}.
 * 
 * @see JobClient
 */
public class ClusterStatus implements Writable {

  private int numActiveTrackers;
  private Collection activeTrackers = new ArrayList();
  private Collection blacklistedTrackers = new ArrayList();
  private int numBlacklistedTrackers;
  private int numExcludedNodes;
  private long ttExpiryInterval;
  private int map_tasks;
  private int reduce_tasks;
  private int max_map_tasks;
  private int max_reduce_tasks;
  private int total_map_tasks;
  private int total_reduce_tasks;
  private JobTracker.State state;
  private long used_memory;
  private long max_memory;
  private Collection taskTrackersDetails =
         new ArrayList();
  // This is the map between task tracker name and the list of all tasks
  // that belong to currently running jobs and are/were executed
  // on this tasktracker
  private Map> taskTrackerExtendedTasks =
         new HashMap>();

  ClusterStatus() {}
  
  /**
   * Construct a new cluster status.
   * 
   * @param trackers no. of tasktrackers in the cluster
   * @param maps no. of currently running map-tasks in the cluster
   * @param reduces no. of currently running reduce-tasks in the cluster
   * @param maxMaps the maximum no. of map tasks in the cluster
   * @param maxReduces the maximum no. of reduce tasks in the cluster
   * @param state the {@link JobTracker.State} of the JobTracker
   * @deprecated 
   */
  @Deprecated
  ClusterStatus(int trackers, int maps, int reduces, int maxMaps,
                int maxReduces, JobTracker.State state) {
    this(trackers, 0, JobTracker.TASKTRACKER_EXPIRY_INTERVAL, maps, reduces,
        maxMaps, maxReduces, state);
  }
  
  /**
   * Construct a new cluster status.
   * 
   * @param trackers no. of tasktrackers in the cluster
   * @param blacklists no of blacklisted task trackers in the cluster
   * @param ttExpiryInterval the tasktracker expiry interval
   * @param maps no. of currently running map-tasks in the cluster
   * @param reduces no. of currently running reduce-tasks in the cluster
   * @param maxMaps the maximum no. of map tasks in the cluster
   * @param maxReduces the maximum no. of reduce tasks in the cluster
   * @param state the {@link JobTracker.State} of the JobTracker
   */
  ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
                int maps, int reduces,
                int maxMaps, int maxReduces, JobTracker.State state) {
    this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps, 
         maxReduces, state, 0);
  }

  /**
   * @param numDecommissionedNodes number of decommission trackers
   */
  ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
                int maps, int reduces, int maxMaps, int maxReduces, 
                JobTracker.State state, int numDecommissionedNodes) {
    numActiveTrackers = trackers;
    numBlacklistedTrackers = blacklists;
    this.numExcludedNodes = numDecommissionedNodes;
    this.ttExpiryInterval = ttExpiryInterval;
    map_tasks = maps;
    reduce_tasks = reduces;
    max_map_tasks = maxMaps;
    max_reduce_tasks = maxReduces;
    this.state = state;

    MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
    MemoryUsage status = memoryMXBean.getHeapMemoryUsage();
    used_memory = status.getUsed();
    max_memory = status.getMax();
  }

  /**
   * Construct a new cluster status.
   * 
   * @param activeTrackers active tasktrackers in the cluster
   * @param blacklistedTrackers blacklisted tasktrackers in the cluster
   * @param ttExpiryInterval the tasktracker expiry interval
   * @param maps no. of currently running map-tasks in the cluster
   * @param reduces no. of currently running reduce-tasks in the cluster
   * @param maxMaps the maximum no. of map tasks in the cluster
   * @param maxReduces the maximum no. of reduce tasks in the cluster
   * @param state the {@link JobTracker.State} of the JobTracker
   */
  ClusterStatus(Collection activeTrackers, 
      Collection blacklistedTrackers,
      long ttExpiryInterval,
      int maps, int reduces, int maxMaps, int maxReduces, 
      JobTracker.State state) {
    this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces, 
         maxMaps, maxReduces, state, 0);
  }

  /**
   * @param numDecommissionNodes number of decommission trackers
   */
  ClusterStatus(Collection activeTrackers, 
                Collection blacklistedTrackers, long ttExpiryInterval,
                int maps, int reduces, int maxMaps, int maxReduces, 
                JobTracker.State state, int numDecommissionNodes) {
    this(activeTrackers.size(), blacklistedTrackers.size(), ttExpiryInterval, 
        maps, reduces, maxMaps, maxReduces, state, numDecommissionNodes);
    this.activeTrackers = activeTrackers;
    this.blacklistedTrackers = blacklistedTrackers;
  }


  /**
   * Get the number of task trackers in the cluster.
   * 
   * @return the number of task trackers in the cluster.
   */
  public int getTaskTrackers() {
    return numActiveTrackers;
  }
  
  /**
   * Get the names of task trackers in the cluster.
   * 
   * @return the active task trackers in the cluster.
   */
  public Collection getActiveTrackerNames() {
    return activeTrackers;
  }

  /**
   * Get the names of task trackers in the cluster.
   * 
   * @return the blacklisted task trackers in the cluster.
   */
  public Collection getBlacklistedTrackerNames() {
    return blacklistedTrackers;
  }
  
  /**
   * Get the number of blacklisted task trackers in the cluster.
   * 
   * @return the number of blacklisted task trackers in the cluster.
   */
  public int getBlacklistedTrackers() {
    return numBlacklistedTrackers;
  }
  
  /**
   * Get the number of excluded hosts in the cluster.
   * @return the number of excluded hosts in the cluster.
   */
  public int getNumExcludedNodes() {
    return numExcludedNodes;
  }
  
  /**
   * Get the tasktracker expiry interval for the cluster
   * @return the expiry interval in msec
   */
  public long getTTExpiryInterval() {
    return ttExpiryInterval;
  }
  
  /**
   * Get the number of currently running map tasks in the cluster.
   * 
   * @return the number of currently running map tasks in the cluster.
   */
  public int getMapTasks() {
    return map_tasks;
  }
  
  /**
   * Get the number of currently running reduce tasks in the cluster.
   * 
   * @return the number of currently running reduce tasks in the cluster.
   */
  public int getReduceTasks() {
    return reduce_tasks;
  }


  public int getTotalMapTasks() {
    return total_map_tasks;
  }

  public int getTotalReduceTasks() {
    return total_reduce_tasks;
  }
  
  /**
   * Get the maximum capacity for running map tasks in the cluster.
   * 
   * @return the maximum capacity for running map tasks in the cluster.
   */
  public int getMaxMapTasks() {
    return max_map_tasks;
  }

  /**
   * Get the maximum capacity for running reduce tasks in the cluster.
   * 
   * @return the maximum capacity for running reduce tasks in the cluster.
   */
  public int getMaxReduceTasks() {
    return max_reduce_tasks;
  }
  
  /**
   * Get the current state of the JobTracker, 
   * as {@link JobTracker.State}
   * 
   * @return the current state of the JobTracker.
   */
  public JobTracker.State getJobTrackerState() {
    return state;
  }

  /**
   * Get the total heap memory used by the JobTracker
   * 
   * @return the size of heap memory used by the JobTracker
   */
  public long getUsedMemory() {
    return used_memory;
  }

  ClusterStatus(Collection activeTrackers,
          Collection blackListedTrackerInfo,
          Collection taskTrackersInfo,
          Collection runningJobs,
          long ttExpiryInterval,
          int maps, int reduces, int maxMaps, int maxReduces,
          JobTracker.State state, int numDecommissionNodes) {
    this(activeTrackers, blackListedTrackerInfo, ttExpiryInterval,
            maps, reduces, maxMaps, maxReduces, state, numDecommissionNodes);
    this.taskTrackersDetails = taskTrackersInfo;
    initTrackersToTasksMap(runningJobs);
  }


  /**
   * Get the TaskTrackerStatus for each task tracker in the cluster
   *
   * @return the collection of all task tracker statuses
   */
  public Collection getTaskTrackersDetails() {
    return taskTrackersDetails;
  }

  /**
   * Goes through the list of TaskStatus objects for each of the running jobs
   * on the cluster and associates them with the name of the task tracker
   * they are or were running on.
   */
  private void initTrackersToTasksMap(Collection jobsInProgress) {
    for (TaskTrackerStatus tracker : taskTrackersDetails) {
      taskTrackerExtendedTasks.put(tracker.getTrackerName(),
              new ArrayList());
    }
    for (JobInProgress job : jobsInProgress) {
      total_map_tasks += job.getTasks(TaskType.MAP).length;
      total_reduce_tasks += job.getTasks(TaskType.REDUCE).length;
      for (TaskInProgress task : job.getTasks(TaskType.REDUCE)) {

        TaskStatus[] taskStatuses = task.getTaskStatuses();
        for (TaskStatus status : taskStatuses) {
          Collection trackerTasks =
                  taskTrackerExtendedTasks.get(status.getTaskTracker());
          if (trackerTasks == null) {
            trackerTasks = new ArrayList();
            taskTrackerExtendedTasks.put(status.getTaskTracker(),
                    trackerTasks);
          }
          trackerTasks.add(status);
        }
      }
      for (TaskInProgress task : job.getTasks(TaskType.MAP)) {
        TaskStatus[] taskStatuses = task.getTaskStatuses();
        for (TaskStatus status : taskStatuses) {
          Collection trackerTasks =
                  taskTrackerExtendedTasks.get(status.getTaskTracker());
          if (trackerTasks == null) {
            trackerTasks = new ArrayList();
            taskTrackerExtendedTasks.put(status.getTaskTracker(),
                    trackerTasks);
          }
          trackerTasks.add(status);
        }
      }
    }
  }

  /**
   * Get the collection of TaskStatus for all tasks that are running on this
   * task tracker or have run on this task tracker and are part of the still
   * running job.
   *
   * @param ttName TaskTracker name
   * @return Collection of TaskStatus objects
   */
  public Collection getTaskTrackerTasksStatuses(String ttName) {
    return taskTrackerExtendedTasks.get(ttName);
  }

  /**
   * Get the maximum configured heap memory that can be used by the JobTracker
   *
   * @return the configured size of max heap memory that can be used by the JobTracker
   */
  public long getMaxMemory() {
    return max_memory;
  }

  public void write(DataOutput out) throws IOException {
    if (activeTrackers.size() == 0) {
      out.writeInt(numActiveTrackers);
      out.writeInt(0);
    } else {
      out.writeInt(activeTrackers.size());
      out.writeInt(activeTrackers.size());
      for (String tracker : activeTrackers) {
        Text.writeString(out, tracker);
      }
    }
    if (blacklistedTrackers.size() == 0) {
      out.writeInt(numBlacklistedTrackers);
      out.writeInt(0);
    } else {
      out.writeInt(blacklistedTrackers.size());
      out.writeInt(blacklistedTrackers.size());
      for (String tracker : blacklistedTrackers) {
        Text.writeString(out, tracker);
      }
    }
    out.writeInt(numExcludedNodes);
    out.writeLong(ttExpiryInterval);
    out.writeInt(map_tasks);
    out.writeInt(reduce_tasks);
    out.writeInt(max_map_tasks);
    out.writeInt(max_reduce_tasks);
    out.writeLong(used_memory);
    out.writeLong(max_memory);
    WritableUtils.writeEnum(out, state);
    out.writeInt(total_map_tasks);
    out.writeInt(total_reduce_tasks);
    out.writeInt(taskTrackersDetails.size());
    for (TaskTrackerStatus status : taskTrackersDetails) {
      status.write(out);
    }

    out.writeInt(taskTrackerExtendedTasks.size());
    for (Map.Entry> trackerTasks :
            taskTrackerExtendedTasks.entrySet()) {
      Text.writeString(out, trackerTasks.getKey());

      Collection tasks = trackerTasks.getValue();
      out.writeInt(tasks.size());

      for (TaskStatus task : tasks) {
        TaskStatus.writeTaskStatus(out, task);
      }
    }
  }

  public void readFields(DataInput in) throws IOException {
    numActiveTrackers = in.readInt();
    int numTrackerNames = in.readInt();
    if (numTrackerNames > 0) {
      for (int i = 0; i < numTrackerNames; i++) {
        String name = Text.readString(in);
        activeTrackers.add(name);
      }
    }
    numBlacklistedTrackers = in.readInt();
    numTrackerNames = in.readInt();
    if (numTrackerNames > 0) {
      for (int i = 0; i < numTrackerNames; i++) {
        String name = Text.readString(in);
        blacklistedTrackers.add(name);
      }
    }
    numExcludedNodes = in.readInt();
    ttExpiryInterval = in.readLong();
    map_tasks = in.readInt();
    reduce_tasks = in.readInt();
    max_map_tasks = in.readInt();
    max_reduce_tasks = in.readInt();
    used_memory = in.readLong();
    max_memory = in.readLong();
    state = WritableUtils.readEnum(in, JobTracker.State.class);

    total_map_tasks = in.readInt();
    total_reduce_tasks = in.readInt();
    int taskTrackers = in.readInt();
    for (int i = 0; i < taskTrackers; i++) {
      TaskTrackerStatus status = new TaskTrackerStatus();
      status.readFields(in);
      taskTrackersDetails.add(status);
    }
    int mapSize = in.readInt();
    for (int i = 0; i < mapSize; i++) {
      String trackerName = Text.readString(in);
      int numTasks = in.readInt();
      Collection tasks = new ArrayList(numTasks);
      for (int j = 0; j < numTasks; j++) {
        TaskStatus status = TaskStatus.readTaskStatus(in);
        tasks.add(status);
      }

      taskTrackerExtendedTasks.put(trackerName, tasks);
    }
  }
}