org.apache.hadoop.mapred.ClusterStatus Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.util.StringInterner;
/**
* Status information on the current state of the Map-Reduce cluster.
*
* ClusterStatus provides clients with information such as:
*
* -
* Size of the cluster.
*
* -
* Name of the trackers.
*
* -
* Task capacity of the cluster.
*
* -
* The number of currently running map and reduce tasks.
*
* -
* State of the
JobTracker.
*
* -
* Details regarding black listed trackers.
*
*
*
* Clients can query for the latest ClusterStatus, via
* {@link JobClient#getClusterStatus()}.
*
* @see JobClient
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class ClusterStatus implements Writable {
/**
* Class which encapsulates information about a blacklisted tasktracker.
*
* The information includes the tasktracker's name and reasons for
* getting blacklisted. The toString method of the class will print
* the information in a whitespace separated fashion to enable parsing.
*/
public static class BlackListInfo implements Writable {
private String trackerName;
private String reasonForBlackListing;
private String blackListReport;
BlackListInfo() {
}
/**
* Gets the blacklisted tasktracker's name.
*
* @return tracker's name.
*/
public String getTrackerName() {
return trackerName;
}
/**
* Gets the reason for which the tasktracker was blacklisted.
*
* @return reason which tracker was blacklisted
*/
public String getReasonForBlackListing() {
return reasonForBlackListing;
}
/**
* Sets the blacklisted tasktracker's name.
*
* @param trackerName of the tracker.
*/
void setTrackerName(String trackerName) {
this.trackerName = trackerName;
}
/**
* Sets the reason for which the tasktracker was blacklisted.
*
* @param reasonForBlackListing
*/
void setReasonForBlackListing(String reasonForBlackListing) {
this.reasonForBlackListing = reasonForBlackListing;
}
/**
* Gets a descriptive report about why the tasktracker was blacklisted.
*
* @return report describing why the tasktracker was blacklisted.
*/
public String getBlackListReport() {
return blackListReport;
}
/**
* Sets a descriptive report about why the tasktracker was blacklisted.
* @param blackListReport report describing why the tasktracker
* was blacklisted.
*/
void setBlackListReport(String blackListReport) {
this.blackListReport = blackListReport;
}
@Override
public void readFields(DataInput in) throws IOException {
trackerName = StringInterner.weakIntern(Text.readString(in));
reasonForBlackListing = StringInterner.weakIntern(Text.readString(in));
blackListReport = StringInterner.weakIntern(Text.readString(in));
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, trackerName);
Text.writeString(out, reasonForBlackListing);
Text.writeString(out, blackListReport);
}
@Override
/**
* Print information related to the blacklisted tasktracker in a
* whitespace separated fashion.
*
* The method changes any newlines in the report describing why
* the tasktracker was blacklisted to a ':' for enabling better
* parsing.
*/
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(trackerName);
sb.append("\t");
sb.append(reasonForBlackListing);
sb.append("\t");
sb.append(blackListReport.replace("\n", ":"));
return sb.toString();
}
@Override
public int hashCode() {
int result = trackerName != null ? trackerName.hashCode() : 0;
result = 31 * result + (reasonForBlackListing != null ?
reasonForBlackListing.hashCode() : 0);
result = 31 * result + (blackListReport != null ?
blackListReport.hashCode() : 0);
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
final BlackListInfo that = (BlackListInfo) obj;
if (trackerName == null ? that.trackerName != null :
!trackerName.equals(that.trackerName)) {
return false;
}
if (reasonForBlackListing == null ? that.reasonForBlackListing != null :
!reasonForBlackListing.equals(that.reasonForBlackListing)) {
return false;
}
if (blackListReport == null ? that.blackListReport != null :
!blackListReport.equals(that.blackListReport)) {
return false;
}
return true;
}
}
public static final long UNINITIALIZED_MEMORY_VALUE = -1;
private int numActiveTrackers;
private Collection activeTrackers = new ArrayList();
private int numBlacklistedTrackers;
private int numExcludedNodes;
private long ttExpiryInterval;
private int map_tasks;
private int reduce_tasks;
private int max_map_tasks;
private int max_reduce_tasks;
private JobTrackerStatus status;
private Collection blacklistedTrackersInfo =
new ArrayList();
private int grayListedTrackers;
ClusterStatus() {}
/**
* Construct a new cluster status.
*
* @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the JobTracker
*/
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
int maps, int reduces,
int maxMaps, int maxReduces, JobTrackerStatus status) {
this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
maxReduces, status, 0);
}
/**
* Construct a new cluster status.
*
* @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the JobTracker
* @param numDecommissionedNodes number of decommission trackers
*/
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
int numDecommissionedNodes) {
this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
maxReduces, status, numDecommissionedNodes, 0);
}
/**
* Construct a new cluster status.
*
* @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the JobTracker
* @param numDecommissionedNodes number of decommission trackers
* @param numGrayListedTrackers number of graylisted trackers
*/
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
int numDecommissionedNodes, int numGrayListedTrackers) {
numActiveTrackers = trackers;
numBlacklistedTrackers = blacklists;
this.numExcludedNodes = numDecommissionedNodes;
this.ttExpiryInterval = ttExpiryInterval;
map_tasks = maps;
reduce_tasks = reduces;
max_map_tasks = maxMaps;
max_reduce_tasks = maxReduces;
this.status = status;
this.grayListedTrackers = numGrayListedTrackers;
}
/**
* Construct a new cluster status.
*
* @param activeTrackers active tasktrackers in the cluster
* @param blacklistedTrackers blacklisted tasktrackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the JobTracker
*/
ClusterStatus(Collection activeTrackers,
Collection blacklistedTrackers,
long ttExpiryInterval,
int maps, int reduces, int maxMaps, int maxReduces,
JobTrackerStatus status) {
this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces,
maxMaps, maxReduces, status, 0);
}
/**
* Construct a new cluster status.
*
* @param activeTrackers active tasktrackers in the cluster
* @param blackListedTrackerInfo blacklisted tasktrackers information
* in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param status the {@link JobTrackerStatus} of the JobTracker
* @param numDecommissionNodes number of decommission trackers
*/
ClusterStatus(Collection activeTrackers,
Collection blackListedTrackerInfo, long ttExpiryInterval,
int maps, int reduces, int maxMaps, int maxReduces,
JobTrackerStatus status, int numDecommissionNodes) {
this(activeTrackers.size(), blackListedTrackerInfo.size(),
ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status,
numDecommissionNodes);
this.activeTrackers = activeTrackers;
this.blacklistedTrackersInfo = blackListedTrackerInfo;
}
/**
* Get the number of task trackers in the cluster.
*
* @return the number of task trackers in the cluster.
*/
public int getTaskTrackers() {
return numActiveTrackers;
}
/**
* Get the names of task trackers in the cluster.
*
* @return the active task trackers in the cluster.
*/
public Collection getActiveTrackerNames() {
return activeTrackers;
}
/**
* Get the names of task trackers in the cluster.
*
* @return the blacklisted task trackers in the cluster.
*/
public Collection getBlacklistedTrackerNames() {
ArrayList blacklistedTrackers = new ArrayList();
for(BlackListInfo bi : blacklistedTrackersInfo) {
blacklistedTrackers.add(bi.getTrackerName());
}
return blacklistedTrackers;
}
/**
* Get the names of graylisted task trackers in the cluster.
*
* The gray list of trackers is no longer available on M/R 2.x. The function
* is kept to be compatible with M/R 1.x applications.
*
* @return an empty graylisted task trackers in the cluster.
*/
@Deprecated
public Collection getGraylistedTrackerNames() {
return Collections.emptySet();
}
/**
* Get the number of graylisted task trackers in the cluster.
*
* The gray list of trackers is no longer available on M/R 2.x. The function
* is kept to be compatible with M/R 1.x applications.
*
* @return 0 graylisted task trackers in the cluster.
*/
@Deprecated
public int getGraylistedTrackers() {
return grayListedTrackers;
}
/**
* Get the number of blacklisted task trackers in the cluster.
*
* @return the number of blacklisted task trackers in the cluster.
*/
public int getBlacklistedTrackers() {
return numBlacklistedTrackers;
}
/**
* Get the number of excluded hosts in the cluster.
* @return the number of excluded hosts in the cluster.
*/
public int getNumExcludedNodes() {
return numExcludedNodes;
}
/**
* Get the tasktracker expiry interval for the cluster
* @return the expiry interval in msec
*/
public long getTTExpiryInterval() {
return ttExpiryInterval;
}
/**
* Get the number of currently running map tasks in the cluster.
*
* @return the number of currently running map tasks in the cluster.
*/
public int getMapTasks() {
return map_tasks;
}
/**
* Get the number of currently running reduce tasks in the cluster.
*
* @return the number of currently running reduce tasks in the cluster.
*/
public int getReduceTasks() {
return reduce_tasks;
}
/**
* Get the maximum capacity for running map tasks in the cluster.
*
* @return the maximum capacity for running map tasks in the cluster.
*/
public int getMaxMapTasks() {
return max_map_tasks;
}
/**
* Get the maximum capacity for running reduce tasks in the cluster.
*
* @return the maximum capacity for running reduce tasks in the cluster.
*/
public int getMaxReduceTasks() {
return max_reduce_tasks;
}
/**
* Get the JobTracker's status.
*
* @return {@link JobTrackerStatus} of the JobTracker
*/
public JobTrackerStatus getJobTrackerStatus() {
return status;
}
/**
* Returns UNINITIALIZED_MEMORY_VALUE (-1)
*/
@Deprecated
public long getMaxMemory() {
return UNINITIALIZED_MEMORY_VALUE;
}
/**
* Returns UNINITIALIZED_MEMORY_VALUE (-1)
*/
@Deprecated
public long getUsedMemory() {
return UNINITIALIZED_MEMORY_VALUE;
}
/**
* Gets the list of blacklisted trackers along with reasons for blacklisting.
*
* @return the collection of {@link BlackListInfo} objects.
*
*/
public Collection getBlackListedTrackersInfo() {
return blacklistedTrackersInfo;
}
/**
* Get the current state of the JobTracker,
* as {@link JobTracker.State}
*
* {@link JobTracker.State} should no longer be used on M/R 2.x. The function
* is kept to be compatible with M/R 1.x applications.
*
* @return the invalid state of the JobTracker.
*/
@Deprecated
public JobTracker.State getJobTrackerState() {
return JobTracker.State.RUNNING;
}
public void write(DataOutput out) throws IOException {
if (activeTrackers.size() == 0) {
out.writeInt(numActiveTrackers);
out.writeInt(0);
} else {
out.writeInt(activeTrackers.size());
out.writeInt(activeTrackers.size());
for (String tracker : activeTrackers) {
Text.writeString(out, tracker);
}
}
if (blacklistedTrackersInfo.size() == 0) {
out.writeInt(numBlacklistedTrackers);
out.writeInt(blacklistedTrackersInfo.size());
} else {
out.writeInt(blacklistedTrackersInfo.size());
out.writeInt(blacklistedTrackersInfo.size());
for (BlackListInfo tracker : blacklistedTrackersInfo) {
tracker.write(out);
}
}
out.writeInt(numExcludedNodes);
out.writeLong(ttExpiryInterval);
out.writeInt(map_tasks);
out.writeInt(reduce_tasks);
out.writeInt(max_map_tasks);
out.writeInt(max_reduce_tasks);
WritableUtils.writeEnum(out, status);
out.writeInt(grayListedTrackers);
}
public void readFields(DataInput in) throws IOException {
numActiveTrackers = in.readInt();
int numTrackerNames = in.readInt();
if (numTrackerNames > 0) {
for (int i = 0; i < numTrackerNames; i++) {
String name = StringInterner.weakIntern(Text.readString(in));
activeTrackers.add(name);
}
}
numBlacklistedTrackers = in.readInt();
int blackListTrackerInfoSize = in.readInt();
if(blackListTrackerInfoSize > 0) {
for (int i = 0; i < blackListTrackerInfoSize; i++) {
BlackListInfo info = new BlackListInfo();
info.readFields(in);
blacklistedTrackersInfo.add(info);
}
}
numExcludedNodes = in.readInt();
ttExpiryInterval = in.readLong();
map_tasks = in.readInt();
reduce_tasks = in.readInt();
max_map_tasks = in.readInt();
max_reduce_tasks = in.readInt();
status = WritableUtils.readEnum(in, JobTrackerStatus.class);
grayListedTrackers = in.readInt();
}
}