001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.mapred;
020
021 import java.io.DataInput;
022 import java.io.DataOutput;
023 import java.io.IOException;
024 import java.util.ArrayList;
025 import java.util.Collection;
026
027 import org.apache.hadoop.classification.InterfaceAudience;
028 import org.apache.hadoop.classification.InterfaceStability;
029 import org.apache.hadoop.io.Text;
030 import org.apache.hadoop.io.Writable;
031 import org.apache.hadoop.io.WritableUtils;
032 import org.apache.hadoop.mapreduce.ClusterMetrics;
033 import org.apache.hadoop.mapreduce.TaskTrackerInfo;
034 import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
035 import org.apache.hadoop.util.StringInterner;
036
037 /**
038 * Status information on the current state of the Map-Reduce cluster.
039 *
040 * <p><code>ClusterStatus</code> provides clients with information such as:
041 * <ol>
042 * <li>
043 * Size of the cluster.
044 * </li>
045 * <li>
046 * Name of the trackers.
047 * </li>
048 * <li>
049 * Task capacity of the cluster.
050 * </li>
051 * <li>
052 * The number of currently running map & reduce tasks.
053 * </li>
054 * <li>
055 * State of the <code>JobTracker</code>.
056 * </li>
057 * <li>
058 * Details regarding black listed trackers.
059 * </li>
060 * </ol></p>
061 *
062 * <p>Clients can query for the latest <code>ClusterStatus</code>, via
063 * {@link JobClient#getClusterStatus()}.</p>
064 *
065 * @see JobClient
066 */
067 @InterfaceAudience.Public
068 @InterfaceStability.Stable
069 public class ClusterStatus implements Writable {
070 /**
071 * Class which encapsulates information about a blacklisted tasktracker.
072 *
073 * The information includes the tasktracker's name and reasons for
074 * getting blacklisted. The toString method of the class will print
075 * the information in a whitespace separated fashion to enable parsing.
076 */
077 public static class BlackListInfo implements Writable {
078
079 private String trackerName;
080
081 private String reasonForBlackListing;
082
083 private String blackListReport;
084
085 BlackListInfo() {
086 }
087
088
089 /**
090 * Gets the blacklisted tasktracker's name.
091 *
092 * @return tracker's name.
093 */
094 public String getTrackerName() {
095 return trackerName;
096 }
097
098 /**
099 * Gets the reason for which the tasktracker was blacklisted.
100 *
101 * @return reason which tracker was blacklisted
102 */
103 public String getReasonForBlackListing() {
104 return reasonForBlackListing;
105 }
106
107 /**
108 * Sets the blacklisted tasktracker's name.
109 *
110 * @param trackerName of the tracker.
111 */
112 void setTrackerName(String trackerName) {
113 this.trackerName = trackerName;
114 }
115
116 /**
117 * Sets the reason for which the tasktracker was blacklisted.
118 *
119 * @param reasonForBlackListing
120 */
121 void setReasonForBlackListing(String reasonForBlackListing) {
122 this.reasonForBlackListing = reasonForBlackListing;
123 }
124
125 /**
126 * Gets a descriptive report about why the tasktracker was blacklisted.
127 *
128 * @return report describing why the tasktracker was blacklisted.
129 */
130 public String getBlackListReport() {
131 return blackListReport;
132 }
133
134 /**
135 * Sets a descriptive report about why the tasktracker was blacklisted.
136 * @param blackListReport report describing why the tasktracker
137 * was blacklisted.
138 */
139 void setBlackListReport(String blackListReport) {
140 this.blackListReport = blackListReport;
141 }
142
143 @Override
144 public void readFields(DataInput in) throws IOException {
145 trackerName = StringInterner.weakIntern(Text.readString(in));
146 reasonForBlackListing = StringInterner.weakIntern(Text.readString(in));
147 blackListReport = StringInterner.weakIntern(Text.readString(in));
148 }
149
150 @Override
151 public void write(DataOutput out) throws IOException {
152 Text.writeString(out, trackerName);
153 Text.writeString(out, reasonForBlackListing);
154 Text.writeString(out, blackListReport);
155 }
156
157 @Override
158 /**
159 * Print information related to the blacklisted tasktracker in a
160 * whitespace separated fashion.
161 *
162 * The method changes any newlines in the report describing why
163 * the tasktracker was blacklisted to a ':' for enabling better
164 * parsing.
165 */
166 public String toString() {
167 StringBuilder sb = new StringBuilder();
168 sb.append(trackerName);
169 sb.append("\t");
170 sb.append(reasonForBlackListing);
171 sb.append("\t");
172 sb.append(blackListReport.replace("\n", ":"));
173 return sb.toString();
174 }
175
176 }
177
178 public static final int UNINITIALIZED_MEMORY_VALUE = -1;
179
180 private int numActiveTrackers;
181 private Collection<String> activeTrackers = new ArrayList<String>();
182 private int numBlacklistedTrackers;
183 private int numExcludedNodes;
184 private long ttExpiryInterval;
185 private int map_tasks;
186 private int reduce_tasks;
187 private int max_map_tasks;
188 private int max_reduce_tasks;
189 private JobTrackerStatus status;
190 private Collection<BlackListInfo> blacklistedTrackersInfo =
191 new ArrayList<BlackListInfo>();
192
193 ClusterStatus() {}
194
195 /**
196 * Construct a new cluster status.
197 *
198 * @param trackers no. of tasktrackers in the cluster
199 * @param blacklists no of blacklisted task trackers in the cluster
200 * @param ttExpiryInterval the tasktracker expiry interval
201 * @param maps no. of currently running map-tasks in the cluster
202 * @param reduces no. of currently running reduce-tasks in the cluster
203 * @param maxMaps the maximum no. of map tasks in the cluster
204 * @param maxReduces the maximum no. of reduce tasks in the cluster
205 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
206 */
207 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
208 int maps, int reduces,
209 int maxMaps, int maxReduces, JobTrackerStatus status) {
210 this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
211 maxReduces, status, 0);
212 }
213
214 /**
215 * Construct a new cluster status.
216 *
217 * @param trackers no. of tasktrackers in the cluster
218 * @param blacklists no of blacklisted task trackers in the cluster
219 * @param ttExpiryInterval the tasktracker expiry interval
220 * @param maps no. of currently running map-tasks in the cluster
221 * @param reduces no. of currently running reduce-tasks in the cluster
222 * @param maxMaps the maximum no. of map tasks in the cluster
223 * @param maxReduces the maximum no. of reduce tasks in the cluster
224 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
225 * @param numDecommissionedNodes number of decommission trackers
226 */
227 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
228 int maps, int reduces, int maxMaps, int maxReduces,
229 JobTrackerStatus status, int numDecommissionedNodes) {
230 numActiveTrackers = trackers;
231 numBlacklistedTrackers = blacklists;
232 this.numExcludedNodes = numDecommissionedNodes;
233 this.ttExpiryInterval = ttExpiryInterval;
234 map_tasks = maps;
235 reduce_tasks = reduces;
236 max_map_tasks = maxMaps;
237 max_reduce_tasks = maxReduces;
238 this.status = status;
239 }
240
241 /**
242 * Construct a new cluster status.
243 *
244 * @param activeTrackers active tasktrackers in the cluster
245 * @param blacklistedTrackers blacklisted tasktrackers in the cluster
246 * @param ttExpiryInterval the tasktracker expiry interval
247 * @param maps no. of currently running map-tasks in the cluster
248 * @param reduces no. of currently running reduce-tasks in the cluster
249 * @param maxMaps the maximum no. of map tasks in the cluster
250 * @param maxReduces the maximum no. of reduce tasks in the cluster
251 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
252 */
253 ClusterStatus(Collection<String> activeTrackers,
254 Collection<BlackListInfo> blacklistedTrackers,
255 long ttExpiryInterval,
256 int maps, int reduces, int maxMaps, int maxReduces,
257 JobTrackerStatus status) {
258 this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces,
259 maxMaps, maxReduces, status, 0);
260 }
261
262
263 /**
264 * Construct a new cluster status.
265 *
266 * @param activeTrackers active tasktrackers in the cluster
267 * @param blackListedTrackerInfo blacklisted tasktrackers information
268 * in the cluster
269 * @param ttExpiryInterval the tasktracker expiry interval
270 * @param maps no. of currently running map-tasks in the cluster
271 * @param reduces no. of currently running reduce-tasks in the cluster
272 * @param maxMaps the maximum no. of map tasks in the cluster
273 * @param maxReduces the maximum no. of reduce tasks in the cluster
274 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
275 * @param numDecommissionNodes number of decommission trackers
276 */
277
278 ClusterStatus(Collection<String> activeTrackers,
279 Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval,
280 int maps, int reduces, int maxMaps, int maxReduces,
281 JobTrackerStatus status, int numDecommissionNodes) {
282 this(activeTrackers.size(), blackListedTrackerInfo.size(),
283 ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status,
284 numDecommissionNodes);
285 this.activeTrackers = activeTrackers;
286 this.blacklistedTrackersInfo = blackListedTrackerInfo;
287 }
288
289 /**
290 * Get the number of task trackers in the cluster.
291 *
292 * @return the number of task trackers in the cluster.
293 */
294 public int getTaskTrackers() {
295 return numActiveTrackers;
296 }
297
298 /**
299 * Get the names of task trackers in the cluster.
300 *
301 * @return the active task trackers in the cluster.
302 */
303 public Collection<String> getActiveTrackerNames() {
304 return activeTrackers;
305 }
306
307 /**
308 * Get the names of task trackers in the cluster.
309 *
310 * @return the blacklisted task trackers in the cluster.
311 */
312 public Collection<String> getBlacklistedTrackerNames() {
313 ArrayList<String> blacklistedTrackers = new ArrayList<String>();
314 for(BlackListInfo bi : blacklistedTrackersInfo) {
315 blacklistedTrackers.add(bi.getTrackerName());
316 }
317 return blacklistedTrackers;
318 }
319
320 /**
321 * Get the number of blacklisted task trackers in the cluster.
322 *
323 * @return the number of blacklisted task trackers in the cluster.
324 */
325 public int getBlacklistedTrackers() {
326 return numBlacklistedTrackers;
327 }
328
329 /**
330 * Get the number of excluded hosts in the cluster.
331 * @return the number of excluded hosts in the cluster.
332 */
333 public int getNumExcludedNodes() {
334 return numExcludedNodes;
335 }
336
337 /**
338 * Get the tasktracker expiry interval for the cluster
339 * @return the expiry interval in msec
340 */
341 public long getTTExpiryInterval() {
342 return ttExpiryInterval;
343 }
344
345 /**
346 * Get the number of currently running map tasks in the cluster.
347 *
348 * @return the number of currently running map tasks in the cluster.
349 */
350 public int getMapTasks() {
351 return map_tasks;
352 }
353
354 /**
355 * Get the number of currently running reduce tasks in the cluster.
356 *
357 * @return the number of currently running reduce tasks in the cluster.
358 */
359 public int getReduceTasks() {
360 return reduce_tasks;
361 }
362
363 /**
364 * Get the maximum capacity for running map tasks in the cluster.
365 *
366 * @return the maximum capacity for running map tasks in the cluster.
367 */
368 public int getMaxMapTasks() {
369 return max_map_tasks;
370 }
371
372 /**
373 * Get the maximum capacity for running reduce tasks in the cluster.
374 *
375 * @return the maximum capacity for running reduce tasks in the cluster.
376 */
377 public int getMaxReduceTasks() {
378 return max_reduce_tasks;
379 }
380
381 /**
382 * Get the JobTracker's status.
383 *
384 * @return {@link JobTrackerStatus} of the JobTracker
385 */
386 public JobTrackerStatus getJobTrackerStatus() {
387 return status;
388 }
389
390 /**
391 * Returns UNINITIALIZED_MEMORY_VALUE (-1)
392 */
393 @Deprecated
394 public long getMaxMemory() {
395 return UNINITIALIZED_MEMORY_VALUE;
396 }
397
398 /**
399 * Returns UNINITIALIZED_MEMORY_VALUE (-1)
400 */
401 @Deprecated
402 public long getUsedMemory() {
403 return UNINITIALIZED_MEMORY_VALUE;
404 }
405
406 /**
407 * Gets the list of blacklisted trackers along with reasons for blacklisting.
408 *
409 * @return the collection of {@link BlackListInfo} objects.
410 *
411 */
412 public Collection<BlackListInfo> getBlackListedTrackersInfo() {
413 return blacklistedTrackersInfo;
414 }
415
416 public void write(DataOutput out) throws IOException {
417 if (activeTrackers.size() == 0) {
418 out.writeInt(numActiveTrackers);
419 out.writeInt(0);
420 } else {
421 out.writeInt(activeTrackers.size());
422 out.writeInt(activeTrackers.size());
423 for (String tracker : activeTrackers) {
424 Text.writeString(out, tracker);
425 }
426 }
427 if (blacklistedTrackersInfo.size() == 0) {
428 out.writeInt(numBlacklistedTrackers);
429 out.writeInt(blacklistedTrackersInfo.size());
430 } else {
431 out.writeInt(blacklistedTrackersInfo.size());
432 out.writeInt(blacklistedTrackersInfo.size());
433 for (BlackListInfo tracker : blacklistedTrackersInfo) {
434 tracker.write(out);
435 }
436 }
437 out.writeInt(numExcludedNodes);
438 out.writeLong(ttExpiryInterval);
439 out.writeInt(map_tasks);
440 out.writeInt(reduce_tasks);
441 out.writeInt(max_map_tasks);
442 out.writeInt(max_reduce_tasks);
443 WritableUtils.writeEnum(out, status);
444 }
445
446 public void readFields(DataInput in) throws IOException {
447 numActiveTrackers = in.readInt();
448 int numTrackerNames = in.readInt();
449 if (numTrackerNames > 0) {
450 for (int i = 0; i < numTrackerNames; i++) {
451 String name = StringInterner.weakIntern(Text.readString(in));
452 activeTrackers.add(name);
453 }
454 }
455 numBlacklistedTrackers = in.readInt();
456 int blackListTrackerInfoSize = in.readInt();
457 if(blackListTrackerInfoSize > 0) {
458 for (int i = 0; i < blackListTrackerInfoSize; i++) {
459 BlackListInfo info = new BlackListInfo();
460 info.readFields(in);
461 blacklistedTrackersInfo.add(info);
462 }
463 }
464 numExcludedNodes = in.readInt();
465 ttExpiryInterval = in.readLong();
466 map_tasks = in.readInt();
467 reduce_tasks = in.readInt();
468 max_map_tasks = in.readInt();
469 max_reduce_tasks = in.readInt();
470 status = WritableUtils.readEnum(in, JobTrackerStatus.class);
471 }
472 }