001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.mapred;
020
021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023
024 import java.io.DataInput;
025 import java.io.DataOutput;
026 import java.io.IOException;
027 import java.text.ParseException;
028 import java.util.Collection;
029 import java.util.Iterator;
030
031 import org.apache.commons.collections.IteratorUtils;
032 import org.apache.commons.logging.Log;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.classification.InterfaceStability;
035 import org.apache.hadoop.mapreduce.FileSystemCounter;
036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037 import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042 import org.apache.hadoop.mapreduce.counters.GenericCounter;
043 import org.apache.hadoop.mapreduce.counters.Limits;
044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045 import org.apache.hadoop.mapreduce.util.CountersStrings;
046
047 import com.google.common.collect.Iterators;
048
049 /**
050 * A set of named counters.
051 *
052 * <p><code>Counters</code> represent global counters, defined either by the
053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054 * any {@link Enum} type.</p>
055 *
056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057 * counters from a particular <code>Enum</code> class.
058 */
059 @InterfaceAudience.Public
060 @InterfaceStability.Stable
061 public class Counters
062 extends AbstractCounters<Counters.Counter, Counters.Group> {
063
064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065
066 public Counters() {
067 super(groupFactory);
068 }
069
070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071 super(newCounters, groupFactory);
072 }
073
074 /**
075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076 * @param newCounters new Counters
077 * @return old Counters instance corresponding to newCounters
078 */
079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080 return new Counters(newCounters);
081 }
082
083 public synchronized Group getGroup(String groupName) {
084 return super.getGroup(groupName);
085 }
086
087 @SuppressWarnings("unchecked")
088 public synchronized Collection<String> getGroupNames() {
089 return IteratorUtils.toList(super.getGroupNames().iterator());
090 }
091
092 public synchronized String makeCompactString() {
093 StringBuilder builder = new StringBuilder();
094 boolean first = true;
095 for(Group group: this){
096 for(Counter counter: group) {
097 if (first) {
098 first = false;
099 } else {
100 builder.append(',');
101 }
102 builder.append(group.getDisplayName());
103 builder.append('.');
104 builder.append(counter.getDisplayName());
105 builder.append(':');
106 builder.append(counter.getCounter());
107 }
108 }
109 return builder.toString();
110 }
111
112 /**
113 * A counter record, comprising its name and value.
114 */
115 @InterfaceAudience.Public
116 @InterfaceStability.Stable
117 public static class Counter implements org.apache.hadoop.mapreduce.Counter {
118 org.apache.hadoop.mapreduce.Counter realCounter;
119
120 Counter(org.apache.hadoop.mapreduce.Counter counter) {
121 this.realCounter = counter;
122 }
123
124 public Counter() {
125 this(new GenericCounter());
126 }
127
128 @SuppressWarnings("deprecation")
129 @Override
130 public void setDisplayName(String displayName) {
131 realCounter.setDisplayName(displayName);
132 }
133
134 @Override
135 public String getName() {
136 return realCounter.getName();
137 }
138
139 @Override
140 public String getDisplayName() {
141 return realCounter.getDisplayName();
142 }
143
144 @Override
145 public long getValue() {
146 return realCounter.getValue();
147 }
148
149 @Override
150 public void setValue(long value) {
151 realCounter.setValue(value);
152 }
153
154 @Override
155 public void increment(long incr) {
156 realCounter.increment(incr);
157 }
158
159 @Override
160 public void write(DataOutput out) throws IOException {
161 realCounter.write(out);
162 }
163
164 @Override
165 public void readFields(DataInput in) throws IOException {
166 realCounter.readFields(in);
167 }
168
169 /**
170 * Returns the compact stringified version of the counter in the format
171 * [(actual-name)(display-name)(value)]
172 * @return the stringified result
173 */
174 public String makeEscapedCompactString() {
175 return toEscapedCompactString(realCounter);
176 }
177
178 /**
179 * Checks for (content) equality of two (basic) counters
180 * @param counter to compare
181 * @return true if content equals
182 * @deprecated
183 */
184 @Deprecated
185 public boolean contentEquals(Counter counter) {
186 return realCounter.equals(counter.getUnderlyingCounter());
187 }
188
189 /**
190 * @return the value of the counter
191 */
192 public long getCounter() {
193 return realCounter.getValue();
194 }
195
196 @Override
197 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
198 return realCounter;
199 }
200
201 @Override
202 public synchronized boolean equals(Object genericRight) {
203 if (genericRight instanceof Counter) {
204 synchronized (genericRight) {
205 Counter right = (Counter) genericRight;
206 return getName().equals(right.getName()) &&
207 getDisplayName().equals(right.getDisplayName()) &&
208 getValue() == right.getValue();
209 }
210 }
211 return false;
212 }
213
214 @Override
215 public int hashCode() {
216 return realCounter.hashCode();
217 }
218 }
219
220
221 /**
222 * <code>Group</code> of counters, comprising of counters from a particular
223 * counter {@link Enum} class.
224 *
225 * <p><code>Group</code>handles localization of the class name and the
226 * counter names.</p>
227 */
228 @InterfaceAudience.Public
229 @InterfaceStability.Stable
230 public static class Group implements CounterGroupBase<Counter> {
231 private CounterGroupBase<Counter> realGroup;
232
233 protected Group() {
234 realGroup = null;
235 }
236
237 Group(GenericGroup group) {
238 this.realGroup = group;
239 }
240 Group(FSGroupImpl group) {
241 this.realGroup = group;
242 }
243
244 @SuppressWarnings({ "unchecked", "rawtypes" })
245 Group(FrameworkGroupImpl group) {
246 this.realGroup = group;
247 }
248
249 /**
250 * @param counterName the name of the counter
251 * @return the value of the specified counter, or 0 if the counter does
252 * not exist.
253 */
254 public long getCounter(String counterName) {
255 return getCounterValue(realGroup, counterName);
256 }
257
258 /**
259 * @return the compact stringified version of the group in the format
260 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
261 * for the counters within.
262 */
263 public String makeEscapedCompactString() {
264 return toEscapedCompactString(realGroup);
265 }
266
267 /**
268 * Get the counter for the given id and create it if it doesn't exist.
269 * @param id the numeric id of the counter within the group
270 * @param name the internal counter name
271 * @return the counter
272 * @deprecated use {@link #findCounter(String)} instead
273 */
274 @Deprecated
275 public Counter getCounter(int id, String name) {
276 return findCounter(name);
277 }
278
279 /**
280 * Get the counter for the given name and create it if it doesn't exist.
281 * @param name the internal counter name
282 * @return the counter
283 */
284 public Counter getCounterForName(String name) {
285 return findCounter(name);
286 }
287
288 @Override
289 public void write(DataOutput out) throws IOException {
290 realGroup.write(out);
291 }
292
293 @Override
294 public void readFields(DataInput in) throws IOException {
295 realGroup.readFields(in);
296 }
297
298 @Override
299 public Iterator<Counter> iterator() {
300 return realGroup.iterator();
301 }
302
303 @Override
304 public String getName() {
305 return realGroup.getName();
306 }
307
308 @Override
309 public String getDisplayName() {
310 return realGroup.getDisplayName();
311 }
312
313 @Override
314 public void setDisplayName(String displayName) {
315 realGroup.setDisplayName(displayName);
316 }
317
318 @Override
319 public void addCounter(Counter counter) {
320 realGroup.addCounter(counter);
321 }
322
323 @Override
324 public Counter addCounter(String name, String displayName, long value) {
325 return realGroup.addCounter(name, displayName, value);
326 }
327
328 @Override
329 public Counter findCounter(String counterName, String displayName) {
330 return realGroup.findCounter(counterName, displayName);
331 }
332
333 @Override
334 public Counter findCounter(String counterName, boolean create) {
335 return realGroup.findCounter(counterName, create);
336 }
337
338 @Override
339 public Counter findCounter(String counterName) {
340 return realGroup.findCounter(counterName);
341 }
342
343 @Override
344 public int size() {
345 return realGroup.size();
346 }
347
348 @Override
349 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
350 realGroup.incrAllCounters(rightGroup);
351 }
352
353 @Override
354 public CounterGroupBase<Counter> getUnderlyingGroup() {
355 return realGroup;
356 }
357
358 @Override
359 public synchronized boolean equals(Object genericRight) {
360 if (genericRight instanceof CounterGroupBase<?>) {
361 @SuppressWarnings("unchecked")
362 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>)
363 genericRight).getUnderlyingGroup();
364 return Iterators.elementsEqual(iterator(), right.iterator());
365 }
366 return false;
367 }
368
369 @Override
370 public int hashCode() {
371 return realGroup.hashCode();
372 }
373 }
374
375 // All the group impls need this for legacy group interface
376 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
377 Counter counter = group.findCounter(counterName, false);
378 if (counter != null) return counter.getValue();
379 return 0L;
380 }
381
382 // Mix the generic group implementation into the Group interface
383 private static class GenericGroup extends AbstractCounterGroup<Counter> {
384
385 GenericGroup(String name, String displayName, Limits limits) {
386 super(name, displayName, limits);
387 }
388
389 @Override
390 protected Counter newCounter(String counterName, String displayName,
391 long value) {
392 return new Counter(new GenericCounter(counterName, displayName, value));
393 }
394
395 @Override
396 protected Counter newCounter() {
397 return new Counter();
398 }
399
400 @Override
401 public CounterGroupBase<Counter> getUnderlyingGroup() {
402 return this;
403 }
404 }
405
406 // Mix the framework group implementation into the Group interface
407 private static class FrameworkGroupImpl<T extends Enum<T>>
408 extends FrameworkCounterGroup<T, Counter> {
409
410 FrameworkGroupImpl(Class<T> cls) {
411 super(cls);
412 }
413
414 @Override
415 protected Counter newCounter(T key) {
416 return new Counter(new FrameworkCounter<T>(key, getName()));
417 }
418
419 @Override
420 public CounterGroupBase<Counter> getUnderlyingGroup() {
421 return this;
422 }
423 }
424
425 // Mix the file system counter group implementation into the Group interface
426 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
427
428 @Override
429 protected Counter newCounter(String scheme, FileSystemCounter key) {
430 return new Counter(new FSCounter(scheme, key));
431 }
432
433 @Override
434 public CounterGroupBase<Counter> getUnderlyingGroup() {
435 return this;
436 }
437 }
438
439 public synchronized Counter findCounter(String group, String name) {
440 if (name.equals("MAP_INPUT_BYTES")) {
441 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
442 "Use FileInputFormatCounters as group name and " +
443 " BYTES_READ as counter name instead");
444 return findCounter(FileInputFormatCounter.BYTES_READ);
445 }
446 return getGroup(group).getCounterForName(name);
447 }
448
449 /**
450 * Provide factory methods for counter group factory implementation.
451 * See also the GroupFactory in
452 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
453 */
454 static class GroupFactory extends CounterGroupFactory<Counter, Group> {
455
456 @Override
457 protected <T extends Enum<T>>
458 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
459 return new FrameworkGroupFactory<Group>() {
460 @Override public Group newGroup(String name) {
461 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
462 }
463 };
464 }
465
466 @Override
467 protected Group newGenericGroup(String name, String displayName,
468 Limits limits) {
469 return new Group(new GenericGroup(name, displayName, limits));
470 }
471
472 @Override
473 protected Group newFileSystemGroup() {
474 return new Group(new FSGroupImpl());
475 }
476 }
477
478 private static final GroupFactory groupFactory = new GroupFactory();
479
480 /**
481 * Find a counter by using strings
482 * @param group the name of the group
483 * @param id the id of the counter within the group (0 to N-1)
484 * @param name the internal name of the counter
485 * @return the counter for that name
486 * @deprecated use {@link #findCounter(String, String)} instead
487 */
488 @Deprecated
489 public Counter findCounter(String group, int id, String name) {
490 return findCounter(group, name);
491 }
492
493 /**
494 * Increments the specified counter by the specified amount, creating it if
495 * it didn't already exist.
496 * @param key identifies a counter
497 * @param amount amount by which counter is to be incremented
498 */
499 public void incrCounter(Enum<?> key, long amount) {
500 findCounter(key).increment(amount);
501 }
502
503 /**
504 * Increments the specified counter by the specified amount, creating it if
505 * it didn't already exist.
506 * @param group the name of the group
507 * @param counter the internal name of the counter
508 * @param amount amount by which counter is to be incremented
509 */
510 public void incrCounter(String group, String counter, long amount) {
511 findCounter(group, counter).increment(amount);
512 }
513
514 /**
515 * Returns current value of the specified counter, or 0 if the counter
516 * does not exist.
517 * @param key the counter enum to lookup
518 * @return the counter value or 0 if counter not found
519 */
520 public synchronized long getCounter(Enum<?> key) {
521 return findCounter(key).getValue();
522 }
523
524 /**
525 * Increments multiple counters by their amounts in another Counters
526 * instance.
527 * @param other the other Counters instance
528 */
529 public synchronized void incrAllCounters(Counters other) {
530 for (Group otherGroup: other) {
531 Group group = getGroup(otherGroup.getName());
532 group.setDisplayName(otherGroup.getDisplayName());
533 for (Counter otherCounter : otherGroup) {
534 Counter counter = group.getCounterForName(otherCounter.getName());
535 counter.setDisplayName(otherCounter.getDisplayName());
536 counter.increment(otherCounter.getValue());
537 }
538 }
539 }
540
541 /**
542 * @return the total number of counters
543 * @deprecated use {@link #countCounters()} instead
544 */
545 public int size() {
546 return countCounters();
547 }
548
549 /**
550 * Convenience method for computing the sum of two sets of counters.
551 * @param a the first counters
552 * @param b the second counters
553 * @return a new summed counters object
554 */
555 public static Counters sum(Counters a, Counters b) {
556 Counters counters = new Counters();
557 counters.incrAllCounters(a);
558 counters.incrAllCounters(b);
559 return counters;
560 }
561
562 /**
563 * Logs the current counter values.
564 * @param log The log to use.
565 */
566 public void log(Log log) {
567 log.info("Counters: " + size());
568 for(Group group: this) {
569 log.info(" " + group.getDisplayName());
570 for (Counter counter: group) {
571 log.info(" " + counter.getDisplayName() + "=" +
572 counter.getCounter());
573 }
574 }
575 }
576
577 /**
578 * Represent the counter in a textual format that can be converted back to
579 * its object form
580 * @return the string in the following format
581 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
582 */
583 public String makeEscapedCompactString() {
584 return toEscapedCompactString(this);
585 }
586
587 /**
588 * Convert a stringified (by {@link #makeEscapedCompactString()} counter
589 * representation into a counter object.
590 * @param compactString to parse
591 * @return a new counters object
592 * @throws ParseException
593 */
594 public static Counters fromEscapedCompactString(String compactString)
595 throws ParseException {
596 return parseEscapedCompactString(compactString, new Counters());
597 }
598 }