001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import java.text.ParseException; 022 023 import org.apache.commons.logging.Log; 024 import org.apache.hadoop.classification.InterfaceAudience; 025 import org.apache.hadoop.classification.InterfaceStability; 026 import org.apache.hadoop.io.IntWritable; 027 import org.apache.hadoop.io.Text; 028 import org.apache.hadoop.io.Writable; 029 import org.apache.hadoop.io.WritableUtils; 030 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; 031 import org.apache.hadoop.util.StringUtils; 032 import org.apache.hadoop.mapreduce.FileSystemCounter; 033 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; 034 import org.apache.hadoop.mapreduce.counters.AbstractCounters; 035 import org.apache.hadoop.mapreduce.counters.CounterGroupBase; 036 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; 037 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; 038 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; 039 import org.apache.hadoop.mapreduce.counters.GenericCounter; 040 import org.apache.hadoop.mapreduce.counters.Limits; 041 import static org.apache.hadoop.mapreduce.util.CountersStrings.*; 042 043 /** 044 * A set of named counters. 045 * 046 * <p><code>Counters</code> represent global counters, defined either by the 047 * Map-Reduce framework or applications. Each <code>Counter</code> can be of 048 * any {@link Enum} type.</p> 049 * 050 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of 051 * counters from a particular <code>Enum</code> class. 052 * @deprecated Use {@link org.apache.hadoop.mapreduce.Counters} instead. 053 */ 054 @Deprecated 055 @InterfaceAudience.Public 056 @InterfaceStability.Stable 057 public class Counters 058 extends AbstractCounters<Counters.Counter, Counters.Group> { 059 060 public Counters() { 061 super(groupFactory); 062 } 063 064 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { 065 super(newCounters, groupFactory); 066 } 067 068 /** 069 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters 070 * @param newCounters new Counters 071 * @return old Counters instance corresponding to newCounters 072 */ 073 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { 074 return new Counters(newCounters); 075 } 076 077 /** 078 * A counter record, comprising its name and value. 079 */ 080 public interface Counter extends org.apache.hadoop.mapreduce.Counter { 081 082 /** 083 * Returns the compact stringified version of the counter in the format 084 * [(actual-name)(display-name)(value)] 085 * @return the stringified result 086 */ 087 String makeEscapedCompactString(); 088 089 /** 090 * Checks for (content) equality of two (basic) counters 091 * @param counter to compare 092 * @return true if content equals 093 * @deprecated 094 */ 095 @Deprecated 096 boolean contentEquals(Counter counter); 097 098 /** 099 * @return the value of the counter 100 */ 101 long getCounter(); 102 } 103 104 static class OldCounterImpl extends GenericCounter implements Counter { 105 106 OldCounterImpl() { 107 } 108 109 OldCounterImpl(String name, String displayName, long value) { 110 super(name, displayName, value); 111 } 112 113 @Override 114 public synchronized String makeEscapedCompactString() { 115 return toEscapedCompactString(this); 116 } 117 118 @Override @Deprecated 119 public boolean contentEquals(Counter counter) { 120 return equals(counter); 121 } 122 123 @Override 124 public long getCounter() { 125 return getValue(); 126 } 127 } 128 129 /** 130 * <code>Group</code> of counters, comprising of counters from a particular 131 * counter {@link Enum} class. 132 * 133 * <p><code>Group</code>handles localization of the class name and the 134 * counter names.</p> 135 */ 136 public static interface Group extends CounterGroupBase<Counter> { 137 138 /** 139 * @param counterName the name of the counter 140 * @return the value of the specified counter, or 0 if the counter does 141 * not exist. 142 */ 143 long getCounter(String counterName); 144 145 /** 146 * @return the compact stringified version of the group in the format 147 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings 148 * for the counters within. 149 */ 150 String makeEscapedCompactString(); 151 152 /** 153 * Get the counter for the given id and create it if it doesn't exist. 154 * @param id the numeric id of the counter within the group 155 * @param name the internal counter name 156 * @return the counter 157 * @deprecated use {@link #findCounter(String)} instead 158 */ 159 @Deprecated 160 Counter getCounter(int id, String name); 161 162 /** 163 * Get the counter for the given name and create it if it doesn't exist. 164 * @param name the internal counter name 165 * @return the counter 166 */ 167 Counter getCounterForName(String name); 168 } 169 170 // All the group impls need this for legacy group interface 171 static long getCounterValue(Group group, String counterName) { 172 Counter counter = group.findCounter(counterName, false); 173 if (counter != null) return counter.getValue(); 174 return 0L; 175 } 176 177 // Mix the generic group implementation into the Group interface 178 private static class GenericGroup extends AbstractCounterGroup<Counter> 179 implements Group { 180 181 GenericGroup(String name, String displayName, Limits limits) { 182 super(name, displayName, limits); 183 } 184 185 @Override 186 public long getCounter(String counterName) { 187 return getCounterValue(this, counterName); 188 } 189 190 @Override 191 public String makeEscapedCompactString() { 192 return toEscapedCompactString(this); 193 } 194 195 @Override 196 public Counter getCounter(int id, String name) { 197 return findCounter(name); 198 } 199 200 @Override 201 public Counter getCounterForName(String name) { 202 return findCounter(name); 203 } 204 205 @Override 206 protected Counter newCounter(String counterName, String displayName, 207 long value) { 208 return new OldCounterImpl(counterName, displayName, value); 209 } 210 211 @Override 212 protected Counter newCounter() { 213 return new OldCounterImpl(); 214 } 215 } 216 217 // Mix the framework group implementation into the Group interface 218 private static class FrameworkGroupImpl<T extends Enum<T>> 219 extends FrameworkCounterGroup<T, Counter> implements Group { 220 221 // Mix the framework counter implmementation into the Counter interface 222 class FrameworkCounterImpl extends FrameworkCounter implements Counter { 223 224 FrameworkCounterImpl(T key) { 225 super(key); 226 } 227 228 @Override 229 public String makeEscapedCompactString() { 230 return toEscapedCompactString(this); 231 } 232 233 @Override 234 public boolean contentEquals(Counter counter) { 235 return equals(counter); 236 } 237 238 @Override 239 public long getCounter() { 240 return getValue(); 241 } 242 } 243 244 FrameworkGroupImpl(Class<T> cls) { 245 super(cls); 246 } 247 248 @Override 249 public long getCounter(String counterName) { 250 return getCounterValue(this, counterName); 251 } 252 253 @Override 254 public String makeEscapedCompactString() { 255 return toEscapedCompactString(this); 256 } 257 258 @Override @Deprecated 259 public Counter getCounter(int id, String name) { 260 return findCounter(name); 261 } 262 263 @Override 264 public Counter getCounterForName(String name) { 265 return findCounter(name); 266 } 267 268 @Override 269 protected Counter newCounter(T key) { 270 return new FrameworkCounterImpl(key); 271 } 272 } 273 274 // Mix the file system counter group implementation into the Group interface 275 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> 276 implements Group { 277 278 private class FSCounterImpl extends FSCounter implements Counter { 279 280 FSCounterImpl(String scheme, FileSystemCounter key) { 281 super(scheme, key); 282 } 283 284 @Override 285 public String makeEscapedCompactString() { 286 return toEscapedCompactString(this); 287 } 288 289 @Override @Deprecated 290 public boolean contentEquals(Counter counter) { 291 throw new UnsupportedOperationException("Not supported yet."); 292 } 293 294 @Override 295 public long getCounter() { 296 return getValue(); 297 } 298 299 } 300 301 @Override 302 protected Counter newCounter(String scheme, FileSystemCounter key) { 303 return new FSCounterImpl(scheme, key); 304 } 305 306 @Override 307 public long getCounter(String counterName) { 308 return getCounterValue(this, counterName); 309 } 310 311 @Override 312 public String makeEscapedCompactString() { 313 return toEscapedCompactString(this); 314 } 315 316 @Override @Deprecated 317 public Counter getCounter(int id, String name) { 318 return findCounter(name); 319 } 320 321 @Override 322 public Counter getCounterForName(String name) { 323 return findCounter(name); 324 } 325 326 } 327 328 public synchronized Counter findCounter(String group, String name) { 329 if (name.equals("MAP_INPUT_BYTES")) { 330 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + 331 "Use FileInputFormatCounters as group name and " + 332 " BYTES_READ as counter name instead"); 333 return findCounter(FileInputFormatCounter.BYTES_READ); 334 } 335 return getGroup(group).getCounterForName(name); 336 } 337 338 /** 339 * Provide factory methods for counter group factory implementation. 340 * See also the GroupFactory in 341 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} 342 */ 343 static class GroupFactory extends CounterGroupFactory<Counter, Group> { 344 345 @Override 346 protected <T extends Enum<T>> 347 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) { 348 return new FrameworkGroupFactory<Group>() { 349 @Override public Group newGroup(String name) { 350 return new FrameworkGroupImpl<T>(cls); // impl in this package 351 } 352 }; 353 } 354 355 @Override 356 protected Group newGenericGroup(String name, String displayName, 357 Limits limits) { 358 return new GenericGroup(name, displayName, limits); 359 } 360 361 @Override 362 protected Group newFileSystemGroup() { 363 return new FSGroupImpl(); 364 } 365 } 366 367 private static final GroupFactory groupFactory = new GroupFactory(); 368 369 /** 370 * Find a counter by using strings 371 * @param group the name of the group 372 * @param id the id of the counter within the group (0 to N-1) 373 * @param name the internal name of the counter 374 * @return the counter for that name 375 * @deprecated use {@link #findCounter(String, String)} instead 376 */ 377 @Deprecated 378 public Counter findCounter(String group, int id, String name) { 379 return findCounter(group, name); 380 } 381 382 /** 383 * Increments the specified counter by the specified amount, creating it if 384 * it didn't already exist. 385 * @param key identifies a counter 386 * @param amount amount by which counter is to be incremented 387 */ 388 public void incrCounter(Enum<?> key, long amount) { 389 findCounter(key).increment(amount); 390 } 391 392 /** 393 * Increments the specified counter by the specified amount, creating it if 394 * it didn't already exist. 395 * @param group the name of the group 396 * @param counter the internal name of the counter 397 * @param amount amount by which counter is to be incremented 398 */ 399 public void incrCounter(String group, String counter, long amount) { 400 findCounter(group, counter).increment(amount); 401 } 402 403 /** 404 * Returns current value of the specified counter, or 0 if the counter 405 * does not exist. 406 * @param key the counter enum to lookup 407 * @return the counter value or 0 if counter not found 408 */ 409 public synchronized long getCounter(Enum<?> key) { 410 return findCounter(key).getValue(); 411 } 412 413 /** 414 * Increments multiple counters by their amounts in another Counters 415 * instance. 416 * @param other the other Counters instance 417 */ 418 public synchronized void incrAllCounters(Counters other) { 419 for (Group otherGroup: other) { 420 Group group = getGroup(otherGroup.getName()); 421 group.setDisplayName(otherGroup.getDisplayName()); 422 for (Counter otherCounter : otherGroup) { 423 Counter counter = group.getCounterForName(otherCounter.getName()); 424 counter.setDisplayName(otherCounter.getDisplayName()); 425 counter.increment(otherCounter.getValue()); 426 } 427 } 428 } 429 430 /** 431 * @return the total number of counters 432 * @deprecated use {@link #countCounters()} instead 433 */ 434 public int size() { 435 return countCounters(); 436 } 437 438 /** 439 * Convenience method for computing the sum of two sets of counters. 440 * @param a the first counters 441 * @param b the second counters 442 * @return a new summed counters object 443 */ 444 public static Counters sum(Counters a, Counters b) { 445 Counters counters = new Counters(); 446 counters.incrAllCounters(a); 447 counters.incrAllCounters(b); 448 return counters; 449 } 450 451 /** 452 * Logs the current counter values. 453 * @param log The log to use. 454 */ 455 public void log(Log log) { 456 log.info("Counters: " + size()); 457 for(Group group: this) { 458 log.info(" " + group.getDisplayName()); 459 for (Counter counter: group) { 460 log.info(" " + counter.getDisplayName() + "=" + 461 counter.getCounter()); 462 } 463 } 464 } 465 466 /** 467 * Represent the counter in a textual format that can be converted back to 468 * its object form 469 * @return the string in the following format 470 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* 471 */ 472 public String makeEscapedCompactString() { 473 return toEscapedCompactString(this); 474 } 475 476 /** 477 * Convert a stringified (by {@link #makeEscapedCompactString()} counter 478 * representation into a counter object. 479 * @param compactString to parse 480 * @return a new counters object 481 * @throws ParseException 482 */ 483 public static Counters fromEscapedCompactString(String compactString) 484 throws ParseException { 485 return parseEscapedCompactString(compactString, new Counters()); 486 } 487 }