001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import java.text.ParseException;
022    
023    import org.apache.commons.logging.Log;
024    import org.apache.hadoop.classification.InterfaceAudience;
025    import org.apache.hadoop.classification.InterfaceStability;
026    import org.apache.hadoop.io.IntWritable;
027    import org.apache.hadoop.io.Text;
028    import org.apache.hadoop.io.Writable;
029    import org.apache.hadoop.io.WritableUtils;
030    import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
031    import org.apache.hadoop.util.StringUtils;
032    import org.apache.hadoop.mapreduce.FileSystemCounter;
033    import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
034    import org.apache.hadoop.mapreduce.counters.AbstractCounters;
035    import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
036    import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
037    import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
038    import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
039    import org.apache.hadoop.mapreduce.counters.GenericCounter;
040    import org.apache.hadoop.mapreduce.counters.Limits;
041    import static org.apache.hadoop.mapreduce.util.CountersStrings.*;
042    
043    /**
044     * A set of named counters.
045     *
046     * <p><code>Counters</code> represent global counters, defined either by the
047     * Map-Reduce framework or applications. Each <code>Counter</code> can be of
048     * any {@link Enum} type.</p>
049     *
050     * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
051     * counters from a particular <code>Enum</code> class.
052     * @deprecated Use {@link org.apache.hadoop.mapreduce.Counters} instead.
053     */
054    @Deprecated
055    @InterfaceAudience.Public
056    @InterfaceStability.Stable
057    public class Counters
058        extends AbstractCounters<Counters.Counter, Counters.Group> {
059    
060      public Counters() {
061        super(groupFactory);
062      }
063    
064      public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
065        super(newCounters, groupFactory);
066      }
067    
068      /**
069       * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
070       * @param newCounters new Counters
071       * @return old Counters instance corresponding to newCounters
072       */
073      static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
074        return new Counters(newCounters);
075      }
076    
077      /**
078       * A counter record, comprising its name and value.
079       */
080      public interface Counter extends org.apache.hadoop.mapreduce.Counter {
081    
082        /**
083         * Returns the compact stringified version of the counter in the format
084         * [(actual-name)(display-name)(value)]
085         * @return the stringified result
086         */
087        String makeEscapedCompactString();
088    
089        /**
090         * Checks for (content) equality of two (basic) counters
091         * @param counter to compare
092         * @return true if content equals
093         * @deprecated
094         */
095        @Deprecated
096        boolean contentEquals(Counter counter);
097    
098        /**
099         * @return the value of the counter
100         */
101        long getCounter();
102      }
103    
104      static class OldCounterImpl extends GenericCounter implements Counter {
105    
106        OldCounterImpl() {
107        }
108    
109        OldCounterImpl(String name, String displayName, long value) {
110          super(name, displayName, value);
111        }
112    
113        @Override
114        public synchronized String makeEscapedCompactString() {
115          return toEscapedCompactString(this);
116        }
117    
118        @Override @Deprecated
119        public boolean contentEquals(Counter counter) {
120          return equals(counter);
121        }
122    
123        @Override
124        public long getCounter() {
125          return getValue();
126        }
127      }
128    
129      /**
130       *  <code>Group</code> of counters, comprising of counters from a particular
131       *  counter {@link Enum} class.
132       *
133       *  <p><code>Group</code>handles localization of the class name and the
134       *  counter names.</p>
135       */
136      public static interface Group extends CounterGroupBase<Counter> {
137    
138        /**
139         * @param counterName the name of the counter
140         * @return the value of the specified counter, or 0 if the counter does
141         * not exist.
142         */
143        long getCounter(String counterName);
144    
145        /**
146         * @return the compact stringified version of the group in the format
147         * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
148         * for the counters within.
149         */
150        String makeEscapedCompactString();
151    
152        /**
153         * Get the counter for the given id and create it if it doesn't exist.
154         * @param id the numeric id of the counter within the group
155         * @param name the internal counter name
156         * @return the counter
157         * @deprecated use {@link #findCounter(String)} instead
158         */
159        @Deprecated
160        Counter getCounter(int id, String name);
161    
162        /**
163         * Get the counter for the given name and create it if it doesn't exist.
164         * @param name the internal counter name
165         * @return the counter
166         */
167        Counter getCounterForName(String name);
168      }
169    
170      // All the group impls need this for legacy group interface
171      static long getCounterValue(Group group, String counterName) {
172        Counter counter = group.findCounter(counterName, false);
173        if (counter != null) return counter.getValue();
174        return 0L;
175      }
176    
177      // Mix the generic group implementation into the Group interface
178      private static class GenericGroup extends AbstractCounterGroup<Counter>
179                                        implements Group {
180    
181        GenericGroup(String name, String displayName, Limits limits) {
182          super(name, displayName, limits);
183        }
184    
185        @Override
186        public long getCounter(String counterName) {
187          return getCounterValue(this, counterName);
188        }
189    
190        @Override
191        public String makeEscapedCompactString() {
192          return toEscapedCompactString(this);
193        }
194    
195        @Override
196        public Counter getCounter(int id, String name) {
197          return findCounter(name);
198        }
199    
200        @Override
201        public Counter getCounterForName(String name) {
202          return findCounter(name);
203        }
204    
205        @Override
206        protected Counter newCounter(String counterName, String displayName,
207                                     long value) {
208          return new OldCounterImpl(counterName, displayName, value);
209        }
210    
211        @Override
212        protected Counter newCounter() {
213          return new OldCounterImpl();
214        }
215      }
216    
217      // Mix the framework group implementation into the Group interface
218      private static class FrameworkGroupImpl<T extends Enum<T>>
219          extends FrameworkCounterGroup<T, Counter> implements Group {
220    
221        // Mix the framework counter implmementation into the Counter interface
222        class FrameworkCounterImpl extends FrameworkCounter implements Counter {
223    
224          FrameworkCounterImpl(T key) {
225            super(key);
226          }
227    
228          @Override
229          public String makeEscapedCompactString() {
230            return toEscapedCompactString(this);
231          }
232    
233          @Override
234          public boolean contentEquals(Counter counter) {
235            return equals(counter);
236          }
237    
238          @Override
239          public long getCounter() {
240            return getValue();
241          }
242        }
243    
244        FrameworkGroupImpl(Class<T> cls) {
245          super(cls);
246        }
247    
248        @Override
249        public long getCounter(String counterName) {
250          return getCounterValue(this, counterName);
251        }
252    
253        @Override
254        public String makeEscapedCompactString() {
255          return toEscapedCompactString(this);
256        }
257    
258        @Override @Deprecated
259        public Counter getCounter(int id, String name) {
260          return findCounter(name);
261        }
262    
263        @Override
264        public Counter getCounterForName(String name) {
265          return findCounter(name);
266        }
267    
268        @Override
269        protected Counter newCounter(T key) {
270          return new FrameworkCounterImpl(key);
271        }
272      }
273    
274      // Mix the file system counter group implementation into the Group interface
275      private static class FSGroupImpl extends FileSystemCounterGroup<Counter>
276                                       implements Group {
277    
278        private class FSCounterImpl extends FSCounter implements Counter {
279    
280          FSCounterImpl(String scheme, FileSystemCounter key) {
281            super(scheme, key);
282          }
283    
284          @Override
285          public String makeEscapedCompactString() {
286            return toEscapedCompactString(this);
287          }
288    
289          @Override @Deprecated
290          public boolean contentEquals(Counter counter) {
291            throw new UnsupportedOperationException("Not supported yet.");
292          }
293    
294          @Override
295          public long getCounter() {
296            return getValue();
297          }
298    
299        }
300    
301        @Override
302        protected Counter newCounter(String scheme, FileSystemCounter key) {
303          return new FSCounterImpl(scheme, key);
304        }
305    
306        @Override
307        public long getCounter(String counterName) {
308          return getCounterValue(this, counterName);
309        }
310    
311        @Override
312        public String makeEscapedCompactString() {
313          return toEscapedCompactString(this);
314        }
315    
316        @Override @Deprecated
317        public Counter getCounter(int id, String name) {
318          return findCounter(name);
319        }
320    
321        @Override
322        public Counter getCounterForName(String name) {
323          return findCounter(name);
324        }
325    
326      }
327    
328      public synchronized Counter findCounter(String group, String name) {
329        if (name.equals("MAP_INPUT_BYTES")) {
330          LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
331                   "Use FileInputFormatCounters as group name and " +
332                   " BYTES_READ as counter name instead");
333          return findCounter(FileInputFormatCounter.BYTES_READ);
334        }
335        return getGroup(group).getCounterForName(name);
336      }
337    
338      /**
339       * Provide factory methods for counter group factory implementation.
340       * See also the GroupFactory in
341       *  {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
342       */
343      static class GroupFactory extends CounterGroupFactory<Counter, Group> {
344    
345        @Override
346        protected <T extends Enum<T>>
347        FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
348          return new FrameworkGroupFactory<Group>() {
349            @Override public Group newGroup(String name) {
350              return new FrameworkGroupImpl<T>(cls); // impl in this package
351            }
352          };
353        }
354    
355        @Override
356        protected Group newGenericGroup(String name, String displayName,
357                                        Limits limits) {
358          return new GenericGroup(name, displayName, limits);
359        }
360    
361        @Override
362        protected Group newFileSystemGroup() {
363          return new FSGroupImpl();
364        }
365      }
366    
367      private static final GroupFactory groupFactory = new GroupFactory();
368    
369      /**
370       * Find a counter by using strings
371       * @param group the name of the group
372       * @param id the id of the counter within the group (0 to N-1)
373       * @param name the internal name of the counter
374       * @return the counter for that name
375       * @deprecated use {@link #findCounter(String, String)} instead
376       */
377      @Deprecated
378      public Counter findCounter(String group, int id, String name) {
379        return findCounter(group, name);
380      }
381    
382      /**
383       * Increments the specified counter by the specified amount, creating it if
384       * it didn't already exist.
385       * @param key identifies a counter
386       * @param amount amount by which counter is to be incremented
387       */
388      public void incrCounter(Enum<?> key, long amount) {
389        findCounter(key).increment(amount);
390      }
391    
392      /**
393       * Increments the specified counter by the specified amount, creating it if
394       * it didn't already exist.
395       * @param group the name of the group
396       * @param counter the internal name of the counter
397       * @param amount amount by which counter is to be incremented
398       */
399      public void incrCounter(String group, String counter, long amount) {
400        findCounter(group, counter).increment(amount);
401      }
402    
403      /**
404       * Returns current value of the specified counter, or 0 if the counter
405       * does not exist.
406       * @param key the counter enum to lookup
407       * @return the counter value or 0 if counter not found
408       */
409      public synchronized long getCounter(Enum<?> key) {
410        return findCounter(key).getValue();
411      }
412    
413      /**
414       * Increments multiple counters by their amounts in another Counters
415       * instance.
416       * @param other the other Counters instance
417       */
418      public synchronized void incrAllCounters(Counters other) {
419        for (Group otherGroup: other) {
420          Group group = getGroup(otherGroup.getName());
421          group.setDisplayName(otherGroup.getDisplayName());
422          for (Counter otherCounter : otherGroup) {
423            Counter counter = group.getCounterForName(otherCounter.getName());
424            counter.setDisplayName(otherCounter.getDisplayName());
425            counter.increment(otherCounter.getValue());
426          }
427        }
428      }
429    
430      /**
431       * @return the total number of counters
432       * @deprecated use {@link #countCounters()} instead
433       */
434      public int size() {
435        return countCounters();
436      }
437    
438      /**
439       * Convenience method for computing the sum of two sets of counters.
440       * @param a the first counters
441       * @param b the second counters
442       * @return a new summed counters object
443       */
444      public static Counters sum(Counters a, Counters b) {
445        Counters counters = new Counters();
446        counters.incrAllCounters(a);
447        counters.incrAllCounters(b);
448        return counters;
449      }
450    
451      /**
452       * Logs the current counter values.
453       * @param log The log to use.
454       */
455      public void log(Log log) {
456        log.info("Counters: " + size());
457        for(Group group: this) {
458          log.info("  " + group.getDisplayName());
459          for (Counter counter: group) {
460            log.info("    " + counter.getDisplayName() + "=" +
461                     counter.getCounter());
462          }
463        }
464      }
465    
466      /**
467       * Represent the counter in a textual format that can be converted back to
468       * its object form
469       * @return the string in the following format
470       * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
471       */
472      public String makeEscapedCompactString() {
473        return toEscapedCompactString(this);
474      }
475    
476      /**
477       * Convert a stringified (by {@link #makeEscapedCompactString()} counter
478       * representation into a counter object.
479       * @param compactString to parse
480       * @return a new counters object
481       * @throws ParseException
482       */
483      public static Counters fromEscapedCompactString(String compactString)
484          throws ParseException {
485        return parseEscapedCompactString(compactString, new Counters());
486      }
487    }