001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.conf;
020    
021    import java.io.BufferedInputStream;
022    import java.io.DataInput;
023    import java.io.DataOutput;
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.InputStreamReader;
029    import java.io.OutputStream;
030    import java.io.OutputStreamWriter;
031    import java.io.Reader;
032    import java.io.Writer;
033    import java.net.URL;
034    import java.util.ArrayList;
035    import java.util.Collection;
036    import java.util.Collections;
037    import java.util.Enumeration;
038    import java.util.HashMap;
039    import java.util.HashSet;
040    import java.util.Iterator;
041    import java.util.List;
042    import java.util.ListIterator;
043    import java.util.Map;
044    import java.util.Properties;
045    import java.util.Set;
046    import java.util.StringTokenizer;
047    import java.util.WeakHashMap;
048    import java.util.concurrent.CopyOnWriteArrayList;
049    import java.util.regex.Matcher;
050    import java.util.regex.Pattern;
051    import java.util.regex.PatternSyntaxException;
052    
053    import javax.xml.parsers.DocumentBuilder;
054    import javax.xml.parsers.DocumentBuilderFactory;
055    import javax.xml.parsers.ParserConfigurationException;
056    import javax.xml.transform.Transformer;
057    import javax.xml.transform.TransformerException;
058    import javax.xml.transform.TransformerFactory;
059    import javax.xml.transform.dom.DOMSource;
060    import javax.xml.transform.stream.StreamResult;
061    
062    import org.apache.commons.logging.Log;
063    import org.apache.commons.logging.LogFactory;
064    import org.apache.hadoop.classification.InterfaceAudience;
065    import org.apache.hadoop.classification.InterfaceStability;
066    import org.apache.hadoop.fs.FileSystem;
067    import org.apache.hadoop.fs.Path;
068    import org.apache.hadoop.fs.CommonConfigurationKeys;
069    import org.apache.hadoop.io.Writable;
070    import org.apache.hadoop.io.WritableUtils;
071    import org.apache.hadoop.util.ReflectionUtils;
072    import org.apache.hadoop.util.StringUtils;
073    import org.codehaus.jackson.JsonFactory;
074    import org.codehaus.jackson.JsonGenerator;
075    import org.w3c.dom.Comment;
076    import org.w3c.dom.DOMException;
077    import org.w3c.dom.Document;
078    import org.w3c.dom.Element;
079    import org.w3c.dom.Node;
080    import org.w3c.dom.NodeList;
081    import org.w3c.dom.Text;
082    import org.xml.sax.SAXException;
083    
084    /** 
085     * Provides access to configuration parameters.
086     *
087     * <h4 id="Resources">Resources</h4>
088     *
089     * <p>Configurations are specified by resources. A resource contains a set of
090     * name/value pairs as XML data. Each resource is named by either a 
091     * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
092     * then the classpath is examined for a file with that name.  If named by a 
093     * <code>Path</code>, then the local filesystem is examined directly, without 
094     * referring to the classpath.
095     *
096     * <p>Unless explicitly turned off, Hadoop by default specifies two 
097     * resources, loaded in-order from the classpath: <ol>
098     * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
099     * </tt>: Read-only defaults for hadoop.</li>
100     * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
101     * installation.</li>
102     * </ol>
103     * Applications may add additional resources, which are loaded
104     * subsequent to these resources in the order they are added.
105     * 
106     * <h4 id="FinalParams">Final Parameters</h4>
107     *
108     * <p>Configuration parameters may be declared <i>final</i>. 
109     * Once a resource declares a value final, no subsequently-loaded 
110     * resource can alter that value.  
111     * For example, one might define a final parameter with:
112     * <tt><pre>
113     *  &lt;property&gt;
114     *    &lt;name&gt;dfs.client.buffer.dir&lt;/name&gt;
115     *    &lt;value&gt;/tmp/hadoop/dfs/client&lt;/value&gt;
116     *    <b>&lt;final&gt;true&lt;/final&gt;</b>
117     *  &lt;/property&gt;</pre></tt>
118     *
119     * Administrators typically define parameters as final in 
120     * <tt>core-site.xml</tt> for values that user applications may not alter.
121     *
122     * <h4 id="VariableExpansion">Variable Expansion</h4>
123     *
124     * <p>Value strings are first processed for <i>variable expansion</i>. The
125     * available properties are:<ol>
126     * <li>Other properties defined in this Configuration; and, if a name is
127     * undefined here,</li>
128     * <li>Properties in {@link System#getProperties()}.</li>
129     * </ol>
130     *
131     * <p>For example, if a configuration resource contains the following property
132     * definitions: 
133     * <tt><pre>
134     *  &lt;property&gt;
135     *    &lt;name&gt;basedir&lt;/name&gt;
136     *    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
137     *  &lt;/property&gt;
138     *  
139     *  &lt;property&gt;
140     *    &lt;name&gt;tempdir&lt;/name&gt;
141     *    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
142     *  &lt;/property&gt;</pre></tt>
143     *
144     * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
145     * will be resolved to another property in this Configuration, while
146     * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
147     * of the System property with that name.
148     */
149    @InterfaceAudience.Public
150    @InterfaceStability.Stable
151    public class Configuration implements Iterable<Map.Entry<String,String>>,
152                                          Writable {
153      private static final Log LOG =
154        LogFactory.getLog(Configuration.class);
155    
156      private boolean quietmode = true;
157      
158      /**
159       * List of configuration resources.
160       */
161      private ArrayList<Object> resources = new ArrayList<Object>();
162    
163      /**
164       * The value reported as the setting resource when a key is set
165       * by code rather than a file resource.
166       */
167      static final String UNKNOWN_RESOURCE = "Unknown";
168    
169      /**
170       * List of configuration parameters marked <b>final</b>. 
171       */
172      private Set<String> finalParameters = new HashSet<String>();
173      
174      private boolean loadDefaults = true;
175      
176      /**
177       * Configuration objects
178       */
179      private static final WeakHashMap<Configuration,Object> REGISTRY = 
180        new WeakHashMap<Configuration,Object>();
181      
182      /**
183       * List of default Resources. Resources are loaded in the order of the list 
184       * entries
185       */
186      private static final CopyOnWriteArrayList<String> defaultResources =
187        new CopyOnWriteArrayList<String>();
188    
189      private static final Map<ClassLoader, Map<String, Class<?>>>
190        CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, Class<?>>>();
191    
192      /**
193       * Stores the mapping of key to the resource which modifies or loads 
194       * the key most recently
195       */
196      private HashMap<String, String> updatingResource;
197     
198      /**
199       * Class to keep the information about the keys which replace the deprecated
200       * ones.
201       * 
202       * This class stores the new keys which replace the deprecated keys and also
203       * gives a provision to have a custom message for each of the deprecated key
204       * that is being replaced. It also provides method to get the appropriate
205       * warning message which can be logged whenever the deprecated key is used.
206       */
207      private static class DeprecatedKeyInfo {
208        private String[] newKeys;
209        private String customMessage;
210        private boolean accessed;
211        DeprecatedKeyInfo(String[] newKeys, String customMessage) {
212          this.newKeys = newKeys;
213          this.customMessage = customMessage;
214          accessed = false;
215        }
216    
217        /**
218         * Method to provide the warning message. It gives the custom message if
219         * non-null, and default message otherwise.
220         * @param key the associated deprecated key.
221         * @return message that is to be logged when a deprecated key is used.
222         */
223        private final String getWarningMessage(String key) {
224          String warningMessage;
225          if(customMessage == null) {
226            StringBuilder message = new StringBuilder(key);
227            String deprecatedKeySuffix = " is deprecated. Instead, use ";
228            message.append(deprecatedKeySuffix);
229            for (int i = 0; i < newKeys.length; i++) {
230              message.append(newKeys[i]);
231              if(i != newKeys.length-1) {
232                message.append(", ");
233              }
234            }
235            warningMessage = message.toString();
236          }
237          else {
238            warningMessage = customMessage;
239          }
240          accessed = true;
241          return warningMessage;
242        }
243      }
244      
245      /**
246       * Stores the deprecated keys, the new keys which replace the deprecated keys
247       * and custom message(if any provided).
248       */
249      private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap = 
250          new HashMap<String, DeprecatedKeyInfo>();
251      
252      /**
253       * Stores a mapping from superseding keys to the keys which they deprecate.
254       */
255      private static Map<String, String> reverseDeprecatedKeyMap =
256          new HashMap<String, String>();
257    
258      /**
259       * Adds the deprecated key to the deprecation map.
260       * It does not override any existing entries in the deprecation map.
261       * This is to be used only by the developers in order to add deprecation of
262       * keys, and attempts to call this method after loading resources once,
263       * would lead to <tt>UnsupportedOperationException</tt>
264       * @param key
265       * @param newKeys
266       * @param customMessage
267       */
268      public synchronized static void addDeprecation(String key, String[] newKeys,
269          String customMessage) {
270        if (key == null || key.length() == 0 ||
271            newKeys == null || newKeys.length == 0) {
272          throw new IllegalArgumentException();
273        }
274        if (!isDeprecated(key)) {
275          DeprecatedKeyInfo newKeyInfo;
276          newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
277          deprecatedKeyMap.put(key, newKeyInfo);
278          for (String newKey : newKeys) {
279            reverseDeprecatedKeyMap.put(newKey, key);
280          }
281        }
282      }
283    
284      /**
285       * Adds the deprecated key to the deprecation map when no custom message
286       * is provided.
287       * It does not override any existing entries in the deprecation map.
288       * This is to be used only by the developers in order to add deprecation of
289       * keys, and attempts to call this method after loading resources once,
290       * would lead to <tt>UnsupportedOperationException</tt>
291       * 
292       * @param key Key that is to be deprecated
293       * @param newKeys list of keys that take up the values of deprecated key
294       */
295      public synchronized static void addDeprecation(String key, String[] newKeys) {
296        addDeprecation(key, newKeys, null);
297      }
298      
299      /**
300       * checks whether the given <code>key</code> is deprecated.
301       * 
302       * @param key the parameter which is to be checked for deprecation
303       * @return <code>true</code> if the key is deprecated and 
304       *         <code>false</code> otherwise.
305       */
306      private static boolean isDeprecated(String key) {
307        return deprecatedKeyMap.containsKey(key);
308      }
309     
310      /**
311       * Checks for the presence of the property <code>name</code> in the
312       * deprecation map. Returns the first of the list of new keys if present
313       * in the deprecation map or the <code>name</code> itself. If the property
314       * is not presently set but the property map contains an entry for the
315       * deprecated key, the value of the deprecated key is set as the value for
316       * the provided property name.
317       *
318       * @param name the property name
319       * @return the first property in the list of properties mapping
320       *         the <code>name</code> or the <code>name</code> itself.
321       */
322      private String handleDeprecation(String name) {
323        if (isDeprecated(name)) {
324          DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
325          if (!keyInfo.accessed) {
326            LOG.warn(keyInfo.getWarningMessage(name));
327          }
328          for (String newKey : keyInfo.newKeys) {
329            if(newKey != null) {
330              name = newKey;
331              break;
332            }
333          }
334        }
335        String deprecatedKey = reverseDeprecatedKeyMap.get(name);
336        if (deprecatedKey != null && !getOverlay().containsKey(name) &&
337            getOverlay().containsKey(deprecatedKey)) {
338          getProps().setProperty(name, getOverlay().getProperty(deprecatedKey));
339          getOverlay().setProperty(name, getOverlay().getProperty(deprecatedKey));
340          
341          DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(deprecatedKey);
342          if (!keyInfo.accessed) {
343            LOG.warn(keyInfo.getWarningMessage(deprecatedKey));
344          }
345        }
346        return name;
347      }
348      
349      static{
350        //print deprecation warning if hadoop-site.xml is found in classpath
351        ClassLoader cL = Thread.currentThread().getContextClassLoader();
352        if (cL == null) {
353          cL = Configuration.class.getClassLoader();
354        }
355        if(cL.getResource("hadoop-site.xml")!=null) {
356          LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
357              "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
358              + "mapred-site.xml and hdfs-site.xml to override properties of " +
359              "core-default.xml, mapred-default.xml and hdfs-default.xml " +
360              "respectively");
361        }
362        addDefaultResource("core-default.xml");
363        addDefaultResource("core-site.xml");
364        //Add code for managing deprecated key mapping
365        //for example
366        //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
367        //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
368        //so get or set of oldKey1 will correctly populate/access values of 
369        //newkey1 and newkey2
370        addDeprecatedKeys();
371      }
372      
373      private Properties properties;
374      private Properties overlay;
375      private ClassLoader classLoader;
376      {
377        classLoader = Thread.currentThread().getContextClassLoader();
378        if (classLoader == null) {
379          classLoader = Configuration.class.getClassLoader();
380        }
381      }
382      
383      /** A new configuration. */
384      public Configuration() {
385        this(true);
386      }
387    
388      /** A new configuration where the behavior of reading from the default 
389       * resources can be turned off.
390       * 
391       * If the parameter {@code loadDefaults} is false, the new instance
392       * will not load resources from the default files. 
393       * @param loadDefaults specifies whether to load from the default files
394       */
395      public Configuration(boolean loadDefaults) {
396        this.loadDefaults = loadDefaults;
397        updatingResource = new HashMap<String, String>();
398        synchronized(Configuration.class) {
399          REGISTRY.put(this, null);
400        }
401      }
402      
403      /** 
404       * A new configuration with the same settings cloned from another.
405       * 
406       * @param other the configuration from which to clone settings.
407       */
408      @SuppressWarnings("unchecked")
409      public Configuration(Configuration other) {
410       this.resources = (ArrayList)other.resources.clone();
411       synchronized(other) {
412         if (other.properties != null) {
413           this.properties = (Properties)other.properties.clone();
414         }
415    
416         if (other.overlay!=null) {
417           this.overlay = (Properties)other.overlay.clone();
418         }
419    
420         this.updatingResource = new HashMap<String, String>(other.updatingResource);
421       }
422       
423        this.finalParameters = new HashSet<String>(other.finalParameters);
424        synchronized(Configuration.class) {
425          REGISTRY.put(this, null);
426        }
427        this.classLoader = other.classLoader;
428        this.loadDefaults = other.loadDefaults;
429        setQuietMode(other.getQuietMode());
430      }
431      
432      /**
433       * Add a default resource. Resources are loaded in the order of the resources 
434       * added.
435       * @param name file name. File should be present in the classpath.
436       */
437      public static synchronized void addDefaultResource(String name) {
438        if(!defaultResources.contains(name)) {
439          defaultResources.add(name);
440          for(Configuration conf : REGISTRY.keySet()) {
441            if(conf.loadDefaults) {
442              conf.reloadConfiguration();
443            }
444          }
445        }
446      }
447    
448      /**
449       * Add a configuration resource. 
450       * 
451       * The properties of this resource will override properties of previously 
452       * added resources, unless they were marked <a href="#Final">final</a>. 
453       * 
454       * @param name resource to be added, the classpath is examined for a file 
455       *             with that name.
456       */
457      public void addResource(String name) {
458        addResourceObject(name);
459      }
460    
461      /**
462       * Add a configuration resource. 
463       * 
464       * The properties of this resource will override properties of previously 
465       * added resources, unless they were marked <a href="#Final">final</a>. 
466       * 
467       * @param url url of the resource to be added, the local filesystem is 
468       *            examined directly to find the resource, without referring to 
469       *            the classpath.
470       */
471      public void addResource(URL url) {
472        addResourceObject(url);
473      }
474    
475      /**
476       * Add a configuration resource. 
477       * 
478       * The properties of this resource will override properties of previously 
479       * added resources, unless they were marked <a href="#Final">final</a>. 
480       * 
481       * @param file file-path of resource to be added, the local filesystem is
482       *             examined directly to find the resource, without referring to 
483       *             the classpath.
484       */
485      public void addResource(Path file) {
486        addResourceObject(file);
487      }
488    
489      /**
490       * Add a configuration resource. 
491       * 
492       * The properties of this resource will override properties of previously 
493       * added resources, unless they were marked <a href="#Final">final</a>. 
494       * 
495       * @param in InputStream to deserialize the object from. 
496       */
497      public void addResource(InputStream in) {
498        addResourceObject(in);
499      }
500      
501      
502      /**
503       * Reload configuration from previously added resources.
504       *
505       * This method will clear all the configuration read from the added 
506       * resources, and final parameters. This will make the resources to 
507       * be read again before accessing the values. Values that are added
508       * via set methods will overlay values read from the resources.
509       */
510      public synchronized void reloadConfiguration() {
511        properties = null;                            // trigger reload
512        finalParameters.clear();                      // clear site-limits
513      }
514      
515      private synchronized void addResourceObject(Object resource) {
516        resources.add(resource);                      // add to resources
517        reloadConfiguration();
518      }
519      
520      private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
521      private static int MAX_SUBST = 20;
522    
523      private String substituteVars(String expr) {
524        if (expr == null) {
525          return null;
526        }
527        Matcher match = varPat.matcher("");
528        String eval = expr;
529        for(int s=0; s<MAX_SUBST; s++) {
530          match.reset(eval);
531          if (!match.find()) {
532            return eval;
533          }
534          String var = match.group();
535          var = var.substring(2, var.length()-1); // remove ${ .. }
536          String val = null;
537          try {
538            val = System.getProperty(var);
539          } catch(SecurityException se) {
540            LOG.warn("Unexpected SecurityException in Configuration", se);
541          }
542          if (val == null) {
543            val = getRaw(var);
544          }
545          if (val == null) {
546            return eval; // return literal ${var}: var is unbound
547          }
548          // substitute
549          eval = eval.substring(0, match.start())+val+eval.substring(match.end());
550        }
551        throw new IllegalStateException("Variable substitution depth too large: " 
552                                        + MAX_SUBST + " " + expr);
553      }
554      
555      /**
556       * Get the value of the <code>name</code> property, <code>null</code> if
557       * no such property exists. If the key is deprecated, it returns the value of
558       * the first key which replaces the deprecated key and is not null
559       * 
560       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
561       * before being returned. 
562       * 
563       * @param name the property name.
564       * @return the value of the <code>name</code> or its replacing property, 
565       *         or null if no such property exists.
566       */
567      public String get(String name) {
568        name = handleDeprecation(name);
569        return substituteVars(getProps().getProperty(name));
570      }
571      
572      /**
573       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
574       * <code>null</code> if no such property exists. 
575       * If the key is deprecated, it returns the value of
576       * the first key which replaces the deprecated key and is not null
577       * 
578       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
579       * before being returned. 
580       * 
581       * @param name the property name.
582       * @return the value of the <code>name</code> or its replacing property, 
583       *         or null if no such property exists.
584       */
585      public String getTrimmed(String name) {
586        String value = get(name);
587        
588        if (null == value) {
589          return null;
590        } else {
591          return value.trim();
592        }
593      }
594    
595      /**
596       * Get the value of the <code>name</code> property, without doing
597       * <a href="#VariableExpansion">variable expansion</a>.If the key is 
598       * deprecated, it returns the value of the first key which replaces 
599       * the deprecated key and is not null.
600       * 
601       * @param name the property name.
602       * @return the value of the <code>name</code> property or 
603       *         its replacing property and null if no such property exists.
604       */
605      public String getRaw(String name) {
606        name = handleDeprecation(name);
607        return getProps().getProperty(name);
608      }
609    
610      /** 
611       * Set the <code>value</code> of the <code>name</code> property. If 
612       * <code>name</code> is deprecated, it sets the <code>value</code> to the keys
613       * that replace the deprecated key.
614       * 
615       * @param name property name.
616       * @param value property value.
617       */
618      public void set(String name, String value) {
619        if (deprecatedKeyMap.isEmpty()) {
620          getProps();
621        }
622        if (!isDeprecated(name)) {
623          getOverlay().setProperty(name, value);
624          getProps().setProperty(name, value);
625          updatingResource.put(name, UNKNOWN_RESOURCE);
626        }
627        else {
628          DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
629          LOG.warn(keyInfo.getWarningMessage(name));
630          for (String newKey : keyInfo.newKeys) {
631            getOverlay().setProperty(newKey, value);
632            getProps().setProperty(newKey, value);
633          }
634        }
635      }
636      
637      /**
638       * Unset a previously set property.
639       */
640      public synchronized void unset(String name) {
641        name = handleDeprecation(name);
642    
643        getOverlay().remove(name);
644        getProps().remove(name);
645      }
646    
647      /**
648       * Sets a property if it is currently unset.
649       * @param name the property name
650       * @param value the new value
651       */
652      public synchronized void setIfUnset(String name, String value) {
653        if (get(name) == null) {
654          set(name, value);
655        }
656      }
657      
658      private synchronized Properties getOverlay() {
659        if (overlay==null){
660          overlay=new Properties();
661        }
662        return overlay;
663      }
664    
665      /** 
666       * Get the value of the <code>name</code>. If the key is deprecated,
667       * it returns the value of the first key which replaces the deprecated key
668       * and is not null.
669       * If no such property exists,
670       * then <code>defaultValue</code> is returned.
671       * 
672       * @param name property name.
673       * @param defaultValue default value.
674       * @return property value, or <code>defaultValue</code> if the property 
675       *         doesn't exist.                    
676       */
677      public String get(String name, String defaultValue) {
678        name = handleDeprecation(name);
679        return substituteVars(getProps().getProperty(name, defaultValue));
680      }
681        
682      /** 
683       * Get the value of the <code>name</code> property as an <code>int</code>.
684       *   
685       * If no such property exists, the provided default value is returned,
686       * or if the specified value is not a valid <code>int</code>,
687       * then an error is thrown.
688       * 
689       * @param name property name.
690       * @param defaultValue default value.
691       * @throws NumberFormatException when the value is invalid
692       * @return property value as an <code>int</code>, 
693       *         or <code>defaultValue</code>. 
694       */
695      public int getInt(String name, int defaultValue) {
696        String valueString = getTrimmed(name);
697        if (valueString == null)
698          return defaultValue;
699        String hexString = getHexDigits(valueString);
700        if (hexString != null) {
701          return Integer.parseInt(hexString, 16);
702        }
703        return Integer.parseInt(valueString);
704      }
705    
706      /** 
707       * Set the value of the <code>name</code> property to an <code>int</code>.
708       * 
709       * @param name property name.
710       * @param value <code>int</code> value of the property.
711       */
712      public void setInt(String name, int value) {
713        set(name, Integer.toString(value));
714      }
715    
716    
717      /** 
718       * Get the value of the <code>name</code> property as a <code>long</code>.  
719       * If no such property exists, the provided default value is returned,
720       * or if the specified value is not a valid <code>long</code>,
721       * then an error is thrown.
722       * 
723       * @param name property name.
724       * @param defaultValue default value.
725       * @throws NumberFormatException when the value is invalid
726       * @return property value as a <code>long</code>, 
727       *         or <code>defaultValue</code>. 
728       */
729      public long getLong(String name, long defaultValue) {
730        String valueString = getTrimmed(name);
731        if (valueString == null)
732          return defaultValue;
733        String hexString = getHexDigits(valueString);
734        if (hexString != null) {
735          return Long.parseLong(hexString, 16);
736        }
737        return Long.parseLong(valueString);
738      }
739    
740      private String getHexDigits(String value) {
741        boolean negative = false;
742        String str = value;
743        String hexString = null;
744        if (value.startsWith("-")) {
745          negative = true;
746          str = value.substring(1);
747        }
748        if (str.startsWith("0x") || str.startsWith("0X")) {
749          hexString = str.substring(2);
750          if (negative) {
751            hexString = "-" + hexString;
752          }
753          return hexString;
754        }
755        return null;
756      }
757      
758      /** 
759       * Set the value of the <code>name</code> property to a <code>long</code>.
760       * 
761       * @param name property name.
762       * @param value <code>long</code> value of the property.
763       */
764      public void setLong(String name, long value) {
765        set(name, Long.toString(value));
766      }
767    
768      /** 
769       * Get the value of the <code>name</code> property as a <code>float</code>.  
770       * If no such property exists, the provided default value is returned,
771       * or if the specified value is not a valid <code>float</code>,
772       * then an error is thrown.
773       *
774       * @param name property name.
775       * @param defaultValue default value.
776       * @throws NumberFormatException when the value is invalid
777       * @return property value as a <code>float</code>, 
778       *         or <code>defaultValue</code>. 
779       */
780      public float getFloat(String name, float defaultValue) {
781        String valueString = getTrimmed(name);
782        if (valueString == null)
783          return defaultValue;
784        return Float.parseFloat(valueString);
785      }
786      /**
787       * Set the value of the <code>name</code> property to a <code>float</code>.
788       * 
789       * @param name property name.
790       * @param value property value.
791       */
792      public void setFloat(String name, float value) {
793        set(name,Float.toString(value));
794      }
795     
796      /** 
797       * Get the value of the <code>name</code> property as a <code>boolean</code>.  
798       * If no such property is specified, or if the specified value is not a valid
799       * <code>boolean</code>, then <code>defaultValue</code> is returned.
800       * 
801       * @param name property name.
802       * @param defaultValue default value.
803       * @return property value as a <code>boolean</code>, 
804       *         or <code>defaultValue</code>. 
805       */
806      public boolean getBoolean(String name, boolean defaultValue) {
807        String valueString = getTrimmed(name);
808        if ("true".equals(valueString))
809          return true;
810        else if ("false".equals(valueString))
811          return false;
812        else return defaultValue;
813      }
814    
815      /** 
816       * Set the value of the <code>name</code> property to a <code>boolean</code>.
817       * 
818       * @param name property name.
819       * @param value <code>boolean</code> value of the property.
820       */
821      public void setBoolean(String name, boolean value) {
822        set(name, Boolean.toString(value));
823      }
824    
825      /**
826       * Set the given property, if it is currently unset.
827       * @param name property name
828       * @param value new value
829       */
830      public void setBooleanIfUnset(String name, boolean value) {
831        setIfUnset(name, Boolean.toString(value));
832      }
833    
834      /**
835       * Set the value of the <code>name</code> property to the given type. This
836       * is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
837       * @param name property name
838       * @param value new value
839       */
840      public <T extends Enum<T>> void setEnum(String name, T value) {
841        set(name, value.toString());
842      }
843    
844      /**
845       * Return value matching this enumerated type.
846       * @param name Property name
847       * @param defaultValue Value returned if no mapping exists
848       * @throws IllegalArgumentException If mapping is illegal for the type
849       * provided
850       */
851      public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
852        final String val = get(name);
853        return null == val
854          ? defaultValue
855          : Enum.valueOf(defaultValue.getDeclaringClass(), val);
856      }
857    
858      /**
859       * Get the value of the <code>name</code> property as a <code>Pattern</code>.
860       * If no such property is specified, or if the specified value is not a valid
861       * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
862       *
863       * @param name property name
864       * @param defaultValue default value
865       * @return property value as a compiled Pattern, or defaultValue
866       */
867      public Pattern getPattern(String name, Pattern defaultValue) {
868        String valString = get(name);
869        if (null == valString || "".equals(valString)) {
870          return defaultValue;
871        }
872        try {
873          return Pattern.compile(valString);
874        } catch (PatternSyntaxException pse) {
875          LOG.warn("Regular expression '" + valString + "' for property '" +
876                   name + "' not valid. Using default", pse);
877          return defaultValue;
878        }
879      }
880    
881      /**
882       * Set the given property to <code>Pattern</code>.
883       * If the pattern is passed as null, sets the empty pattern which results in
884       * further calls to getPattern(...) returning the default value.
885       *
886       * @param name property name
887       * @param pattern new value
888       */
889      public void setPattern(String name, Pattern pattern) {
890        if (null == pattern) {
891          set(name, null);
892        } else {
893          set(name, pattern.pattern());
894        }
895      }
896    
897      /**
898       * A class that represents a set of positive integer ranges. It parses 
899       * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
900       * the lower/upper bounds are separated by dash. Either the lower or upper 
901       * bound may be omitted meaning all values up to or over. So the string 
902       * above means 2, 3, 5, and 7, 8, 9, ...
903       */
904      public static class IntegerRanges {
905        private static class Range {
906          int start;
907          int end;
908        }
909    
910        List<Range> ranges = new ArrayList<Range>();
911        
912        public IntegerRanges() {
913        }
914        
915        public IntegerRanges(String newValue) {
916          StringTokenizer itr = new StringTokenizer(newValue, ",");
917          while (itr.hasMoreTokens()) {
918            String rng = itr.nextToken().trim();
919            String[] parts = rng.split("-", 3);
920            if (parts.length < 1 || parts.length > 2) {
921              throw new IllegalArgumentException("integer range badly formed: " + 
922                                                 rng);
923            }
924            Range r = new Range();
925            r.start = convertToInt(parts[0], 0);
926            if (parts.length == 2) {
927              r.end = convertToInt(parts[1], Integer.MAX_VALUE);
928            } else {
929              r.end = r.start;
930            }
931            if (r.start > r.end) {
932              throw new IllegalArgumentException("IntegerRange from " + r.start + 
933                                                 " to " + r.end + " is invalid");
934            }
935            ranges.add(r);
936          }
937        }
938    
939        /**
940         * Convert a string to an int treating empty strings as the default value.
941         * @param value the string value
942         * @param defaultValue the value for if the string is empty
943         * @return the desired integer
944         */
945        private static int convertToInt(String value, int defaultValue) {
946          String trim = value.trim();
947          if (trim.length() == 0) {
948            return defaultValue;
949          }
950          return Integer.parseInt(trim);
951        }
952    
953        /**
954         * Is the given value in the set of ranges
955         * @param value the value to check
956         * @return is the value in the ranges?
957         */
958        public boolean isIncluded(int value) {
959          for(Range r: ranges) {
960            if (r.start <= value && value <= r.end) {
961              return true;
962            }
963          }
964          return false;
965        }
966        
967        @Override
968        public String toString() {
969          StringBuilder result = new StringBuilder();
970          boolean first = true;
971          for(Range r: ranges) {
972            if (first) {
973              first = false;
974            } else {
975              result.append(',');
976            }
977            result.append(r.start);
978            result.append('-');
979            result.append(r.end);
980          }
981          return result.toString();
982        }
983      }
984    
985      /**
986       * Parse the given attribute as a set of integer ranges
987       * @param name the attribute name
988       * @param defaultValue the default value if it is not set
989       * @return a new set of ranges from the configured value
990       */
991      public IntegerRanges getRange(String name, String defaultValue) {
992        return new IntegerRanges(get(name, defaultValue));
993      }
994    
995      /** 
996       * Get the comma delimited values of the <code>name</code> property as 
997       * a collection of <code>String</code>s.  
998       * If no such property is specified then empty collection is returned.
999       * <p>
1000       * This is an optimized version of {@link #getStrings(String)}
1001       * 
1002       * @param name property name.
1003       * @return property value as a collection of <code>String</code>s. 
1004       */
1005      public Collection<String> getStringCollection(String name) {
1006        String valueString = get(name);
1007        return StringUtils.getStringCollection(valueString);
1008      }
1009    
1010      /** 
1011       * Get the comma delimited values of the <code>name</code> property as 
1012       * an array of <code>String</code>s.  
1013       * If no such property is specified then <code>null</code> is returned.
1014       * 
1015       * @param name property name.
1016       * @return property value as an array of <code>String</code>s, 
1017       *         or <code>null</code>. 
1018       */
1019      public String[] getStrings(String name) {
1020        String valueString = get(name);
1021        return StringUtils.getStrings(valueString);
1022      }
1023    
1024      /** 
1025       * Get the comma delimited values of the <code>name</code> property as 
1026       * an array of <code>String</code>s.  
1027       * If no such property is specified then default value is returned.
1028       * 
1029       * @param name property name.
1030       * @param defaultValue The default value
1031       * @return property value as an array of <code>String</code>s, 
1032       *         or default value. 
1033       */
1034      public String[] getStrings(String name, String... defaultValue) {
1035        String valueString = get(name);
1036        if (valueString == null) {
1037          return defaultValue;
1038        } else {
1039          return StringUtils.getStrings(valueString);
1040        }
1041      }
1042      
1043      /** 
1044       * Get the comma delimited values of the <code>name</code> property as 
1045       * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
1046       * If no such property is specified then empty <code>Collection</code> is returned.
1047       *
1048       * @param name property name.
1049       * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 
1050       */
1051      public Collection<String> getTrimmedStringCollection(String name) {
1052        String valueString = get(name);
1053        if (null == valueString) {
1054          Collection<String> empty = new ArrayList<String>();
1055          return empty;
1056        }
1057        return StringUtils.getTrimmedStringCollection(valueString);
1058      }
1059      
1060      /** 
1061       * Get the comma delimited values of the <code>name</code> property as 
1062       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1063       * If no such property is specified then an empty array is returned.
1064       * 
1065       * @param name property name.
1066       * @return property value as an array of trimmed <code>String</code>s, 
1067       *         or empty array. 
1068       */
1069      public String[] getTrimmedStrings(String name) {
1070        String valueString = get(name);
1071        return StringUtils.getTrimmedStrings(valueString);
1072      }
1073    
1074      /** 
1075       * Get the comma delimited values of the <code>name</code> property as 
1076       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1077       * If no such property is specified then default value is returned.
1078       * 
1079       * @param name property name.
1080       * @param defaultValue The default value
1081       * @return property value as an array of trimmed <code>String</code>s, 
1082       *         or default value. 
1083       */
1084      public String[] getTrimmedStrings(String name, String... defaultValue) {
1085        String valueString = get(name);
1086        if (null == valueString) {
1087          return defaultValue;
1088        } else {
1089          return StringUtils.getTrimmedStrings(valueString);
1090        }
1091      }
1092    
1093      /** 
1094       * Set the array of string values for the <code>name</code> property as 
1095       * as comma delimited values.  
1096       * 
1097       * @param name property name.
1098       * @param values The values
1099       */
1100      public void setStrings(String name, String... values) {
1101        set(name, StringUtils.arrayToString(values));
1102      }
1103    
1104      /**
1105       * Load a class by name.
1106       * 
1107       * @param name the class name.
1108       * @return the class object.
1109       * @throws ClassNotFoundException if the class is not found.
1110       */
1111      public Class<?> getClassByName(String name) throws ClassNotFoundException {
1112        Map<String, Class<?>> map;
1113        
1114        synchronized (CACHE_CLASSES) {
1115          map = CACHE_CLASSES.get(classLoader);
1116          if (map == null) {
1117            map = Collections.synchronizedMap(
1118              new WeakHashMap<String, Class<?>>());
1119            CACHE_CLASSES.put(classLoader, map);
1120          }
1121        }
1122    
1123        Class<?> clazz = map.get(name);
1124        if (clazz == null) {
1125          clazz = Class.forName(name, true, classLoader);
1126          if (clazz != null) {
1127            // two putters can race here, but they'll put the same class
1128            map.put(name, clazz);
1129          }
1130        }
1131    
1132        return clazz;
1133      }
1134    
1135      /** 
1136       * Get the value of the <code>name</code> property
1137       * as an array of <code>Class</code>.
1138       * The value of the property specifies a list of comma separated class names.  
1139       * If no such property is specified, then <code>defaultValue</code> is 
1140       * returned.
1141       * 
1142       * @param name the property name.
1143       * @param defaultValue default value.
1144       * @return property value as a <code>Class[]</code>, 
1145       *         or <code>defaultValue</code>. 
1146       */
1147      public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1148        String[] classnames = getTrimmedStrings(name);
1149        if (classnames == null)
1150          return defaultValue;
1151        try {
1152          Class<?>[] classes = new Class<?>[classnames.length];
1153          for(int i = 0; i < classnames.length; i++) {
1154            classes[i] = getClassByName(classnames[i]);
1155          }
1156          return classes;
1157        } catch (ClassNotFoundException e) {
1158          throw new RuntimeException(e);
1159        }
1160      }
1161    
1162      /** 
1163       * Get the value of the <code>name</code> property as a <code>Class</code>.  
1164       * If no such property is specified, then <code>defaultValue</code> is 
1165       * returned.
1166       * 
1167       * @param name the class name.
1168       * @param defaultValue default value.
1169       * @return property value as a <code>Class</code>, 
1170       *         or <code>defaultValue</code>. 
1171       */
1172      public Class<?> getClass(String name, Class<?> defaultValue) {
1173        String valueString = getTrimmed(name);
1174        if (valueString == null)
1175          return defaultValue;
1176        try {
1177          return getClassByName(valueString);
1178        } catch (ClassNotFoundException e) {
1179          throw new RuntimeException(e);
1180        }
1181      }
1182    
1183      /** 
1184       * Get the value of the <code>name</code> property as a <code>Class</code>
1185       * implementing the interface specified by <code>xface</code>.
1186       *   
1187       * If no such property is specified, then <code>defaultValue</code> is 
1188       * returned.
1189       * 
1190       * An exception is thrown if the returned class does not implement the named
1191       * interface. 
1192       * 
1193       * @param name the class name.
1194       * @param defaultValue default value.
1195       * @param xface the interface implemented by the named class.
1196       * @return property value as a <code>Class</code>, 
1197       *         or <code>defaultValue</code>.
1198       */
1199      public <U> Class<? extends U> getClass(String name, 
1200                                             Class<? extends U> defaultValue, 
1201                                             Class<U> xface) {
1202        try {
1203          Class<?> theClass = getClass(name, defaultValue);
1204          if (theClass != null && !xface.isAssignableFrom(theClass))
1205            throw new RuntimeException(theClass+" not "+xface.getName());
1206          else if (theClass != null)
1207            return theClass.asSubclass(xface);
1208          else
1209            return null;
1210        } catch (Exception e) {
1211          throw new RuntimeException(e);
1212        }
1213      }
1214    
1215      /**
1216       * Get the value of the <code>name</code> property as a <code>List</code>
1217       * of objects implementing the interface specified by <code>xface</code>.
1218       * 
1219       * An exception is thrown if any of the classes does not exist, or if it does
1220       * not implement the named interface.
1221       * 
1222       * @param name the property name.
1223       * @param xface the interface implemented by the classes named by
1224       *        <code>name</code>.
1225       * @return a <code>List</code> of objects implementing <code>xface</code>.
1226       */
1227      @SuppressWarnings("unchecked")
1228      public <U> List<U> getInstances(String name, Class<U> xface) {
1229        List<U> ret = new ArrayList<U>();
1230        Class<?>[] classes = getClasses(name);
1231        for (Class<?> cl: classes) {
1232          if (!xface.isAssignableFrom(cl)) {
1233            throw new RuntimeException(cl + " does not implement " + xface);
1234          }
1235          ret.add((U)ReflectionUtils.newInstance(cl, this));
1236        }
1237        return ret;
1238      }
1239    
1240      /** 
1241       * Set the value of the <code>name</code> property to the name of a 
1242       * <code>theClass</code> implementing the given interface <code>xface</code>.
1243       * 
1244       * An exception is thrown if <code>theClass</code> does not implement the 
1245       * interface <code>xface</code>. 
1246       * 
1247       * @param name property name.
1248       * @param theClass property value.
1249       * @param xface the interface implemented by the named class.
1250       */
1251      public void setClass(String name, Class<?> theClass, Class<?> xface) {
1252        if (!xface.isAssignableFrom(theClass))
1253          throw new RuntimeException(theClass+" not "+xface.getName());
1254        set(name, theClass.getName());
1255      }
1256    
1257      /** 
1258       * Get a local file under a directory named by <i>dirsProp</i> with
1259       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1260       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1261       * directory does not exist, an attempt is made to create it.
1262       * 
1263       * @param dirsProp directory in which to locate the file.
1264       * @param path file-path.
1265       * @return local file under the directory with the given path.
1266       */
1267      public Path getLocalPath(String dirsProp, String path)
1268        throws IOException {
1269        String[] dirs = getTrimmedStrings(dirsProp);
1270        int hashCode = path.hashCode();
1271        FileSystem fs = FileSystem.getLocal(this);
1272        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1273          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1274          Path file = new Path(dirs[index], path);
1275          Path dir = file.getParent();
1276          if (fs.mkdirs(dir) || fs.exists(dir)) {
1277            return file;
1278          }
1279        }
1280        LOG.warn("Could not make " + path + 
1281                 " in local directories from " + dirsProp);
1282        for(int i=0; i < dirs.length; i++) {
1283          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1284          LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1285        }
1286        throw new IOException("No valid local directories in property: "+dirsProp);
1287      }
1288    
1289      /** 
1290       * Get a local file name under a directory named in <i>dirsProp</i> with
1291       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1292       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1293       * directory does not exist, an attempt is made to create it.
1294       * 
1295       * @param dirsProp directory in which to locate the file.
1296       * @param path file-path.
1297       * @return local file under the directory with the given path.
1298       */
1299      public File getFile(String dirsProp, String path)
1300        throws IOException {
1301        String[] dirs = getTrimmedStrings(dirsProp);
1302        int hashCode = path.hashCode();
1303        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1304          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1305          File file = new File(dirs[index], path);
1306          File dir = file.getParentFile();
1307          if (dir.exists() || dir.mkdirs()) {
1308            return file;
1309          }
1310        }
1311        throw new IOException("No valid local directories in property: "+dirsProp);
1312      }
1313    
1314      /** 
1315       * Get the {@link URL} for the named resource.
1316       * 
1317       * @param name resource name.
1318       * @return the url for the named resource.
1319       */
1320      public URL getResource(String name) {
1321        return classLoader.getResource(name);
1322      }
1323      
1324      /** 
1325       * Get an input stream attached to the configuration resource with the
1326       * given <code>name</code>.
1327       * 
1328       * @param name configuration resource name.
1329       * @return an input stream attached to the resource.
1330       */
1331      public InputStream getConfResourceAsInputStream(String name) {
1332        try {
1333          URL url= getResource(name);
1334    
1335          if (url == null) {
1336            LOG.info(name + " not found");
1337            return null;
1338          } else {
1339            LOG.info("found resource " + name + " at " + url);
1340          }
1341    
1342          return url.openStream();
1343        } catch (Exception e) {
1344          return null;
1345        }
1346      }
1347    
1348      /** 
1349       * Get a {@link Reader} attached to the configuration resource with the
1350       * given <code>name</code>.
1351       * 
1352       * @param name configuration resource name.
1353       * @return a reader attached to the resource.
1354       */
1355      public Reader getConfResourceAsReader(String name) {
1356        try {
1357          URL url= getResource(name);
1358    
1359          if (url == null) {
1360            LOG.info(name + " not found");
1361            return null;
1362          } else {
1363            LOG.info("found resource " + name + " at " + url);
1364          }
1365    
1366          return new InputStreamReader(url.openStream());
1367        } catch (Exception e) {
1368          return null;
1369        }
1370      }
1371    
1372      protected synchronized Properties getProps() {
1373        if (properties == null) {
1374          properties = new Properties();
1375          loadResources(properties, resources, quietmode);
1376          if (overlay!= null) {
1377            properties.putAll(overlay);
1378            for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1379              updatingResource.put((String) item.getKey(), UNKNOWN_RESOURCE);
1380            }
1381          }
1382        }
1383        return properties;
1384      }
1385    
1386      /**
1387       * Return the number of keys in the configuration.
1388       *
1389       * @return number of keys in the configuration.
1390       */
1391      public int size() {
1392        return getProps().size();
1393      }
1394    
1395      /**
1396       * Clears all keys from the configuration.
1397       */
1398      public void clear() {
1399        getProps().clear();
1400        getOverlay().clear();
1401      }
1402    
1403      /**
1404       * Get an {@link Iterator} to go through the list of <code>String</code> 
1405       * key-value pairs in the configuration.
1406       * 
1407       * @return an iterator over the entries.
1408       */
1409      public Iterator<Map.Entry<String, String>> iterator() {
1410        // Get a copy of just the string to string pairs. After the old object
1411        // methods that allow non-strings to be put into configurations are removed,
1412        // we could replace properties with a Map<String,String> and get rid of this
1413        // code.
1414        Map<String,String> result = new HashMap<String,String>();
1415        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1416          if (item.getKey() instanceof String && 
1417              item.getValue() instanceof String) {
1418            result.put((String) item.getKey(), (String) item.getValue());
1419          }
1420        }
1421        return result.entrySet().iterator();
1422      }
1423    
1424      private void loadResources(Properties properties,
1425                                 ArrayList resources,
1426                                 boolean quiet) {
1427        if(loadDefaults) {
1428          for (String resource : defaultResources) {
1429            loadResource(properties, resource, quiet);
1430          }
1431        
1432          //support the hadoop-site.xml as a deprecated case
1433          if(getResource("hadoop-site.xml")!=null) {
1434            loadResource(properties, "hadoop-site.xml", quiet);
1435          }
1436        }
1437        
1438        for (Object resource : resources) {
1439          loadResource(properties, resource, quiet);
1440        }
1441      }
1442      
1443      private void loadResource(Properties properties, Object name, boolean quiet) {
1444        try {
1445          DocumentBuilderFactory docBuilderFactory 
1446            = DocumentBuilderFactory.newInstance();
1447          //ignore all comments inside the xml file
1448          docBuilderFactory.setIgnoringComments(true);
1449    
1450          //allow includes in the xml file
1451          docBuilderFactory.setNamespaceAware(true);
1452          try {
1453              docBuilderFactory.setXIncludeAware(true);
1454          } catch (UnsupportedOperationException e) {
1455            LOG.error("Failed to set setXIncludeAware(true) for parser "
1456                    + docBuilderFactory
1457                    + ":" + e,
1458                    e);
1459          }
1460          DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
1461          Document doc = null;
1462          Element root = null;
1463    
1464          if (name instanceof URL) {                  // an URL resource
1465            URL url = (URL)name;
1466            if (url != null) {
1467              if (!quiet) {
1468                LOG.info("parsing " + url);
1469              }
1470              doc = builder.parse(url.toString());
1471            }
1472          } else if (name instanceof String) {        // a CLASSPATH resource
1473            URL url = getResource((String)name);
1474            if (url != null) {
1475              if (!quiet) {
1476                LOG.info("parsing " + url);
1477              }
1478              doc = builder.parse(url.toString());
1479            }
1480          } else if (name instanceof Path) {          // a file resource
1481            // Can't use FileSystem API or we get an infinite loop
1482            // since FileSystem uses Configuration API.  Use java.io.File instead.
1483            File file = new File(((Path)name).toUri().getPath())
1484              .getAbsoluteFile();
1485            if (file.exists()) {
1486              if (!quiet) {
1487                LOG.info("parsing " + file);
1488              }
1489              InputStream in = new BufferedInputStream(new FileInputStream(file));
1490              try {
1491                doc = builder.parse(in);
1492              } finally {
1493                in.close();
1494              }
1495            }
1496          } else if (name instanceof InputStream) {
1497            try {
1498              doc = builder.parse((InputStream)name);
1499            } finally {
1500              ((InputStream)name).close();
1501            }
1502          } else if (name instanceof Element) {
1503            root = (Element)name;
1504          }
1505    
1506          if (doc == null && root == null) {
1507            if (quiet)
1508              return;
1509            throw new RuntimeException(name + " not found");
1510          }
1511    
1512          if (root == null) {
1513            root = doc.getDocumentElement();
1514          }
1515          if (!"configuration".equals(root.getTagName()))
1516            LOG.fatal("bad conf file: top-level element not <configuration>");
1517          NodeList props = root.getChildNodes();
1518          for (int i = 0; i < props.getLength(); i++) {
1519            Node propNode = props.item(i);
1520            if (!(propNode instanceof Element))
1521              continue;
1522            Element prop = (Element)propNode;
1523            if ("configuration".equals(prop.getTagName())) {
1524              loadResource(properties, prop, quiet);
1525              continue;
1526            }
1527            if (!"property".equals(prop.getTagName()))
1528              LOG.warn("bad conf file: element not <property>");
1529            NodeList fields = prop.getChildNodes();
1530            String attr = null;
1531            String value = null;
1532            boolean finalParameter = false;
1533            for (int j = 0; j < fields.getLength(); j++) {
1534              Node fieldNode = fields.item(j);
1535              if (!(fieldNode instanceof Element))
1536                continue;
1537              Element field = (Element)fieldNode;
1538              if ("name".equals(field.getTagName()) && field.hasChildNodes())
1539                attr = ((Text)field.getFirstChild()).getData().trim();
1540              if ("value".equals(field.getTagName()) && field.hasChildNodes())
1541                value = ((Text)field.getFirstChild()).getData();
1542              if ("final".equals(field.getTagName()) && field.hasChildNodes())
1543                finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
1544            }
1545            
1546            // Ignore this parameter if it has already been marked as 'final'
1547            if (attr != null) {
1548              if (deprecatedKeyMap.containsKey(attr)) {
1549                DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
1550                keyInfo.accessed = false;
1551                for (String key:keyInfo.newKeys) {
1552                  // update new keys with deprecated key's value 
1553                  loadProperty(properties, name, key, value, finalParameter);
1554                }
1555              }
1556              else {
1557                loadProperty(properties, name, attr, value, finalParameter);
1558              }
1559            }
1560          }
1561            
1562        } catch (IOException e) {
1563          LOG.fatal("error parsing conf file: " + e);
1564          throw new RuntimeException(e);
1565        } catch (DOMException e) {
1566          LOG.fatal("error parsing conf file: " + e);
1567          throw new RuntimeException(e);
1568        } catch (SAXException e) {
1569          LOG.fatal("error parsing conf file: " + e);
1570          throw new RuntimeException(e);
1571        } catch (ParserConfigurationException e) {
1572          LOG.fatal("error parsing conf file: " + e);
1573          throw new RuntimeException(e);
1574        }
1575      }
1576    
1577      private void loadProperty(Properties properties, Object name, String attr,
1578          String value, boolean finalParameter) {
1579        if (value != null) {
1580          if (!finalParameters.contains(attr)) {
1581            properties.setProperty(attr, value);
1582            updatingResource.put(attr, name.toString());
1583          } else {
1584            LOG.warn(name+":an attempt to override final parameter: "+attr
1585                +";  Ignoring.");
1586          }
1587        }
1588        if (finalParameter) {
1589          finalParameters.add(attr);
1590        }
1591      }
1592    
1593      /** 
1594       * Write out the non-default properties in this configuration to the given
1595       * {@link OutputStream}.
1596       * 
1597       * @param out the output stream to write to.
1598       */
1599      public void writeXml(OutputStream out) throws IOException {
1600        writeXml(new OutputStreamWriter(out));
1601      }
1602    
1603      /** 
1604       * Write out the non-default properties in this configuration to the given
1605       * {@link Writer}.
1606       * 
1607       * @param out the writer to write to.
1608       */
1609      public void writeXml(Writer out) throws IOException {
1610        Document doc = asXmlDocument();
1611    
1612        try {
1613          DOMSource source = new DOMSource(doc);
1614          StreamResult result = new StreamResult(out);
1615          TransformerFactory transFactory = TransformerFactory.newInstance();
1616          Transformer transformer = transFactory.newTransformer();
1617    
1618          // Important to not hold Configuration log while writing result, since
1619          // 'out' may be an HDFS stream which needs to lock this configuration
1620          // from another thread.
1621          transformer.transform(source, result);
1622        } catch (TransformerException te) {
1623          throw new IOException(te);
1624        }
1625      }
1626    
1627      /**
1628       * Return the XML DOM corresponding to this Configuration.
1629       */
1630      private synchronized Document asXmlDocument() throws IOException {
1631        Document doc;
1632        try {
1633          doc =
1634            DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
1635        } catch (ParserConfigurationException pe) {
1636          throw new IOException(pe);
1637        }
1638        Element conf = doc.createElement("configuration");
1639        doc.appendChild(conf);
1640        conf.appendChild(doc.createTextNode("\n"));
1641        getProps(); // ensure properties is set
1642        for (Enumeration e = properties.keys(); e.hasMoreElements();) {
1643          String name = (String)e.nextElement();
1644          Object object = properties.get(name);
1645          String value = null;
1646          if (object instanceof String) {
1647            value = (String) object;
1648          }else {
1649            continue;
1650          }
1651          Element propNode = doc.createElement("property");
1652          conf.appendChild(propNode);
1653    
1654          if (updatingResource != null) {
1655            Comment commentNode = doc.createComment(
1656              "Loaded from " + updatingResource.get(name));
1657            propNode.appendChild(commentNode);
1658          }
1659          Element nameNode = doc.createElement("name");
1660          nameNode.appendChild(doc.createTextNode(name));
1661          propNode.appendChild(nameNode);
1662    
1663          Element valueNode = doc.createElement("value");
1664          valueNode.appendChild(doc.createTextNode(value));
1665          propNode.appendChild(valueNode);
1666    
1667          conf.appendChild(doc.createTextNode("\n"));
1668        }
1669        return doc;
1670      }
1671    
1672      /**
1673       *  Writes out all the parameters and their properties (final and resource) to
1674       *  the given {@link Writer}
1675       *  The format of the output would be 
1676       *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
1677       *  key2.isFinal,key2.resource}... ] } 
1678       *  It does not output the parameters of the configuration object which is 
1679       *  loaded from an input stream.
1680       * @param out the Writer to write to
1681       * @throws IOException
1682       */
1683      public static void dumpConfiguration(Configuration config,
1684          Writer out) throws IOException {
1685        JsonFactory dumpFactory = new JsonFactory();
1686        JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
1687        dumpGenerator.writeStartObject();
1688        dumpGenerator.writeFieldName("properties");
1689        dumpGenerator.writeStartArray();
1690        dumpGenerator.flush();
1691        synchronized (config) {
1692          for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
1693            dumpGenerator.writeStartObject();
1694            dumpGenerator.writeStringField("key", (String) item.getKey());
1695            dumpGenerator.writeStringField("value", 
1696                                           config.get((String) item.getKey()));
1697            dumpGenerator.writeBooleanField("isFinal",
1698                                            config.finalParameters.contains(item.getKey()));
1699            dumpGenerator.writeStringField("resource",
1700                                           config.updatingResource.get(item.getKey()));
1701            dumpGenerator.writeEndObject();
1702          }
1703        }
1704        dumpGenerator.writeEndArray();
1705        dumpGenerator.writeEndObject();
1706        dumpGenerator.flush();
1707      }
1708      
1709      /**
1710       * Get the {@link ClassLoader} for this job.
1711       * 
1712       * @return the correct class loader.
1713       */
1714      public ClassLoader getClassLoader() {
1715        return classLoader;
1716      }
1717      
1718      /**
1719       * Set the class loader that will be used to load the various objects.
1720       * 
1721       * @param classLoader the new class loader.
1722       */
1723      public void setClassLoader(ClassLoader classLoader) {
1724        this.classLoader = classLoader;
1725      }
1726      
1727      @Override
1728      public String toString() {
1729        StringBuilder sb = new StringBuilder();
1730        sb.append("Configuration: ");
1731        if(loadDefaults) {
1732          toString(defaultResources, sb);
1733          if(resources.size()>0) {
1734            sb.append(", ");
1735          }
1736        }
1737        toString(resources, sb);
1738        return sb.toString();
1739      }
1740    
1741      private <T> void toString(List<T> resources, StringBuilder sb) {
1742        ListIterator<T> i = resources.listIterator();
1743        while (i.hasNext()) {
1744          if (i.nextIndex() != 0) {
1745            sb.append(", ");
1746          }
1747          sb.append(i.next());
1748        }
1749      }
1750    
1751      /** 
1752       * Set the quietness-mode. 
1753       * 
1754       * In the quiet-mode, error and informational messages might not be logged.
1755       * 
1756       * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
1757       *              to turn it off.
1758       */
1759      public synchronized void setQuietMode(boolean quietmode) {
1760        this.quietmode = quietmode;
1761      }
1762    
1763      synchronized boolean getQuietMode() {
1764        return this.quietmode;
1765      }
1766      
1767      /** For debugging.  List non-default properties to the terminal and exit. */
1768      public static void main(String[] args) throws Exception {
1769        new Configuration().writeXml(System.out);
1770      }
1771    
1772      @Override
1773      public void readFields(DataInput in) throws IOException {
1774        clear();
1775        int size = WritableUtils.readVInt(in);
1776        for(int i=0; i < size; ++i) {
1777          set(org.apache.hadoop.io.Text.readString(in), 
1778              org.apache.hadoop.io.Text.readString(in));
1779        }
1780      }
1781    
1782      //@Override
1783      public void write(DataOutput out) throws IOException {
1784        Properties props = getProps();
1785        WritableUtils.writeVInt(out, props.size());
1786        for(Map.Entry<Object, Object> item: props.entrySet()) {
1787          org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
1788          org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
1789        }
1790      }
1791      
1792      /**
1793       * get keys matching the the regex 
1794       * @param regex
1795       * @return Map<String,String> with matching keys
1796       */
1797      public Map<String,String> getValByRegex(String regex) {
1798        Pattern p = Pattern.compile(regex);
1799    
1800        Map<String,String> result = new HashMap<String,String>();
1801        Matcher m;
1802    
1803        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1804          if (item.getKey() instanceof String && 
1805              item.getValue() instanceof String) {
1806            m = p.matcher((String)item.getKey());
1807            if(m.find()) { // match
1808              result.put((String) item.getKey(), (String) item.getValue());
1809            }
1810          }
1811        }
1812        return result;
1813      }
1814    
1815      //Load deprecated keys in common
1816      private static void addDeprecatedKeys() {
1817        Configuration.addDeprecation("topology.script.file.name", 
1818                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
1819        Configuration.addDeprecation("topology.script.number.args", 
1820                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
1821        Configuration.addDeprecation("hadoop.configured.node.mapping", 
1822                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
1823        Configuration.addDeprecation("topology.node.switch.mapping.impl", 
1824                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
1825        Configuration.addDeprecation("dfs.df.interval", 
1826                   new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
1827        Configuration.addDeprecation("dfs.client.buffer.dir", 
1828                   new String[]{CommonConfigurationKeys.FS_CLIENT_BUFFER_DIR_KEY});
1829        Configuration.addDeprecation("hadoop.native.lib", 
1830                   new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
1831        Configuration.addDeprecation("fs.default.name", 
1832                   new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
1833      }
1834    }