001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.conf; 020 021 import java.io.BufferedInputStream; 022 import java.io.DataInput; 023 import java.io.DataOutput; 024 import java.io.File; 025 import java.io.FileInputStream; 026 import java.io.IOException; 027 import java.io.InputStream; 028 import java.io.InputStreamReader; 029 import java.io.OutputStream; 030 import java.io.OutputStreamWriter; 031 import java.io.Reader; 032 import java.io.Writer; 033 import java.net.URL; 034 import java.util.ArrayList; 035 import java.util.Collection; 036 import java.util.Collections; 037 import java.util.Enumeration; 038 import java.util.HashMap; 039 import java.util.HashSet; 040 import java.util.Iterator; 041 import java.util.List; 042 import java.util.ListIterator; 043 import java.util.Map; 044 import java.util.Properties; 045 import java.util.Set; 046 import java.util.StringTokenizer; 047 import java.util.WeakHashMap; 048 import java.util.concurrent.CopyOnWriteArrayList; 049 import java.util.regex.Matcher; 050 import java.util.regex.Pattern; 051 import java.util.regex.PatternSyntaxException; 052 053 import javax.xml.parsers.DocumentBuilder; 054 import javax.xml.parsers.DocumentBuilderFactory; 055 import javax.xml.parsers.ParserConfigurationException; 056 import javax.xml.transform.Transformer; 057 import javax.xml.transform.TransformerException; 058 import javax.xml.transform.TransformerFactory; 059 import javax.xml.transform.dom.DOMSource; 060 import javax.xml.transform.stream.StreamResult; 061 062 import org.apache.commons.logging.Log; 063 import org.apache.commons.logging.LogFactory; 064 import org.apache.hadoop.classification.InterfaceAudience; 065 import org.apache.hadoop.classification.InterfaceStability; 066 import org.apache.hadoop.fs.FileSystem; 067 import org.apache.hadoop.fs.Path; 068 import org.apache.hadoop.fs.CommonConfigurationKeys; 069 import org.apache.hadoop.io.Writable; 070 import org.apache.hadoop.io.WritableUtils; 071 import org.apache.hadoop.util.ReflectionUtils; 072 import org.apache.hadoop.util.StringUtils; 073 import org.codehaus.jackson.JsonFactory; 074 import org.codehaus.jackson.JsonGenerator; 075 import org.w3c.dom.Comment; 076 import org.w3c.dom.DOMException; 077 import org.w3c.dom.Document; 078 import org.w3c.dom.Element; 079 import org.w3c.dom.Node; 080 import org.w3c.dom.NodeList; 081 import org.w3c.dom.Text; 082 import org.xml.sax.SAXException; 083 084 /** 085 * Provides access to configuration parameters. 086 * 087 * <h4 id="Resources">Resources</h4> 088 * 089 * <p>Configurations are specified by resources. A resource contains a set of 090 * name/value pairs as XML data. Each resource is named by either a 091 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 092 * then the classpath is examined for a file with that name. If named by a 093 * <code>Path</code>, then the local filesystem is examined directly, without 094 * referring to the classpath. 095 * 096 * <p>Unless explicitly turned off, Hadoop by default specifies two 097 * resources, loaded in-order from the classpath: <ol> 098 * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a> 099 * </tt>: Read-only defaults for hadoop.</li> 100 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop 101 * installation.</li> 102 * </ol> 103 * Applications may add additional resources, which are loaded 104 * subsequent to these resources in the order they are added. 105 * 106 * <h4 id="FinalParams">Final Parameters</h4> 107 * 108 * <p>Configuration parameters may be declared <i>final</i>. 109 * Once a resource declares a value final, no subsequently-loaded 110 * resource can alter that value. 111 * For example, one might define a final parameter with: 112 * <tt><pre> 113 * <property> 114 * <name>dfs.client.buffer.dir</name> 115 * <value>/tmp/hadoop/dfs/client</value> 116 * <b><final>true</final></b> 117 * </property></pre></tt> 118 * 119 * Administrators typically define parameters as final in 120 * <tt>core-site.xml</tt> for values that user applications may not alter. 121 * 122 * <h4 id="VariableExpansion">Variable Expansion</h4> 123 * 124 * <p>Value strings are first processed for <i>variable expansion</i>. The 125 * available properties are:<ol> 126 * <li>Other properties defined in this Configuration; and, if a name is 127 * undefined here,</li> 128 * <li>Properties in {@link System#getProperties()}.</li> 129 * </ol> 130 * 131 * <p>For example, if a configuration resource contains the following property 132 * definitions: 133 * <tt><pre> 134 * <property> 135 * <name>basedir</name> 136 * <value>/user/${<i>user.name</i>}</value> 137 * </property> 138 * 139 * <property> 140 * <name>tempdir</name> 141 * <value>${<i>basedir</i>}/tmp</value> 142 * </property></pre></tt> 143 * 144 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt> 145 * will be resolved to another property in this Configuration, while 146 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value 147 * of the System property with that name. 148 */ 149 @InterfaceAudience.Public 150 @InterfaceStability.Stable 151 public class Configuration implements Iterable<Map.Entry<String,String>>, 152 Writable { 153 private static final Log LOG = 154 LogFactory.getLog(Configuration.class); 155 156 private boolean quietmode = true; 157 158 /** 159 * List of configuration resources. 160 */ 161 private ArrayList<Object> resources = new ArrayList<Object>(); 162 163 /** 164 * The value reported as the setting resource when a key is set 165 * by code rather than a file resource. 166 */ 167 static final String UNKNOWN_RESOURCE = "Unknown"; 168 169 /** 170 * List of configuration parameters marked <b>final</b>. 171 */ 172 private Set<String> finalParameters = new HashSet<String>(); 173 174 private boolean loadDefaults = true; 175 176 /** 177 * Configuration objects 178 */ 179 private static final WeakHashMap<Configuration,Object> REGISTRY = 180 new WeakHashMap<Configuration,Object>(); 181 182 /** 183 * List of default Resources. Resources are loaded in the order of the list 184 * entries 185 */ 186 private static final CopyOnWriteArrayList<String> defaultResources = 187 new CopyOnWriteArrayList<String>(); 188 189 private static final Map<ClassLoader, Map<String, Class<?>>> 190 CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, Class<?>>>(); 191 192 /** 193 * Stores the mapping of key to the resource which modifies or loads 194 * the key most recently 195 */ 196 private HashMap<String, String> updatingResource; 197 198 /** 199 * Class to keep the information about the keys which replace the deprecated 200 * ones. 201 * 202 * This class stores the new keys which replace the deprecated keys and also 203 * gives a provision to have a custom message for each of the deprecated key 204 * that is being replaced. It also provides method to get the appropriate 205 * warning message which can be logged whenever the deprecated key is used. 206 */ 207 private static class DeprecatedKeyInfo { 208 private String[] newKeys; 209 private String customMessage; 210 private boolean accessed; 211 DeprecatedKeyInfo(String[] newKeys, String customMessage) { 212 this.newKeys = newKeys; 213 this.customMessage = customMessage; 214 accessed = false; 215 } 216 217 /** 218 * Method to provide the warning message. It gives the custom message if 219 * non-null, and default message otherwise. 220 * @param key the associated deprecated key. 221 * @return message that is to be logged when a deprecated key is used. 222 */ 223 private final String getWarningMessage(String key) { 224 String warningMessage; 225 if(customMessage == null) { 226 StringBuilder message = new StringBuilder(key); 227 String deprecatedKeySuffix = " is deprecated. Instead, use "; 228 message.append(deprecatedKeySuffix); 229 for (int i = 0; i < newKeys.length; i++) { 230 message.append(newKeys[i]); 231 if(i != newKeys.length-1) { 232 message.append(", "); 233 } 234 } 235 warningMessage = message.toString(); 236 } 237 else { 238 warningMessage = customMessage; 239 } 240 accessed = true; 241 return warningMessage; 242 } 243 } 244 245 /** 246 * Stores the deprecated keys, the new keys which replace the deprecated keys 247 * and custom message(if any provided). 248 */ 249 private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap = 250 new HashMap<String, DeprecatedKeyInfo>(); 251 252 /** 253 * Stores a mapping from superseding keys to the keys which they deprecate. 254 */ 255 private static Map<String, String> reverseDeprecatedKeyMap = 256 new HashMap<String, String>(); 257 258 /** 259 * Adds the deprecated key to the deprecation map. 260 * It does not override any existing entries in the deprecation map. 261 * This is to be used only by the developers in order to add deprecation of 262 * keys, and attempts to call this method after loading resources once, 263 * would lead to <tt>UnsupportedOperationException</tt> 264 * @param key 265 * @param newKeys 266 * @param customMessage 267 */ 268 public synchronized static void addDeprecation(String key, String[] newKeys, 269 String customMessage) { 270 if (key == null || key.length() == 0 || 271 newKeys == null || newKeys.length == 0) { 272 throw new IllegalArgumentException(); 273 } 274 if (!isDeprecated(key)) { 275 DeprecatedKeyInfo newKeyInfo; 276 newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage); 277 deprecatedKeyMap.put(key, newKeyInfo); 278 for (String newKey : newKeys) { 279 reverseDeprecatedKeyMap.put(newKey, key); 280 } 281 } 282 } 283 284 /** 285 * Adds the deprecated key to the deprecation map when no custom message 286 * is provided. 287 * It does not override any existing entries in the deprecation map. 288 * This is to be used only by the developers in order to add deprecation of 289 * keys, and attempts to call this method after loading resources once, 290 * would lead to <tt>UnsupportedOperationException</tt> 291 * 292 * @param key Key that is to be deprecated 293 * @param newKeys list of keys that take up the values of deprecated key 294 */ 295 public synchronized static void addDeprecation(String key, String[] newKeys) { 296 addDeprecation(key, newKeys, null); 297 } 298 299 /** 300 * checks whether the given <code>key</code> is deprecated. 301 * 302 * @param key the parameter which is to be checked for deprecation 303 * @return <code>true</code> if the key is deprecated and 304 * <code>false</code> otherwise. 305 */ 306 private static boolean isDeprecated(String key) { 307 return deprecatedKeyMap.containsKey(key); 308 } 309 310 /** 311 * Checks for the presence of the property <code>name</code> in the 312 * deprecation map. Returns the first of the list of new keys if present 313 * in the deprecation map or the <code>name</code> itself. If the property 314 * is not presently set but the property map contains an entry for the 315 * deprecated key, the value of the deprecated key is set as the value for 316 * the provided property name. 317 * 318 * @param name the property name 319 * @return the first property in the list of properties mapping 320 * the <code>name</code> or the <code>name</code> itself. 321 */ 322 private String handleDeprecation(String name) { 323 if (isDeprecated(name)) { 324 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); 325 if (!keyInfo.accessed) { 326 LOG.warn(keyInfo.getWarningMessage(name)); 327 } 328 for (String newKey : keyInfo.newKeys) { 329 if(newKey != null) { 330 name = newKey; 331 break; 332 } 333 } 334 } 335 String deprecatedKey = reverseDeprecatedKeyMap.get(name); 336 if (deprecatedKey != null && !getOverlay().containsKey(name) && 337 getOverlay().containsKey(deprecatedKey)) { 338 getProps().setProperty(name, getOverlay().getProperty(deprecatedKey)); 339 getOverlay().setProperty(name, getOverlay().getProperty(deprecatedKey)); 340 341 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(deprecatedKey); 342 if (!keyInfo.accessed) { 343 LOG.warn(keyInfo.getWarningMessage(deprecatedKey)); 344 } 345 } 346 return name; 347 } 348 349 static{ 350 //print deprecation warning if hadoop-site.xml is found in classpath 351 ClassLoader cL = Thread.currentThread().getContextClassLoader(); 352 if (cL == null) { 353 cL = Configuration.class.getClassLoader(); 354 } 355 if(cL.getResource("hadoop-site.xml")!=null) { 356 LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + 357 "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " 358 + "mapred-site.xml and hdfs-site.xml to override properties of " + 359 "core-default.xml, mapred-default.xml and hdfs-default.xml " + 360 "respectively"); 361 } 362 addDefaultResource("core-default.xml"); 363 addDefaultResource("core-site.xml"); 364 //Add code for managing deprecated key mapping 365 //for example 366 //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"}); 367 //adds deprecation for oldKey1 to two new keys(newkey1, newkey2). 368 //so get or set of oldKey1 will correctly populate/access values of 369 //newkey1 and newkey2 370 addDeprecatedKeys(); 371 } 372 373 private Properties properties; 374 private Properties overlay; 375 private ClassLoader classLoader; 376 { 377 classLoader = Thread.currentThread().getContextClassLoader(); 378 if (classLoader == null) { 379 classLoader = Configuration.class.getClassLoader(); 380 } 381 } 382 383 /** A new configuration. */ 384 public Configuration() { 385 this(true); 386 } 387 388 /** A new configuration where the behavior of reading from the default 389 * resources can be turned off. 390 * 391 * If the parameter {@code loadDefaults} is false, the new instance 392 * will not load resources from the default files. 393 * @param loadDefaults specifies whether to load from the default files 394 */ 395 public Configuration(boolean loadDefaults) { 396 this.loadDefaults = loadDefaults; 397 updatingResource = new HashMap<String, String>(); 398 synchronized(Configuration.class) { 399 REGISTRY.put(this, null); 400 } 401 } 402 403 /** 404 * A new configuration with the same settings cloned from another. 405 * 406 * @param other the configuration from which to clone settings. 407 */ 408 @SuppressWarnings("unchecked") 409 public Configuration(Configuration other) { 410 this.resources = (ArrayList)other.resources.clone(); 411 synchronized(other) { 412 if (other.properties != null) { 413 this.properties = (Properties)other.properties.clone(); 414 } 415 416 if (other.overlay!=null) { 417 this.overlay = (Properties)other.overlay.clone(); 418 } 419 420 this.updatingResource = new HashMap<String, String>(other.updatingResource); 421 } 422 423 this.finalParameters = new HashSet<String>(other.finalParameters); 424 synchronized(Configuration.class) { 425 REGISTRY.put(this, null); 426 } 427 this.classLoader = other.classLoader; 428 this.loadDefaults = other.loadDefaults; 429 setQuietMode(other.getQuietMode()); 430 } 431 432 /** 433 * Add a default resource. Resources are loaded in the order of the resources 434 * added. 435 * @param name file name. File should be present in the classpath. 436 */ 437 public static synchronized void addDefaultResource(String name) { 438 if(!defaultResources.contains(name)) { 439 defaultResources.add(name); 440 for(Configuration conf : REGISTRY.keySet()) { 441 if(conf.loadDefaults) { 442 conf.reloadConfiguration(); 443 } 444 } 445 } 446 } 447 448 /** 449 * Add a configuration resource. 450 * 451 * The properties of this resource will override properties of previously 452 * added resources, unless they were marked <a href="#Final">final</a>. 453 * 454 * @param name resource to be added, the classpath is examined for a file 455 * with that name. 456 */ 457 public void addResource(String name) { 458 addResourceObject(name); 459 } 460 461 /** 462 * Add a configuration resource. 463 * 464 * The properties of this resource will override properties of previously 465 * added resources, unless they were marked <a href="#Final">final</a>. 466 * 467 * @param url url of the resource to be added, the local filesystem is 468 * examined directly to find the resource, without referring to 469 * the classpath. 470 */ 471 public void addResource(URL url) { 472 addResourceObject(url); 473 } 474 475 /** 476 * Add a configuration resource. 477 * 478 * The properties of this resource will override properties of previously 479 * added resources, unless they were marked <a href="#Final">final</a>. 480 * 481 * @param file file-path of resource to be added, the local filesystem is 482 * examined directly to find the resource, without referring to 483 * the classpath. 484 */ 485 public void addResource(Path file) { 486 addResourceObject(file); 487 } 488 489 /** 490 * Add a configuration resource. 491 * 492 * The properties of this resource will override properties of previously 493 * added resources, unless they were marked <a href="#Final">final</a>. 494 * 495 * @param in InputStream to deserialize the object from. 496 */ 497 public void addResource(InputStream in) { 498 addResourceObject(in); 499 } 500 501 502 /** 503 * Reload configuration from previously added resources. 504 * 505 * This method will clear all the configuration read from the added 506 * resources, and final parameters. This will make the resources to 507 * be read again before accessing the values. Values that are added 508 * via set methods will overlay values read from the resources. 509 */ 510 public synchronized void reloadConfiguration() { 511 properties = null; // trigger reload 512 finalParameters.clear(); // clear site-limits 513 } 514 515 private synchronized void addResourceObject(Object resource) { 516 resources.add(resource); // add to resources 517 reloadConfiguration(); 518 } 519 520 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); 521 private static int MAX_SUBST = 20; 522 523 private String substituteVars(String expr) { 524 if (expr == null) { 525 return null; 526 } 527 Matcher match = varPat.matcher(""); 528 String eval = expr; 529 for(int s=0; s<MAX_SUBST; s++) { 530 match.reset(eval); 531 if (!match.find()) { 532 return eval; 533 } 534 String var = match.group(); 535 var = var.substring(2, var.length()-1); // remove ${ .. } 536 String val = null; 537 try { 538 val = System.getProperty(var); 539 } catch(SecurityException se) { 540 LOG.warn("Unexpected SecurityException in Configuration", se); 541 } 542 if (val == null) { 543 val = getRaw(var); 544 } 545 if (val == null) { 546 return eval; // return literal ${var}: var is unbound 547 } 548 // substitute 549 eval = eval.substring(0, match.start())+val+eval.substring(match.end()); 550 } 551 throw new IllegalStateException("Variable substitution depth too large: " 552 + MAX_SUBST + " " + expr); 553 } 554 555 /** 556 * Get the value of the <code>name</code> property, <code>null</code> if 557 * no such property exists. If the key is deprecated, it returns the value of 558 * the first key which replaces the deprecated key and is not null 559 * 560 * Values are processed for <a href="#VariableExpansion">variable expansion</a> 561 * before being returned. 562 * 563 * @param name the property name. 564 * @return the value of the <code>name</code> or its replacing property, 565 * or null if no such property exists. 566 */ 567 public String get(String name) { 568 name = handleDeprecation(name); 569 return substituteVars(getProps().getProperty(name)); 570 } 571 572 /** 573 * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 574 * <code>null</code> if no such property exists. 575 * If the key is deprecated, it returns the value of 576 * the first key which replaces the deprecated key and is not null 577 * 578 * Values are processed for <a href="#VariableExpansion">variable expansion</a> 579 * before being returned. 580 * 581 * @param name the property name. 582 * @return the value of the <code>name</code> or its replacing property, 583 * or null if no such property exists. 584 */ 585 public String getTrimmed(String name) { 586 String value = get(name); 587 588 if (null == value) { 589 return null; 590 } else { 591 return value.trim(); 592 } 593 } 594 595 /** 596 * Get the value of the <code>name</code> property, without doing 597 * <a href="#VariableExpansion">variable expansion</a>.If the key is 598 * deprecated, it returns the value of the first key which replaces 599 * the deprecated key and is not null. 600 * 601 * @param name the property name. 602 * @return the value of the <code>name</code> property or 603 * its replacing property and null if no such property exists. 604 */ 605 public String getRaw(String name) { 606 name = handleDeprecation(name); 607 return getProps().getProperty(name); 608 } 609 610 /** 611 * Set the <code>value</code> of the <code>name</code> property. If 612 * <code>name</code> is deprecated, it sets the <code>value</code> to the keys 613 * that replace the deprecated key. 614 * 615 * @param name property name. 616 * @param value property value. 617 */ 618 public void set(String name, String value) { 619 if (deprecatedKeyMap.isEmpty()) { 620 getProps(); 621 } 622 if (!isDeprecated(name)) { 623 getOverlay().setProperty(name, value); 624 getProps().setProperty(name, value); 625 updatingResource.put(name, UNKNOWN_RESOURCE); 626 } 627 else { 628 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name); 629 LOG.warn(keyInfo.getWarningMessage(name)); 630 for (String newKey : keyInfo.newKeys) { 631 getOverlay().setProperty(newKey, value); 632 getProps().setProperty(newKey, value); 633 } 634 } 635 } 636 637 /** 638 * Unset a previously set property. 639 */ 640 public synchronized void unset(String name) { 641 name = handleDeprecation(name); 642 643 getOverlay().remove(name); 644 getProps().remove(name); 645 } 646 647 /** 648 * Sets a property if it is currently unset. 649 * @param name the property name 650 * @param value the new value 651 */ 652 public synchronized void setIfUnset(String name, String value) { 653 if (get(name) == null) { 654 set(name, value); 655 } 656 } 657 658 private synchronized Properties getOverlay() { 659 if (overlay==null){ 660 overlay=new Properties(); 661 } 662 return overlay; 663 } 664 665 /** 666 * Get the value of the <code>name</code>. If the key is deprecated, 667 * it returns the value of the first key which replaces the deprecated key 668 * and is not null. 669 * If no such property exists, 670 * then <code>defaultValue</code> is returned. 671 * 672 * @param name property name. 673 * @param defaultValue default value. 674 * @return property value, or <code>defaultValue</code> if the property 675 * doesn't exist. 676 */ 677 public String get(String name, String defaultValue) { 678 name = handleDeprecation(name); 679 return substituteVars(getProps().getProperty(name, defaultValue)); 680 } 681 682 /** 683 * Get the value of the <code>name</code> property as an <code>int</code>. 684 * 685 * If no such property exists, the provided default value is returned, 686 * or if the specified value is not a valid <code>int</code>, 687 * then an error is thrown. 688 * 689 * @param name property name. 690 * @param defaultValue default value. 691 * @throws NumberFormatException when the value is invalid 692 * @return property value as an <code>int</code>, 693 * or <code>defaultValue</code>. 694 */ 695 public int getInt(String name, int defaultValue) { 696 String valueString = getTrimmed(name); 697 if (valueString == null) 698 return defaultValue; 699 String hexString = getHexDigits(valueString); 700 if (hexString != null) { 701 return Integer.parseInt(hexString, 16); 702 } 703 return Integer.parseInt(valueString); 704 } 705 706 /** 707 * Set the value of the <code>name</code> property to an <code>int</code>. 708 * 709 * @param name property name. 710 * @param value <code>int</code> value of the property. 711 */ 712 public void setInt(String name, int value) { 713 set(name, Integer.toString(value)); 714 } 715 716 717 /** 718 * Get the value of the <code>name</code> property as a <code>long</code>. 719 * If no such property exists, the provided default value is returned, 720 * or if the specified value is not a valid <code>long</code>, 721 * then an error is thrown. 722 * 723 * @param name property name. 724 * @param defaultValue default value. 725 * @throws NumberFormatException when the value is invalid 726 * @return property value as a <code>long</code>, 727 * or <code>defaultValue</code>. 728 */ 729 public long getLong(String name, long defaultValue) { 730 String valueString = getTrimmed(name); 731 if (valueString == null) 732 return defaultValue; 733 String hexString = getHexDigits(valueString); 734 if (hexString != null) { 735 return Long.parseLong(hexString, 16); 736 } 737 return Long.parseLong(valueString); 738 } 739 740 private String getHexDigits(String value) { 741 boolean negative = false; 742 String str = value; 743 String hexString = null; 744 if (value.startsWith("-")) { 745 negative = true; 746 str = value.substring(1); 747 } 748 if (str.startsWith("0x") || str.startsWith("0X")) { 749 hexString = str.substring(2); 750 if (negative) { 751 hexString = "-" + hexString; 752 } 753 return hexString; 754 } 755 return null; 756 } 757 758 /** 759 * Set the value of the <code>name</code> property to a <code>long</code>. 760 * 761 * @param name property name. 762 * @param value <code>long</code> value of the property. 763 */ 764 public void setLong(String name, long value) { 765 set(name, Long.toString(value)); 766 } 767 768 /** 769 * Get the value of the <code>name</code> property as a <code>float</code>. 770 * If no such property exists, the provided default value is returned, 771 * or if the specified value is not a valid <code>float</code>, 772 * then an error is thrown. 773 * 774 * @param name property name. 775 * @param defaultValue default value. 776 * @throws NumberFormatException when the value is invalid 777 * @return property value as a <code>float</code>, 778 * or <code>defaultValue</code>. 779 */ 780 public float getFloat(String name, float defaultValue) { 781 String valueString = getTrimmed(name); 782 if (valueString == null) 783 return defaultValue; 784 return Float.parseFloat(valueString); 785 } 786 /** 787 * Set the value of the <code>name</code> property to a <code>float</code>. 788 * 789 * @param name property name. 790 * @param value property value. 791 */ 792 public void setFloat(String name, float value) { 793 set(name,Float.toString(value)); 794 } 795 796 /** 797 * Get the value of the <code>name</code> property as a <code>boolean</code>. 798 * If no such property is specified, or if the specified value is not a valid 799 * <code>boolean</code>, then <code>defaultValue</code> is returned. 800 * 801 * @param name property name. 802 * @param defaultValue default value. 803 * @return property value as a <code>boolean</code>, 804 * or <code>defaultValue</code>. 805 */ 806 public boolean getBoolean(String name, boolean defaultValue) { 807 String valueString = getTrimmed(name); 808 if ("true".equals(valueString)) 809 return true; 810 else if ("false".equals(valueString)) 811 return false; 812 else return defaultValue; 813 } 814 815 /** 816 * Set the value of the <code>name</code> property to a <code>boolean</code>. 817 * 818 * @param name property name. 819 * @param value <code>boolean</code> value of the property. 820 */ 821 public void setBoolean(String name, boolean value) { 822 set(name, Boolean.toString(value)); 823 } 824 825 /** 826 * Set the given property, if it is currently unset. 827 * @param name property name 828 * @param value new value 829 */ 830 public void setBooleanIfUnset(String name, boolean value) { 831 setIfUnset(name, Boolean.toString(value)); 832 } 833 834 /** 835 * Set the value of the <code>name</code> property to the given type. This 836 * is equivalent to <code>set(<name>, value.toString())</code>. 837 * @param name property name 838 * @param value new value 839 */ 840 public <T extends Enum<T>> void setEnum(String name, T value) { 841 set(name, value.toString()); 842 } 843 844 /** 845 * Return value matching this enumerated type. 846 * @param name Property name 847 * @param defaultValue Value returned if no mapping exists 848 * @throws IllegalArgumentException If mapping is illegal for the type 849 * provided 850 */ 851 public <T extends Enum<T>> T getEnum(String name, T defaultValue) { 852 final String val = get(name); 853 return null == val 854 ? defaultValue 855 : Enum.valueOf(defaultValue.getDeclaringClass(), val); 856 } 857 858 /** 859 * Get the value of the <code>name</code> property as a <code>Pattern</code>. 860 * If no such property is specified, or if the specified value is not a valid 861 * <code>Pattern</code>, then <code>DefaultValue</code> is returned. 862 * 863 * @param name property name 864 * @param defaultValue default value 865 * @return property value as a compiled Pattern, or defaultValue 866 */ 867 public Pattern getPattern(String name, Pattern defaultValue) { 868 String valString = get(name); 869 if (null == valString || "".equals(valString)) { 870 return defaultValue; 871 } 872 try { 873 return Pattern.compile(valString); 874 } catch (PatternSyntaxException pse) { 875 LOG.warn("Regular expression '" + valString + "' for property '" + 876 name + "' not valid. Using default", pse); 877 return defaultValue; 878 } 879 } 880 881 /** 882 * Set the given property to <code>Pattern</code>. 883 * If the pattern is passed as null, sets the empty pattern which results in 884 * further calls to getPattern(...) returning the default value. 885 * 886 * @param name property name 887 * @param pattern new value 888 */ 889 public void setPattern(String name, Pattern pattern) { 890 if (null == pattern) { 891 set(name, null); 892 } else { 893 set(name, pattern.pattern()); 894 } 895 } 896 897 /** 898 * A class that represents a set of positive integer ranges. It parses 899 * strings of the form: "2-3,5,7-" where ranges are separated by comma and 900 * the lower/upper bounds are separated by dash. Either the lower or upper 901 * bound may be omitted meaning all values up to or over. So the string 902 * above means 2, 3, 5, and 7, 8, 9, ... 903 */ 904 public static class IntegerRanges { 905 private static class Range { 906 int start; 907 int end; 908 } 909 910 List<Range> ranges = new ArrayList<Range>(); 911 912 public IntegerRanges() { 913 } 914 915 public IntegerRanges(String newValue) { 916 StringTokenizer itr = new StringTokenizer(newValue, ","); 917 while (itr.hasMoreTokens()) { 918 String rng = itr.nextToken().trim(); 919 String[] parts = rng.split("-", 3); 920 if (parts.length < 1 || parts.length > 2) { 921 throw new IllegalArgumentException("integer range badly formed: " + 922 rng); 923 } 924 Range r = new Range(); 925 r.start = convertToInt(parts[0], 0); 926 if (parts.length == 2) { 927 r.end = convertToInt(parts[1], Integer.MAX_VALUE); 928 } else { 929 r.end = r.start; 930 } 931 if (r.start > r.end) { 932 throw new IllegalArgumentException("IntegerRange from " + r.start + 933 " to " + r.end + " is invalid"); 934 } 935 ranges.add(r); 936 } 937 } 938 939 /** 940 * Convert a string to an int treating empty strings as the default value. 941 * @param value the string value 942 * @param defaultValue the value for if the string is empty 943 * @return the desired integer 944 */ 945 private static int convertToInt(String value, int defaultValue) { 946 String trim = value.trim(); 947 if (trim.length() == 0) { 948 return defaultValue; 949 } 950 return Integer.parseInt(trim); 951 } 952 953 /** 954 * Is the given value in the set of ranges 955 * @param value the value to check 956 * @return is the value in the ranges? 957 */ 958 public boolean isIncluded(int value) { 959 for(Range r: ranges) { 960 if (r.start <= value && value <= r.end) { 961 return true; 962 } 963 } 964 return false; 965 } 966 967 @Override 968 public String toString() { 969 StringBuilder result = new StringBuilder(); 970 boolean first = true; 971 for(Range r: ranges) { 972 if (first) { 973 first = false; 974 } else { 975 result.append(','); 976 } 977 result.append(r.start); 978 result.append('-'); 979 result.append(r.end); 980 } 981 return result.toString(); 982 } 983 } 984 985 /** 986 * Parse the given attribute as a set of integer ranges 987 * @param name the attribute name 988 * @param defaultValue the default value if it is not set 989 * @return a new set of ranges from the configured value 990 */ 991 public IntegerRanges getRange(String name, String defaultValue) { 992 return new IntegerRanges(get(name, defaultValue)); 993 } 994 995 /** 996 * Get the comma delimited values of the <code>name</code> property as 997 * a collection of <code>String</code>s. 998 * If no such property is specified then empty collection is returned. 999 * <p> 1000 * This is an optimized version of {@link #getStrings(String)} 1001 * 1002 * @param name property name. 1003 * @return property value as a collection of <code>String</code>s. 1004 */ 1005 public Collection<String> getStringCollection(String name) { 1006 String valueString = get(name); 1007 return StringUtils.getStringCollection(valueString); 1008 } 1009 1010 /** 1011 * Get the comma delimited values of the <code>name</code> property as 1012 * an array of <code>String</code>s. 1013 * If no such property is specified then <code>null</code> is returned. 1014 * 1015 * @param name property name. 1016 * @return property value as an array of <code>String</code>s, 1017 * or <code>null</code>. 1018 */ 1019 public String[] getStrings(String name) { 1020 String valueString = get(name); 1021 return StringUtils.getStrings(valueString); 1022 } 1023 1024 /** 1025 * Get the comma delimited values of the <code>name</code> property as 1026 * an array of <code>String</code>s. 1027 * If no such property is specified then default value is returned. 1028 * 1029 * @param name property name. 1030 * @param defaultValue The default value 1031 * @return property value as an array of <code>String</code>s, 1032 * or default value. 1033 */ 1034 public String[] getStrings(String name, String... defaultValue) { 1035 String valueString = get(name); 1036 if (valueString == null) { 1037 return defaultValue; 1038 } else { 1039 return StringUtils.getStrings(valueString); 1040 } 1041 } 1042 1043 /** 1044 * Get the comma delimited values of the <code>name</code> property as 1045 * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace. 1046 * If no such property is specified then empty <code>Collection</code> is returned. 1047 * 1048 * @param name property name. 1049 * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 1050 */ 1051 public Collection<String> getTrimmedStringCollection(String name) { 1052 String valueString = get(name); 1053 if (null == valueString) { 1054 Collection<String> empty = new ArrayList<String>(); 1055 return empty; 1056 } 1057 return StringUtils.getTrimmedStringCollection(valueString); 1058 } 1059 1060 /** 1061 * Get the comma delimited values of the <code>name</code> property as 1062 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace. 1063 * If no such property is specified then an empty array is returned. 1064 * 1065 * @param name property name. 1066 * @return property value as an array of trimmed <code>String</code>s, 1067 * or empty array. 1068 */ 1069 public String[] getTrimmedStrings(String name) { 1070 String valueString = get(name); 1071 return StringUtils.getTrimmedStrings(valueString); 1072 } 1073 1074 /** 1075 * Get the comma delimited values of the <code>name</code> property as 1076 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace. 1077 * If no such property is specified then default value is returned. 1078 * 1079 * @param name property name. 1080 * @param defaultValue The default value 1081 * @return property value as an array of trimmed <code>String</code>s, 1082 * or default value. 1083 */ 1084 public String[] getTrimmedStrings(String name, String... defaultValue) { 1085 String valueString = get(name); 1086 if (null == valueString) { 1087 return defaultValue; 1088 } else { 1089 return StringUtils.getTrimmedStrings(valueString); 1090 } 1091 } 1092 1093 /** 1094 * Set the array of string values for the <code>name</code> property as 1095 * as comma delimited values. 1096 * 1097 * @param name property name. 1098 * @param values The values 1099 */ 1100 public void setStrings(String name, String... values) { 1101 set(name, StringUtils.arrayToString(values)); 1102 } 1103 1104 /** 1105 * Load a class by name. 1106 * 1107 * @param name the class name. 1108 * @return the class object. 1109 * @throws ClassNotFoundException if the class is not found. 1110 */ 1111 public Class<?> getClassByName(String name) throws ClassNotFoundException { 1112 Map<String, Class<?>> map; 1113 1114 synchronized (CACHE_CLASSES) { 1115 map = CACHE_CLASSES.get(classLoader); 1116 if (map == null) { 1117 map = Collections.synchronizedMap( 1118 new WeakHashMap<String, Class<?>>()); 1119 CACHE_CLASSES.put(classLoader, map); 1120 } 1121 } 1122 1123 Class<?> clazz = map.get(name); 1124 if (clazz == null) { 1125 clazz = Class.forName(name, true, classLoader); 1126 if (clazz != null) { 1127 // two putters can race here, but they'll put the same class 1128 map.put(name, clazz); 1129 } 1130 } 1131 1132 return clazz; 1133 } 1134 1135 /** 1136 * Get the value of the <code>name</code> property 1137 * as an array of <code>Class</code>. 1138 * The value of the property specifies a list of comma separated class names. 1139 * If no such property is specified, then <code>defaultValue</code> is 1140 * returned. 1141 * 1142 * @param name the property name. 1143 * @param defaultValue default value. 1144 * @return property value as a <code>Class[]</code>, 1145 * or <code>defaultValue</code>. 1146 */ 1147 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { 1148 String[] classnames = getTrimmedStrings(name); 1149 if (classnames == null) 1150 return defaultValue; 1151 try { 1152 Class<?>[] classes = new Class<?>[classnames.length]; 1153 for(int i = 0; i < classnames.length; i++) { 1154 classes[i] = getClassByName(classnames[i]); 1155 } 1156 return classes; 1157 } catch (ClassNotFoundException e) { 1158 throw new RuntimeException(e); 1159 } 1160 } 1161 1162 /** 1163 * Get the value of the <code>name</code> property as a <code>Class</code>. 1164 * If no such property is specified, then <code>defaultValue</code> is 1165 * returned. 1166 * 1167 * @param name the class name. 1168 * @param defaultValue default value. 1169 * @return property value as a <code>Class</code>, 1170 * or <code>defaultValue</code>. 1171 */ 1172 public Class<?> getClass(String name, Class<?> defaultValue) { 1173 String valueString = getTrimmed(name); 1174 if (valueString == null) 1175 return defaultValue; 1176 try { 1177 return getClassByName(valueString); 1178 } catch (ClassNotFoundException e) { 1179 throw new RuntimeException(e); 1180 } 1181 } 1182 1183 /** 1184 * Get the value of the <code>name</code> property as a <code>Class</code> 1185 * implementing the interface specified by <code>xface</code>. 1186 * 1187 * If no such property is specified, then <code>defaultValue</code> is 1188 * returned. 1189 * 1190 * An exception is thrown if the returned class does not implement the named 1191 * interface. 1192 * 1193 * @param name the class name. 1194 * @param defaultValue default value. 1195 * @param xface the interface implemented by the named class. 1196 * @return property value as a <code>Class</code>, 1197 * or <code>defaultValue</code>. 1198 */ 1199 public <U> Class<? extends U> getClass(String name, 1200 Class<? extends U> defaultValue, 1201 Class<U> xface) { 1202 try { 1203 Class<?> theClass = getClass(name, defaultValue); 1204 if (theClass != null && !xface.isAssignableFrom(theClass)) 1205 throw new RuntimeException(theClass+" not "+xface.getName()); 1206 else if (theClass != null) 1207 return theClass.asSubclass(xface); 1208 else 1209 return null; 1210 } catch (Exception e) { 1211 throw new RuntimeException(e); 1212 } 1213 } 1214 1215 /** 1216 * Get the value of the <code>name</code> property as a <code>List</code> 1217 * of objects implementing the interface specified by <code>xface</code>. 1218 * 1219 * An exception is thrown if any of the classes does not exist, or if it does 1220 * not implement the named interface. 1221 * 1222 * @param name the property name. 1223 * @param xface the interface implemented by the classes named by 1224 * <code>name</code>. 1225 * @return a <code>List</code> of objects implementing <code>xface</code>. 1226 */ 1227 @SuppressWarnings("unchecked") 1228 public <U> List<U> getInstances(String name, Class<U> xface) { 1229 List<U> ret = new ArrayList<U>(); 1230 Class<?>[] classes = getClasses(name); 1231 for (Class<?> cl: classes) { 1232 if (!xface.isAssignableFrom(cl)) { 1233 throw new RuntimeException(cl + " does not implement " + xface); 1234 } 1235 ret.add((U)ReflectionUtils.newInstance(cl, this)); 1236 } 1237 return ret; 1238 } 1239 1240 /** 1241 * Set the value of the <code>name</code> property to the name of a 1242 * <code>theClass</code> implementing the given interface <code>xface</code>. 1243 * 1244 * An exception is thrown if <code>theClass</code> does not implement the 1245 * interface <code>xface</code>. 1246 * 1247 * @param name property name. 1248 * @param theClass property value. 1249 * @param xface the interface implemented by the named class. 1250 */ 1251 public void setClass(String name, Class<?> theClass, Class<?> xface) { 1252 if (!xface.isAssignableFrom(theClass)) 1253 throw new RuntimeException(theClass+" not "+xface.getName()); 1254 set(name, theClass.getName()); 1255 } 1256 1257 /** 1258 * Get a local file under a directory named by <i>dirsProp</i> with 1259 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories, 1260 * then one is chosen based on <i>path</i>'s hash code. If the selected 1261 * directory does not exist, an attempt is made to create it. 1262 * 1263 * @param dirsProp directory in which to locate the file. 1264 * @param path file-path. 1265 * @return local file under the directory with the given path. 1266 */ 1267 public Path getLocalPath(String dirsProp, String path) 1268 throws IOException { 1269 String[] dirs = getTrimmedStrings(dirsProp); 1270 int hashCode = path.hashCode(); 1271 FileSystem fs = FileSystem.getLocal(this); 1272 for (int i = 0; i < dirs.length; i++) { // try each local dir 1273 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 1274 Path file = new Path(dirs[index], path); 1275 Path dir = file.getParent(); 1276 if (fs.mkdirs(dir) || fs.exists(dir)) { 1277 return file; 1278 } 1279 } 1280 LOG.warn("Could not make " + path + 1281 " in local directories from " + dirsProp); 1282 for(int i=0; i < dirs.length; i++) { 1283 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 1284 LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]); 1285 } 1286 throw new IOException("No valid local directories in property: "+dirsProp); 1287 } 1288 1289 /** 1290 * Get a local file name under a directory named in <i>dirsProp</i> with 1291 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories, 1292 * then one is chosen based on <i>path</i>'s hash code. If the selected 1293 * directory does not exist, an attempt is made to create it. 1294 * 1295 * @param dirsProp directory in which to locate the file. 1296 * @param path file-path. 1297 * @return local file under the directory with the given path. 1298 */ 1299 public File getFile(String dirsProp, String path) 1300 throws IOException { 1301 String[] dirs = getTrimmedStrings(dirsProp); 1302 int hashCode = path.hashCode(); 1303 for (int i = 0; i < dirs.length; i++) { // try each local dir 1304 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 1305 File file = new File(dirs[index], path); 1306 File dir = file.getParentFile(); 1307 if (dir.exists() || dir.mkdirs()) { 1308 return file; 1309 } 1310 } 1311 throw new IOException("No valid local directories in property: "+dirsProp); 1312 } 1313 1314 /** 1315 * Get the {@link URL} for the named resource. 1316 * 1317 * @param name resource name. 1318 * @return the url for the named resource. 1319 */ 1320 public URL getResource(String name) { 1321 return classLoader.getResource(name); 1322 } 1323 1324 /** 1325 * Get an input stream attached to the configuration resource with the 1326 * given <code>name</code>. 1327 * 1328 * @param name configuration resource name. 1329 * @return an input stream attached to the resource. 1330 */ 1331 public InputStream getConfResourceAsInputStream(String name) { 1332 try { 1333 URL url= getResource(name); 1334 1335 if (url == null) { 1336 LOG.info(name + " not found"); 1337 return null; 1338 } else { 1339 LOG.info("found resource " + name + " at " + url); 1340 } 1341 1342 return url.openStream(); 1343 } catch (Exception e) { 1344 return null; 1345 } 1346 } 1347 1348 /** 1349 * Get a {@link Reader} attached to the configuration resource with the 1350 * given <code>name</code>. 1351 * 1352 * @param name configuration resource name. 1353 * @return a reader attached to the resource. 1354 */ 1355 public Reader getConfResourceAsReader(String name) { 1356 try { 1357 URL url= getResource(name); 1358 1359 if (url == null) { 1360 LOG.info(name + " not found"); 1361 return null; 1362 } else { 1363 LOG.info("found resource " + name + " at " + url); 1364 } 1365 1366 return new InputStreamReader(url.openStream()); 1367 } catch (Exception e) { 1368 return null; 1369 } 1370 } 1371 1372 protected synchronized Properties getProps() { 1373 if (properties == null) { 1374 properties = new Properties(); 1375 loadResources(properties, resources, quietmode); 1376 if (overlay!= null) { 1377 properties.putAll(overlay); 1378 for (Map.Entry<Object,Object> item: overlay.entrySet()) { 1379 updatingResource.put((String) item.getKey(), UNKNOWN_RESOURCE); 1380 } 1381 } 1382 } 1383 return properties; 1384 } 1385 1386 /** 1387 * Return the number of keys in the configuration. 1388 * 1389 * @return number of keys in the configuration. 1390 */ 1391 public int size() { 1392 return getProps().size(); 1393 } 1394 1395 /** 1396 * Clears all keys from the configuration. 1397 */ 1398 public void clear() { 1399 getProps().clear(); 1400 getOverlay().clear(); 1401 } 1402 1403 /** 1404 * Get an {@link Iterator} to go through the list of <code>String</code> 1405 * key-value pairs in the configuration. 1406 * 1407 * @return an iterator over the entries. 1408 */ 1409 public Iterator<Map.Entry<String, String>> iterator() { 1410 // Get a copy of just the string to string pairs. After the old object 1411 // methods that allow non-strings to be put into configurations are removed, 1412 // we could replace properties with a Map<String,String> and get rid of this 1413 // code. 1414 Map<String,String> result = new HashMap<String,String>(); 1415 for(Map.Entry<Object,Object> item: getProps().entrySet()) { 1416 if (item.getKey() instanceof String && 1417 item.getValue() instanceof String) { 1418 result.put((String) item.getKey(), (String) item.getValue()); 1419 } 1420 } 1421 return result.entrySet().iterator(); 1422 } 1423 1424 private void loadResources(Properties properties, 1425 ArrayList resources, 1426 boolean quiet) { 1427 if(loadDefaults) { 1428 for (String resource : defaultResources) { 1429 loadResource(properties, resource, quiet); 1430 } 1431 1432 //support the hadoop-site.xml as a deprecated case 1433 if(getResource("hadoop-site.xml")!=null) { 1434 loadResource(properties, "hadoop-site.xml", quiet); 1435 } 1436 } 1437 1438 for (Object resource : resources) { 1439 loadResource(properties, resource, quiet); 1440 } 1441 } 1442 1443 private void loadResource(Properties properties, Object name, boolean quiet) { 1444 try { 1445 DocumentBuilderFactory docBuilderFactory 1446 = DocumentBuilderFactory.newInstance(); 1447 //ignore all comments inside the xml file 1448 docBuilderFactory.setIgnoringComments(true); 1449 1450 //allow includes in the xml file 1451 docBuilderFactory.setNamespaceAware(true); 1452 try { 1453 docBuilderFactory.setXIncludeAware(true); 1454 } catch (UnsupportedOperationException e) { 1455 LOG.error("Failed to set setXIncludeAware(true) for parser " 1456 + docBuilderFactory 1457 + ":" + e, 1458 e); 1459 } 1460 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); 1461 Document doc = null; 1462 Element root = null; 1463 1464 if (name instanceof URL) { // an URL resource 1465 URL url = (URL)name; 1466 if (url != null) { 1467 if (!quiet) { 1468 LOG.info("parsing " + url); 1469 } 1470 doc = builder.parse(url.toString()); 1471 } 1472 } else if (name instanceof String) { // a CLASSPATH resource 1473 URL url = getResource((String)name); 1474 if (url != null) { 1475 if (!quiet) { 1476 LOG.info("parsing " + url); 1477 } 1478 doc = builder.parse(url.toString()); 1479 } 1480 } else if (name instanceof Path) { // a file resource 1481 // Can't use FileSystem API or we get an infinite loop 1482 // since FileSystem uses Configuration API. Use java.io.File instead. 1483 File file = new File(((Path)name).toUri().getPath()) 1484 .getAbsoluteFile(); 1485 if (file.exists()) { 1486 if (!quiet) { 1487 LOG.info("parsing " + file); 1488 } 1489 InputStream in = new BufferedInputStream(new FileInputStream(file)); 1490 try { 1491 doc = builder.parse(in); 1492 } finally { 1493 in.close(); 1494 } 1495 } 1496 } else if (name instanceof InputStream) { 1497 try { 1498 doc = builder.parse((InputStream)name); 1499 } finally { 1500 ((InputStream)name).close(); 1501 } 1502 } else if (name instanceof Element) { 1503 root = (Element)name; 1504 } 1505 1506 if (doc == null && root == null) { 1507 if (quiet) 1508 return; 1509 throw new RuntimeException(name + " not found"); 1510 } 1511 1512 if (root == null) { 1513 root = doc.getDocumentElement(); 1514 } 1515 if (!"configuration".equals(root.getTagName())) 1516 LOG.fatal("bad conf file: top-level element not <configuration>"); 1517 NodeList props = root.getChildNodes(); 1518 for (int i = 0; i < props.getLength(); i++) { 1519 Node propNode = props.item(i); 1520 if (!(propNode instanceof Element)) 1521 continue; 1522 Element prop = (Element)propNode; 1523 if ("configuration".equals(prop.getTagName())) { 1524 loadResource(properties, prop, quiet); 1525 continue; 1526 } 1527 if (!"property".equals(prop.getTagName())) 1528 LOG.warn("bad conf file: element not <property>"); 1529 NodeList fields = prop.getChildNodes(); 1530 String attr = null; 1531 String value = null; 1532 boolean finalParameter = false; 1533 for (int j = 0; j < fields.getLength(); j++) { 1534 Node fieldNode = fields.item(j); 1535 if (!(fieldNode instanceof Element)) 1536 continue; 1537 Element field = (Element)fieldNode; 1538 if ("name".equals(field.getTagName()) && field.hasChildNodes()) 1539 attr = ((Text)field.getFirstChild()).getData().trim(); 1540 if ("value".equals(field.getTagName()) && field.hasChildNodes()) 1541 value = ((Text)field.getFirstChild()).getData(); 1542 if ("final".equals(field.getTagName()) && field.hasChildNodes()) 1543 finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); 1544 } 1545 1546 // Ignore this parameter if it has already been marked as 'final' 1547 if (attr != null) { 1548 if (deprecatedKeyMap.containsKey(attr)) { 1549 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr); 1550 keyInfo.accessed = false; 1551 for (String key:keyInfo.newKeys) { 1552 // update new keys with deprecated key's value 1553 loadProperty(properties, name, key, value, finalParameter); 1554 } 1555 } 1556 else { 1557 loadProperty(properties, name, attr, value, finalParameter); 1558 } 1559 } 1560 } 1561 1562 } catch (IOException e) { 1563 LOG.fatal("error parsing conf file: " + e); 1564 throw new RuntimeException(e); 1565 } catch (DOMException e) { 1566 LOG.fatal("error parsing conf file: " + e); 1567 throw new RuntimeException(e); 1568 } catch (SAXException e) { 1569 LOG.fatal("error parsing conf file: " + e); 1570 throw new RuntimeException(e); 1571 } catch (ParserConfigurationException e) { 1572 LOG.fatal("error parsing conf file: " + e); 1573 throw new RuntimeException(e); 1574 } 1575 } 1576 1577 private void loadProperty(Properties properties, Object name, String attr, 1578 String value, boolean finalParameter) { 1579 if (value != null) { 1580 if (!finalParameters.contains(attr)) { 1581 properties.setProperty(attr, value); 1582 updatingResource.put(attr, name.toString()); 1583 } else { 1584 LOG.warn(name+":an attempt to override final parameter: "+attr 1585 +"; Ignoring."); 1586 } 1587 } 1588 if (finalParameter) { 1589 finalParameters.add(attr); 1590 } 1591 } 1592 1593 /** 1594 * Write out the non-default properties in this configuration to the given 1595 * {@link OutputStream}. 1596 * 1597 * @param out the output stream to write to. 1598 */ 1599 public void writeXml(OutputStream out) throws IOException { 1600 writeXml(new OutputStreamWriter(out)); 1601 } 1602 1603 /** 1604 * Write out the non-default properties in this configuration to the given 1605 * {@link Writer}. 1606 * 1607 * @param out the writer to write to. 1608 */ 1609 public void writeXml(Writer out) throws IOException { 1610 Document doc = asXmlDocument(); 1611 1612 try { 1613 DOMSource source = new DOMSource(doc); 1614 StreamResult result = new StreamResult(out); 1615 TransformerFactory transFactory = TransformerFactory.newInstance(); 1616 Transformer transformer = transFactory.newTransformer(); 1617 1618 // Important to not hold Configuration log while writing result, since 1619 // 'out' may be an HDFS stream which needs to lock this configuration 1620 // from another thread. 1621 transformer.transform(source, result); 1622 } catch (TransformerException te) { 1623 throw new IOException(te); 1624 } 1625 } 1626 1627 /** 1628 * Return the XML DOM corresponding to this Configuration. 1629 */ 1630 private synchronized Document asXmlDocument() throws IOException { 1631 Document doc; 1632 try { 1633 doc = 1634 DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); 1635 } catch (ParserConfigurationException pe) { 1636 throw new IOException(pe); 1637 } 1638 Element conf = doc.createElement("configuration"); 1639 doc.appendChild(conf); 1640 conf.appendChild(doc.createTextNode("\n")); 1641 getProps(); // ensure properties is set 1642 for (Enumeration e = properties.keys(); e.hasMoreElements();) { 1643 String name = (String)e.nextElement(); 1644 Object object = properties.get(name); 1645 String value = null; 1646 if (object instanceof String) { 1647 value = (String) object; 1648 }else { 1649 continue; 1650 } 1651 Element propNode = doc.createElement("property"); 1652 conf.appendChild(propNode); 1653 1654 if (updatingResource != null) { 1655 Comment commentNode = doc.createComment( 1656 "Loaded from " + updatingResource.get(name)); 1657 propNode.appendChild(commentNode); 1658 } 1659 Element nameNode = doc.createElement("name"); 1660 nameNode.appendChild(doc.createTextNode(name)); 1661 propNode.appendChild(nameNode); 1662 1663 Element valueNode = doc.createElement("value"); 1664 valueNode.appendChild(doc.createTextNode(value)); 1665 propNode.appendChild(valueNode); 1666 1667 conf.appendChild(doc.createTextNode("\n")); 1668 } 1669 return doc; 1670 } 1671 1672 /** 1673 * Writes out all the parameters and their properties (final and resource) to 1674 * the given {@link Writer} 1675 * The format of the output would be 1676 * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, 1677 * key2.isFinal,key2.resource}... ] } 1678 * It does not output the parameters of the configuration object which is 1679 * loaded from an input stream. 1680 * @param out the Writer to write to 1681 * @throws IOException 1682 */ 1683 public static void dumpConfiguration(Configuration config, 1684 Writer out) throws IOException { 1685 JsonFactory dumpFactory = new JsonFactory(); 1686 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); 1687 dumpGenerator.writeStartObject(); 1688 dumpGenerator.writeFieldName("properties"); 1689 dumpGenerator.writeStartArray(); 1690 dumpGenerator.flush(); 1691 synchronized (config) { 1692 for (Map.Entry<Object,Object> item: config.getProps().entrySet()) { 1693 dumpGenerator.writeStartObject(); 1694 dumpGenerator.writeStringField("key", (String) item.getKey()); 1695 dumpGenerator.writeStringField("value", 1696 config.get((String) item.getKey())); 1697 dumpGenerator.writeBooleanField("isFinal", 1698 config.finalParameters.contains(item.getKey())); 1699 dumpGenerator.writeStringField("resource", 1700 config.updatingResource.get(item.getKey())); 1701 dumpGenerator.writeEndObject(); 1702 } 1703 } 1704 dumpGenerator.writeEndArray(); 1705 dumpGenerator.writeEndObject(); 1706 dumpGenerator.flush(); 1707 } 1708 1709 /** 1710 * Get the {@link ClassLoader} for this job. 1711 * 1712 * @return the correct class loader. 1713 */ 1714 public ClassLoader getClassLoader() { 1715 return classLoader; 1716 } 1717 1718 /** 1719 * Set the class loader that will be used to load the various objects. 1720 * 1721 * @param classLoader the new class loader. 1722 */ 1723 public void setClassLoader(ClassLoader classLoader) { 1724 this.classLoader = classLoader; 1725 } 1726 1727 @Override 1728 public String toString() { 1729 StringBuilder sb = new StringBuilder(); 1730 sb.append("Configuration: "); 1731 if(loadDefaults) { 1732 toString(defaultResources, sb); 1733 if(resources.size()>0) { 1734 sb.append(", "); 1735 } 1736 } 1737 toString(resources, sb); 1738 return sb.toString(); 1739 } 1740 1741 private <T> void toString(List<T> resources, StringBuilder sb) { 1742 ListIterator<T> i = resources.listIterator(); 1743 while (i.hasNext()) { 1744 if (i.nextIndex() != 0) { 1745 sb.append(", "); 1746 } 1747 sb.append(i.next()); 1748 } 1749 } 1750 1751 /** 1752 * Set the quietness-mode. 1753 * 1754 * In the quiet-mode, error and informational messages might not be logged. 1755 * 1756 * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code> 1757 * to turn it off. 1758 */ 1759 public synchronized void setQuietMode(boolean quietmode) { 1760 this.quietmode = quietmode; 1761 } 1762 1763 synchronized boolean getQuietMode() { 1764 return this.quietmode; 1765 } 1766 1767 /** For debugging. List non-default properties to the terminal and exit. */ 1768 public static void main(String[] args) throws Exception { 1769 new Configuration().writeXml(System.out); 1770 } 1771 1772 @Override 1773 public void readFields(DataInput in) throws IOException { 1774 clear(); 1775 int size = WritableUtils.readVInt(in); 1776 for(int i=0; i < size; ++i) { 1777 set(org.apache.hadoop.io.Text.readString(in), 1778 org.apache.hadoop.io.Text.readString(in)); 1779 } 1780 } 1781 1782 //@Override 1783 public void write(DataOutput out) throws IOException { 1784 Properties props = getProps(); 1785 WritableUtils.writeVInt(out, props.size()); 1786 for(Map.Entry<Object, Object> item: props.entrySet()) { 1787 org.apache.hadoop.io.Text.writeString(out, (String) item.getKey()); 1788 org.apache.hadoop.io.Text.writeString(out, (String) item.getValue()); 1789 } 1790 } 1791 1792 /** 1793 * get keys matching the the regex 1794 * @param regex 1795 * @return Map<String,String> with matching keys 1796 */ 1797 public Map<String,String> getValByRegex(String regex) { 1798 Pattern p = Pattern.compile(regex); 1799 1800 Map<String,String> result = new HashMap<String,String>(); 1801 Matcher m; 1802 1803 for(Map.Entry<Object,Object> item: getProps().entrySet()) { 1804 if (item.getKey() instanceof String && 1805 item.getValue() instanceof String) { 1806 m = p.matcher((String)item.getKey()); 1807 if(m.find()) { // match 1808 result.put((String) item.getKey(), (String) item.getValue()); 1809 } 1810 } 1811 } 1812 return result; 1813 } 1814 1815 //Load deprecated keys in common 1816 private static void addDeprecatedKeys() { 1817 Configuration.addDeprecation("topology.script.file.name", 1818 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY}); 1819 Configuration.addDeprecation("topology.script.number.args", 1820 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY}); 1821 Configuration.addDeprecation("hadoop.configured.node.mapping", 1822 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY}); 1823 Configuration.addDeprecation("topology.node.switch.mapping.impl", 1824 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY}); 1825 Configuration.addDeprecation("dfs.df.interval", 1826 new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY}); 1827 Configuration.addDeprecation("dfs.client.buffer.dir", 1828 new String[]{CommonConfigurationKeys.FS_CLIENT_BUFFER_DIR_KEY}); 1829 Configuration.addDeprecation("hadoop.native.lib", 1830 new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY}); 1831 Configuration.addDeprecation("fs.default.name", 1832 new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY}); 1833 } 1834 }