From 19654b90fa67491e5b4a6c51f67b5452d62b754f Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@cloudera.com>
Date: Wed, 28 Oct 2009 11:10:41 -0700
Subject: [PATCH] HADOOP-6184. Provide a configuration dump in json format.

---
 ivy.xml                                            |    4 +
 src/core/org/apache/hadoop/conf/Configuration.java |   83 +++++++++-
 .../org/apache/hadoop/conf/TestConfiguration.java  |  180 ++++++++++++++++++++
 3 files changed, 265 insertions(+), 2 deletions(-)

diff --git a/ivy.xml b/ivy.xml
index 3f81fdb..29c32fe 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -256,6 +256,10 @@
       rev="${slf4j-log4j12.version}"
       conf="common->master">
     </dependency>
+    <dependency org="org.codehaus.jackson"
+      name="jackson-mapper-asl"
+      rev="1.0.1"
+      conf="common->default"/>
     </dependencies>
   
 </ivy-module>
diff --git a/src/core/org/apache/hadoop/conf/Configuration.java b/src/core/org/apache/hadoop/conf/Configuration.java
index 8ae90cb..58243aa 100644
--- a/src/core/org/apache/hadoop/conf/Configuration.java
+++ b/src/core/org/apache/hadoop/conf/Configuration.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.io.Writer;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -63,6 +64,8 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -156,7 +159,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private boolean loadDefaults = true;
   
   /**
-   * Configurtion objects
+   * Configuration objects
    */
   private static final WeakHashMap<Configuration,Object> REGISTRY = 
     new WeakHashMap<Configuration,Object>();
@@ -171,6 +174,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private static final ConcurrentMap<ClassLoader, Map<String, Class<?>>>
     CACHE_CLASSES = new ConcurrentHashMap<ClassLoader, Map<String, Class<?>>>();
   
+  /**
+   * Flag to indicate if the storage of resource which updates a key needs 
+   * to be stored for each key
+   */
+  private boolean storeResource;
+  
+  /**
+   * Stores the mapping of key to the resource which modifies or loads 
+   * the key most recently
+   */
+  private HashMap<String, String> updatingResource;
+  
   static{
     //print deprecation warning if hadoop-site.xml is found in classpath
     ClassLoader cL = Thread.currentThread().getContextClassLoader();
@@ -218,6 +233,24 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     synchronized(Configuration.class) {
       REGISTRY.put(this, null);
     }
+    this.storeResource = false;
+  }
+  
+  /**
+   * A new configuration with the same settings and additional facility for
+   * storage of resource to each key which loads or updates 
+   * the key most recently
+   * @param other the configuration from which to clone settings
+   * @param storeResource flag to indicate if the storage of resource to 
+   * each key is to be stored
+   */
+  private Configuration(Configuration other, boolean storeResource) {
+    this(other);
+    this.loadDefaults = other.loadDefaults;
+    this.storeResource = storeResource;
+    if (storeResource) {
+      updatingResource = new HashMap<String, String>();
+    }
   }
   
   /** 
@@ -1037,8 +1070,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     if (properties == null) {
       properties = new Properties();
       loadResources(properties, resources, quietmode);
-      if (overlay!= null)
+      if (overlay!= null) {
         properties.putAll(overlay);
+        if (storeResource) {
+          for (Map.Entry<Object,Object> item: overlay.entrySet()) {
+            updatingResource.put((String) item.getKey(), "Unknown");
+          }
+        }
+      }
     }
     return properties;
   }
@@ -1212,6 +1251,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         if (attr != null && value != null) {
           if (!finalParameters.contains(attr)) {
             properties.setProperty(attr, value);
+            if (storeResource) {
+              updatingResource.put(attr, name.toString());
+            }
             if (finalParameter)
               finalParameters.add(attr);
           } else {
@@ -1284,6 +1326,43 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   }
 
   /**
+   *  Writes out all the parameters and their properties (final and resource) to
+   *  the given {@link Writer}
+   *  The format of the output would be 
+   *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+   *  key2.isFinal,key2.resource}... ] } 
+   *  It does not output the parameters of the configuration object which is 
+   *  loaded from an input stream.
+   * @param out the Writer to write to
+   * @throws IOException
+   */
+  public static void dumpConfiguration(Configuration conf, 
+      Writer out) throws IOException {
+    Configuration config = new Configuration(conf,true);
+    config.reloadConfiguration();
+    JsonFactory dumpFactory = new JsonFactory();
+    JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+    dumpGenerator.writeStartObject();
+    dumpGenerator.writeFieldName("properties");
+    dumpGenerator.writeStartArray();
+    dumpGenerator.flush();
+    for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+      dumpGenerator.writeStartObject();
+      dumpGenerator.writeStringField("key", (String) item.getKey());
+      dumpGenerator.writeStringField("value", 
+          config.get((String) item.getKey()));
+      dumpGenerator.writeBooleanField("isFinal",
+          config.finalParameters.contains(item.getKey()));
+      dumpGenerator.writeStringField("resource",
+          config.updatingResource.get(item.getKey()));
+      dumpGenerator.writeEndObject();
+    }
+    dumpGenerator.writeEndArray();
+    dumpGenerator.writeEndObject();
+    dumpGenerator.flush();
+  }
+  
+  /**
    * Get the {@link ClassLoader} for this job.
    * 
    * @return the correct class loader.
diff --git a/src/test/org/apache/hadoop/conf/TestConfiguration.java b/src/test/org/apache/hadoop/conf/TestConfiguration.java
index e509fd3..0d01dac 100644
--- a/src/test/org/apache/hadoop/conf/TestConfiguration.java
+++ b/src/test/org/apache/hadoop/conf/TestConfiguration.java
@@ -25,12 +25,15 @@ import java.io.DataInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.DataOutputStream;
+import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Random;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.fs.Path;
+import org.codehaus.jackson.map.ObjectMapper;
 
 
 public class TestConfiguration extends TestCase {
@@ -389,4 +392,181 @@ public class TestConfiguration extends TestCase {
       TestConfiguration.class.getName()
     });
   }
+
+  static class JsonConfiguration {
+    JsonProperty[] properties;
+
+    public JsonProperty[] getProperties() {
+      return properties;
+    }
+
+    public void setProperties(JsonProperty[] properties) {
+      this.properties = properties;
+    }
+  }
+  
+  static class JsonProperty {
+    String key;
+    public String getKey() {
+      return key;
+    }
+    public void setKey(String key) {
+      this.key = key;
+    }
+    public String getValue() {
+      return value;
+    }
+    public void setValue(String value) {
+      this.value = value;
+    }
+    public boolean getIsFinal() {
+      return isFinal;
+    }
+    public void setIsFinal(boolean isFinal) {
+      this.isFinal = isFinal;
+    }
+    public String getResource() {
+      return resource;
+    }
+    public void setResource(String resource) {
+      this.resource = resource;
+    }
+    String value;
+    boolean isFinal;
+    String resource;
+  }
+  
+  public void testDumpConfiguration () throws IOException {
+    StringWriter outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    String jsonStr = outWriter.toString();
+    ObjectMapper mapper = new ObjectMapper();
+    JsonConfiguration jconf = 
+      mapper.readValue(jsonStr, JsonConfiguration.class);
+    int defaultLength = jconf.getProperties().length;
+    
+    // add 3 keys to the existing configuration properties
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendProperty("test.key1", "value1");
+    appendProperty("test.key2", "value2",true);
+    appendProperty("test.key3", "value3");
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    conf.addResource(fileResource);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    int length = jconf.getProperties().length;
+    // check for consistency in the number of properties parsed in Json format.
+    assertEquals(length, defaultLength+3);
+    
+    //change few keys in another resource file
+    out=new BufferedWriter(new FileWriter(CONFIG2));
+    startConfig();
+    appendProperty("test.key1", "newValue1");
+    appendProperty("test.key2", "newValue2");
+    endConfig();
+    Path fileResource1 = new Path(CONFIG2);
+    conf.addResource(fileResource1);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    // put the keys and their corresponding attributes into a hashmap for their 
+    // efficient retrieval
+    HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
+    for(JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    // check if the value and resource of test.key1 is changed
+    assertEquals("newValue1", confDump.get("test.key1").getValue());
+    assertEquals(false, confDump.get("test.key1").getIsFinal());
+    assertEquals(fileResource1.toString(),
+        confDump.get("test.key1").getResource());
+    // check if final parameter test.key2 is not changed, since it is first 
+    // loaded as final parameter
+    assertEquals("value2", confDump.get("test.key2").getValue());
+    assertEquals(true, confDump.get("test.key2").getIsFinal());
+    assertEquals(fileResource.toString(),
+        confDump.get("test.key2").getResource());
+    // check for other keys which are not modified later
+    assertEquals("value3", confDump.get("test.key3").getValue());
+    assertEquals(false, confDump.get("test.key3").getIsFinal());
+    assertEquals(fileResource.toString(),
+        confDump.get("test.key3").getResource());
+    // check for resource to be "Unknown" for keys which are loaded using 'set' 
+    // and expansion of properties
+    conf.set("test.key4", "value4");
+    conf.set("test.key5", "value5");
+    conf.set("test.key6", "${test.key5}");
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(conf, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    confDump = new HashMap<String, JsonProperty>();
+    for(JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    assertEquals("value5",confDump.get("test.key6").getValue());
+    assertEquals("Unknown", confDump.get("test.key4").getResource());
+    outWriter.close();
+  }
+  
+  public void testDumpConfiguratioWithoutDefaults() throws IOException {
+    // check for case when default resources are not loaded
+    Configuration config = new Configuration(false);
+    StringWriter outWriter = new StringWriter();
+    Configuration.dumpConfiguration(config, outWriter);
+    String jsonStr = outWriter.toString();
+    ObjectMapper mapper = new ObjectMapper();
+    JsonConfiguration jconf = 
+      mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    //ensure that no properties are loaded.
+    assertEquals(0, jconf.getProperties().length);
+    
+    // add 2 keys
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendProperty("test.key1", "value1");
+    appendProperty("test.key2", "value2",true);
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    config.addResource(fileResource);
+    out.close();
+    
+    outWriter = new StringWriter();
+    Configuration.dumpConfiguration(config, outWriter);
+    jsonStr = outWriter.toString();
+    mapper = new ObjectMapper();
+    jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+    
+    HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
+    for (JsonProperty prop : jconf.getProperties()) {
+      confDump.put(prop.getKey(), prop);
+    }
+    //ensure only 2 keys are loaded
+    assertEquals(2,jconf.getProperties().length);
+    //ensure the values are consistent
+    assertEquals(confDump.get("test.key1").getValue(),"value1");
+    assertEquals(confDump.get("test.key2").getValue(),"value2");
+    //check the final tag
+    assertEquals(false, confDump.get("test.key1").getIsFinal());
+    assertEquals(true, confDump.get("test.key2").getIsFinal());
+    //check the resource for each property
+    for (JsonProperty prop : jconf.getProperties()) {
+      assertEquals(fileResource.toString(),prop.getResource());
+    }
+  }
 }
+
-- 
1.6.0.4

