From 402e34faf0202092b65f944df3ff3e104fdd9d39 Mon Sep 17 00:00:00 2001
From: Philip Zeyliger <philip@cloudera.com>
Date: Tue, 15 Sep 2009 01:19:54 -0700
Subject: [PATCH] MAPREDUCE-987 (nee HDFS-621). Add ability to start MiniClusters from the command line

---
 src/test/org/apache/hadoop/test/AllTestDriver.java |    2 +
 .../hadoop/test/MiniHadoopClusterManager.java      |  310 ++++++++++++++++++++
 2 files changed, 312 insertions(+), 0 deletions(-)
 create mode 100644 src/test/org/apache/hadoop/test/MiniHadoopClusterManager.java

diff --git a/src/test/org/apache/hadoop/test/AllTestDriver.java b/src/test/org/apache/hadoop/test/AllTestDriver.java
index 3c86a50..d78a3f3 100644
--- a/src/test/org/apache/hadoop/test/AllTestDriver.java
+++ b/src/test/org/apache/hadoop/test/AllTestDriver.java
@@ -78,6 +78,8 @@ public class AllTestDriver {
       pgd.addClass("MRReliabilityTest", ReliabilityTest.class,
           "A program that tests the reliability of the MR framework by " +
           "injecting faults/failures");
+      pgd.addClass("minicluster", MiniHadoopClusterManager.class,
+          "Single process HDFS and MR cluster.");
       pgd.driver(argv);
     } catch(Throwable e) {
       e.printStackTrace();
diff --git a/src/test/org/apache/hadoop/test/MiniHadoopClusterManager.java b/src/test/org/apache/hadoop/test/MiniHadoopClusterManager.java
new file mode 100644
index 0000000..e97acd6
--- /dev/null
+++ b/src/test/org/apache/hadoop/test/MiniHadoopClusterManager.java
@@ -0,0 +1,310 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.mortbay.util.ajax.JSON;
+
+/**
+ * This class drives the creation of a mini-cluster on the local machine. By
+ * default, a MiniDFSCluster and MiniMRCluster are spawned on the first
+ * available ports that are found.
+ *
+ * A series of command line flags controls the startup cluster options.
+ *
+ * This class can dump a Hadoop configuration and some basic metadata (in JSON)
+ * into a textfile.
+ *
+ * To shutdown the cluster, kill the process.
+ */
+public class MiniHadoopClusterManager {
+  private static final Log LOG = 
+    LogFactory.getLog(MiniHadoopClusterManager.class);
+  
+  private MiniMRCluster mrc;
+  private MiniDFSCluster dfs;
+  private String writeDetails;
+  private int numTaskTrackers;
+  private int numDataNodes;
+  private int nameNodePort;
+  private int jobTrackerPort;
+  private int taskTrackerPort;
+  private StartupOption dfsOpts;
+  private boolean noDFS;
+  private boolean noMR;
+  private boolean useLoopBackHosts;
+  private String fs;
+  private String writeConfig;
+  private JobConf conf;
+
+  /**
+   * Creates configuration options object.
+   */
+  @SuppressWarnings("static-access")
+  private Options makeOptions() {
+    Options options = new Options();
+    options
+        .addOption("nodfs", false, "Don't start a mini DFS cluster")
+        .addOption("nomr", false, "Don't start a mini MR cluster")
+        .addOption("useloopbackhosts", false, "Set TaskTracker hostnames to 127.0.1.N")
+        .addOption("tasktrackers", true,
+            "How many tasktrackers to start (default 1)")
+        .addOption("datanodes", true, "How many datanodes to start (default 1)")
+        .addOption("format", false, "Format the DFS (default false)")
+        .addOption("cmdport", true,
+            "Which port to listen on for commands (default 0--we choose)")
+        .addOption("nnport", true, "NameNode port (default 0--we choose)")
+        .addOption("namenode", true, "URL of the namenode (default "
+            + "is either the DFS cluster or a temporary dir)")
+        .addOption(
+            "tasktrackerport",
+            true,
+            "Port number to start the tasktracker on (default 0--we choose)")
+        .addOption(
+            "jobtrackerport",
+            true,
+            "JobTracker port (default 0--we choose)")
+        .addOption(OptionBuilder
+            .hasArgs()
+            .withArgName("property=value")
+            .withDescription("Options to pass into configuration object")
+            .create("D"))
+        .addOption(OptionBuilder
+            .hasArg()
+            .withArgName("path")
+            .withDescription("Save configuration to this XML file.")
+            .create("writeConfig"))
+         .addOption(OptionBuilder
+            .hasArg()
+            .withArgName("path")
+            .withDescription("Write basic information to this JSON file.")
+            .create("writeDetails"))
+        .addOption(OptionBuilder.withDescription("Prints option help.")
+            .create("help"));
+    return options;
+  }
+
+  /**
+   * Main entry-point.
+   */
+  public void run(String[] args) throws IOException {
+    if (!parseArguments(args)) {
+      return;
+    }
+    start();
+    sleepForever();
+  }
+
+  private void sleepForever() {
+    while (true) {
+      try {
+        Thread.sleep(1000*60);
+      } catch (InterruptedException _) {
+        // nothing
+      }
+    }
+  }
+
+  /**
+   * Starts DFS and MR clusters, as specified in member-variable
+   * options.  Also writes out configuration and details, if requested.
+   * 
+   * @throws IOException
+   * @throws FileNotFoundException
+   */
+  public void start() throws IOException, FileNotFoundException {
+    if (!noDFS) {
+      dfs = new MiniDFSCluster(nameNodePort, conf, numDataNodes,
+          true, true, dfsOpts, null, null);
+      LOG.info("Started MiniDFSCluster -- namenode on port "
+          + dfs.getNameNodePort());
+    }
+    if (!noMR) {
+      if (fs == null && dfs != null) {
+        fs = dfs.getFileSystem().getUri().toString();
+      } else if (fs == null) {
+        fs = "file:///tmp/minimr-" + System.nanoTime();
+      }
+
+      String hosts[] = null;
+      if (useLoopBackHosts) {
+        hosts = new String[numTaskTrackers];
+        for (int i = 0; i < numTaskTrackers; i++) {
+          hosts[i] = "127.0.1." + (i + 1);
+        }
+      }
+
+      mrc = new MiniMRCluster(
+        jobTrackerPort, taskTrackerPort, numTaskTrackers,
+        fs, 1 /* numDir */, null /* racks */, hosts, null /*ugi*/,
+        conf);
+      LOG.info("Started MiniMRCluster  -- jobtracker on port "
+          + mrc.getJobTrackerPort());
+    }
+
+    if (writeConfig != null) {
+      FileOutputStream fos = new FileOutputStream(new File(writeConfig));
+      conf.writeXml(fos);
+      fos.close();
+    }
+
+    if (writeDetails != null) {
+      Map<String, Object> map = new TreeMap<String, Object>();
+      if (dfs != null) {
+        map.put("namenode_port", dfs.getNameNodePort());
+      }
+      if (mrc != null) {
+        map.put("jobtracker_port", mrc.getJobTrackerPort());
+      }
+      FileWriter fw = new FileWriter(new File(writeDetails));
+      fw.write(new JSON().toJSON(map));
+      fw.close();
+    }
+  }
+  
+  /**
+   * Shuts down in-process clusters.
+   */
+  public void stop() {
+    if (mrc != null) {
+      mrc.shutdown();
+    }
+    if (dfs != null) {
+      dfs.shutdown();
+    }
+  }
+
+  /**
+   * Parses arguments and fills out the member variables.
+   * @param args Command-line arguments.
+   * @return true on successful parse; false to indicate that the 
+   * program should exit.
+   */
+  private boolean parseArguments(String[] args) {
+    Options options = makeOptions();
+    CommandLine cli;
+    try {
+      CommandLineParser parser = new GnuParser();
+      cli = parser.parse(options, args);
+    } catch(ParseException e) {
+      LOG.warn("options parsing failed:  "+e.getMessage());
+      new HelpFormatter().printHelp("...", options);
+      return false;
+    }
+
+    if (cli.hasOption("help")) {
+      new HelpFormatter().printHelp("...", options);
+      return false;
+    }
+    if (cli.getArgs().length > 0) {
+      for (String arg : cli.getArgs()) {
+        System.err.println("Unrecognized option: " + arg);
+        new HelpFormatter().printHelp("...", options);
+        return false;
+      }
+    }
+
+    // MR
+    noMR = cli.hasOption("nomr");
+    numTaskTrackers = intArgument(cli, "tasktrackers", 1);
+    jobTrackerPort = intArgument(cli, "jobtrackerport", 0);
+    taskTrackerPort = intArgument(cli, "tasktrackerport", 0);
+    fs = cli.getOptionValue("namenode");
+
+    useLoopBackHosts = cli.hasOption("useloopbackhosts");
+
+    // HDFS
+    noDFS = cli.hasOption("nodfs");
+    numDataNodes = intArgument(cli, "datanodes", 1);
+    nameNodePort = intArgument(cli, "nnport", 0);
+    dfsOpts = cli.hasOption("format") ?
+        StartupOption.FORMAT : StartupOption.REGULAR;
+
+    // Runner
+    writeDetails = cli.getOptionValue("writeDetails");
+    writeConfig = cli.getOptionValue("writeConfig");
+
+    // General
+    conf = new JobConf();
+    updateConfiguration(conf, cli.getOptionValues("D"));
+    
+    return true;
+  }
+
+  /**
+   * Updates configuration based on what's given on the command line.
+   *
+   * @param conf The configuration object
+   * @param keyvalues An array of interleaved key value pairs.
+   */
+  private void updateConfiguration(JobConf conf, String[] keyvalues) {
+    int num_confs_updated = 0;
+    if (keyvalues != null) {
+      for (String prop : keyvalues) {
+        String[] keyval = prop.split("=", 2);
+        if (keyval.length == 2) {
+          conf.set(keyval[0], keyval[1]);
+          num_confs_updated++;
+        } else {
+          LOG.warn("Ignoring -D option " + prop);
+        }
+      }
+    }
+    LOG.info("Updated " + num_confs_updated +
+        " configuration settings from command line.");
+  }
+
+  /**
+   * Extracts an integer argument with specified default value.
+   */
+  private int intArgument(CommandLine cli, String argName, int default_) {
+    String o = cli.getOptionValue(argName);
+    if (o == null) {
+      return default_;
+    } else {
+      return Integer.parseInt(o);
+    }
+  }
+
+  /**
+   * Starts a MiniHadoopCluster.
+   */
+  public static void main(String[] args) throws IOException {
+    new MiniHadoopClusterManager().run(args);
+  }
+}
-- 
1.6.0.4

