From 139bea6660193cc73852832e03fe570437343e96 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Fri, 12 Mar 2010 15:02:55 -0800
Subject: [PATCH] HDFS-528. Add ability for safemode to wait for a minimum number of live datanodes

Description: When starting up a fresh cluster programatically, users often want to wait until DFS is "writable" before continuing in a script. "dfsadmin -safemode wait" doesn't quite work for this on a completely fresh cluster, since when there are 0 blocks on the system, 100% of them are accounted for before any DNs have reported.

<p>This JIRA is to add a command which waits until a certain number of DNs have reported as alive to the NN.</p>
Reason: New feature
Author: Todd Lipcon
Ref: UNKNOWN
---
 src/hdfs/hdfs-default.xml                          |   17 ++-
 .../hadoop/hdfs/server/namenode/FSNamesystem.java  |   53 ++++++--
 src/test/org/apache/hadoop/hdfs/TestSafeMode.java  |  111 --------------
 .../hadoop/hdfs/server/namenode/TestSafeMode.java  |  157 ++++++++++++++++++++
 4 files changed, 216 insertions(+), 122 deletions(-)
 delete mode 100644 src/test/org/apache/hadoop/hdfs/TestSafeMode.java
 create mode 100644 src/test/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java

diff --git a/src/hdfs/hdfs-default.xml b/src/hdfs/hdfs-default.xml
index 5073306..f32a07d 100644
--- a/src/hdfs/hdfs-default.xml
+++ b/src/hdfs/hdfs-default.xml
@@ -272,12 +272,27 @@ creations/deletions), or "all".</description>
   <description>
     Specifies the percentage of blocks that should satisfy 
     the minimal replication requirement defined by dfs.replication.min.
-    Values less than or equal to 0 mean not to start in safe mode.
+    Values less than or equal to 0 mean not to wait for any particular
+    percentage of blocks before exiting safemode.
     Values greater than 1 will make safe mode permanent.
   </description>
 </property>
 
 <property>
+  <name>dfs.safemode.min.datanodes</name>
+  <value>0</value>
+  <description>
+    Specifies the number of datanodes that must be considered alive
+    before the name node exits safemode.
+    Values less than or equal to 0 mean not to take the number of live
+    datanodes into account when deciding whether to remain in safe mode
+    during startup.
+    Values greater than the number of datanodes in the cluster
+    will make safe mode permanent.
+  </description>
+</property>
+
+<property>
   <name>dfs.safemode.extension</name>
   <value>30000</value>
   <description>
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1534932..9cf8aae 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2129,6 +2129,10 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
     resolveNetworkLocation(nodeDescr);
     unprotectedAddDatanode(nodeDescr);
     clusterMap.add(nodeDescr);
+
+    if (safeMode != null) {
+      safeMode.checkMode();
+    }
       
     // also treat the registration message as a heartbeat
     synchronized(heartbeats) {
@@ -2779,6 +2783,10 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
     }
     unprotectedRemoveDatanode(nodeInfo);
     clusterMap.remove(nodeInfo);
+
+    if (safeMode != null) {
+      safeMode.checkMode();
+    }
   }
 
   void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) {
@@ -3986,6 +3994,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
     // configuration fields
     /** Safe mode threshold condition %.*/
     private double threshold;
+    /** Safe mode minimum number of datanodes alive */
+    private int datanodeThreshold;
     /** Safe mode extension after the threshold. */
     private int extension;
     /** Min replication required by safe mode. */
@@ -4015,6 +4025,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
      */
     SafeModeInfo(Configuration conf) {
       this.threshold = conf.getFloat("dfs.safemode.threshold.pct", 0.95f);
+      this.datanodeThreshold = conf.getInt("dfs.safemode.min.datanodes", 0);
       this.extension = conf.getInt("dfs.safemode.extension", 0);
       this.safeReplication = conf.getInt("dfs.replication.min", 1);
       this.blockTotal = 0; 
@@ -4031,6 +4042,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
      */
     private SafeModeInfo() {
       this.threshold = 1.5f;  // this threshold can never be reached
+      this.datanodeThreshold = Integer.MAX_VALUE;
       this.extension = Integer.MAX_VALUE;
       this.safeReplication = Short.MAX_VALUE + 1; // more than maxReplication
       this.blockTotal = -1;
@@ -4123,13 +4135,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
      * if DFS is empty or {@link #threshold} == 0
      */
     boolean needEnter() {
-      return threshold != 0 && blockSafe < blockThreshold;
+      return (threshold != 0 && blockSafe < blockThreshold) ||
+        (numLiveDataNodes() < datanodeThreshold);
     }
       
     /**
      * Check and trigger safe mode if needed. 
      */
-    private void checkMode() {
+    private synchronized void checkMode() {
       if (needEnter()) {
         enter();
         reportStatus("STATE* Safe mode ON.", false);
@@ -4213,17 +4226,37 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
       }
       if(blockTotal < 0)
         return leaveMsg + ".";
-      
-      String msg = null;
+
+      int numLive = numLiveDataNodes();
+      String msg = "";
       if (reached == 0) {
-        msg = String.format("The reported blocks %d needs additional %d"
-            + " blocks to reach the threshold %.4f of total blocks %d. %s",
-            blockSafe, (blockThreshold - blockSafe), threshold, blockTotal,
-            leaveMsg);
+        if (blockSafe < blockThreshold) {
+          msg += String.format(
+            "The reported blocks %d needs additional %d"
+            + " blocks to reach the threshold %.4f of total blocks %d.",
+            blockSafe, (blockThreshold - blockSafe), threshold, blockTotal);
+        }
+        if (numLive < datanodeThreshold) {
+          if (!"".equals(msg)) {
+            msg += "\n";
+          }
+          msg += String.format(
+            "The number of live datanodes %d needs an additional %d live "
+            + "datanodes to reach the minimum number %d.",
+            numLive, datanodeThreshold - numLive, datanodeThreshold);
+        }
+        msg += " " + leaveMsg;
       } else {
         msg = String.format("The reported blocks %d has reached the threshold"
-            + " %.4f of total blocks %d. %s", blockSafe, threshold, 
-            blockTotal, leaveMsg);
+            + " %.4f of total blocks %d.", blockSafe, threshold, 
+            blockTotal);
+
+        if (datanodeThreshold > 0) {
+          msg += String.format(" The number of live datanodes %d has reached "
+                               + "the minimum number %d.",
+                               numLive, datanodeThreshold);
+        }
+        msg += " " + leaveMsg;
       }
       if(reached == 0 || isManual()) {  // threshold is not reached or manual       
         return msg + ".";
diff --git a/src/test/org/apache/hadoop/hdfs/TestSafeMode.java b/src/test/org/apache/hadoop/hdfs/TestSafeMode.java
deleted file mode 100644
index 14f0a59..0000000
--- a/src/test/org/apache/hadoop/hdfs/TestSafeMode.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
-
-import junit.framework.TestCase;
-
-/**
- * Tests to verify safe mode correctness.
- */
-public class TestSafeMode extends TestCase {
-  
-  static Log LOG = LogFactory.getLog(TestSafeMode.class);
-
-  /**
-   * This test verifies that if SafeMode is manually entered, name-node does not
-   * come out of safe mode even after the startup safe mode conditions are met.
-   * <ol>
-   * <li>Start cluster with 1 data-node.</li>
-   * <li>Create 2 files with replication 1.</li>
-   * <li>Re-start cluster with 0 data-nodes. 
-   * Name-node should stay in automatic safe-mode.</li>
-   * <li>Enter safe mode manually.</li>
-   * <li>Start the data-node.</li>
-   * <li>Wait longer than <tt>dfs.safemode.extension</tt> and 
-   * verify that the name-node is still in safe mode.</li>
-   * </ol>
-   *  
-   * @throws IOException
-   */
-  public void testManualSafeMode() throws IOException {
-    MiniDFSCluster cluster = null;
-    DistributedFileSystem fs = null;
-    try {
-      Configuration conf = new Configuration();
-      // disable safemode extension to make the test run faster.
-      conf.set("dfs.safemode.extension", "1");
-      cluster = new MiniDFSCluster(conf, 1, true, null);
-      cluster.waitActive();
-      
-      fs = (DistributedFileSystem)cluster.getFileSystem();
-      Path file1 = new Path("/tmp/testManualSafeMode/file1");
-      Path file2 = new Path("/tmp/testManualSafeMode/file2");
-      
-      LOG.info("Created file1 and file2.");
-      
-      // create two files with one block each.
-      DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0);
-      DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0);
-      fs.close();
-      cluster.shutdown();
-      
-      // now bring up just the NameNode.
-      cluster = new MiniDFSCluster(conf, 0, false, null);
-      cluster.waitActive();
-      fs = (DistributedFileSystem)cluster.getFileSystem();
-      
-      LOG.info("Restarted cluster with just the NameNode");
-      
-      assertTrue("No datanode is started. Should be in SafeMode", 
-                 fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
-      
-      // manually set safemode.
-      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
-      
-      // now bring up the datanode and wait for it to be active.
-      cluster.startDataNodes(conf, 1, true, null, null);
-      cluster.waitActive();
-      
-      LOG.info("Datanode is started.");
-
-      // wait longer than dfs.safemode.extension
-      try {
-        Thread.sleep(2000);
-      } catch (InterruptedException ignored) {}
-      
-      assertTrue("should still be in SafeMode",
-          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
-      
-      fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
-      assertFalse("should not be in SafeMode",
-          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
-    } finally {
-      if(fs != null) fs.close();
-      if(cluster!= null) cluster.shutdown();
-    }
-  }
-}
diff --git a/src/test/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java b/src/test/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java
new file mode 100644
index 0000000..97a48ed
--- /dev/null
+++ b/src/test/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import junit.framework.TestCase;
+
+/**
+ * Tests to verify safe mode correctness.
+ */
+public class TestSafeMode extends TestCase {
+  
+  static Log LOG = LogFactory.getLog(TestSafeMode.class);
+
+  /**
+   * This test verifies that if SafeMode is manually entered, name-node does not
+   * come out of safe mode even after the startup safe mode conditions are met.
+   * <ol>
+   * <li>Start cluster with 1 data-node.</li>
+   * <li>Create 2 files with replication 1.</li>
+   * <li>Re-start cluster with 0 data-nodes. 
+   * Name-node should stay in automatic safe-mode.</li>
+   * <li>Enter safe mode manually.</li>
+   * <li>Start the data-node.</li>
+   * <li>Wait longer than <tt>dfs.safemode.extension</tt> and 
+   * verify that the name-node is still in safe mode.</li>
+   * </ol>
+   *  
+   * @throws IOException
+   */
+  public void testManualSafeMode() throws IOException {
+    MiniDFSCluster cluster = null;
+    DistributedFileSystem fs = null;
+    try {
+      Configuration conf = new Configuration();
+      // disable safemode extension to make the test run faster.
+      conf.set("dfs.safemode.extension", "1");
+      cluster = new MiniDFSCluster(conf, 1, true, null);
+      cluster.waitActive();
+      
+      fs = (DistributedFileSystem)cluster.getFileSystem();
+      Path file1 = new Path("/tmp/testManualSafeMode/file1");
+      Path file2 = new Path("/tmp/testManualSafeMode/file2");
+      
+      LOG.info("Created file1 and file2.");
+      
+      // create two files with one block each.
+      DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0);
+      DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0);
+      fs.close();
+      cluster.shutdown();
+      
+      // now bring up just the NameNode.
+      cluster = new MiniDFSCluster(conf, 0, false, null);
+      cluster.waitActive();
+      fs = (DistributedFileSystem)cluster.getFileSystem();
+      
+      LOG.info("Restarted cluster with just the NameNode");
+      
+      assertTrue("No datanode is started. Should be in SafeMode", 
+                 fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
+      
+      // manually set safemode.
+      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+      
+      // now bring up the datanode and wait for it to be active.
+      cluster.startDataNodes(conf, 1, true, null, null);
+      cluster.waitActive();
+      
+      LOG.info("Datanode is started.");
+
+      // wait longer than dfs.safemode.extension
+      try {
+        Thread.sleep(2000);
+      } catch (InterruptedException ignored) {}
+      
+      assertTrue("should still be in SafeMode",
+          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
+      
+      fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+      assertFalse("should not be in SafeMode",
+          fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
+    } finally {
+      if(fs != null) fs.close();
+      if(cluster!= null) cluster.shutdown();
+    }
+  }
+
+
+  /**
+   * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min
+   * is set to a number greater than the number of live datanodes.
+   */
+  public void testDatanodeThreshold() throws IOException {
+    MiniDFSCluster cluster = null;
+    DistributedFileSystem fs = null;
+    try {
+      Configuration conf = new Configuration();
+      conf.set("dfs.safemode.extension", "0");
+      conf.set("dfs.safemode.min.datanodes", "1");
+
+      // bring up a cluster with no datanodes
+      cluster = new MiniDFSCluster(conf, 0, false, null);
+      cluster.waitActive();
+      fs = (DistributedFileSystem)cluster.getFileSystem();
+
+      assertTrue("No datanode started, but we require one - safemode expected",
+                 fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
+
+      String tipMsg = cluster.getNameNode().getNamesystem().getSafeModeTip();
+      assertTrue("Safemode tip message looks right",
+                 tipMsg.contains("The number of live datanodes 0 needs an " +
+                                 "additional 1 live"));
+
+      // Start a datanode
+      cluster.startDataNodes(conf, 1, true, null, null);
+
+      // Wait long enough for safemode check to refire
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException ignored) {}
+
+      // We now should be out of safe mode.
+      assertFalse(
+        "Out of safe mode after starting datanode.",
+        fs.setSafeMode(SafeModeAction.SAFEMODE_GET));
+    } finally {
+      if (fs != null) fs.close();
+      if (cluster != null) cluster.shutdown();
+    }
+  }
+}
-- 
1.6.0.4

