From f9d68f074e5fad2bfe66912564f656b319625d51 Mon Sep 17 00:00:00 2001
From: Eli Collins <eli@cloudera.com>
Date: Tue, 18 May 2010 14:30:04 -0700
Subject: [PATCH] HDFS-1161. Make DN minimum valid volumes configurable

Description: This change adds a dfs.datanode.failed.volumes.tolerated parameter so that users can configure the number of volumes that are allowed to fail before a datanode stops offering service. By default any volume failure will cause a datanode to shutdown.

Reason: Improvement
Author: Eli Collins
Ref: CDH-1081
---
 src/hdfs/hdfs-default.xml                          |    9 +++++++
 .../apache/hadoop/hdfs/protocol/FSConstants.java   |    1 -
 .../hadoop/hdfs/server/datanode/DataNode.java      |   26 +++++++------------
 .../hadoop/hdfs/server/datanode/FSDataset.java     |   21 ++++++++++++----
 .../hdfs/server/datanode/FSDatasetInterface.java   |    2 +-
 .../hdfs/server/datanode/SimulatedFSDataset.java   |    2 +-
 6 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/src/hdfs/hdfs-default.xml b/src/hdfs/hdfs-default.xml
index f32a07d..1219e9e 100644
--- a/src/hdfs/hdfs-default.xml
+++ b/src/hdfs/hdfs-default.xml
@@ -391,4 +391,13 @@ creations/deletions), or "all".</description>
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.failed.volumes.tolerated</name>
+  <value>0</value>
+  <description>The number of volumes that are allowed to
+  fail before a datanode stops offering service. By default
+  any volume failure will cause a datanode to shutdown.
+  </description>
+</property>
+
 </configuration>
diff --git a/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java b/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
index bf59c7b..044dc44 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
@@ -56,7 +56,6 @@ public interface FSConstants {
   public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
 
   public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
-  public static final int MIN_NUM_OF_VALID_VOLUMES = 1;// for a DN to run
 
   // SafeMode actions
   public enum SafeModeAction{ SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; }
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 4ee710e..1f50a9e 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -679,25 +679,19 @@ public class DataNode extends Configured
   }
   
   private void handleDiskError(String errMsgr) {
-    boolean hasEnoughResource = data.hasEnoughResource();
-    LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResource);
-    
-    //if hasEnoughtResource = true - more volumes are available, so we don't want 
-    // to shutdown DN completely and don't want NN to remove it.
-    int dp_error = DatanodeProtocol.DISK_ERROR;
-    if(hasEnoughResource == false) {
-      // DN will be shutdown and NN should remove it
-      dp_error = DatanodeProtocol.FATAL_DISK_ERROR;
-    }
-    //inform NameNode
+    final boolean hasEnoughResources = data.hasEnoughResources();
+    LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResources);
+          
+    // If we have enough active valid volumes then we do not want to 
+    // shutdown the DN completely.
+    int dpError = hasEnoughResources ? DatanodeProtocol.DISK_ERROR  
+                                     : DatanodeProtocol.FATAL_DISK_ERROR; 
     try {
-      namenode.errorReport(
-                           dnRegistration, dp_error, errMsgr);
-    } catch(IOException ignored) {              
+      namenode.errorReport(dnRegistration, dpError, errMsgr);
+    } catch(IOException ignored) {
     }
     
-    
-    if(hasEnoughResource) {
+    if (hasEnoughResources) {
       scheduleBlockReport(0);
       return; // do not shutdown
     }
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
index 962f35e..3d911ef 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
@@ -746,12 +746,23 @@ public class FSDataset implements FSConstants, FSDatasetInterface {
   private int maxBlocksPerDir = 0;
   private HashMap<Block,DatanodeBlockInfo> volumeMap = null;
   static  Random random = new Random();
-  
+  private int validVolsRequired;
+
   /**
    * An FSDataset has a directory where it loads its data files.
    */
   public FSDataset(DataStorage storage, Configuration conf) throws IOException {
     this.maxBlocksPerDir = conf.getInt("dfs.datanode.numblocks", 64);
+    // The number of volumes required for operation is the total number 
+    // of volumes minus the number of failed volumes we can tolerate.
+    final int volFailuresTolerated =
+      conf.getInt("dfs.datanode.failed.volumes.tolerated", 0);
+    this.validVolsRequired = storage.getNumStorageDirs() - volFailuresTolerated; 
+    if (validVolsRequired < 1 ||
+        validVolsRequired > storage.getNumStorageDirs()) {
+      DataNode.LOG.error("Invalid value " + volFailuresTolerated + " for " +
+          "dfs.datanode.failed.volumes.tolerated");
+    }  
     FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()];
     for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
       volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(), conf);
@@ -768,12 +779,12 @@ public class FSDataset implements FSConstants, FSDatasetInterface {
   public long getDfsUsed() throws IOException {
     return volumes.getDfsUsed();
   }
+
   /**
-   * Return true - if there are still valid volumes 
-   * on the DataNode
+   * Return true - if there are still valid volumes on the DataNode. 
    */
-  public boolean hasEnoughResource(){
-    return volumes.numberOfVolumes() >= MIN_NUM_OF_VALID_VOLUMES;
+  public boolean hasEnoughResources() {
+    return volumes.numberOfVolumes() >= validVolsRequired; 
   }
 
   /**
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java
index 60be87a..bda66b4 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java
@@ -269,5 +269,5 @@ public interface FSDatasetInterface extends FSDatasetMBean {
    * checks how many valid storage volumes are there in the DataNode
    * @return true if more then minimum valid volumes left in the FSDataSet
    */
-  public boolean hasEnoughResource();
+  public boolean hasEnoughResources();
 }
diff --git a/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 62f3a6d..c5c6440 100644
--- a/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -656,7 +656,7 @@ public class SimulatedFSDataset  implements FSConstants, FSDatasetInterface, Con
     return "Simulated FSDataset-" + storageId;
   }
   
-  public boolean hasEnoughResource() {
+  public boolean hasEnoughResources() {
     return true;
   }
 }
-- 
1.6.0.4

