From df32e6f306f03e8bc4a561710b3caecbf4b244e3 Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@cloudera.com>
Date: Fri, 28 May 2010 12:21:56 -0700
Subject: [PATCH] Amend MAPREDUCE-551. Fix potential deadlock in fair scheduler task accounting

Reason: The previous bug fix introduced a synchronization cycle between the
        Fair Scheduler and the JobInProgress.
Test plan: Added fair scheduler system test which enables preemption and
           very fast update interval to catch race conditions and
           deadlocks.
Author: Todd Lipcon
Ref: CDH-1150
---
 src/contrib/build-contrib.xml                      |    3 +
 .../org/apache/hadoop/mapred/FairScheduler.java    |    2 +-
 .../hadoop/mapred/TestFairSchedulerSystem.java     |   84 ++++++++++++++++++++
 3 files changed, 88 insertions(+), 1 deletions(-)
 create mode 100644 src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java

diff --git a/src/contrib/build-contrib.xml b/src/contrib/build-contrib.xml
index da4c79f..b2d83f3 100644
--- a/src/contrib/build-contrib.xml
+++ b/src/contrib/build-contrib.xml
@@ -49,6 +49,7 @@
             value="${os.name}-${os.arch}-${sun.arch.data.model}"/>
   <property name="build.test" location="${build.dir}/test"/>
   <property name="build.examples" location="${build.dir}/examples"/>
+  <property name="hadoop.build.examples" location="${hadoop.root}/build/examples"/>
   <property name="hadoop.log.dir" location="${build.dir}/test/logs"/>
   <!-- all jars together -->
   <property name="javac.deprecation" value="off"/>
@@ -103,6 +104,7 @@
     <pathelement location="${hadoop.root}/src/contrib/test"/>
     <pathelement location="${conf.dir}"/>
     <pathelement location="${hadoop.root}/build"/>
+    <pathelement location="${hadoop.build.examples}"/>
     <pathelement location="${build.examples}"/>
     <path refid="${ant.project.name}.test-classpath"/>
     <path refid="contrib-classpath"/>
@@ -121,6 +123,7 @@
     <mkdir dir="${build.dir}"/>
     <mkdir dir="${build.classes}"/>
     <mkdir dir="${build.test}"/>
+    <mkdir dir="${hadoop.build.examples}"/>
     <mkdir dir="${build.examples}"/>
     <mkdir dir="${hadoop.log.dir}"/>
     <antcall target="init-contrib"/>
diff --git a/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java b/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
index 79626be..03ac5e8 100644
--- a/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
+++ b/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
@@ -536,7 +536,7 @@ public class FairScheduler extends TaskScheduler {
         }
       }
       info.runningReduces = runningReduces;
-      if (job.scheduleReduces()) {
+      if (finishedMaps >= job.completedMapsForReduceSlowstart) {
         info.neededReduces = (totalReduces - runningReduces - finishedReduces 
             + taskSelector.neededSpeculativeReduces(job));
       } else {
diff --git a/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java b/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
new file mode 100644
index 0000000..8a01c9a
--- /dev/null
+++ b/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.examples.SleepJob;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.conf.Configuration;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * System tests for the fair scheduler. These run slower than the
+ * mock-based tests in TestFairScheduler but have a better chance
+ * of catching synchronization bugs with the real JT.
+ */
+public class TestFairSchedulerSystem {
+  static final int NUM_THREADS=50;
+
+  private void runSleepJob(JobConf conf) throws Exception {
+    String[] args = { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
+    ToolRunner.run(conf, new SleepJob(), args);
+  }
+
+  /**
+   * Bump up the frequency of preemption updates to test against
+   * deadlocks, etc.
+   */
+  @Test
+  public void testPreemptionUpdates() throws Exception {
+    MiniMRCluster mr = null;
+    try {
+      final int taskTrackers = 1;
+
+      Configuration conf = new Configuration();
+      conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
+      conf.set("mapred.fairscheduler.update.interval", "0");
+      conf.set("mapred.fairscheduler.preemption.interval", "0");
+      conf.set("mapred.fairscheduler.preemption", "true");
+      mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, new JobConf(conf));
+      final MiniMRCluster finalMR = mr;
+      ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
+      List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
+      for (int i = 0; i < NUM_THREADS; i++) {
+        futures.add(exec.submit(new Callable<Void>() {
+          public Void call() throws Exception {
+            JobConf jobConf = finalMR.createJobConf();
+            runSleepJob(jobConf);
+            return null;
+          }
+        }));
+      }
+
+      for (Future<Void> future : futures) {
+        future.get();
+      }
+    } finally {
+      if (mr != null) { mr.shutdown();
+      }
+    }
+  
+  }
+}
-- 
1.6.0.4

