From d3db49b4e5a4bef64615f9229a2784472529bf28 Mon Sep 17 00:00:00 2001
From: Todd Lipcon <todd@cloudera.com>
Date: Fri, 16 Apr 2010 14:54:59 -0700
Subject: [PATCH] Ammend MAPREDUCE-551. Fix FairScheduler preemption when slow start is configured to non-default value

Description: The Fair Scheduler assumed that slow start was always configured to the default,
rather than looking at the actual configured value. This patch changes it to ask the
JobInProgress whether reduces are schedulable rather than running its own calculations.

This fix does not directly correspond to an Apache JIRA, since it affects only the
version of preemption that is in CDH2. The version in CDH3 and in any Apache releases
after 0.20 does not have this bug (was fixed by MAPREDUCE-706)

Reason: Fixes incorrect preemption for jobs with slow start enabled.
Author: Todd Lipcon
Ref: CDH-948
---
 .../org/apache/hadoop/mapred/FairScheduler.java    |   15 +----------
 .../apache/hadoop/mapred/TestFairScheduler.java    |   26 --------------------
 2 files changed, 1 insertions(+), 40 deletions(-)

diff --git a/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java b/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
index 4c188f4..79626be 100644
--- a/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
+++ b/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
@@ -70,7 +70,6 @@ public class FairScheduler extends TaskScheduler {
   protected boolean useFifo;      // Set if we want to revert to FIFO behavior
   protected boolean assignMultiple; // Simultaneously assign map and reduce?
   protected boolean sizeBasedWeight; // Give larger weights to larger jobs
-  protected boolean waitForMapsBeforeLaunchingReduces = true;
   protected boolean preemptionEnabled;
   protected boolean onlyLogPreemption; // Only log when tasks should be killed
 
@@ -537,7 +536,7 @@ public class FairScheduler extends TaskScheduler {
         }
       }
       info.runningReduces = runningReduces;
-      if (enoughMapsFinishedToRunReduces(finishedMaps, totalMaps)) {
+      if (job.scheduleReduces()) {
         info.neededReduces = (totalReduces - runningReduces - finishedReduces 
             + taskSelector.neededSpeculativeReduces(job));
       } else {
@@ -553,18 +552,6 @@ public class FairScheduler extends TaskScheduler {
     }
   }
 
-  /**
-   * Has a job finished enough maps to allow launching its reduces?
-   */
-  protected boolean enoughMapsFinishedToRunReduces(
-      int finishedMaps, int totalMaps) {
-    if (waitForMapsBeforeLaunchingReduces) {
-      return finishedMaps >= Math.max(1, totalMaps * 0.05);
-    } else {
-      return true;
-    }
-  }
-
   private void updateWeights() {
     // First, calculate raw weights for each job
     for (Map.Entry<JobInProgress, JobInfo> entry: infos.entrySet()) {
diff --git a/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java b/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
index 2e88a38..184cb3f 100644
--- a/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
+++ b/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
@@ -441,7 +441,6 @@ public class TestFairScheduler extends TestCase {
     taskTrackerManager = new FakeTaskTrackerManager(numTaskTrackers);
     clock = new FakeClock();
     scheduler = new FairScheduler(clock, true);
-    scheduler.waitForMapsBeforeLaunchingReduces = false;
     scheduler.setConf(conf);
     scheduler.setTaskTrackerManager(taskTrackerManager);
     scheduler.start();
@@ -1297,31 +1296,6 @@ public class TestFairScheduler extends TestCase {
                scheduler.infos.get(job2).reduceFairShare);
   }
   
-  public void testWaitForMapsBeforeLaunchingReduces() {
-    // We have set waitForMapsBeforeLaunchingReduces to false by default in
-    // this class, so this should return true
-    assertTrue(scheduler.enoughMapsFinishedToRunReduces(0, 100));
-    
-    // However, if we set waitForMapsBeforeLaunchingReduces to true, we should
-    // now no longer be able to assign reduces until 5 have finished
-    scheduler.waitForMapsBeforeLaunchingReduces = true;
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(0, 100));
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(1, 100));
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(2, 100));
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(3, 100));
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(4, 100));
-    assertTrue(scheduler.enoughMapsFinishedToRunReduces(5, 100));
-    assertTrue(scheduler.enoughMapsFinishedToRunReduces(6, 100));
-    
-    // Also test some jobs that have very few maps, in which case we will
-    // wait for at least 1 map to finish
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(0, 5));
-    assertTrue(scheduler.enoughMapsFinishedToRunReduces(1, 5));
-    assertFalse(scheduler.enoughMapsFinishedToRunReduces(0, 1));
-    assertTrue(scheduler.enoughMapsFinishedToRunReduces(1, 1));
-  }
-  
-
   /**
    * This test submits jobs in three pools: poolA, which has a weight
    * of 2.0; poolB, which has a weight of 0.5; and the default pool, which
-- 
1.6.0.4

