From 338fea8b2ae9d7c6ef2d0bbea7965ccd450e88f7 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Thu, 21 Jan 2010 11:45:04 -0800
Subject: [PATCH] MAPREDUCE-433. TestReduceFetch failed.

---
 .../org/apache/hadoop/mapred/ReduceTask.java       |    6 +-
 .../org/apache/hadoop/mapred/TestReduceFetch.java  |   49 ++++++++++++--------
 2 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/src/mapred/org/apache/hadoop/mapred/ReduceTask.java b/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
index 06ca77a..0711783 100644
--- a/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
+++ b/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
@@ -1006,9 +1006,9 @@ class ReduceTask extends Task {
           throw new IOException("mapred.job.shuffle.input.buffer.percent" +
                                 maxInMemCopyUse);
         }
-        maxSize = Math.min(
-            (long) (Runtime.getRuntime().maxMemory() * maxInMemCopyUse),
-            Long.MAX_VALUE);
+        maxSize = (long)(conf.getLong("mapred.job.reduce.total.mem.bytes",
+            (long)Math.min(Runtime.getRuntime().maxMemory(), Long.MAX_VALUE))
+          * maxInMemCopyUse);
         maxSingleShuffleLimit = (long)(maxSize * MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
         LOG.info("ShuffleRamManager: MemoryLimit=" + maxSize + 
                  ", MaxSingleShuffleLimit=" + maxSingleShuffleLimit);
diff --git a/src/test/org/apache/hadoop/mapred/TestReduceFetch.java b/src/test/org/apache/hadoop/mapred/TestReduceFetch.java
index 2864587..ae49a82 100644
--- a/src/test/org/apache/hadoop/mapred/TestReduceFetch.java
+++ b/src/test/org/apache/hadoop/mapred/TestReduceFetch.java
@@ -101,48 +101,57 @@ public class TestReduceFetch extends TestCase {
   }
 
   public void testReduceFromDisk() throws Exception {
+    final int MAP_TASKS = 8;
     JobConf job = mrCluster.createJobConf();
     job.set("mapred.job.reduce.input.buffer.percent", "0.0");
-    job.setNumMapTasks(3);
+    job.setNumMapTasks(MAP_TASKS);
+    job.set("mapred.child.java.opts", "-Xmx128m");
+    job.setInt("mapred.job.reduce.total.mem.bytes", 128 << 20);
+    job.set("mapred.job.shuffle.input.buffer.percent", "0.14");
+    job.setInt("io.sort.factor", 2);
+    job.setInt("mapred.inmem.merge.threshold", 4);
     Counters c = runJob(job);
-    final long hdfsWritten = c.findCounter(Task.FILESYSTEM_COUNTER_GROUP, 
-        Task.getFileSystemCounterNames("hdfs")[1]).getCounter();
-    final long localRead = c.findCounter(Task.FILESYSTEM_COUNTER_GROUP, 
-        Task.getFileSystemCounterNames("file")[0]).getCounter();
-    assertTrue("Expected more bytes read from local (" +
-        localRead + ") than written to HDFS (" + hdfsWritten + ")",
-        hdfsWritten <= localRead);
+    final long spill = c.findCounter(Task.Counter.SPILLED_RECORDS).getCounter();
+    final long out = c.findCounter(Task.Counter.MAP_OUTPUT_RECORDS).getCounter();
+    assertTrue("Expected all records spilled during reduce (" + spill + ")",
+        spill >= 2 * out); // all records spill at map, reduce
+    assertTrue("Expected intermediate merges (" + spill + ")",
+        spill >= 2 * out + (out / MAP_TASKS)); // some records hit twice
   }
 
   public void testReduceFromPartialMem() throws Exception {
+    final int MAP_TASKS = 5;
     JobConf job = mrCluster.createJobConf();
-    job.setNumMapTasks(5);
+    job.setNumMapTasks(MAP_TASKS);
     job.setInt("mapred.inmem.merge.threshold", 0);
     job.set("mapred.job.reduce.input.buffer.percent", "1.0");
     job.setInt("mapred.reduce.parallel.copies", 1);
     job.setInt("io.sort.mb", 10);
     job.set("mapred.child.java.opts", "-Xmx128m");
+    job.setInt("mapred.job.reduce.total.mem.bytes", 128 << 20);
     job.set("mapred.job.shuffle.input.buffer.percent", "0.14");
     job.setNumTasksToExecutePerJvm(1);
     job.set("mapred.job.shuffle.merge.percent", "1.0");
     Counters c = runJob(job);
-    final long hdfsWritten = c.findCounter(Task.FILESYSTEM_COUNTER_GROUP, 
-        Task.getFileSystemCounterNames("hdfs")[1]).getCounter();
-    final long localRead = c.findCounter(Task.FILESYSTEM_COUNTER_GROUP, 
-        Task.getFileSystemCounterNames("file")[0]).getCounter();
-    assertTrue("Expected at least 1MB fewer bytes read from local (" +
-        localRead + ") than written to HDFS (" + hdfsWritten + ")",
-        hdfsWritten >= localRead + 1024 * 1024);
+    final long out = c.findCounter(Task.Counter.MAP_OUTPUT_RECORDS).getCounter();
+    final long spill = c.findCounter(Task.Counter.SPILLED_RECORDS).getCounter();
+    final long twiceOut = 2 * out;
+    assertTrue("Expected some records not spilled during reduce (spill="
+        + spill + ", 2*out=" + twiceOut + ")",
+        spill < twiceOut); // spilled map records, some records at the reduce
   }
 
   public void testReduceFromMem() throws Exception {
+    final int MAP_TASKS = 3;
     JobConf job = mrCluster.createJobConf();
+    job.set("mapred.job.shuffle.input.buffer.percent", "1.0");
+    job.setInt("mapred.job.reduce.total.mem.bytes", 128 << 20);
+    job.setNumMapTasks(MAP_TASKS);
     job.set("mapred.job.reduce.input.buffer.percent", "1.0");
-    job.setNumMapTasks(3);
     Counters c = runJob(job);
-    final long localRead = c.findCounter(Task.FILESYSTEM_COUNTER_GROUP, 
-        Task.getFileSystemCounterNames("file")[0]).getCounter();
-    assertTrue("Non-zero read from local: " + localRead, localRead == 0);
+    final long spill = c.findCounter(Task.Counter.SPILLED_RECORDS).getCounter();
+    final long out = c.findCounter(Task.Counter.MAP_OUTPUT_RECORDS).getCounter();
+    assertEquals("Spilled records: " + spill, out, spill); // no reduce spill
   }
 
 }
-- 
1.6.0.4

