From 8e662cb065be1c4bc61c55e6bff161e09c1d36f3 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Fri, 29 Jan 2010 14:45:49 -0800
Subject: [PATCH] MAPREDUCE-1395. Sqoop does not check return value of Job.waitForCompletion()

---
 .../org/apache/hadoop/sqoop/ImportOptions.java     |   23 ++++-
 .../src/java/org/apache/hadoop/sqoop/Sqoop.java    |    4 +
 .../sqoop/mapreduce/DataDrivenImportJob.java       |   36 +++++-
 .../org/apache/hadoop/sqoop/util/ImportError.java  |    6 +
 .../apache/hadoop/sqoop/hive/TestHiveImport.java   |    2 +-
 .../apache/hadoop/sqoop/manager/MySQLAuthTest.java |   90 +++++++++------
 .../hadoop/sqoop/mapreduce/MapreduceTests.java     |    1 +
 .../hadoop/sqoop/mapreduce/TestImportJob.java      |  117 ++++++++++++++++++++
 .../hadoop/sqoop/testutil/ImportJobTestCase.java   |    2 +-
 9 files changed, 234 insertions(+), 47 deletions(-)
 create mode 100644 src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/TestImportJob.java

diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
index fec6b4a..cc44407 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
@@ -28,6 +28,7 @@ import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ToolRunner;
 
 /**
@@ -117,12 +118,18 @@ public class ImportOptions {
 
   private boolean areDelimsManuallySet;
 
+  private Configuration conf;
+
   public static final int DEFAULT_NUM_MAPPERS = 4;
 
   private static final String DEFAULT_CONFIG_FILE = "sqoop.properties";
 
   public ImportOptions() {
-    initDefaults();
+    initDefaults(null);
+  }
+
+  public ImportOptions(Configuration conf) {
+    initDefaults(conf);
   }
 
   /**
@@ -132,7 +139,7 @@ public class ImportOptions {
    * @param table Table to read
    */
   public ImportOptions(final String connect, final String table) {
-    initDefaults();
+    initDefaults(null);
 
     this.connectString = connect;
     this.tableName = table;
@@ -209,7 +216,7 @@ public class ImportOptions {
     return this.tmpDir;
   }
 
-  private void initDefaults() {
+  private void initDefaults(Configuration baseConfiguration) {
     // first, set the true defaults if nothing else happens.
     // default action is to run the full pipeline.
     this.action = ControlAction.FullImport;
@@ -249,6 +256,12 @@ public class ImportOptions {
     this.useCompression = false;
     this.directSplitSize = 0;
 
+    if (null == baseConfiguration) {
+      this.conf = new Configuration();
+    } else {
+      this.conf = new Configuration(baseConfiguration);
+    }
+
     loadFromProperties();
   }
 
@@ -873,4 +886,8 @@ public class ImportOptions {
   public long getDirectSplitSize() {
     return this.directSplitSize;
   }
+
+  public Configuration getConf() {
+    return conf;
+  }
 }
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
index eb11a53..c6fedf2 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
@@ -59,6 +59,10 @@ public class Sqoop extends Configured implements Tool {
   public Sqoop() {
   }
 
+  public Sqoop(Configuration conf) {
+    setConf(conf);
+  }
+
   public ImportOptions getOptions() {
     return options;
   }
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapreduce/DataDrivenImportJob.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapreduce/DataDrivenImportJob.java
index b52ae2b..ebc3f1e 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapreduce/DataDrivenImportJob.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapreduce/DataDrivenImportJob.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
@@ -43,6 +44,7 @@ import org.apache.hadoop.sqoop.ImportOptions;
 import org.apache.hadoop.sqoop.manager.ConnManager;
 import org.apache.hadoop.sqoop.orm.TableClassName;
 import org.apache.hadoop.sqoop.util.ClassLoaderStack;
+import org.apache.hadoop.sqoop.util.ImportError;
 import org.apache.hadoop.sqoop.util.PerfCounters;
 
 /**
@@ -53,12 +55,22 @@ public class DataDrivenImportJob {
 
   public static final Log LOG = LogFactory.getLog(DataDrivenImportJob.class.getName());
 
-  private ImportOptions options;
+  private final ImportOptions options;
+  private final Class<Mapper> mapperClass;
 
+  // For dependency-injection purposes, we can specify a mapper class
+  // to use during tests.
+  public final static String DATA_DRIVEN_MAPPER_KEY =
+      "sqoop.data.driven.mapper.class";
+
+  @SuppressWarnings("unchecked")
   public DataDrivenImportJob(final ImportOptions opts) {
     this.options = opts;
+    this.mapperClass = (Class<Mapper>) opts.getConf().getClass(
+        DATA_DRIVEN_MAPPER_KEY, null);
   }
 
+
   /**
    * Run an import job to read a table in to HDFS
    *
@@ -66,9 +78,11 @@ public class DataDrivenImportJob {
    * @param ormJarFile the Jar file to insert into the dcache classpath. (may be null)
    * @param splitByCol the column of the database table to use to split the import
    * @param conf A fresh Hadoop Configuration to use to build an MR job.
+   * @throws IOException if the job encountered an IO problem
+   * @throws ImportError if the job failed unexpectedly or was misconfigured.
    */
   public void runImport(String tableName, String ormJarFile, String splitByCol,
-      Configuration conf) throws IOException {
+      Configuration conf) throws IOException, ImportError {
 
     LOG.info("Beginning data-driven import of " + tableName);
 
@@ -102,7 +116,11 @@ public class DataDrivenImportJob {
 
       if (options.getFileLayout() == ImportOptions.FileLayout.TextFile) {
         job.setOutputFormatClass(RawKeyTextOutputFormat.class);
-        job.setMapperClass(TextImportMapper.class);
+        if (null == mapperClass) {
+          job.setMapperClass(TextImportMapper.class);
+        } else {
+          job.setMapperClass(mapperClass);
+        }
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(NullWritable.class);
         if (options.shouldUseCompression()) {
@@ -111,7 +129,11 @@ public class DataDrivenImportJob {
         }
       } else if (options.getFileLayout() == ImportOptions.FileLayout.SequenceFile) {
         job.setOutputFormatClass(SequenceFileOutputFormat.class);
-        job.setMapperClass(AutoProgressMapper.class);
+        if (null == mapperClass) {
+          job.setMapperClass(AutoProgressMapper.class);
+        } else {
+          job.setMapperClass(mapperClass);
+        }
         if (options.shouldUseCompression()) {
           SequenceFileOutputFormat.setCompressOutput(job, true);
           SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
@@ -170,8 +192,9 @@ public class DataDrivenImportJob {
       PerfCounters counters = new PerfCounters();
       counters.startClock();
 
+      boolean success;
       try {
-        job.waitForCompletion(false);
+        success = job.waitForCompletion(false);
       } catch (InterruptedException ie) {
         throw new IOException(ie);
       } catch (ClassNotFoundException cnfe) {
@@ -182,6 +205,9 @@ public class DataDrivenImportJob {
       counters.addBytes(job.getCounters().getGroup("FileSystemCounters")
           .findCounter("HDFS_BYTES_WRITTEN").getValue());
       LOG.info("Transferred " + counters.toString());
+      if (!success) {
+        throw new ImportError("Import job failed!");
+      }
     } finally {
       if (isLocal && null != prevClassLoader) {
         // unload the special classloader for this jar.
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/util/ImportError.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/util/ImportError.java
index ec03890..47996cd 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/util/ImportError.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/util/ImportError.java
@@ -41,4 +41,10 @@ public class ImportError extends Exception {
   public ImportError(final String message, final Throwable cause) {
     super(message, cause);
   }
+
+  @Override
+  public String toString() {
+    String msg = getMessage();
+    return (null == msg) ? "ImportError" : msg;
+  }
 }
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/hive/TestHiveImport.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/hive/TestHiveImport.java
index 11d5854..d48fe0c 100644
--- a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/hive/TestHiveImport.java
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/hive/TestHiveImport.java
@@ -43,7 +43,7 @@ public class TestHiveImport extends ImportJobTestCase {
    * Create the argv to pass to Sqoop
    * @return the argv as an array of strings.
    */
-  private String [] getArgv(boolean includeHadoopFlags, String [] moreArgs) {
+  protected String [] getArgv(boolean includeHadoopFlags, String [] moreArgs) {
     ArrayList<String> args = new ArrayList<String>();
 
     if (includeHadoopFlags) {
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
index 12942d6..1292ce3 100644
--- a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
@@ -256,52 +256,68 @@ public class MySQLAuthTest extends ImportJobTestCase {
   public void doZeroTimestampTest(int testNum, boolean expectSuccess,
       String connectString) throws IOException, SQLException {
 
-    final String tableName = "mysqlTimestampTable" + Integer.toString(testNum);
+    LOG.info("Beginning zero-timestamp test #" + testNum);
 
-    // Create a table containing a full-zeros timestamp.
-    ImportOptions options = new ImportOptions(connectString, tableName);
-    options.setUsername(AUTH_TEST_USER);
-    options.setPassword(AUTH_TEST_PASS);
+    try {
+      final String tableName = "mysqlTimestampTable" + Integer.toString(testNum);
 
-    manager = new LocalMySQLManager(options);
+      // Create a table containing a full-zeros timestamp.
+      ImportOptions options = new ImportOptions(connectString, tableName);
+      options.setUsername(AUTH_TEST_USER);
+      options.setPassword(AUTH_TEST_PASS);
 
-    Connection connection = null;
-    Statement st = null;
+      manager = new LocalMySQLManager(options);
 
-    connection = manager.getConnection();
-    connection.setAutoCommit(false);
-    st = connection.createStatement();
+      Connection connection = null;
+      Statement st = null;
 
-    // create the database table and populate it with data. 
-    st.executeUpdate("DROP TABLE IF EXISTS " + tableName);
-    st.executeUpdate("CREATE TABLE " + tableName + " ("
-        + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
-        + "ts TIMESTAMP NOT NULL)");
+      connection = manager.getConnection();
+      connection.setAutoCommit(false);
+      st = connection.createStatement();
 
-    st.executeUpdate("INSERT INTO " + tableName + " VALUES("
-        + "NULL,'0000-00-00 00:00:00.0')");
-    connection.commit();
-    st.close();
-    connection.close();
+      // create the database table and populate it with data. 
+      st.executeUpdate("DROP TABLE IF EXISTS " + tableName);
+      st.executeUpdate("CREATE TABLE " + tableName + " ("
+          + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+          + "ts TIMESTAMP NOT NULL)");
 
-    // Run the import.
-    String [] argv = getArgv(true, false, connectString, tableName);
-    runImport(argv);
+      st.executeUpdate("INSERT INTO " + tableName + " VALUES("
+          + "NULL,'0000-00-00 00:00:00.0')");
+      connection.commit();
+      st.close();
+      connection.close();
 
-    // Make sure the result file is there.
-    Path warehousePath = new Path(this.getWarehouseDir());
-    Path tablePath = new Path(warehousePath, tableName);
-    Path filePath = new Path(tablePath, "part-m-00000");
+      // Run the import.
+      String [] argv = getArgv(true, false, connectString, tableName);
+      try {
+        runImport(argv);
+      } catch (Exception e) {
+        if (expectSuccess) {
+          // This is unexpected. rethrow.
+          throw new RuntimeException(e);
+        } else {
+          // We expected an error.
+          LOG.info("Got exception running import (expected). msg: " + e);
+        }
+      }
 
-    File f = new File(filePath.toString());
-    if (expectSuccess) {
-      assertTrue("Could not find imported data file", f.exists());
-      BufferedReader r = new BufferedReader(new InputStreamReader(
-          new FileInputStream(f)));
-      assertEquals("1,null", r.readLine());
-      IOUtils.closeStream(r);
-    } else {
-      assertFalse("Imported data when expected failure", f.exists());
+      // Make sure the result file is there.
+      Path warehousePath = new Path(this.getWarehouseDir());
+      Path tablePath = new Path(warehousePath, tableName);
+      Path filePath = new Path(tablePath, "part-m-00000");
+
+      File f = new File(filePath.toString());
+      if (expectSuccess) {
+        assertTrue("Could not find imported data file", f.exists());
+        BufferedReader r = new BufferedReader(new InputStreamReader(
+            new FileInputStream(f)));
+        assertEquals("1,null", r.readLine());
+        IOUtils.closeStream(r);
+      } else {
+        assertFalse("Imported data when expected failure", f.exists());
+      }
+    } finally {
+      LOG.info("Finished zero timestamp test #" + testNum);
     }
   }
 }
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/MapreduceTests.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/MapreduceTests.java
index 6ec9270..441fac9 100644
--- a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/MapreduceTests.java
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/MapreduceTests.java
@@ -31,6 +31,7 @@ public final class MapreduceTests {
   public static Test suite() {
     TestSuite suite = new TestSuite("Tests for org.apache.hadoop.sqoop.mapreduce");
     suite.addTestSuite(TestTextImportMapper.class);
+    suite.addTestSuite(TestImportJob.class);
     return suite;
   }
 }
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/TestImportJob.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/TestImportJob.java
new file mode 100644
index 0000000..c6572eb
--- /dev/null
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/mapreduce/TestImportJob.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.mapreduce;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Before;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.db.DBWritable;
+import org.apache.hadoop.sqoop.Sqoop;
+import org.apache.hadoop.sqoop.mapreduce.AutoProgressMapper;
+import org.apache.hadoop.sqoop.testutil.CommonArgs;
+import org.apache.hadoop.sqoop.testutil.HsqldbTestServer;
+import org.apache.hadoop.sqoop.testutil.ImportJobTestCase;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Test aspects of the DataDrivenImportJob class
+ */
+public class TestImportJob extends ImportJobTestCase {
+
+  public void testFailedImportDueToIOException() throws IOException {
+    // Make sure that if a MapReduce job to do the import fails due
+    // to an IOException, we tell the user about it.
+
+    // Create a table to attempt to import.
+    createTableForColType("VARCHAR(32)", "'meep'");
+
+    // Make the output dir exist so we know the job will fail via IOException.
+    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
+    FileSystem fs = FileSystem.getLocal(new Configuration());
+    fs.mkdirs(outputPath);
+
+    assertTrue(fs.exists(outputPath));
+
+    String [] argv = getArgv(true, new String [] { "DATA_COL0" });
+
+    Sqoop importer = new Sqoop();
+    try {
+      ToolRunner.run(importer, argv);
+      fail("Expected IOException running this job.");
+    } catch (Exception e) {
+      // In debug mode, IOException is wrapped in RuntimeException.
+      LOG.info("Got exceptional return (expected: ok). msg is: " + e);
+    }
+  }
+
+  // A mapper that is guaranteed to cause the task to fail.
+  public static class NullDereferenceMapper
+      extends AutoProgressMapper<LongWritable, DBWritable, Text, NullWritable> {
+
+    public void map(LongWritable key, DBWritable val, Context c)
+        throws IOException, InterruptedException {
+      String s = null;
+      s.length(); // This will throw a NullPointerException.
+    }
+  }
+
+  public void testFailedImportDueToJobFail() throws IOException {
+    // Test that if the job returns 'false' it still fails and informs
+    // the user.
+
+    // Create a table to attempt to import.
+    createTableForColType("VARCHAR(32)", "'meep'");
+
+    String [] argv = getArgv(true, new String [] { "DATA_COL0" });
+
+    // Use dependency injection to specify a mapper that we know
+    // will fail.
+    Configuration conf = new Configuration();
+    conf.setClass(DataDrivenImportJob.DATA_DRIVEN_MAPPER_KEY,
+        NullDereferenceMapper.class,
+        Mapper.class);
+
+    Sqoop importer = new Sqoop(conf);
+    try {
+      ToolRunner.run(importer, argv);
+      fail("Expected ImportError running this job.");
+    } catch (Exception e) {
+      // In debug mode, ImportError is wrapped in RuntimeException.
+      LOG.info("Got exceptional return (expected: ok). msg is: " + e);
+    }
+  }
+
+}
+
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ImportJobTestCase.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ImportJobTestCase.java
index 090cf3a..bc05589 100644
--- a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ImportJobTestCase.java
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ImportJobTestCase.java
@@ -272,7 +272,7 @@ public class ImportJobTestCase extends TestCase {
    * @param colNames the columns to import. If null, all columns are used.
    * @return the argv as an array of strings.
    */
-  private String [] getArgv(boolean includeHadoopFlags, String [] colNames) {
+  protected String [] getArgv(boolean includeHadoopFlags, String [] colNames) {
     if (null == colNames) {
       colNames = getColNames();
     }
-- 
1.6.0.4

