From c1d48d5d45d2262cd410d240b056ddc1a31a6880 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Mon, 14 Sep 2009 17:48:35 -0700
Subject: [PATCH] MAPREDUCE-750: Extensible ConnManager factory API

---
 src/contrib/sqoop/conf/sqoop-default.xml           |   33 +++++
 .../java/org/apache/hadoop/sqoop/ConnFactory.java  |  101 ++++++++------
 .../src/java/org/apache/hadoop/sqoop/Sqoop.java    |    8 +-
 .../sqoop/manager/DefaultManagerFactory.java       |   76 +++++++++++
 .../hadoop/sqoop/manager/ManagerFactory.java       |   33 +++++
 .../org/apache/hadoop/sqoop/mapred/ImportJob.java  |    2 +-
 .../src/test/org/apache/hadoop/sqoop/AllTests.java |    1 +
 .../org/apache/hadoop/sqoop/TestConnFactory.java   |  137 ++++++++++++++++++++
 8 files changed, 346 insertions(+), 45 deletions(-)
 create mode 100644 src/contrib/sqoop/conf/sqoop-default.xml
 create mode 100644 src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/DefaultManagerFactory.java
 create mode 100644 src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ManagerFactory.java
 create mode 100644 src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestConnFactory.java

diff --git a/src/contrib/sqoop/conf/sqoop-default.xml b/src/contrib/sqoop/conf/sqoop-default.xml
new file mode 100644
index 0000000..07906dc
--- /dev/null
+++ b/src/contrib/sqoop/conf/sqoop-default.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- Put Sqoop-specific properties in this file. -->
+
+<configuration>
+
+  <property>
+    <name>sqoop.connection.factories</name>
+    <value>org.apache.hadoop.sqoop.manager.DefaultManagerFactory</value>
+    <description>A comma-delimited list of ManagerFactory implementations
+      which are consulted, in order, to instantiate ConnManager instances
+      used to drive connections to databases.
+    </description>
+  </property>
+
+</configuration>
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
index 6315444..7d3f4a8 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
@@ -18,72 +18,87 @@
 
 package org.apache.hadoop.sqoop;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.sqoop.manager.ConnManager;
-import org.apache.hadoop.sqoop.manager.GenericJdbcManager;
-import org.apache.hadoop.sqoop.manager.HsqldbManager;
-import org.apache.hadoop.sqoop.manager.LocalMySQLManager;
-import org.apache.hadoop.sqoop.manager.MySQLManager;
-import org.apache.hadoop.sqoop.manager.OracleManager;
+import org.apache.hadoop.sqoop.manager.DefaultManagerFactory;
+import org.apache.hadoop.sqoop.manager.ManagerFactory;
+import org.apache.hadoop.util.ReflectionUtils;
 
 import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * Static factory class to create the ConnManager type required
+ * Factory class to create the ConnManager type required
  * for the current import job.
+ *
+ * This class delegates the actual responsibility for instantiating
+ * ConnManagers to one or more instances of ManagerFactory. ManagerFactories
+ * are consulted in the order specified in sqoop-site.xml (sqoop.connection.factories).
  */
-public final class ConnFactory {
+public class ConnFactory {
 
   public static final Log LOG = LogFactory.getLog(ConnFactory.class.getName());
 
-  private ConnFactory() { }
+  public ConnFactory(Configuration conf) {
+    factories = new LinkedList<ManagerFactory>();
+    instantiateFactories(conf);
+  }
 
-  /**
-   * Factory method to get a ConnManager for the given JDBC connect string
-   * @param opts The parsed command-line options
-   * @return a ConnManager instance for the appropriate database
-   * @throws IOException if it cannot find a ConnManager for this schema
+  /** The sqoop-site.xml configuration property used to set the list of 
+   * available ManagerFactories.
    */
-  public static ConnManager getManager(ImportOptions opts) throws IOException {
-
-    String manualDriver = opts.getDriverClassName();
-    if (manualDriver != null) {
-      // User has manually specified JDBC implementation with --driver.
-      // Just use GenericJdbcManager.
-      return new GenericJdbcManager(manualDriver, opts);
-    }
+  public final static String FACTORY_CLASS_NAMES_KEY = "sqoop.connection.factories";
 
-    String connectStr = opts.getConnectString();
+  // The default value for sqoop.connection.factories is the name of the DefaultManagerFactory.
+  final static String DEFAULT_FACTORY_CLASS_NAMES = DefaultManagerFactory.class.getName(); 
 
-    int schemeStopIdx = connectStr.indexOf("//");
-    if (-1 == schemeStopIdx) {
-      // no scheme component?
-      throw new IOException("Malformed connect string: " + connectStr);
-    }
+  /** The list of ManagerFactory instances consulted by getManager().
+   */
+  private List<ManagerFactory> factories;
 
-    String scheme = connectStr.substring(0, schemeStopIdx);
+  /**
+   * Create the ManagerFactory instances that should populate
+   * the factories list.
+   */
+  private void instantiateFactories(Configuration conf) {
+    String [] classNameArray =
+        conf.getStrings(FACTORY_CLASS_NAMES_KEY, DEFAULT_FACTORY_CLASS_NAMES);
 
-    if (null == scheme) {
-      // We don't know if this is a mysql://, hsql://, etc.
-      // Can't do anything with this.
-      throw new IOException("Null scheme associated with connect string.");
+    for (String className : classNameArray) {
+      try {
+        className = className.trim(); // Ignore leading/trailing whitespace.
+        ManagerFactory factory = ReflectionUtils.newInstance(
+            (Class<ManagerFactory>) conf.getClassByName(className), conf);
+        LOG.debug("Loaded manager factory: " + className);
+        factories.add(factory);
+      } catch (ClassNotFoundException cnfe) {
+        LOG.error("Could not load ManagerFactory " + className + " (not found)");
+      }
     }
+  }
 
-    if (scheme.equals("jdbc:mysql:")) {
-      if (opts.isDirect()) {
-        return new LocalMySQLManager(opts);
-      } else {
-        return new MySQLManager(opts);
+  /**
+   * Factory method to get a ConnManager for the given JDBC connect string.
+   * @param opts The parsed command-line options
+   * @return a ConnManager instance for the appropriate database
+   * @throws IOException if it cannot find a ConnManager for this schema
+   */
+  public ConnManager getManager(ImportOptions opts) throws IOException {
+    // Try all the available manager factories.
+    for (ManagerFactory factory : factories) {
+      LOG.debug("Trying ManagerFactory: " + factory.getClass().getName());
+      ConnManager mgr = factory.accept(opts);
+      if (null != mgr) {
+        LOG.debug("Instantiated ConnManager.");
+        return mgr;
       }
-    } else if (scheme.equals("jdbc:hsqldb:hsql:")) {
-      return new HsqldbManager(opts);
-    } else if (scheme.startsWith("jdbc:oracle:")) {
-      return new OracleManager(opts);
-    } else {
-      throw new IOException("Unknown connection scheme: " + scheme);
     }
+
+    throw new IOException("No manager for connect string: " + opts.getConnectString());
   }
 }
 
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
index a4a4963..89939c7 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -41,6 +42,11 @@ public class Sqoop extends Configured implements Tool {
 
   public static final Log LOG = LogFactory.getLog(Sqoop.class.getName());
 
+  static {
+    Configuration.addDefaultResource("sqoop-default.xml");
+    Configuration.addDefaultResource("sqoop-site.xml");
+  }
+
   private ImportOptions options;
   private ConnManager manager;
   private HiveImport hiveImport;
@@ -103,7 +109,7 @@ public class Sqoop extends Configured implements Tool {
 
     // Get the connection to the database
     try {
-      manager = ConnFactory.getManager(options);
+      manager = new ConnFactory(getConf()).getManager(options);
     } catch (Exception e) {
       LOG.error("Got error creating database manager: " + e.toString());
       return 1;
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/DefaultManagerFactory.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/DefaultManagerFactory.java
new file mode 100644
index 0000000..a9a5650
--- /dev/null
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/DefaultManagerFactory.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.manager;
+
+import org.apache.hadoop.sqoop.ImportOptions;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Contains instantiation code for all ConnManager implementations
+ * shipped and enabled by default in Sqoop.
+ */
+public final class DefaultManagerFactory implements ManagerFactory {
+
+  public static final Log LOG = LogFactory.getLog(DefaultManagerFactory.class.getName());
+
+  public ConnManager accept(ImportOptions options) {
+    String manualDriver = options.getDriverClassName();
+    if (manualDriver != null) {
+      // User has manually specified JDBC implementation with --driver.
+      // Just use GenericJdbcManager.
+      return new GenericJdbcManager(manualDriver, options);
+    }
+
+    String connectStr = options.getConnectString();
+
+    int schemeStopIdx = connectStr.indexOf("//");
+    if (-1 == schemeStopIdx) {
+      // no scheme component?
+      LOG.warn("Could not parse connect string: [" + connectStr
+          + "]; this may be malformed.");
+      return null;
+    }
+
+    String scheme = connectStr.substring(0, schemeStopIdx);
+
+    if (null == scheme) {
+      // We don't know if this is a mysql://, hsql://, etc.
+      // Can't do anything with this.
+      LOG.warn("Null scheme associated with connect string.");
+      return null;
+    }
+
+    if (scheme.equals("jdbc:mysql:")) {
+      if (options.isDirect()) {
+        return new LocalMySQLManager(options);
+      } else {
+        return new MySQLManager(options);
+      }
+    } else if (scheme.startsWith("jdbc:hsqldb:")) {
+      return new HsqldbManager(options);
+    } else if (scheme.startsWith("jdbc:oracle:")) {
+      return new OracleManager(options);
+    } else {
+      return null;
+    }
+  }
+}
+
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ManagerFactory.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ManagerFactory.java
new file mode 100644
index 0000000..d898a1f
--- /dev/null
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ManagerFactory.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.manager;
+
+import org.apache.hadoop.sqoop.ImportOptions;
+
+/**
+ * Interface for factory classes for ConnManager implementations.
+ * ManagerFactories are instantiated by o.a.h.s.ConnFactory and
+ * stored in an ordered list. The ConnFactory.getManager() implementation
+ * calls the accept() method of each ManagerFactory, in order until
+ * one such call returns a non-null ConnManager instance.
+ */
+public interface ManagerFactory {
+  ConnManager accept(ImportOptions options);
+}
+
diff --git a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapred/ImportJob.java b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapred/ImportJob.java
index 050652a..b13105b 100644
--- a/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapred/ImportJob.java
+++ b/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/mapred/ImportJob.java
@@ -117,7 +117,7 @@ public class ImportJob {
 
       FileOutputFormat.setOutputPath(job, outputPath);
 
-      ConnManager mgr = ConnFactory.getManager(options);
+      ConnManager mgr = new ConnFactory(conf).getManager(options);
       String username = options.getUsername();
       if (null == username || username.length() == 0) {
         DBConfiguration.configureDB(job, mgr.getDriverClass(), options.getConnectString());
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/AllTests.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/AllTests.java
index d86ae10..303bf31 100644
--- a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/AllTests.java
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/AllTests.java
@@ -54,6 +54,7 @@ public final class AllTests  {
     suite.addTestSuite(TestFieldFormatter.class);
     suite.addTestSuite(TestImportOptions.class);
     suite.addTestSuite(TestParseMethods.class);
+    suite.addTestSuite(TestConnFactory.class);
     suite.addTest(ThirdPartyTests.suite());
 
     return suite;
diff --git a/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestConnFactory.java b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestConnFactory.java
new file mode 100644
index 0000000..119a3ab
--- /dev/null
+++ b/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestConnFactory.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.sqoop.manager.ConnManager;
+import org.apache.hadoop.sqoop.manager.ManagerFactory;
+
+import junit.framework.TestCase;
+
+import java.io.IOException; 
+import java.util.Map;
+import java.sql.Connection;
+import java.sql.ResultSet;
+
+/**
+ * Test the ConnFactory implementation and its ability to delegate to multiple
+ * different ManagerFactory implementations using reflection.
+ */
+public class TestConnFactory extends TestCase {
+
+  public void testCustomFactory() throws IOException {
+    Configuration conf = new Configuration();
+    conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, AlwaysDummyFactory.class.getName());
+
+    ConnFactory factory = new ConnFactory(conf);
+    ConnManager manager = factory.getManager(new ImportOptions());
+    assertNotNull("No manager returned", manager);
+    assertTrue("Expected a DummyManager", manager instanceof DummyManager);
+  }
+
+  public void testExceptionForNoManager() {
+    Configuration conf = new Configuration();
+    conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, EmptyFactory.class.getName());
+
+    ConnFactory factory = new ConnFactory(conf);
+    try {
+      ConnManager manager = factory.getManager(new ImportOptions());
+      fail("factory.getManager() expected to throw IOException");
+    } catch (IOException ioe) {
+      // Expected this. Test passes.
+    }
+  }
+
+  public void testMultipleManagers() throws IOException {
+    Configuration conf = new Configuration();
+    // The AlwaysDummyFactory is second in this list. Nevertheless, since
+    // we know the first factory in the list will return null, we should still
+    // get a DummyManager out.
+    String classNames = EmptyFactory.class.getName()
+        + "," + AlwaysDummyFactory.class.getName();
+    conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, classNames);
+
+    ConnFactory factory = new ConnFactory(conf);
+    ConnManager manager = factory.getManager(new ImportOptions());
+    assertNotNull("No manager returned", manager);
+    assertTrue("Expected a DummyManager", manager instanceof DummyManager);
+  }
+
+  ////// mock classes used for test cases above //////
+
+  public static class AlwaysDummyFactory implements ManagerFactory {
+    public ConnManager accept(ImportOptions opts) {
+      // Always return a new DummyManager
+      return new DummyManager();
+    }
+  }
+
+  public static class EmptyFactory implements ManagerFactory {
+    public ConnManager accept(ImportOptions opts) {
+      // Never instantiate a proper ConnManager;
+      return null;
+    }
+  }
+
+  /**
+   * This implementation doesn't do anything special.
+   */
+  public static class DummyManager implements ConnManager {
+    public void close() {
+    }
+
+    public String [] listDatabases() {
+      return null; 
+    }
+
+    public String [] listTables() {
+      return null;
+    }
+
+    public String [] getColumnNames(String tableName) {
+      return null;
+    }
+
+    public String getPrimaryKey(String tableName) {
+      return null;
+    }
+
+    public Map<String, Integer> getColumnTypes(String tableName) {
+      return null;
+    }
+
+    public ResultSet readTable(String tableName, String [] columns) {
+      return null;
+    }
+
+    public Connection getConnection() {
+      return null;
+    }
+
+    public String getDriverClass() {
+      return null;
+    }
+
+    public void execAndPrint(String s) {
+    }
+
+    public void importTable(String tableName, String jarFile, Configuration conf) {
+    }
+  }
+}
-- 
1.6.0.4

