From ca547d89042fff3a38c0c93b6e0ece78e74ae064 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Fri, 12 Mar 2010 17:11:10 -0800
Subject: [PATCH] HADOOP-4655. FileSystem.CACHE should be ref-counted

Description: FileSystem.CACHE is not ref-counted, and could lead to resource leakage.
Adds new method FileSystem.newInstance() that always returns a newly allocated
FileSystem object.
Reason: Bugfix
Author: dhruba borthakur
Ref: UNKNOWN
---
 src/c++/libhdfs/hdfs.c                            |  188 ++++++++++++++++++++-
 src/c++/libhdfs/hdfs.h                            |    9 +-
 src/c++/libhdfs/hdfs_test.c                       |    6 +-
 src/core/org/apache/hadoop/fs/FileSystem.java     |   76 ++++++++-
 src/test/org/apache/hadoop/fs/TestFileSystem.java |   16 ++
 5 files changed, 288 insertions(+), 7 deletions(-)

diff --git a/src/c++/libhdfs/hdfs.c b/src/c++/libhdfs/hdfs.c
index b8ceaea..bd932f6 100644
--- a/src/c++/libhdfs/hdfs.c
+++ b/src/c++/libhdfs/hdfs.c
@@ -168,10 +168,15 @@ done:
 
 
 hdfsFS hdfsConnect(const char* host, tPort port) {
-  // conect with NULL as user name/groups
+  // connect with NULL as user name/groups
   return hdfsConnectAsUser(host, port, NULL, NULL, 0);
 }
 
+/** Always return a new FileSystem handle */
+hdfsFS hdfsConnectNewInstance(const char* host, tPort port) {
+  // connect with NULL as user name/groups
+  return hdfsConnectAsUserNewInstance(host, port, NULL, NULL, 0);
+}
 
 hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user , const char **groups, int groups_size )
 {
@@ -355,6 +360,187 @@ hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user , const
 }
 
 
+/** Always return a new FileSystem handle */
+hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, const char *user , const char **groups, int groups_size )
+{
+    // JAVA EQUIVALENT:
+    //  FileSystem fs = FileSystem.get(new Configuration());
+    //  return fs;
+
+    JNIEnv *env = 0;
+    jobject jConfiguration = NULL;
+    jobject jFS = NULL;
+    jobject jURI = NULL;
+    jstring jURIString = NULL;
+    jvalue  jVal;
+    jthrowable jExc = NULL;
+    char    *cURI = 0;
+    jobject gFsRef = NULL;
+
+
+    //Get the JNIEnv* corresponding to current thread
+    env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return NULL;
+    }
+
+    //Create the org.apache.hadoop.conf.Configuration object
+    jConfiguration =
+        constructNewObjectOfClass(env, NULL, HADOOP_CONF, "()V");
+
+    if (jConfiguration == NULL) {
+        fprintf(stderr, "Can't construct instance of class "
+                "org.apache.hadoop.conf.Configuration\n");
+        errno = EINTERNAL;
+        return NULL;
+    }
+ 
+    if (user != NULL) {
+
+      if (groups == NULL || groups_size <= 0) {
+        fprintf(stderr, "ERROR: groups must not be empty/null\n");
+        errno = EINVAL;
+        return NULL;
+      }
+
+      jstring jUserString = (*env)->NewStringUTF(env, user);
+      jarray jGroups = constructNewArrayString(env, &jExc, groups, groups_size);
+      if (jGroups == NULL) {
+        errno = EINTERNAL;
+        fprintf(stderr, "ERROR: could not construct groups array\n");
+        return NULL;
+      }
+
+      jobject jUgi;
+      if ((jUgi = constructNewObjectOfClass(env, &jExc, HADOOP_UNIX_USER_GROUP_INFO, JMETHOD2(JPARAM(JAVA_STRING), JARRPARAM(JAVA_STRING), JAVA_VOID), jUserString, jGroups)) == NULL) {
+        fprintf(stderr,"failed to construct hadoop user unix group info object\n");
+        errno = errnoFromException(jExc, env, HADOOP_UNIX_USER_GROUP_INFO,
+                                   "init");
+        destroyLocalReference(env, jConfiguration);
+        destroyLocalReference(env, jUserString);
+        if (jGroups != NULL) {
+          destroyLocalReference(env, jGroups);
+        }          
+        return NULL;
+      }
+#define USE_UUGI
+#ifdef USE_UUGI
+
+      // UnixUserGroupInformation.UGI_PROPERTY_NAME
+      jstring jAttrString = (*env)->NewStringUTF(env,"hadoop.job.ugi");
+      
+      if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, HADOOP_UNIX_USER_GROUP_INFO, "saveToConf",
+                       JMETHOD3(JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), JPARAM(HADOOP_UNIX_USER_GROUP_INFO), JAVA_VOID),
+                       jConfiguration, jAttrString, jUgi) != 0) {
+        errno = errnoFromException(jExc, env, HADOOP_FSPERM,
+                                   "init");
+        destroyLocalReference(env, jConfiguration);
+        destroyLocalReference(env, jUserString);
+        if (jGroups != NULL) {
+          destroyLocalReference(env, jGroups);
+        }          
+        destroyLocalReference(env, jUgi);
+        return NULL;
+      }
+
+      destroyLocalReference(env, jUserString);
+      destroyLocalReference(env, jGroups);
+      destroyLocalReference(env, jUgi);
+    }
+#else
+    
+    // what does "current" mean in the context of libhdfs ? does it mean for the last hdfs connection we used?
+    // that's why this code cannot be activated. We know the above use of the conf object should work well with 
+    // multiple connections.
+      if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, HADOOP_USER_GROUP_INFO, "setCurrentUGI",
+                       JMETHOD1(JPARAM(HADOOP_USER_GROUP_INFO), JAVA_VOID),
+                       jUgi) != 0) {
+        errno = errnoFromException(jExc, env, HADOOP_USER_GROUP_INFO,
+                                   "setCurrentUGI");
+        destroyLocalReference(env, jConfiguration);
+        destroyLocalReference(env, jUserString);
+        if (jGroups != NULL) {
+          destroyLocalReference(env, jGroups);
+        }          
+        destroyLocalReference(env, jUgi);
+        return NULL;
+      }
+
+      destroyLocalReference(env, jUserString);
+      destroyLocalReference(env, jGroups);
+      destroyLocalReference(env, jUgi);
+    }
+#endif      
+    //Check what type of FileSystem the caller wants...
+    if (host == NULL) {
+        // fs = FileSytem::newInstanceLocal(conf);
+        if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, HADOOP_FS, "newInstanceLocal",
+                         JMETHOD1(JPARAM(HADOOP_CONF),
+                                  JPARAM(HADOOP_LOCALFS)),
+                         jConfiguration) != 0) {
+            errno = errnoFromException(jExc, env, "org.apache.hadoop.fs."
+                                       "FileSystem::newInstanceLocal");
+            goto done;
+        }
+        jFS = jVal.l;
+    }
+    else if (!strcmp(host, "default") && port == 0) {
+        //fs = FileSystem::get(conf); 
+        if (invokeMethod(env, &jVal, &jExc, STATIC, NULL,
+                         HADOOP_FS, "newInstance",
+                         JMETHOD1(JPARAM(HADOOP_CONF),
+                                  JPARAM(HADOOP_FS)),
+                         jConfiguration) != 0) {
+            errno = errnoFromException(jExc, env, "org.apache.hadoop.fs."
+                                       "FileSystem::newInstance");
+            goto done;
+        }
+        jFS = jVal.l;
+    }
+    else {
+        // fs = FileSystem::newInstance(URI, conf);
+        cURI = malloc(strlen(host)+16);
+        sprintf(cURI, "hdfs://%s:%d", host, (int)(port));
+
+        jURIString = (*env)->NewStringUTF(env, cURI);
+        if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, JAVA_NET_URI,
+                         "create", "(Ljava/lang/String;)Ljava/net/URI;",
+                         jURIString) != 0) {
+            errno = errnoFromException(jExc, env, "java.net.URI::create");
+            goto done;
+        }
+        jURI = jVal.l;
+
+        if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, HADOOP_FS, "newInstance",
+                         JMETHOD2(JPARAM(JAVA_NET_URI),
+                                  JPARAM(HADOOP_CONF), JPARAM(HADOOP_FS)),
+                         jURI, jConfiguration) != 0) {
+            errno = errnoFromException(jExc, env, "org.apache.hadoop.fs."
+                                       "Filesystem::newInstance(URI, Configuration)");
+            goto done;
+        }
+
+        jFS = jVal.l;
+    }
+
+  done:
+    
+    // Release unnecessary local references
+    destroyLocalReference(env, jConfiguration);
+    destroyLocalReference(env, jURIString);
+    destroyLocalReference(env, jURI);
+
+    if (cURI) free(cURI);
+
+    /* Create a global reference for this fs */
+    if (jFS) {
+        gFsRef = (*env)->NewGlobalRef(env, jFS);
+        destroyLocalReference(env, jFS);
+    }
+
+    return gFsRef;
+}
 
 int hdfsDisconnect(hdfsFS fs)
 {
diff --git a/src/c++/libhdfs/hdfs.h b/src/c++/libhdfs/hdfs.h
index b57b42e..88942cc 100644
--- a/src/c++/libhdfs/hdfs.h
+++ b/src/c++/libhdfs/hdfs.h
@@ -90,7 +90,6 @@ extern  "C" {
     };
     typedef struct hdfsFile_internal* hdfsFile;
       
-
     /** 
      * hdfsConnectAsUser - Connect to a hdfs file system as a specific user
      * Connect to the hdfs.
@@ -121,6 +120,14 @@ extern  "C" {
      hdfsFS hdfsConnect(const char* host, tPort port);
 
 
+    /**
+     * This are the same as hdfsConnectAsUser except that every invocation returns a new FileSystem handle.
+     * Applications should call a hdfsDisconnect for every call to hdfsConnectAsUserNewInstance.
+     */
+     hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, const char *user , const char *groups[], int groups_size );
+     hdfsFS hdfsConnectNewInstance(const char* host, tPort port);
+     hdfsFS hdfsConnectPath(const char* uri);
+
     /** 
      * hdfsDisconnect - Disconnect from the hdfs file system.
      * Disconnect from hdfs.
diff --git a/src/c++/libhdfs/hdfs_test.c b/src/c++/libhdfs/hdfs_test.c
index a26cc82..1d2bd51 100644
--- a/src/c++/libhdfs/hdfs_test.c
+++ b/src/c++/libhdfs/hdfs_test.c
@@ -52,13 +52,13 @@ void permission_disp(short permissions, char *rtr) {
 
 int main(int argc, char **argv) {
 
-    hdfsFS fs = hdfsConnect("default", 0);
+    hdfsFS fs = hdfsConnectNewInstance("default", 0);
     if(!fs) {
         fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
         exit(-1);
     } 
  
-    hdfsFS lfs = hdfsConnect(NULL, 0);
+    hdfsFS lfs = hdfsConnectNewInstance(NULL, 0);
     if(!lfs) {
         fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
         exit(-1);
@@ -401,7 +401,7 @@ int main(int argc, char **argv) {
       groups[0] = "users";
       groups[1] = "nobody";
 
-      fs = hdfsConnectAsUser("default", 0, tuser, groups, 2);
+      fs = hdfsConnectAsUserNewInstance("default", 0, tuser, groups, 2);
       if(!fs) {
         fprintf(stderr, "Oops! Failed to connect to hdfs as user %s!\n",tuser);
         exit(-1);
diff --git a/src/core/org/apache/hadoop/fs/FileSystem.java b/src/core/org/apache/hadoop/fs/FileSystem.java
index 945ad2e..f355605 100644
--- a/src/core/org/apache/hadoop/fs/FileSystem.java
+++ b/src/core/org/apache/hadoop/fs/FileSystem.java
@@ -197,6 +197,58 @@ public abstract class FileSystem extends Configured implements Closeable {
     return CACHE.get(uri, conf);
   }
 
+  /** Returns the FileSystem for this URI's scheme and authority.  The scheme
+   * of the URI determines a configuration property name,
+   * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
+   * The entire URI is passed to the FileSystem instance's initialize method.
+   * This always returns a new FileSystem object.
+   */
+  public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
+    String scheme = uri.getScheme();
+    String authority = uri.getAuthority();
+
+    if (scheme == null) {                       // no scheme: use default FS
+      return newInstance(conf);
+    }
+
+    if (authority == null) {                       // no authority
+      URI defaultUri = getDefaultUri(conf);
+      if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
+          && defaultUri.getAuthority() != null) {  // & default has authority
+        return newInstance(defaultUri, conf);              // return default
+      }
+    }
+    return CACHE.getUnique(uri, conf);
+  }
+
+  /** Returns a unique configured filesystem implementation.
+   * This always returns a new FileSystem object. */
+  public static FileSystem newInstance(Configuration conf) throws IOException {
+    return newInstance(getDefaultUri(conf), conf);
+  }
+
+  /**
+   * Get a unique local file system object
+   * @param conf the configuration to configure the file system with
+   * @return a LocalFileSystem
+   * This always returns a new FileSystem object.
+   */
+  public static LocalFileSystem newInstanceLocal(Configuration conf)
+    throws IOException {
+    return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
+  }
+
+  private static class ClientFinalizer extends Thread {
+    public synchronized void run() {
+      try {
+        FileSystem.closeAll();
+      } catch (IOException e) {
+        LOG.info("FileSystem.closeAll() threw an exception:\n" + e);
+      }
+    }
+  }
+  private static final ClientFinalizer clientFinalizer = new ClientFinalizer();
+
   /**
    * Close all cached filesystems. Be sure those filesystems are not
    * used anymore.
@@ -1376,8 +1428,21 @@ public abstract class FileSystem extends Configured implements Closeable {
     private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
     private final Set<Key> toAutoClose = new HashSet<Key>();
 
+    /** A variable that makes all objects in the cache unique */
+    private static AtomicLong unique = new AtomicLong(1);
+
     synchronized FileSystem get(URI uri, Configuration conf) throws IOException{
       Key key = new Key(uri, conf);
+      return getInternal(uri, conf, key);
+    }
+
+    /** The objects inserted into the cache using this method are all unique */
+    synchronized FileSystem getUnique(URI uri, Configuration conf) throws IOException{
+      Key key = new Key(uri, conf, unique.getAndIncrement());
+      return getInternal(uri, conf, key);
+    }
+
+    private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
       FileSystem fs = map.get(key);
       if (fs == null) {
         fs = createFileSystem(uri, conf);
@@ -1463,10 +1528,16 @@ public abstract class FileSystem extends Configured implements Closeable {
       final String scheme;
       final String authority;
       final String username;
+      final long unique;   // an artificial way to make a key unique
 
       Key(URI uri, Configuration conf) throws IOException {
+        this(uri, conf, 0);
+      }
+
+      Key(URI uri, Configuration conf, long unique) throws IOException {
         scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
         authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
+        this.unique = unique;
         UserGroupInformation ugi = UserGroupInformation.readFrom(conf);
         if (ugi == null) {
           try {
@@ -1480,7 +1551,7 @@ public abstract class FileSystem extends Configured implements Closeable {
 
       /** {@inheritDoc} */
       public int hashCode() {
-        return (scheme + authority + username).hashCode();
+        return (scheme + authority + username).hashCode() + (int)unique;
       }
 
       static boolean isEqual(Object a, Object b) {
@@ -1496,7 +1567,8 @@ public abstract class FileSystem extends Configured implements Closeable {
           Key that = (Key)obj;
           return isEqual(this.scheme, that.scheme)
                  && isEqual(this.authority, that.authority)
-                 && isEqual(this.username, that.username);
+                 && isEqual(this.username, that.username)
+                 && (this.unique == that.unique);
         }
         return false;        
       }
diff --git a/src/test/org/apache/hadoop/fs/TestFileSystem.java b/src/test/org/apache/hadoop/fs/TestFileSystem.java
index 00fe345..4e7e756 100644
--- a/src/test/org/apache/hadoop/fs/TestFileSystem.java
+++ b/src/test/org/apache/hadoop/fs/TestFileSystem.java
@@ -664,4 +664,20 @@ public class TestFileSystem extends TestCase {
       super.close();
     }
   }
+
+  public static void testFsUniqueness(long megaBytes, int numFiles, long seed)
+    throws Exception {
+
+    // multiple invocations of FileSystem.get return the same object.
+    FileSystem fs1 = FileSystem.get(conf);
+    FileSystem fs2 = FileSystem.get(conf);
+    assertTrue(fs1 == fs2);
+
+    // multiple invocations of FileSystem.newInstance return different objects
+    fs1 = FileSystem.newInstance(conf);
+    fs2 = FileSystem.newInstance(conf);
+    assertTrue(fs1 != fs2 && !fs1.equals(fs2));
+    fs1.close();
+    fs2.close();
+  }
 }
-- 
1.6.0.4

