001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs;
019    
020    import java.io.FileNotFoundException;
021    import java.io.IOException;
022    import java.io.InputStream;
023    import java.io.OutputStream;
024    import java.net.URI;
025    import java.security.PrivilegedExceptionAction;
026    import java.util.ArrayList;
027    import java.util.Arrays;
028    import java.util.EnumSet;
029    import java.util.HashSet;
030    import java.util.IdentityHashMap;
031    import java.util.List;
032    import java.util.Map;
033    import java.util.Set;
034    import java.util.Stack;
035    import java.util.TreeSet;
036    import java.util.Map.Entry;
037    
038    import org.apache.commons.logging.Log;
039    import org.apache.commons.logging.LogFactory;
040    import org.apache.hadoop.HadoopIllegalArgumentException;
041    import org.apache.hadoop.classification.InterfaceAudience;
042    import org.apache.hadoop.classification.InterfaceStability;
043    import org.apache.hadoop.conf.Configuration;
044    import org.apache.hadoop.fs.FileSystem.Statistics;
045    import org.apache.hadoop.fs.Options.CreateOpts;
046    import org.apache.hadoop.fs.permission.FsPermission;
047    import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
048    import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
049    import org.apache.hadoop.io.IOUtils;
050    import org.apache.hadoop.ipc.RpcClientException;
051    import org.apache.hadoop.ipc.RpcServerException;
052    import org.apache.hadoop.ipc.UnexpectedServerException;
053    import org.apache.hadoop.fs.InvalidPathException;
054    import org.apache.hadoop.security.AccessControlException;
055    import org.apache.hadoop.security.UserGroupInformation;
056    import org.apache.hadoop.security.token.Token;
057    
058    /**
059     * The FileContext class provides an interface to the application writer for
060     * using the Hadoop file system.
061     * It provides a set of methods for the usual operation: create, open, 
062     * list, etc 
063     * 
064     * <p>
065     * <b> *** Path Names *** </b>
066     * <p>
067     * 
068     * The Hadoop file system supports a URI name space and URI names.
069     * It offers a forest of file systems that can be referenced using fully
070     * qualified URIs.
071     * Two common Hadoop file systems implementations are
072     * <ul>
073     * <li> the local file system: file:///path
074     * <li> the hdfs file system hdfs://nnAddress:nnPort/path
075     * </ul>
076     * 
077     * While URI names are very flexible, it requires knowing the name or address
078     * of the server. For convenience one often wants to access the default system
079     * in one's environment without knowing its name/address. This has an
080     * additional benefit that it allows one to change one's default fs
081     *  (e.g. admin moves application from cluster1 to cluster2).
082     * <p>
083     * 
084     * To facilitate this, Hadoop supports a notion of a default file system.
085     * The user can set his default file system, although this is
086     * typically set up for you in your environment via your default config.
087     * A default file system implies a default scheme and authority; slash-relative
088     * names (such as /for/bar) are resolved relative to that default FS.
089     * Similarly a user can also have working-directory-relative names (i.e. names
090     * not starting with a slash). While the working directory is generally in the
091     * same default FS, the wd can be in a different FS.
092     * <p>
093     *  Hence Hadoop path names can be one of:
094     *  <ul>
095     *  <li> fully qualified URI: scheme://authority/path
096     *  <li> slash relative names: /path relative to the default file system
097     *  <li> wd-relative names: path  relative to the working dir
098     *  </ul>   
099     *  Relative paths with scheme (scheme:foo/bar) are illegal.
100     *  
101     *  <p>
102     *  <b>****The Role of the FileContext and configuration defaults****</b>
103     *  <p>
104     *  The FileContext provides file namespace context for resolving file names;
105     *  it also contains the umask for permissions, In that sense it is like the
106     *  per-process file-related state in Unix system.
107     *  These two properties
108     *  <ul> 
109     *  <li> default file system i.e your slash)
110     *  <li> umask
111     *  </ul>
112     *  in general, are obtained from the default configuration file
113     *  in your environment,  (@see {@link Configuration}).
114     *  
115     *  No other configuration parameters are obtained from the default config as 
116     *  far as the file context layer is concerned. All file system instances
117     *  (i.e. deployments of file systems) have default properties; we call these
118     *  server side (SS) defaults. Operation like create allow one to select many 
119     *  properties: either pass them in as explicit parameters or use
120     *  the SS properties.
121     *  <p>
122     *  The file system related SS defaults are
123     *  <ul>
124     *  <li> the home directory (default is "/user/userName")
125     *  <li> the initial wd (only for local fs)
126     *  <li> replication factor
127     *  <li> block size
128     *  <li> buffer size
129     *  <li> bytesPerChecksum (if used).
130     *  </ul>
131     *
132     * <p>
133     * <b> *** Usage Model for the FileContext class *** </b>
134     * <p>
135     * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
136     *   Unspecified values come from core-defaults.xml in the release jar.
137     *  <ul>  
138     *  <li> myFContext = FileContext.getFileContext(); // uses the default config
139     *                                                // which has your default FS 
140     *  <li>  myFContext.create(path, ...);
141     *  <li>  myFContext.setWorkingDir(path)
142     *  <li>  myFContext.open (path, ...);  
143     *  </ul>  
144     * Example 2: Get a FileContext with a specific URI as the default FS
145     *  <ul>  
146     *  <li> myFContext = FileContext.getFileContext(URI)
147     *  <li> myFContext.create(path, ...);
148     *   ...
149     * </ul> 
150     * Example 3: FileContext with local file system as the default
151     *  <ul> 
152     *  <li> myFContext = FileContext.getLocalFSFileContext()
153     *  <li> myFContext.create(path, ...);
154     *  <li> ...
155     *  </ul> 
156     * Example 4: Use a specific config, ignoring $HADOOP_CONFIG
157     *  Generally you should not need use a config unless you are doing
158     *   <ul> 
159     *   <li> configX = someConfigSomeOnePassedToYou.
160     *   <li> myFContext = getFileContext(configX); // configX is not changed,
161     *                                              // is passed down 
162     *   <li> myFContext.create(path, ...);
163     *   <li>...
164     *  </ul>                                          
165     *    
166     */
167    
168    @InterfaceAudience.Public
169    @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
170    public final class FileContext {
171      
172      public static final Log LOG = LogFactory.getLog(FileContext.class);
173      public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
174      
175      /**
176       * List of files that should be deleted on JVM shutdown.
177       */
178      static final Map<FileContext, Set<Path>> DELETE_ON_EXIT = 
179        new IdentityHashMap<FileContext, Set<Path>>();
180    
181      /** JVM shutdown hook thread. */
182      static final FileContextFinalizer FINALIZER = 
183        new FileContextFinalizer();
184      
185      private static final PathFilter DEFAULT_FILTER = new PathFilter() {
186        public boolean accept(final Path file) {
187          return true;
188        }
189      };
190      
191      /**
192       * The FileContext is defined by.
193       *  1) defaultFS (slash)
194       *  2) wd
195       *  3) umask
196       */   
197      private final AbstractFileSystem defaultFS; //default FS for this FileContext.
198      private Path workingDir;          // Fully qualified
199      private FsPermission umask;
200      private final Configuration conf;
201      private final UserGroupInformation ugi;
202    
203      private FileContext(final AbstractFileSystem defFs,
204        final FsPermission theUmask, final Configuration aConf) {
205        defaultFS = defFs;
206        umask = FsPermission.getUMask(aConf);
207        conf = aConf;
208        try {
209          ugi = UserGroupInformation.getCurrentUser();
210        } catch (IOException e) {
211          LOG.error("Exception in getCurrentUser: ",e);
212          throw new RuntimeException("Failed to get the current user " +
213                    "while creating a FileContext", e);
214        }
215        /*
216         * Init the wd.
217         * WorkingDir is implemented at the FileContext layer 
218         * NOT at the AbstractFileSystem layer. 
219         * If the DefaultFS, such as localFilesystem has a notion of
220         *  builtin WD, we use that as the initial WD.
221         *  Otherwise the WD is initialized to the home directory.
222         */
223        workingDir = defaultFS.getInitialWorkingDirectory();
224        if (workingDir == null) {
225          workingDir = defaultFS.getHomeDirectory();
226        }
227        util = new Util(); // for the inner class
228      }
229     
230      /* 
231       * Remove relative part - return "absolute":
232       * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
233       * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
234       * ("/foo/bar") are returned unchanged.
235       * 
236       * Applications that use FileContext should use #makeQualified() since
237       * they really want a fully qualified URI.
238       * Hence this method is not called makeAbsolute() and 
239       * has been deliberately declared private.
240       */
241      private Path fixRelativePart(Path p) {
242        if (p.isUriPathAbsolute()) {
243          return p;
244        } else {
245          return new Path(workingDir, p);
246        }
247      }
248    
249      /**
250       * Delete all the paths that were marked as delete-on-exit.
251       */
252      static void processDeleteOnExit() {
253        synchronized (DELETE_ON_EXIT) {
254          Set<Entry<FileContext, Set<Path>>> set = DELETE_ON_EXIT.entrySet();
255          for (Entry<FileContext, Set<Path>> entry : set) {
256            FileContext fc = entry.getKey();
257            Set<Path> paths = entry.getValue();
258            for (Path path : paths) {
259              try {
260                fc.delete(path, true);
261              } catch (IOException e) {
262                LOG.warn("Ignoring failure to deleteOnExit for path " + path);
263              }
264            }
265          }
266          DELETE_ON_EXIT.clear();
267        }
268      }
269      
270      /**
271       * Pathnames with scheme and relative path are illegal.
272       * @param path to be checked
273       */
274      private static void checkNotSchemeWithRelative(final Path path) {
275        if (path.toUri().isAbsolute() && !path.isUriPathAbsolute()) {
276          throw new HadoopIllegalArgumentException(
277              "Unsupported name: has scheme but relative path-part");
278        }
279      }
280    
281      /**
282       * Get the file system of supplied path.
283       * 
284       * @param absOrFqPath - absolute or fully qualified path
285       * @return the file system of the path
286       * 
287       * @throws UnsupportedFileSystemException If the file system for
288       *           <code>absOrFqPath</code> is not supported.
289       * @throws IOExcepton If the file system for <code>absOrFqPath</code> could
290       *         not be instantiated.
291       */
292      private AbstractFileSystem getFSofPath(final Path absOrFqPath)
293          throws UnsupportedFileSystemException, IOException {
294        checkNotSchemeWithRelative(absOrFqPath);
295        if (!absOrFqPath.isAbsolute() && absOrFqPath.toUri().getScheme() == null) {
296          throw new HadoopIllegalArgumentException(
297              "FileContext Bug: path is relative");
298        }
299    
300        try { 
301          // Is it the default FS for this FileContext?
302          defaultFS.checkPath(absOrFqPath);
303          return defaultFS;
304        } catch (Exception e) { // it is different FileSystem
305          return getAbstractFileSystem(ugi, absOrFqPath.toUri(), conf);
306        }
307      }
308      
309      private static AbstractFileSystem getAbstractFileSystem(
310          UserGroupInformation user, final URI uri, final Configuration conf)
311          throws UnsupportedFileSystemException, IOException {
312        try {
313          return user.doAs(new PrivilegedExceptionAction<AbstractFileSystem>() {
314            public AbstractFileSystem run() throws UnsupportedFileSystemException {
315              return AbstractFileSystem.get(uri, conf);
316            }
317          });
318        } catch (InterruptedException ex) {
319          LOG.error(ex);
320          throw new IOException("Failed to get the AbstractFileSystem for path: "
321              + uri, ex);
322        }
323      }
324      
325      /**
326       * Protected Static Factory methods for getting a FileContexts
327       * that take a AbstractFileSystem as input. To be used for testing.
328       */
329    
330      /**
331       * Create a FileContext with specified FS as default using the specified
332       * config.
333       * 
334       * @param defFS
335       * @param aConf
336       * @return new FileContext with specifed FS as default.
337       */
338      public static FileContext getFileContext(final AbstractFileSystem defFS,
339                        final Configuration aConf) {
340        return new FileContext(defFS, FsPermission.getUMask(aConf), aConf);
341      }
342      
343      /**
344       * Create a FileContext for specified file system using the default config.
345       * 
346       * @param defaultFS
347       * @return a FileContext with the specified AbstractFileSystem
348       *                 as the default FS.
349       */
350      protected static FileContext getFileContext(
351        final AbstractFileSystem defaultFS) {
352        return getFileContext(defaultFS, new Configuration());
353      }
354     
355      /**
356       * Static Factory methods for getting a FileContext.
357       * Note new file contexts are created for each call.
358       * The only singleton is the local FS context using the default config.
359       * 
360       * Methods that use the default config: the default config read from the
361       * $HADOOP_CONFIG/core.xml,
362       * Unspecified key-values for config are defaulted from core-defaults.xml
363       * in the release jar.
364       * 
365       * The keys relevant to the FileContext layer are extracted at time of
366       * construction. Changes to the config after the call are ignore
367       * by the FileContext layer. 
368       * The conf is passed to lower layers like AbstractFileSystem and HDFS which
369       * pick up their own config variables.
370       */
371    
372      /**
373       * Create a FileContext using the default config read from the
374       * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
375       * from core-defaults.xml in the release jar.
376       * 
377       * @throws UnsupportedFileSystemException If the file system from the default
378       *           configuration is not supported
379       */
380      public static FileContext getFileContext()
381          throws UnsupportedFileSystemException {
382        return getFileContext(new Configuration());
383      }
384    
385      /**
386       * @return a FileContext for the local file system using the default config.
387       * @throws UnsupportedFileSystemException If the file system for
388       *           {@link FsConstants#LOCAL_FS_URI} is not supported.
389       */
390      public static FileContext getLocalFSFileContext()
391          throws UnsupportedFileSystemException {
392        return getFileContext(FsConstants.LOCAL_FS_URI);
393      }
394    
395      /**
396       * Create a FileContext for specified URI using the default config.
397       * 
398       * @param defaultFsUri
399       * @return a FileContext with the specified URI as the default FS.
400       * 
401       * @throws UnsupportedFileSystemException If the file system for
402       *           <code>defaultFsUri</code> is not supported
403       */
404      public static FileContext getFileContext(final URI defaultFsUri)
405          throws UnsupportedFileSystemException {
406        return getFileContext(defaultFsUri, new Configuration());
407      }
408    
409      /**
410       * Create a FileContext for specified default URI using the specified config.
411       * 
412       * @param defaultFsUri
413       * @param aConf
414       * @return new FileContext for specified uri
415       * @throws UnsupportedFileSystemException If the file system with specified is
416       *           not supported
417       * @throws RuntimeException If the file system specified is supported but
418       *         could not be instantiated, or if login fails.
419       */
420      public static FileContext getFileContext(final URI defaultFsUri,
421          final Configuration aConf) throws UnsupportedFileSystemException {
422        UserGroupInformation currentUser = null;
423        AbstractFileSystem defaultAfs = null;
424        try {
425          currentUser = UserGroupInformation.getCurrentUser();
426          defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
427        } catch (UnsupportedFileSystemException ex) {
428          throw ex;
429        } catch (IOException ex) {
430          LOG.error(ex);
431          throw new RuntimeException(ex);
432        }
433        return getFileContext(defaultAfs, aConf);
434      }
435    
436      /**
437       * Create a FileContext using the passed config. Generally it is better to use
438       * {@link #getFileContext(URI, Configuration)} instead of this one.
439       * 
440       * 
441       * @param aConf
442       * @return new FileContext
443       * @throws UnsupportedFileSystemException If file system in the config
444       *           is not supported
445       */
446      public static FileContext getFileContext(final Configuration aConf)
447          throws UnsupportedFileSystemException {
448        return getFileContext(
449          URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)), 
450          aConf);
451      }
452    
453      /**
454       * @param aConf - from which the FileContext is configured
455       * @return a FileContext for the local file system using the specified config.
456       * 
457       * @throws UnsupportedFileSystemException If default file system in the config
458       *           is not supported
459       * 
460       */
461      public static FileContext getLocalFSFileContext(final Configuration aConf)
462          throws UnsupportedFileSystemException {
463        return getFileContext(FsConstants.LOCAL_FS_URI, aConf);
464      }
465    
466      /* This method is needed for tests. */
467      @InterfaceAudience.Private
468      @InterfaceStability.Unstable /* return type will change to AFS once
469                                      HADOOP-6223 is completed */
470      public AbstractFileSystem getDefaultFileSystem() {
471        return defaultFS;
472      }
473      
474      /**
475       * Set the working directory for wd-relative names (such a "foo/bar"). Working
476       * directory feature is provided by simply prefixing relative names with the
477       * working dir. Note this is different from Unix where the wd is actually set
478       * to the inode. Hence setWorkingDir does not follow symlinks etc. This works
479       * better in a distributed environment that has multiple independent roots.
480       * {@link #getWorkingDirectory()} should return what setWorkingDir() set.
481       * 
482       * @param newWDir new working directory
483       * @throws IOException 
484       * <br>
485       *           NewWdir can be one of:
486       *           <ul>
487       *           <li>relative path: "foo/bar";</li>
488       *           <li>absolute without scheme: "/foo/bar"</li>
489       *           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
490       *           </ul>
491       * <br>
492       *           Illegal WDs:
493       *           <ul>
494       *           <li>relative with scheme: "xx:foo/bar"</li>
495       *           <li>non existent directory</li>
496       *           </ul>
497       */
498      public void setWorkingDirectory(final Path newWDir) throws IOException {
499        checkNotSchemeWithRelative(newWDir);
500        /* wd is stored as a fully qualified path. We check if the given 
501         * path is not relative first since resolve requires and returns 
502         * an absolute path.
503         */  
504        final Path newWorkingDir = new Path(workingDir, newWDir);
505        FileStatus status = getFileStatus(newWorkingDir);
506        if (status.isFile()) {
507          throw new FileNotFoundException("Cannot setWD to a file");
508        }
509        workingDir = newWorkingDir;
510      }
511      
512      /**
513       * Gets the working directory for wd-relative names (such a "foo/bar").
514       */
515      public Path getWorkingDirectory() {
516        return workingDir;
517      }
518      
519      /**
520       * Gets the ugi in the file-context
521       * @return UserGroupInformation
522       */
523      public UserGroupInformation getUgi() {
524        return ugi;
525      }
526      
527      /**
528       * Return the current user's home directory in this file system.
529       * The default implementation returns "/user/$USER/".
530       * @return the home directory
531       */
532      public Path getHomeDirectory() {
533        return defaultFS.getHomeDirectory();
534      }
535      
536      /**
537       * 
538       * @return the umask of this FileContext
539       */
540      public FsPermission getUMask() {
541        return umask;
542      }
543      
544      /**
545       * Set umask to the supplied parameter.
546       * @param newUmask  the new umask
547       */
548      public void setUMask(final FsPermission newUmask) {
549        umask = newUmask;
550      }
551      
552      
553      /**
554       * Resolve the path following any symlinks or mount points
555       * @param f to be resolved
556       * @return fully qualified resolved path
557       * 
558       * @throws FileNotFoundException  If <code>f</code> does not exist
559       * @throws AccessControlException if access denied
560       * @throws IOException If an IO Error occurred
561       * 
562       * Exceptions applicable to file systems accessed over RPC:
563       * @throws RpcClientException If an exception occurred in the RPC client
564       * @throws RpcServerException If an exception occurred in the RPC server
565       * @throws UnexpectedServerException If server implementation throws
566       *           undeclared exception to RPC server
567       * 
568       * RuntimeExceptions:
569       * @throws InvalidPathException If path <code>f</code> is not valid
570       */
571      public Path resolvePath(final Path f) throws FileNotFoundException,
572          UnresolvedLinkException, AccessControlException, IOException {
573        return resolve(f);
574      }
575      
576      /**
577       * Make the path fully qualified if it is isn't. 
578       * A Fully-qualified path has scheme and authority specified and an absolute
579       * path.
580       * Use the default file system and working dir in this FileContext to qualify.
581       * @param path
582       * @return qualified path
583       */
584      public Path makeQualified(final Path path) {
585        return path.makeQualified(defaultFS.getUri(), getWorkingDirectory());
586      }
587    
588      /**
589       * Create or overwrite file on indicated path and returns an output stream for
590       * writing into the file.
591       * 
592       * @param f the file name to open
593       * @param createFlag gives the semantics of create; see {@link CreateFlag}
594       * @param opts file creation options; see {@link Options.CreateOpts}.
595       *          <ul>
596       *          <li>Progress - to report progress on the operation - default null
597       *          <li>Permission - umask is applied against permisssion: default is
598       *          FsPermissions:getDefault()
599       * 
600       *          <li>CreateParent - create missing parent path; default is to not
601       *          to create parents
602       *          <li>The defaults for the following are SS defaults of the file
603       *          server implementing the target path. Not all parameters make sense
604       *          for all kinds of file system - eg. localFS ignores Blocksize,
605       *          replication, checksum
606       *          <ul>
607       *          <li>BufferSize - buffersize used in FSDataOutputStream
608       *          <li>Blocksize - block size for file blocks
609       *          <li>ReplicationFactor - replication for blocks
610       *          <li>BytesPerChecksum - bytes per checksum
611       *          </ul>
612       *          </ul>
613       * 
614       * @return {@link FSDataOutputStream} for created file
615       * 
616       * @throws AccessControlException If access is denied
617       * @throws FileAlreadyExistsException If file <code>f</code> already exists
618       * @throws FileNotFoundException If parent of <code>f</code> does not exist
619       *           and <code>createParent</code> is false
620       * @throws ParentNotDirectoryException If parent of <code>f</code> is not a
621       *           directory.
622       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
623       *           not supported
624       * @throws IOException If an I/O error occurred
625       * 
626       * Exceptions applicable to file systems accessed over RPC:
627       * @throws RpcClientException If an exception occurred in the RPC client
628       * @throws RpcServerException If an exception occurred in the RPC server
629       * @throws UnexpectedServerException If server implementation throws
630       *           undeclared exception to RPC server
631       * 
632       * RuntimeExceptions:
633       * @throws InvalidPathException If path <code>f</code> is not valid
634       */
635      public FSDataOutputStream create(final Path f,
636          final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
637          throws AccessControlException, FileAlreadyExistsException,
638          FileNotFoundException, ParentNotDirectoryException,
639          UnsupportedFileSystemException, IOException {
640        Path absF = fixRelativePart(f);
641    
642        // If one of the options is a permission, extract it & apply umask
643        // If not, add a default Perms and apply umask;
644        // AbstractFileSystem#create
645    
646        CreateOpts.Perms permOpt = 
647          (CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts);
648        FsPermission permission = (permOpt != null) ? permOpt.getValue() :
649                                          FsPermission.getDefault();
650        permission = permission.applyUMask(umask);
651    
652        final CreateOpts[] updatedOpts = 
653                          CreateOpts.setOpt(CreateOpts.perms(permission), opts);
654        return new FSLinkResolver<FSDataOutputStream>() {
655          public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 
656            throws IOException {
657            return fs.create(p, createFlag, updatedOpts);
658          }
659        }.resolve(this, absF);
660      }
661    
662      /**
663       * Make(create) a directory and all the non-existent parents.
664       * 
665       * @param dir - the dir to make
666       * @param permission - permissions is set permission&~umask
667       * @param createParent - if true then missing parent dirs are created if false
668       *          then parent must exist
669       * 
670       * @throws AccessControlException If access is denied
671       * @throws FileAlreadyExistsException If directory <code>dir</code> already
672       *           exists
673       * @throws FileNotFoundException If parent of <code>dir</code> does not exist
674       *           and <code>createParent</code> is false
675       * @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
676       *           directory
677       * @throws UnsupportedFileSystemException If file system for <code>dir</code>
678       *         is not supported
679       * @throws IOException If an I/O error occurred
680       * 
681       * Exceptions applicable to file systems accessed over RPC:
682       * @throws RpcClientException If an exception occurred in the RPC client
683       * @throws UnexpectedServerException If server implementation throws 
684       *           undeclared exception to RPC server
685       * 
686       * RuntimeExceptions:
687       * @throws InvalidPathException If path <code>dir</code> is not valid
688       */
689      public void mkdir(final Path dir, final FsPermission permission,
690          final boolean createParent) throws AccessControlException,
691          FileAlreadyExistsException, FileNotFoundException,
692          ParentNotDirectoryException, UnsupportedFileSystemException, 
693          IOException {
694        final Path absDir = fixRelativePart(dir);
695        final FsPermission absFerms = (permission == null ? 
696              FsPermission.getDefault() : permission).applyUMask(umask);
697        new FSLinkResolver<Void>() {
698          public Void next(final AbstractFileSystem fs, final Path p) 
699            throws IOException, UnresolvedLinkException {
700            fs.mkdir(p, absFerms, createParent);
701            return null;
702          }
703        }.resolve(this, absDir);
704      }
705    
706      /**
707       * Delete a file.
708       * @param f the path to delete.
709       * @param recursive if path is a directory and set to 
710       * true, the directory is deleted else throws an exception. In
711       * case of a file the recursive can be set to either true or false.
712       *
713       * @throws AccessControlException If access is denied
714       * @throws FileNotFoundException If <code>f</code> does not exist
715       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
716       *           not supported
717       * @throws IOException If an I/O error occurred
718       * 
719       * Exceptions applicable to file systems accessed over RPC:
720       * @throws RpcClientException If an exception occurred in the RPC client
721       * @throws RpcServerException If an exception occurred in the RPC server
722       * @throws UnexpectedServerException If server implementation throws 
723       *           undeclared exception to RPC server
724       * 
725       * RuntimeExceptions:
726       * @throws InvalidPathException If path <code>f</code> is invalid
727       */
728      public boolean delete(final Path f, final boolean recursive)
729          throws AccessControlException, FileNotFoundException,
730          UnsupportedFileSystemException, IOException {
731        Path absF = fixRelativePart(f);
732        return new FSLinkResolver<Boolean>() {
733          public Boolean next(final AbstractFileSystem fs, final Path p) 
734            throws IOException, UnresolvedLinkException {
735            return Boolean.valueOf(fs.delete(p, recursive));
736          }
737        }.resolve(this, absF);
738      }
739     
740      /**
741       * Opens an FSDataInputStream at the indicated Path using
742       * default buffersize.
743       * @param f the file name to open
744       *
745       * @throws AccessControlException If access is denied
746       * @throws FileNotFoundException If file <code>f</code> does not exist
747       * @throws UnsupportedFileSystemException If file system for <code>f</code>
748       *         is not supported
749       * @throws IOException If an I/O error occurred
750       * 
751       * Exceptions applicable to file systems accessed over RPC:
752       * @throws RpcClientException If an exception occurred in the RPC client
753       * @throws RpcServerException If an exception occurred in the RPC server
754       * @throws UnexpectedServerException If server implementation throws 
755       *           undeclared exception to RPC server
756       */
757      public FSDataInputStream open(final Path f) throws AccessControlException,
758          FileNotFoundException, UnsupportedFileSystemException, IOException {
759        final Path absF = fixRelativePart(f);
760        return new FSLinkResolver<FSDataInputStream>() {
761          public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 
762            throws IOException, UnresolvedLinkException {
763            return fs.open(p);
764          }
765        }.resolve(this, absF);
766      }
767    
768      /**
769       * Opens an FSDataInputStream at the indicated Path.
770       * 
771       * @param f the file name to open
772       * @param bufferSize the size of the buffer to be used.
773       * 
774       * @throws AccessControlException If access is denied
775       * @throws FileNotFoundException If file <code>f</code> does not exist
776       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
777       *           not supported
778       * @throws IOException If an I/O error occurred
779       * 
780       * Exceptions applicable to file systems accessed over RPC:
781       * @throws RpcClientException If an exception occurred in the RPC client
782       * @throws RpcServerException If an exception occurred in the RPC server
783       * @throws UnexpectedServerException If server implementation throws 
784       *           undeclared exception to RPC server
785       */
786      public FSDataInputStream open(final Path f, final int bufferSize)
787          throws AccessControlException, FileNotFoundException,
788          UnsupportedFileSystemException, IOException {
789        final Path absF = fixRelativePart(f);
790        return new FSLinkResolver<FSDataInputStream>() {
791          public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 
792            throws IOException, UnresolvedLinkException {
793            return fs.open(p, bufferSize);
794          }
795        }.resolve(this, absF);
796      }
797    
798      /**
799       * Set replication for an existing file.
800       * 
801       * @param f file name
802       * @param replication new replication
803       *
804       * @return true if successful
805       *
806       * @throws AccessControlException If access is denied
807       * @throws FileNotFoundException If file <code>f</code> does not exist
808       * @throws IOException If an I/O error occurred
809       * 
810       * Exceptions applicable to file systems accessed over RPC:
811       * @throws RpcClientException If an exception occurred in the RPC client
812       * @throws RpcServerException If an exception occurred in the RPC server
813       * @throws UnexpectedServerException If server implementation throws 
814       *           undeclared exception to RPC server
815       */
816      public boolean setReplication(final Path f, final short replication)
817          throws AccessControlException, FileNotFoundException,
818          IOException {
819        final Path absF = fixRelativePart(f);
820        return new FSLinkResolver<Boolean>() {
821          public Boolean next(final AbstractFileSystem fs, final Path p) 
822            throws IOException, UnresolvedLinkException {
823            return Boolean.valueOf(fs.setReplication(p, replication));
824          }
825        }.resolve(this, absF);
826      }
827    
828      /**
829       * Renames Path src to Path dst
830       * <ul>
831       * <li
832       * <li>Fails if src is a file and dst is a directory.
833       * <li>Fails if src is a directory and dst is a file.
834       * <li>Fails if the parent of dst does not exist or is a file.
835       * </ul>
836       * <p>
837       * If OVERWRITE option is not passed as an argument, rename fails if the dst
838       * already exists.
839       * <p>
840       * If OVERWRITE option is passed as an argument, rename overwrites the dst if
841       * it is a file or an empty directory. Rename fails if dst is a non-empty
842       * directory.
843       * <p>
844       * Note that atomicity of rename is dependent on the file system
845       * implementation. Please refer to the file system documentation for details
846       * <p>
847       * 
848       * @param src path to be renamed
849       * @param dst new path after rename
850       * 
851       * @throws AccessControlException If access is denied
852       * @throws FileAlreadyExistsException If <code>dst</code> already exists and
853       *           <code>options</options> has {@link Options.Rename#OVERWRITE} 
854       *           option false.
855       * @throws FileNotFoundException If <code>src</code> does not exist
856       * @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
857       *           directory
858       * @throws UnsupportedFileSystemException If file system for <code>src</code>
859       *           and <code>dst</code> is not supported
860       * @throws IOException If an I/O error occurred
861       * 
862       * Exceptions applicable to file systems accessed over RPC:
863       * @throws RpcClientException If an exception occurred in the RPC client
864       * @throws RpcServerException If an exception occurred in the RPC server
865       * @throws UnexpectedServerException If server implementation throws
866       *           undeclared exception to RPC server
867       */
868      public void rename(final Path src, final Path dst,
869          final Options.Rename... options) throws AccessControlException,
870          FileAlreadyExistsException, FileNotFoundException,
871          ParentNotDirectoryException, UnsupportedFileSystemException,
872          IOException {
873        final Path absSrc = fixRelativePart(src);
874        final Path absDst = fixRelativePart(dst);
875        AbstractFileSystem srcFS = getFSofPath(absSrc);
876        AbstractFileSystem dstFS = getFSofPath(absDst);
877        if(!srcFS.getUri().equals(dstFS.getUri())) {
878          throw new IOException("Renames across AbstractFileSystems not supported");
879        }
880        try {
881          srcFS.rename(absSrc, absDst, options);
882        } catch (UnresolvedLinkException e) {
883          /* We do not know whether the source or the destination path
884           * was unresolved. Resolve the source path up until the final
885           * path component, then fully resolve the destination. 
886           */
887          final Path source = resolveIntermediate(absSrc);    
888          new FSLinkResolver<Void>() {
889            public Void next(final AbstractFileSystem fs, final Path p) 
890              throws IOException, UnresolvedLinkException {
891              fs.rename(source, p, options);
892              return null;
893            }
894          }.resolve(this, absDst);
895        }
896      }
897      
898      /**
899       * Set permission of a path.
900       * @param f
901       * @param permission - the new absolute permission (umask is not applied)
902       *
903       * @throws AccessControlException If access is denied
904       * @throws FileNotFoundException If <code>f</code> does not exist
905       * @throws UnsupportedFileSystemException If file system for <code>f</code>
906       *         is not supported
907       * @throws IOException If an I/O error occurred
908       * 
909       * Exceptions applicable to file systems accessed over RPC:
910       * @throws RpcClientException If an exception occurred in the RPC client
911       * @throws RpcServerException If an exception occurred in the RPC server
912       * @throws UnexpectedServerException If server implementation throws 
913       *           undeclared exception to RPC server
914       */
915      public void setPermission(final Path f, final FsPermission permission)
916          throws AccessControlException, FileNotFoundException,
917          UnsupportedFileSystemException, IOException {
918        final Path absF = fixRelativePart(f);
919        new FSLinkResolver<Void>() {
920          public Void next(final AbstractFileSystem fs, final Path p) 
921            throws IOException, UnresolvedLinkException {
922            fs.setPermission(p, permission);
923            return null;
924          }
925        }.resolve(this, absF);
926      }
927    
928      /**
929       * Set owner of a path (i.e. a file or a directory). The parameters username
930       * and groupname cannot both be null.
931       * 
932       * @param f The path
933       * @param username If it is null, the original username remains unchanged.
934       * @param groupname If it is null, the original groupname remains unchanged.
935       * 
936       * @throws AccessControlException If access is denied
937       * @throws FileNotFoundException If <code>f</code> does not exist
938       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
939       *           not supported
940       * @throws IOException If an I/O error occurred
941       * 
942       * Exceptions applicable to file systems accessed over RPC:
943       * @throws RpcClientException If an exception occurred in the RPC client
944       * @throws RpcServerException If an exception occurred in the RPC server
945       * @throws UnexpectedServerException If server implementation throws 
946       *           undeclared exception to RPC server
947       * 
948       * RuntimeExceptions:
949       * @throws HadoopIllegalArgumentException If <code>username</code> or
950       *           <code>groupname</code> is invalid.
951       */
952      public void setOwner(final Path f, final String username,
953          final String groupname) throws AccessControlException,
954          UnsupportedFileSystemException, FileNotFoundException,
955          IOException {
956        if ((username == null) && (groupname == null)) {
957          throw new HadoopIllegalArgumentException(
958              "username and groupname cannot both be null");
959        }
960        final Path absF = fixRelativePart(f);
961        new FSLinkResolver<Void>() {
962          public Void next(final AbstractFileSystem fs, final Path p) 
963            throws IOException, UnresolvedLinkException {
964            fs.setOwner(p, username, groupname);
965            return null;
966          }
967        }.resolve(this, absF);
968      }
969    
970      /**
971       * Set access time of a file.
972       * @param f The path
973       * @param mtime Set the modification time of this file.
974       *        The number of milliseconds since epoch (Jan 1, 1970). 
975       *        A value of -1 means that this call should not set modification time.
976       * @param atime Set the access time of this file.
977       *        The number of milliseconds since Jan 1, 1970. 
978       *        A value of -1 means that this call should not set access time.
979       *
980       * @throws AccessControlException If access is denied
981       * @throws FileNotFoundException If <code>f</code> does not exist
982       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
983       *           not supported
984       * @throws IOException If an I/O error occurred
985       * 
986       * Exceptions applicable to file systems accessed over RPC:
987       * @throws RpcClientException If an exception occurred in the RPC client
988       * @throws RpcServerException If an exception occurred in the RPC server
989       * @throws UnexpectedServerException If server implementation throws 
990       *           undeclared exception to RPC server
991       */
992      public void setTimes(final Path f, final long mtime, final long atime)
993          throws AccessControlException, FileNotFoundException,
994          UnsupportedFileSystemException, IOException {
995        final Path absF = fixRelativePart(f);
996        new FSLinkResolver<Void>() {
997          public Void next(final AbstractFileSystem fs, final Path p) 
998            throws IOException, UnresolvedLinkException {
999            fs.setTimes(p, mtime, atime);
1000            return null;
1001          }
1002        }.resolve(this, absF);
1003      }
1004    
1005      /**
1006       * Get the checksum of a file.
1007       *
1008       * @param f file path
1009       *
1010       * @return The file checksum.  The default return value is null,
1011       *  which indicates that no checksum algorithm is implemented
1012       *  in the corresponding FileSystem.
1013       *
1014       * @throws AccessControlException If access is denied
1015       * @throws FileNotFoundException If <code>f</code> does not exist
1016       * @throws IOException If an I/O error occurred
1017       * 
1018       * Exceptions applicable to file systems accessed over RPC:
1019       * @throws RpcClientException If an exception occurred in the RPC client
1020       * @throws RpcServerException If an exception occurred in the RPC server
1021       * @throws UnexpectedServerException If server implementation throws 
1022       *           undeclared exception to RPC server
1023       */
1024      public FileChecksum getFileChecksum(final Path f)
1025          throws AccessControlException, FileNotFoundException,
1026          IOException {
1027        final Path absF = fixRelativePart(f);
1028        return new FSLinkResolver<FileChecksum>() {
1029          public FileChecksum next(final AbstractFileSystem fs, final Path p) 
1030            throws IOException, UnresolvedLinkException {
1031            return fs.getFileChecksum(p);
1032          }
1033        }.resolve(this, absF);
1034      }
1035    
1036      /**
1037       * Set the verify checksum flag for the  file system denoted by the path.
1038       * This is only applicable if the 
1039       * corresponding FileSystem supports checksum. By default doesn't do anything.
1040       * @param verifyChecksum
1041       * @param f set the verifyChecksum for the Filesystem containing this path
1042       *
1043       * @throws AccessControlException If access is denied
1044       * @throws FileNotFoundException If <code>f</code> does not exist
1045       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1046       *           not supported
1047       * @throws IOException If an I/O error occurred
1048       * 
1049       * Exceptions applicable to file systems accessed over RPC:
1050       * @throws RpcClientException If an exception occurred in the RPC client
1051       * @throws RpcServerException If an exception occurred in the RPC server
1052       * @throws UnexpectedServerException If server implementation throws 
1053       *           undeclared exception to RPC server
1054       */
1055      public void setVerifyChecksum(final boolean verifyChecksum, final Path f)
1056          throws AccessControlException, FileNotFoundException,
1057          UnsupportedFileSystemException, IOException {
1058        final Path absF = resolve(fixRelativePart(f));
1059        getFSofPath(absF).setVerifyChecksum(verifyChecksum);
1060      }
1061    
1062      /**
1063       * Return a file status object that represents the path.
1064       * @param f The path we want information from
1065       *
1066       * @return a FileStatus object
1067       *
1068       * @throws AccessControlException If access is denied
1069       * @throws FileNotFoundException If <code>f</code> does not exist
1070       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1071       *           not supported
1072       * @throws IOException If an I/O error occurred
1073       * 
1074       * Exceptions applicable to file systems accessed over RPC:
1075       * @throws RpcClientException If an exception occurred in the RPC client
1076       * @throws RpcServerException If an exception occurred in the RPC server
1077       * @throws UnexpectedServerException If server implementation throws 
1078       *           undeclared exception to RPC server
1079       */
1080      public FileStatus getFileStatus(final Path f) throws AccessControlException,
1081          FileNotFoundException, UnsupportedFileSystemException, IOException {
1082        final Path absF = fixRelativePart(f);
1083        return new FSLinkResolver<FileStatus>() {
1084          public FileStatus next(final AbstractFileSystem fs, final Path p) 
1085            throws IOException, UnresolvedLinkException {
1086            return fs.getFileStatus(p);
1087          }
1088        }.resolve(this, absF);
1089      }
1090    
1091      /**
1092       * Return a fully qualified version of the given symlink target if it
1093       * has no scheme and authority. Partially and fully qualified paths 
1094       * are returned unmodified.
1095       * @param linkFS The AbstractFileSystem of link 
1096       * @param link   The path of the symlink
1097       * @param target The symlink's target
1098       * @return Fully qualified version of the target.
1099       */
1100      private Path qualifySymlinkTarget(final AbstractFileSystem linkFS, 
1101          Path link, Path target) {
1102        /* NB: makeQualified uses link's scheme/authority, if specified, 
1103         * and the scheme/authority of linkFS, if not. If link does have
1104         * a scheme and authority they should match those of linkFS since
1105         * resolve updates the path and file system of a path that contains
1106         * links each time a link is encountered.
1107         */
1108        final String linkScheme = link.toUri().getScheme();
1109        final String linkAuth   = link.toUri().getAuthority();
1110        if (linkScheme != null && linkAuth != null) {
1111          assert linkScheme.equals(linkFS.getUri().getScheme());
1112          assert linkAuth.equals(linkFS.getUri().getAuthority());
1113        }
1114        final boolean justPath = (target.toUri().getScheme() == null &&
1115                                  target.toUri().getAuthority() == null);
1116        return justPath ? target.makeQualified(linkFS.getUri(), link.getParent()) 
1117                        : target;
1118      }
1119      
1120      /**
1121       * Return a file status object that represents the path. If the path 
1122       * refers to a symlink then the FileStatus of the symlink is returned.
1123       * The behavior is equivalent to #getFileStatus() if the underlying
1124       * file system does not support symbolic links.
1125       * @param  f The path we want information from.
1126       * @return A FileStatus object
1127       * 
1128       * @throws AccessControlException If access is denied
1129       * @throws FileNotFoundException If <code>f</code> does not exist
1130       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1131       *           not supported
1132       * @throws IOException If an I/O error occurred
1133       */
1134      public FileStatus getFileLinkStatus(final Path f)
1135          throws AccessControlException, FileNotFoundException,
1136          UnsupportedFileSystemException, IOException {
1137        final Path absF = fixRelativePart(f);
1138        return new FSLinkResolver<FileStatus>() {
1139          public FileStatus next(final AbstractFileSystem fs, final Path p) 
1140            throws IOException, UnresolvedLinkException {
1141            FileStatus fi = fs.getFileLinkStatus(p);
1142            if (fi.isSymlink()) {
1143              fi.setSymlink(qualifySymlinkTarget(fs, p, fi.getSymlink()));
1144            }
1145            return fi;
1146          }
1147        }.resolve(this, absF);
1148      }
1149      
1150      /**
1151       * Returns the un-interpreted target of the given symbolic link.
1152       * Transparently resolves all links up to the final path component.
1153       * @param f
1154       * @return The un-interpreted target of the symbolic link.
1155       * 
1156       * @throws AccessControlException If access is denied
1157       * @throws FileNotFoundException If path <code>f</code> does not exist
1158       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1159       *           not supported
1160       * @throws IOException If an I/O error occurred
1161       */
1162      public Path getLinkTarget(final Path f) throws AccessControlException,
1163          FileNotFoundException, UnsupportedFileSystemException, IOException {
1164        final Path absF = fixRelativePart(f);
1165        return new FSLinkResolver<Path>() {
1166          public Path next(final AbstractFileSystem fs, final Path p) 
1167            throws IOException, UnresolvedLinkException {
1168            FileStatus fi = fs.getFileLinkStatus(p);
1169            return fi.getSymlink();
1170          }
1171        }.resolve(this, absF);
1172      }
1173      
1174      /**
1175       * Return blockLocation of the given file for the given offset and len.
1176       *  For a nonexistent file or regions, null will be returned.
1177       *
1178       * This call is most helpful with DFS, where it returns 
1179       * hostnames of machines that contain the given file.
1180       * 
1181       * @param f - get blocklocations of this file
1182       * @param start position (byte offset)
1183       * @param len (in bytes)
1184       *
1185       * @return block locations for given file at specified offset of len
1186       *
1187       * @throws AccessControlException If access is denied
1188       * @throws FileNotFoundException If <code>f</code> does not exist
1189       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1190       *           not supported
1191       * @throws IOException If an I/O error occurred
1192       * 
1193       * Exceptions applicable to file systems accessed over RPC:
1194       * @throws RpcClientException If an exception occurred in the RPC client
1195       * @throws RpcServerException If an exception occurred in the RPC server
1196       * @throws UnexpectedServerException If server implementation throws 
1197       *           undeclared exception to RPC server
1198       * 
1199       * RuntimeExceptions:
1200       * @throws InvalidPathException If path <code>f</code> is invalid
1201       */
1202      @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
1203      @InterfaceStability.Evolving
1204      public BlockLocation[] getFileBlockLocations(final Path f, final long start,
1205          final long len) throws AccessControlException, FileNotFoundException,
1206          UnsupportedFileSystemException, IOException {
1207        final Path absF = fixRelativePart(f);
1208        return new FSLinkResolver<BlockLocation[]>() {
1209          public BlockLocation[] next(final AbstractFileSystem fs, final Path p) 
1210            throws IOException, UnresolvedLinkException {
1211            return fs.getFileBlockLocations(p, start, len);
1212          }
1213        }.resolve(this, absF);
1214      }
1215      
1216      /**
1217       * Returns a status object describing the use and capacity of the
1218       * file system denoted by the Parh argument p.
1219       * If the file system has multiple partitions, the
1220       * use and capacity of the partition pointed to by the specified
1221       * path is reflected.
1222       * 
1223       * @param f Path for which status should be obtained. null means the
1224       * root partition of the default file system. 
1225       *
1226       * @return a FsStatus object
1227       *
1228       * @throws AccessControlException If access is denied
1229       * @throws FileNotFoundException If <code>f</code> does not exist
1230       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1231       *           not supported
1232       * @throws IOException If an I/O error occurred
1233       * 
1234       * Exceptions applicable to file systems accessed over RPC:
1235       * @throws RpcClientException If an exception occurred in the RPC client
1236       * @throws RpcServerException If an exception occurred in the RPC server
1237       * @throws UnexpectedServerException If server implementation throws 
1238       *           undeclared exception to RPC server
1239       */
1240      public FsStatus getFsStatus(final Path f) throws AccessControlException,
1241          FileNotFoundException, UnsupportedFileSystemException, IOException {
1242        if (f == null) {
1243          return defaultFS.getFsStatus();
1244        }
1245        final Path absF = fixRelativePart(f);
1246        return new FSLinkResolver<FsStatus>() {
1247          public FsStatus next(final AbstractFileSystem fs, final Path p) 
1248            throws IOException, UnresolvedLinkException {
1249            return fs.getFsStatus(p);
1250          }
1251        }.resolve(this, absF);
1252      }
1253    
1254      /**
1255       * Creates a symbolic link to an existing file. An exception is thrown if 
1256       * the symlink exits, the user does not have permission to create symlink,
1257       * or the underlying file system does not support symlinks.
1258       * 
1259       * Symlink permissions are ignored, access to a symlink is determined by
1260       * the permissions of the symlink target.
1261       * 
1262       * Symlinks in paths leading up to the final path component are resolved 
1263       * transparently. If the final path component refers to a symlink some 
1264       * functions operate on the symlink itself, these are:
1265       * - delete(f) and deleteOnExit(f) - Deletes the symlink.
1266       * - rename(src, dst) - If src refers to a symlink, the symlink is 
1267       *   renamed. If dst refers to a symlink, the symlink is over-written.
1268       * - getLinkTarget(f) - Returns the target of the symlink. 
1269       * - getFileLinkStatus(f) - Returns a FileStatus object describing
1270       *   the symlink.
1271       * Some functions, create() and mkdir(), expect the final path component
1272       * does not exist. If they are given a path that refers to a symlink that 
1273       * does exist they behave as if the path referred to an existing file or 
1274       * directory. All other functions fully resolve, ie follow, the symlink. 
1275       * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
1276       * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
1277       * getFsStatus, getFileStatus, exists, and listStatus.
1278       * 
1279       * Symlink targets are stored as given to createSymlink, assuming the 
1280       * underlying file system is capable of storign a fully qualified URI. 
1281       * Dangling symlinks are permitted. FileContext supports four types of 
1282       * symlink targets, and resolves them as follows
1283       * <pre>
1284       * Given a path referring to a symlink of form:
1285       * 
1286       *   <---X---> 
1287       *   fs://host/A/B/link 
1288       *   <-----Y----->
1289       * 
1290       * In this path X is the scheme and authority that identify the file system,
1291       * and Y is the path leading up to the final path component "link". If Y is
1292       * a symlink  itself then let Y' be the target of Y and X' be the scheme and
1293       * authority of Y'. Symlink targets may:
1294       * 
1295       * 1. Fully qualified URIs
1296       * 
1297       * fs://hostX/A/B/file  Resolved according to the target file system.
1298       * 
1299       * 2. Partially qualified URIs (eg scheme but no host)
1300       * 
1301       * fs:///A/B/file  Resolved according to the target file sytem. Eg resolving
1302       *                 a symlink to hdfs:///A results in an exception because
1303       *                 HDFS URIs must be fully qualified, while a symlink to 
1304       *                 file:///A will not since Hadoop's local file systems 
1305       *                 require partially qualified URIs.
1306       * 
1307       * 3. Relative paths
1308       * 
1309       * path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
1310       *       is "../B/file" then [Y'][path] is hdfs://host/B/file
1311       * 
1312       * 4. Absolute paths
1313       * 
1314       * path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
1315       *       is "/file" then [X][path] is hdfs://host/file
1316       * </pre>
1317       * 
1318       * @param target the target of the symbolic link
1319       * @param link the path to be created that points to target
1320       * @param createParent if true then missing parent dirs are created if 
1321       *                     false then parent must exist
1322       *
1323       *
1324       * @throws AccessControlException If access is denied
1325       * @throws FileAlreadyExistsException If file <code>linkcode> already exists
1326       * @throws FileNotFoundException If <code>target</code> does not exist
1327       * @throws ParentNotDirectoryException If parent of <code>link</code> is not a
1328       *           directory.
1329       * @throws UnsupportedFileSystemException If file system for 
1330       *           <code>target</code> or <code>link</code> is not supported
1331       * @throws IOException If an I/O error occurred
1332       */
1333      public void createSymlink(final Path target, final Path link,
1334          final boolean createParent) throws AccessControlException,
1335          FileAlreadyExistsException, FileNotFoundException,
1336          ParentNotDirectoryException, UnsupportedFileSystemException, 
1337          IOException { 
1338        final Path nonRelLink = fixRelativePart(link);
1339        new FSLinkResolver<Void>() {
1340          public Void next(final AbstractFileSystem fs, final Path p) 
1341            throws IOException, UnresolvedLinkException {
1342            fs.createSymlink(target, p, createParent);
1343            return null;
1344          }
1345        }.resolve(this, nonRelLink);
1346      }
1347      
1348      /**
1349       * List the statuses of the files/directories in the given path if the path is
1350       * a directory.
1351       * 
1352       * @param f is the path
1353       *
1354       * @return an iterator that traverses statuses of the files/directories 
1355       *         in the given path
1356       *
1357       * @throws AccessControlException If access is denied
1358       * @throws FileNotFoundException If <code>f</code> does not exist
1359       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1360       *           not supported
1361       * @throws IOException If an I/O error occurred
1362       * 
1363       * Exceptions applicable to file systems accessed over RPC:
1364       * @throws RpcClientException If an exception occurred in the RPC client
1365       * @throws RpcServerException If an exception occurred in the RPC server
1366       * @throws UnexpectedServerException If server implementation throws 
1367       *           undeclared exception to RPC server
1368       */
1369      public RemoteIterator<FileStatus> listStatus(final Path f) throws
1370          AccessControlException, FileNotFoundException,
1371          UnsupportedFileSystemException, IOException {
1372        final Path absF = fixRelativePart(f);
1373        return new FSLinkResolver<RemoteIterator<FileStatus>>() {
1374          public RemoteIterator<FileStatus> next(
1375              final AbstractFileSystem fs, final Path p) 
1376            throws IOException, UnresolvedLinkException {
1377            return fs.listStatusIterator(p);
1378          }
1379        }.resolve(this, absF);
1380      }
1381    
1382      /**
1383       * @return an iterator over the corrupt files under the given path
1384       * (may contain duplicates if a file has more than one corrupt block)
1385       * @throws IOException
1386       */
1387      public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1388        throws IOException {
1389        final Path absF = fixRelativePart(path);
1390        return new FSLinkResolver<RemoteIterator<Path>>() {
1391          @Override
1392          public RemoteIterator<Path> next(final AbstractFileSystem fs,
1393                                           final Path p) 
1394            throws IOException, UnresolvedLinkException {
1395            return fs.listCorruptFileBlocks(p);
1396          }
1397        }.resolve(this, absF);
1398      }
1399      
1400      /**
1401       * List the statuses of the files/directories in the given path if the path is
1402       * a directory. 
1403       * Return the file's status and block locations If the path is a file.
1404       * 
1405       * If a returned status is a file, it contains the file's block locations.
1406       * 
1407       * @param f is the path
1408       *
1409       * @return an iterator that traverses statuses of the files/directories 
1410       *         in the given path
1411       * If any IO exception (for example the input directory gets deleted while
1412       * listing is being executed), next() or hasNext() of the returned iterator
1413       * may throw a RuntimeException with the io exception as the cause.
1414       *
1415       * @throws AccessControlException If access is denied
1416       * @throws FileNotFoundException If <code>f</code> does not exist
1417       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1418       *           not supported
1419       * @throws IOException If an I/O error occurred
1420       * 
1421       * Exceptions applicable to file systems accessed over RPC:
1422       * @throws RpcClientException If an exception occurred in the RPC client
1423       * @throws RpcServerException If an exception occurred in the RPC server
1424       * @throws UnexpectedServerException If server implementation throws 
1425       *           undeclared exception to RPC server
1426       */
1427      public RemoteIterator<LocatedFileStatus> listLocatedStatus(
1428          final Path f) throws
1429          AccessControlException, FileNotFoundException,
1430          UnsupportedFileSystemException, IOException {
1431        final Path absF = fixRelativePart(f);
1432        return new FSLinkResolver<RemoteIterator<LocatedFileStatus>>() {
1433          public RemoteIterator<LocatedFileStatus> next(
1434              final AbstractFileSystem fs, final Path p) 
1435            throws IOException, UnresolvedLinkException {
1436            return fs.listLocatedStatus(p);
1437          }
1438        }.resolve(this, absF);
1439      }
1440    
1441      /**
1442       * Mark a path to be deleted on JVM shutdown.
1443       * 
1444       * @param f the existing path to delete.
1445       *
1446       * @return  true if deleteOnExit is successful, otherwise false.
1447       *
1448       * @throws AccessControlException If access is denied
1449       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1450       *           not supported
1451       * @throws IOException If an I/O error occurred
1452       * 
1453       * Exceptions applicable to file systems accessed over RPC:
1454       * @throws RpcClientException If an exception occurred in the RPC client
1455       * @throws RpcServerException If an exception occurred in the RPC server
1456       * @throws UnexpectedServerException If server implementation throws 
1457       *           undeclared exception to RPC server
1458       */
1459      public boolean deleteOnExit(Path f) throws AccessControlException,
1460          IOException {
1461        if (!this.util().exists(f)) {
1462          return false;
1463        }
1464        synchronized (DELETE_ON_EXIT) {
1465          if (DELETE_ON_EXIT.isEmpty() && !FINALIZER.isAlive()) {
1466            Runtime.getRuntime().addShutdownHook(FINALIZER);
1467          }
1468          
1469          Set<Path> set = DELETE_ON_EXIT.get(this);
1470          if (set == null) {
1471            set = new TreeSet<Path>();
1472            DELETE_ON_EXIT.put(this, set);
1473          }
1474          set.add(f);
1475        }
1476        return true;
1477      }
1478      
1479      private final Util util;
1480      public Util util() {
1481        return util;
1482      }
1483      
1484      
1485      /**
1486       * Utility/library methods built over the basic FileContext methods.
1487       * Since this are library functions, the oprtation are not atomic
1488       * and some of them may partially complete if other threads are making
1489       * changes to the same part of the name space.
1490       */
1491      public class Util {
1492        /**
1493         * Does the file exist?
1494         * Note: Avoid using this method if you already have FileStatus in hand.
1495         * Instead reuse the FileStatus 
1496         * @param f the  file or dir to be checked
1497         *
1498         * @throws AccessControlException If access is denied
1499         * @throws IOException If an I/O error occurred
1500         * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1501         *           not supported
1502         * 
1503         * Exceptions applicable to file systems accessed over RPC:
1504         * @throws RpcClientException If an exception occurred in the RPC client
1505         * @throws RpcServerException If an exception occurred in the RPC server
1506         * @throws UnexpectedServerException If server implementation throws 
1507         *           undeclared exception to RPC server
1508         */
1509        public boolean exists(final Path f) throws AccessControlException,
1510          UnsupportedFileSystemException, IOException {
1511          try {
1512            FileStatus fs = FileContext.this.getFileStatus(f);
1513            assert fs != null;
1514            return true;
1515          } catch (FileNotFoundException e) {
1516            return false;
1517          }
1518        }
1519        
1520        /**
1521         * Return a list of file status objects that corresponds to supplied paths
1522         * excluding those non-existent paths.
1523         * 
1524         * @param paths list of paths we want information from
1525         *
1526         * @return a list of FileStatus objects
1527         *
1528         * @throws AccessControlException If access is denied
1529         * @throws IOException If an I/O error occurred
1530         * 
1531         * Exceptions applicable to file systems accessed over RPC:
1532         * @throws RpcClientException If an exception occurred in the RPC client
1533         * @throws RpcServerException If an exception occurred in the RPC server
1534         * @throws UnexpectedServerException If server implementation throws 
1535         *           undeclared exception to RPC server
1536         */
1537        private FileStatus[] getFileStatus(Path[] paths)
1538            throws AccessControlException, IOException {
1539          if (paths == null) {
1540            return null;
1541          }
1542          ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
1543          for (int i = 0; i < paths.length; i++) {
1544            try {
1545              results.add(FileContext.this.getFileStatus(paths[i]));
1546            } catch (FileNotFoundException fnfe) {
1547              // ignoring 
1548            }
1549          }
1550          return results.toArray(new FileStatus[results.size()]);
1551        }
1552        
1553        
1554        /**
1555         * Return the {@link ContentSummary} of path f.
1556         * @param f path
1557         *
1558         * @return the {@link ContentSummary} of path f.
1559         *
1560         * @throws AccessControlException If access is denied
1561         * @throws FileNotFoundException If <code>f</code> does not exist
1562         * @throws UnsupportedFileSystemException If file system for 
1563         *         <code>f</code> is not supported
1564         * @throws IOException If an I/O error occurred
1565         * 
1566         * Exceptions applicable to file systems accessed over RPC:
1567         * @throws RpcClientException If an exception occurred in the RPC client
1568         * @throws RpcServerException If an exception occurred in the RPC server
1569         * @throws UnexpectedServerException If server implementation throws 
1570         *           undeclared exception to RPC server
1571         */
1572        public ContentSummary getContentSummary(Path f)
1573            throws AccessControlException, FileNotFoundException,
1574            UnsupportedFileSystemException, IOException {
1575          FileStatus status = FileContext.this.getFileStatus(f);
1576          if (status.isFile()) {
1577            return new ContentSummary(status.getLen(), 1, 0);
1578          }
1579          long[] summary = {0, 0, 1};
1580          RemoteIterator<FileStatus> statusIterator = 
1581            FileContext.this.listStatus(f);
1582          while(statusIterator.hasNext()) {
1583            FileStatus s = statusIterator.next();
1584            ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1585                                           new ContentSummary(s.getLen(), 1, 0);
1586            summary[0] += c.getLength();
1587            summary[1] += c.getFileCount();
1588            summary[2] += c.getDirectoryCount();
1589          }
1590          return new ContentSummary(summary[0], summary[1], summary[2]);
1591        }
1592        
1593        /**
1594         * See {@link #listStatus(Path[], PathFilter)}
1595         */
1596        public FileStatus[] listStatus(Path[] files) throws AccessControlException,
1597            FileNotFoundException, IOException {
1598          return listStatus(files, DEFAULT_FILTER);
1599        }
1600         
1601        /**
1602         * Filter files/directories in the given path using the user-supplied path
1603         * filter.
1604         * 
1605         * @param f is the path name
1606         * @param filter is the user-supplied path filter
1607         *
1608         * @return an array of FileStatus objects for the files under the given path
1609         *         after applying the filter
1610         *
1611         * @throws AccessControlException If access is denied
1612         * @throws FileNotFoundException If <code>f</code> does not exist
1613         * @throws UnsupportedFileSystemException If file system for 
1614         *         <code>pathPattern</code> is not supported
1615         * @throws IOException If an I/O error occurred
1616         * 
1617         * Exceptions applicable to file systems accessed over RPC:
1618         * @throws RpcClientException If an exception occurred in the RPC client
1619         * @throws RpcServerException If an exception occurred in the RPC server
1620         * @throws UnexpectedServerException If server implementation throws 
1621         *           undeclared exception to RPC server
1622         */
1623        public FileStatus[] listStatus(Path f, PathFilter filter)
1624            throws AccessControlException, FileNotFoundException,
1625            UnsupportedFileSystemException, IOException {
1626          ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1627          listStatus(results, f, filter);
1628          return results.toArray(new FileStatus[results.size()]);
1629        }
1630        
1631        /**
1632         * Filter files/directories in the given list of paths using user-supplied
1633         * path filter.
1634         * 
1635         * @param files is a list of paths
1636         * @param filter is the filter
1637         *
1638         * @return a list of statuses for the files under the given paths after
1639         *         applying the filter
1640         *
1641         * @throws AccessControlException If access is denied
1642         * @throws FileNotFoundException If a file in <code>files</code> does not 
1643         *           exist
1644         * @throws IOException If an I/O error occurred
1645         * 
1646         * Exceptions applicable to file systems accessed over RPC:
1647         * @throws RpcClientException If an exception occurred in the RPC client
1648         * @throws RpcServerException If an exception occurred in the RPC server
1649         * @throws UnexpectedServerException If server implementation throws 
1650         *           undeclared exception to RPC server
1651         */
1652        public FileStatus[] listStatus(Path[] files, PathFilter filter)
1653            throws AccessControlException, FileNotFoundException, IOException {
1654          ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1655          for (int i = 0; i < files.length; i++) {
1656            listStatus(results, files[i], filter);
1657          }
1658          return results.toArray(new FileStatus[results.size()]);
1659        }
1660      
1661        /*
1662         * Filter files/directories in the given path using the user-supplied path
1663         * filter. Results are added to the given array <code>results</code>.
1664         */
1665        private void listStatus(ArrayList<FileStatus> results, Path f,
1666            PathFilter filter) throws AccessControlException,
1667            FileNotFoundException, IOException {
1668          FileStatus[] listing = listStatus(f);
1669          if (listing != null) {
1670            for (int i = 0; i < listing.length; i++) {
1671              if (filter.accept(listing[i].getPath())) {
1672                results.add(listing[i]);
1673              }
1674            }
1675          }
1676        }
1677    
1678        /**
1679         * List the statuses of the files/directories in the given path 
1680         * if the path is a directory.
1681         * 
1682         * @param f is the path
1683         *
1684         * @return an array that contains statuses of the files/directories 
1685         *         in the given path
1686         *
1687         * @throws AccessControlException If access is denied
1688         * @throws FileNotFoundException If <code>f</code> does not exist
1689         * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1690         *           not supported
1691         * @throws IOException If an I/O error occurred
1692         * 
1693         * Exceptions applicable to file systems accessed over RPC:
1694         * @throws RpcClientException If an exception occurred in the RPC client
1695         * @throws RpcServerException If an exception occurred in the RPC server
1696         * @throws UnexpectedServerException If server implementation throws 
1697         *           undeclared exception to RPC server
1698         */
1699        public FileStatus[] listStatus(final Path f) throws AccessControlException,
1700            FileNotFoundException, UnsupportedFileSystemException,
1701            IOException {
1702          final Path absF = fixRelativePart(f);
1703          return new FSLinkResolver<FileStatus[]>() {
1704            public FileStatus[] next(final AbstractFileSystem fs, final Path p) 
1705              throws IOException, UnresolvedLinkException {
1706              return fs.listStatus(p);
1707            }
1708          }.resolve(FileContext.this, absF);
1709        }
1710    
1711        /**
1712         * List the statuses and block locations of the files in the given path.
1713         * 
1714         * If the path is a directory, 
1715         *   if recursive is false, returns files in the directory;
1716         *   if recursive is true, return files in the subtree rooted at the path.
1717         *   The subtree is traversed in the depth-first order.
1718         * If the path is a file, return the file's status and block locations.
1719         * Files across symbolic links are also returned.
1720         * 
1721         * @param f is the path
1722         * @param recursive if the subdirectories need to be traversed recursively
1723         *
1724         * @return an iterator that traverses statuses of the files
1725         * If any IO exception (for example a sub-directory gets deleted while
1726         * listing is being executed), next() or hasNext() of the returned iterator
1727         * may throw a RuntimeException with the IO exception as the cause.
1728         *
1729         * @throws AccessControlException If access is denied
1730         * @throws FileNotFoundException If <code>f</code> does not exist
1731         * @throws UnsupportedFileSystemException If file system for <code>f</code>
1732         *         is not supported
1733         * @throws IOException If an I/O error occurred
1734         * 
1735         * Exceptions applicable to file systems accessed over RPC:
1736         * @throws RpcClientException If an exception occurred in the RPC client
1737         * @throws RpcServerException If an exception occurred in the RPC server
1738         * @throws UnexpectedServerException If server implementation throws 
1739         *           undeclared exception to RPC server
1740         */
1741        public RemoteIterator<LocatedFileStatus> listFiles(
1742            final Path f, final boolean recursive) throws AccessControlException,
1743            FileNotFoundException, UnsupportedFileSystemException, 
1744            IOException {
1745          return new RemoteIterator<LocatedFileStatus>() {
1746            private Stack<RemoteIterator<LocatedFileStatus>> itors = 
1747              new Stack<RemoteIterator<LocatedFileStatus>>();
1748            RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f);
1749            LocatedFileStatus curFile;
1750    
1751            /**
1752             * Returns <tt>true</tt> if the iterator has more files.
1753             *
1754             * @return <tt>true</tt> if the iterator has more files.
1755             * @throws AccessControlException if not allowed to access next
1756             *                                file's status or locations
1757             * @throws FileNotFoundException if next file does not exist any more
1758             * @throws UnsupportedFileSystemException if next file's 
1759             *                                        fs is unsupported
1760             * @throws IOException for all other IO errors
1761             *                     for example, NameNode is not avaialbe or
1762             *                     NameNode throws IOException due to an error
1763             *                     while getting the status or block locations
1764             */
1765            @Override
1766            public boolean hasNext() throws IOException {
1767              while (curFile == null) {
1768                if (curItor.hasNext()) {
1769                  handleFileStat(curItor.next());
1770                } else if (!itors.empty()) {
1771                  curItor = itors.pop();
1772                } else {
1773                  return false;
1774                }
1775              }
1776              return true;
1777            }
1778    
1779            /**
1780             * Process the input stat.
1781             * If it is a file, return the file stat.
1782             * If it is a directory, traverse the directory if recursive is true;
1783             * ignore it if recursive is false.
1784             * If it is a symlink, resolve the symlink first and then process it
1785             * depending on if it is a file or directory.
1786             * @param stat input status
1787             * @throws AccessControlException if access is denied
1788             * @throws FileNotFoundException if file is not found
1789             * @throws UnsupportedFileSystemException if fs is not supported
1790             * @throws IOException for all other IO errors
1791             */
1792            private void handleFileStat(LocatedFileStatus stat)
1793            throws IOException {
1794              if (stat.isFile()) { // file
1795                curFile = stat;
1796              } else if (stat.isSymlink()) { // symbolic link
1797                // resolve symbolic link
1798                FileStatus symstat = FileContext.this.getFileStatus(
1799                    stat.getSymlink());
1800                if (symstat.isFile() || (recursive && symstat.isDirectory())) {
1801                  itors.push(curItor);
1802                  curItor = listLocatedStatus(stat.getPath());
1803                }
1804              } else if (recursive) { // directory
1805                itors.push(curItor);
1806                curItor = listLocatedStatus(stat.getPath());
1807              }
1808            }
1809    
1810            /**
1811             * Returns the next file's status with its block locations
1812             *
1813             * @throws AccessControlException if not allowed to access next
1814             *                                file's status or locations
1815             * @throws FileNotFoundException if next file does not exist any more
1816             * @throws UnsupportedFileSystemException if next file's 
1817             *                                        fs is unsupported
1818             * @throws IOException for all other IO errors
1819             *                     for example, NameNode is not avaialbe or
1820             *                     NameNode throws IOException due to an error
1821             *                     while getting the status or block locations
1822             */
1823            @Override
1824            public LocatedFileStatus next() throws IOException {
1825              if (hasNext()) {
1826                LocatedFileStatus result = curFile;
1827                curFile = null;
1828                return result;
1829              } 
1830              throw new java.util.NoSuchElementException("No more entry in " + f);
1831            }
1832          };
1833        }
1834    
1835        /**
1836         * <p>Return all the files that match filePattern and are not checksum
1837         * files. Results are sorted by their names.
1838         * 
1839         * <p>
1840         * A filename pattern is composed of <i>regular</i> characters and
1841         * <i>special pattern matching</i> characters, which are:
1842         *
1843         * <dl>
1844         *  <dd>
1845         *   <dl>
1846         *    <p>
1847         *    <dt> <tt> ? </tt>
1848         *    <dd> Matches any single character.
1849         *
1850         *    <p>
1851         *    <dt> <tt> * </tt>
1852         *    <dd> Matches zero or more characters.
1853         *
1854         *    <p>
1855         *    <dt> <tt> [<i>abc</i>] </tt>
1856         *    <dd> Matches a single character from character set
1857         *     <tt>{<i>a,b,c</i>}</tt>.
1858         *
1859         *    <p>
1860         *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1861         *    <dd> Matches a single character from the character range
1862         *     <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be
1863         *     lexicographically less than or equal to character <tt><i>b</i></tt>.
1864         *
1865         *    <p>
1866         *    <dt> <tt> [^<i>a</i>] </tt>
1867         *    <dd> Matches a single char that is not from character set or range
1868         *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
1869         *     immediately to the right of the opening bracket.
1870         *
1871         *    <p>
1872         *    <dt> <tt> \<i>c</i> </tt>
1873         *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
1874         *
1875         *    <p>
1876         *    <dt> <tt> {ab,cd} </tt>
1877         *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1878         *    
1879         *    <p>
1880         *    <dt> <tt> {ab,c{de,fh}} </tt>
1881         *    <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt>
1882         *
1883         *   </dl>
1884         *  </dd>
1885         * </dl>
1886         *
1887         * @param pathPattern a regular expression specifying a pth pattern
1888         *
1889         * @return an array of paths that match the path pattern
1890         *
1891         * @throws AccessControlException If access is denied
1892         * @throws UnsupportedFileSystemException If file system for 
1893         *         <code>pathPattern</code> is not supported
1894         * @throws IOException If an I/O error occurred
1895         * 
1896         * Exceptions applicable to file systems accessed over RPC:
1897         * @throws RpcClientException If an exception occurred in the RPC client
1898         * @throws RpcServerException If an exception occurred in the RPC server
1899         * @throws UnexpectedServerException If server implementation throws 
1900         *           undeclared exception to RPC server
1901         */
1902        public FileStatus[] globStatus(Path pathPattern)
1903            throws AccessControlException, UnsupportedFileSystemException,
1904            IOException {
1905          return globStatus(pathPattern, DEFAULT_FILTER);
1906        }
1907        
1908        /**
1909         * Return an array of FileStatus objects whose path names match pathPattern
1910         * and is accepted by the user-supplied path filter. Results are sorted by
1911         * their path names.
1912         * Return null if pathPattern has no glob and the path does not exist.
1913         * Return an empty array if pathPattern has a glob and no path matches it. 
1914         * 
1915         * @param pathPattern regular expression specifying the path pattern
1916         * @param filter user-supplied path filter
1917         *
1918         * @return an array of FileStatus objects
1919         *
1920         * @throws AccessControlException If access is denied
1921         * @throws UnsupportedFileSystemException If file system for 
1922         *         <code>pathPattern</code> is not supported
1923         * @throws IOException If an I/O error occurred
1924         * 
1925         * Exceptions applicable to file systems accessed over RPC:
1926         * @throws RpcClientException If an exception occurred in the RPC client
1927         * @throws RpcServerException If an exception occurred in the RPC server
1928         * @throws UnexpectedServerException If server implementation throws 
1929         *           undeclared exception to RPC server
1930         */
1931        public FileStatus[] globStatus(final Path pathPattern,
1932            final PathFilter filter) throws AccessControlException,
1933            UnsupportedFileSystemException, IOException {
1934          URI uri = getFSofPath(fixRelativePart(pathPattern)).getUri();
1935    
1936          String filename = pathPattern.toUri().getPath();
1937    
1938          List<String> filePatterns = GlobExpander.expand(filename);
1939          if (filePatterns.size() == 1) {
1940            Path absPathPattern = fixRelativePart(pathPattern);
1941            return globStatusInternal(uri, new Path(absPathPattern.toUri()
1942                .getPath()), filter);
1943          } else {
1944            List<FileStatus> results = new ArrayList<FileStatus>();
1945            for (String iFilePattern : filePatterns) {
1946              Path iAbsFilePattern = fixRelativePart(new Path(iFilePattern));
1947              FileStatus[] files = globStatusInternal(uri, iAbsFilePattern, filter);
1948              for (FileStatus file : files) {
1949                results.add(file);
1950              }
1951            }
1952            return results.toArray(new FileStatus[results.size()]);
1953          }
1954        }
1955    
1956        /**
1957         * 
1958         * @param uri for all the inPathPattern
1959         * @param inPathPattern - without the scheme & authority (take from uri)
1960         * @param filter
1961         *
1962         * @return an array of FileStatus objects
1963         *
1964         * @throws AccessControlException If access is denied
1965         * @throws IOException If an I/O error occurred
1966         */
1967        private FileStatus[] globStatusInternal(final URI uri,
1968            final Path inPathPattern, final PathFilter filter)
1969            throws AccessControlException, IOException
1970          {
1971          Path[] parents = new Path[1];
1972          int level = 0;
1973          
1974          assert(inPathPattern.toUri().getScheme() == null &&
1975              inPathPattern.toUri().getAuthority() == null && 
1976              inPathPattern.isUriPathAbsolute());
1977    
1978          
1979          String filename = inPathPattern.toUri().getPath();
1980          
1981          // path has only zero component
1982          if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
1983            Path p = inPathPattern.makeQualified(uri, null);
1984            return getFileStatus(new Path[]{p});
1985          }
1986    
1987          // path has at least one component
1988          String[] components = filename.split(Path.SEPARATOR);
1989          
1990          // Path is absolute, first component is "/" hence first component
1991          // is the uri root
1992          parents[0] = new Path(new Path(uri), new Path("/"));
1993          level = 1;
1994    
1995          // glob the paths that match the parent path, ie. [0, components.length-1]
1996          boolean[] hasGlob = new boolean[]{false};
1997          Path[] relParentPaths = 
1998            globPathsLevel(parents, components, level, hasGlob);
1999          FileStatus[] results;
2000          
2001          if (relParentPaths == null || relParentPaths.length == 0) {
2002            results = null;
2003          } else {
2004            // fix the pathes to be abs
2005            Path[] parentPaths = new Path [relParentPaths.length]; 
2006            for(int i=0; i<relParentPaths.length; i++) {
2007              parentPaths[i] = relParentPaths[i].makeQualified(uri, null);
2008            }
2009            
2010            // Now work on the last component of the path
2011            GlobFilter fp = 
2012                        new GlobFilter(components[components.length - 1], filter);
2013            if (fp.hasPattern()) { // last component has a pattern
2014              // list parent directories and then glob the results
2015              results = listStatus(parentPaths, fp);
2016              hasGlob[0] = true;
2017            } else { // last component does not have a pattern
2018              // get all the path names
2019              ArrayList<Path> filteredPaths = 
2020                                          new ArrayList<Path>(parentPaths.length);
2021              for (int i = 0; i < parentPaths.length; i++) {
2022                parentPaths[i] = new Path(parentPaths[i],
2023                  components[components.length - 1]);
2024                if (fp.accept(parentPaths[i])) {
2025                  filteredPaths.add(parentPaths[i]);
2026                }
2027              }
2028              // get all their statuses
2029              results = getFileStatus(
2030                  filteredPaths.toArray(new Path[filteredPaths.size()]));
2031            }
2032          }
2033    
2034          // Decide if the pathPattern contains a glob or not
2035          if (results == null) {
2036            if (hasGlob[0]) {
2037              results = new FileStatus[0];
2038            }
2039          } else {
2040            if (results.length == 0) {
2041              if (!hasGlob[0]) {
2042                results = null;
2043              }
2044            } else {
2045              Arrays.sort(results);
2046            }
2047          }
2048          return results;
2049        }
2050    
2051        /*
2052         * For a path of N components, return a list of paths that match the
2053         * components [<code>level</code>, <code>N-1</code>].
2054         */
2055        private Path[] globPathsLevel(Path[] parents, String[] filePattern,
2056            int level, boolean[] hasGlob) throws AccessControlException,
2057            FileNotFoundException, IOException {
2058          if (level == filePattern.length - 1) {
2059            return parents;
2060          }
2061          if (parents == null || parents.length == 0) {
2062            return null;
2063          }
2064          GlobFilter fp = new GlobFilter(filePattern[level]);
2065          if (fp.hasPattern()) {
2066            parents = FileUtil.stat2Paths(listStatus(parents, fp));
2067            hasGlob[0] = true;
2068          } else {
2069            for (int i = 0; i < parents.length; i++) {
2070              parents[i] = new Path(parents[i], filePattern[level]);
2071            }
2072          }
2073          return globPathsLevel(parents, filePattern, level + 1, hasGlob);
2074        }
2075    
2076        /**
2077         * Copy file from src to dest. See
2078         * {@link #copy(Path, Path, boolean, boolean)}
2079         */
2080        public boolean copy(final Path src, final Path dst)
2081            throws AccessControlException, FileAlreadyExistsException,
2082            FileNotFoundException, ParentNotDirectoryException,
2083            UnsupportedFileSystemException, IOException {
2084          return copy(src, dst, false, false);
2085        }
2086        
2087        /**
2088         * Copy from src to dst, optionally deleting src and overwriting dst.
2089         * @param src
2090         * @param dst
2091         * @param deleteSource - delete src if true
2092         * @param overwrite  overwrite dst if true; throw IOException if dst exists
2093         *         and overwrite is false.
2094         *
2095         * @return true if copy is successful
2096         *
2097         * @throws AccessControlException If access is denied
2098         * @throws FileAlreadyExistsException If <code>dst</code> already exists
2099         * @throws FileNotFoundException If <code>src</code> does not exist
2100         * @throws ParentNotDirectoryException If parent of <code>dst</code> is not
2101         *           a directory
2102         * @throws UnsupportedFileSystemException If file system for 
2103         *         <code>src</code> or <code>dst</code> is not supported
2104         * @throws IOException If an I/O error occurred
2105         * 
2106         * Exceptions applicable to file systems accessed over RPC:
2107         * @throws RpcClientException If an exception occurred in the RPC client
2108         * @throws RpcServerException If an exception occurred in the RPC server
2109         * @throws UnexpectedServerException If server implementation throws 
2110         *           undeclared exception to RPC server
2111         * 
2112         * RuntimeExceptions:
2113         * @throws InvalidPathException If path <code>dst</code> is invalid
2114         */
2115        public boolean copy(final Path src, final Path dst, boolean deleteSource,
2116            boolean overwrite) throws AccessControlException,
2117            FileAlreadyExistsException, FileNotFoundException,
2118            ParentNotDirectoryException, UnsupportedFileSystemException, 
2119            IOException {
2120          checkNotSchemeWithRelative(src);
2121          checkNotSchemeWithRelative(dst);
2122          Path qSrc = makeQualified(src);
2123          Path qDst = makeQualified(dst);
2124          checkDest(qSrc.getName(), qDst, overwrite);
2125          FileStatus fs = FileContext.this.getFileStatus(qSrc);
2126          if (fs.isDirectory()) {
2127            checkDependencies(qSrc, qDst);
2128            mkdir(qDst, FsPermission.getDefault(), true);
2129            FileStatus[] contents = listStatus(qSrc);
2130            for (FileStatus content : contents) {
2131              copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
2132                  content.getPath().getName())), deleteSource, overwrite);
2133            }
2134          } else {
2135            InputStream in=null;
2136            OutputStream out = null;
2137            try {
2138              in = open(qSrc);
2139              EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
2140                  CreateFlag.CREATE, CreateFlag.OVERWRITE) : 
2141                    EnumSet.of(CreateFlag.CREATE);
2142              out = create(qDst, createFlag);
2143              IOUtils.copyBytes(in, out, conf, true);
2144            } catch (IOException e) {
2145              IOUtils.closeStream(out);
2146              IOUtils.closeStream(in);
2147              throw e;
2148            }
2149          }
2150          if (deleteSource) {
2151            return delete(qSrc, true);
2152          } else {
2153            return true;
2154          }
2155        }
2156      }
2157    
2158      /**
2159       * Check if copying srcName to dst would overwrite an existing 
2160       * file or directory.
2161       * @param srcName File or directory to be copied.
2162       * @param dst Destination to copy srcName to.
2163       * @param overwrite Whether it's ok to overwrite an existing file. 
2164       * @throws AccessControlException If access is denied.
2165       * @throws IOException If dst is an existing directory, or dst is an 
2166       * existing file and the overwrite option is not passed.
2167       */
2168      private void checkDest(String srcName, Path dst, boolean overwrite)
2169          throws AccessControlException, IOException {
2170        try {
2171          FileStatus dstFs = getFileStatus(dst);
2172          if (dstFs.isDirectory()) {
2173            if (null == srcName) {
2174              throw new IOException("Target " + dst + " is a directory");
2175            }
2176            // Recurse to check if dst/srcName exists.
2177            checkDest(null, new Path(dst, srcName), overwrite);
2178          } else if (!overwrite) {
2179            throw new IOException("Target " + new Path(dst, srcName)
2180                + " already exists");
2181          }
2182        } catch (FileNotFoundException e) {
2183          // dst does not exist - OK to copy.
2184        }
2185      }
2186       
2187      //
2188      // If the destination is a subdirectory of the source, then
2189      // generate exception
2190      //
2191      private static void checkDependencies(Path qualSrc, Path qualDst)
2192        throws IOException {
2193        if (isSameFS(qualSrc, qualDst)) {
2194          String srcq = qualSrc.toString() + Path.SEPARATOR;
2195          String dstq = qualDst.toString() + Path.SEPARATOR;
2196          if (dstq.startsWith(srcq)) {
2197            if (srcq.length() == dstq.length()) {
2198              throw new IOException("Cannot copy " + qualSrc + " to itself.");
2199            } else {
2200              throw new IOException("Cannot copy " + qualSrc +
2201                                 " to its subdirectory " + qualDst);
2202            }
2203          }
2204        }
2205      }
2206      
2207      /**
2208       * Are qualSrc and qualDst of the same file system?
2209       * @param qualPath1 - fully qualified path
2210       * @param qualPath2 - fully qualified path
2211       * @return
2212       */
2213      private static boolean isSameFS(Path qualPath1, Path qualPath2) {
2214        URI srcUri = qualPath1.toUri();
2215        URI dstUri = qualPath2.toUri();
2216        return (srcUri.getScheme().equals(dstUri.getScheme()) && 
2217            !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri
2218            .getAuthority().equals(dstUri.getAuthority())));
2219      }
2220    
2221      /**
2222       * Deletes all the paths in deleteOnExit on JVM shutdown.
2223       */
2224      static class FileContextFinalizer extends Thread {
2225        public synchronized void run() {
2226          processDeleteOnExit();
2227        }
2228      }
2229    
2230      /**
2231       * Resolves all symbolic links in the specified path.
2232       * Returns the new path object.
2233       */
2234      protected Path resolve(final Path f) throws FileNotFoundException,
2235          UnresolvedLinkException, AccessControlException, IOException {
2236        return new FSLinkResolver<Path>() {
2237          public Path next(final AbstractFileSystem fs, final Path p) 
2238            throws IOException, UnresolvedLinkException {
2239            return fs.resolvePath(p);
2240          }
2241        }.resolve(this, f);
2242      }
2243    
2244      /**
2245       * Resolves all symbolic links in the specified path leading up 
2246       * to, but not including the final path component.
2247       * @param f path to resolve
2248       * @return the new path object.
2249       */
2250      protected Path resolveIntermediate(final Path f) throws IOException {
2251        return new FSLinkResolver<FileStatus>() {
2252          public FileStatus next(final AbstractFileSystem fs, final Path p) 
2253            throws IOException, UnresolvedLinkException {
2254            return fs.getFileLinkStatus(p);
2255          }
2256        }.resolve(this, f).getPath();
2257      }
2258    
2259      /**
2260       * Returns the list of AbstractFileSystems accessed in the path. The list may
2261       * contain more than one AbstractFileSystems objects in case of symlinks.
2262       * 
2263       * @param f
2264       *          Path which needs to be resolved
2265       * @return List of AbstractFileSystems accessed in the path
2266       * @throws IOException
2267       */
2268      Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f)
2269          throws IOException {
2270        final Path absF = fixRelativePart(f);
2271        final HashSet<AbstractFileSystem> result 
2272          = new HashSet<AbstractFileSystem>();
2273        new FSLinkResolver<Void>() {
2274          public Void next(final AbstractFileSystem fs, final Path p)
2275              throws IOException, UnresolvedLinkException {
2276            result.add(fs);
2277            fs.getFileStatus(p);
2278            return null;
2279          }
2280        }.resolve(this, absF);
2281        return result;
2282      }
2283      
2284      /**
2285       * Class used to perform an operation on and resolve symlinks in a
2286       * path. The operation may potentially span multiple file systems.  
2287       */
2288      protected abstract class FSLinkResolver<T> {
2289        // The maximum number of symbolic link components in a path
2290        private static final int MAX_PATH_LINKS = 32;
2291    
2292        /**
2293         * Generic helper function overridden on instantiation to perform a 
2294         * specific operation on the given file system using the given path
2295         * which may result in an UnresolvedLinkException. 
2296         * @param fs AbstractFileSystem to perform the operation on.
2297         * @param p Path given the file system.
2298         * @return Generic type determined by the specific implementation.
2299         * @throws UnresolvedLinkException If symbolic link <code>path</code> could 
2300         *           not be resolved
2301         * @throws IOException an I/O error occured
2302         */
2303        public abstract T next(final AbstractFileSystem fs, final Path p) 
2304          throws IOException, UnresolvedLinkException;  
2305            
2306        /**
2307         * Performs the operation specified by the next function, calling it
2308         * repeatedly until all symlinks in the given path are resolved.
2309         * @param fc FileContext used to access file systems.
2310         * @param p The path to resolve symlinks in.
2311         * @return Generic type determined by the implementation of next.
2312         * @throws IOException
2313         */
2314        public T resolve(final FileContext fc, Path p) throws IOException {
2315          int count = 0;
2316          T in = null;
2317          Path first = p;
2318          // NB: More than one AbstractFileSystem can match a scheme, eg 
2319          // "file" resolves to LocalFs but could have come by RawLocalFs.
2320          AbstractFileSystem fs = fc.getFSofPath(p);      
2321    
2322          // Loop until all symlinks are resolved or the limit is reached
2323          for (boolean isLink = true; isLink;) {
2324            try {
2325              in = next(fs, p);
2326              isLink = false;
2327            } catch (UnresolvedLinkException e) {
2328              if (count++ > MAX_PATH_LINKS) {
2329                throw new IOException("Possible cyclic loop while " +
2330                                      "following symbolic link " + first);
2331              }
2332              // Resolve the first unresolved path component
2333              p = qualifySymlinkTarget(fs, p, fs.getLinkTarget(p));
2334              fs = fc.getFSofPath(p);
2335            }
2336          }
2337          return in;
2338        }
2339      }
2340      
2341      /**
2342       * Get the statistics for a particular file system
2343       * 
2344       * @param uri
2345       *          the uri to lookup the statistics. Only scheme and authority part
2346       *          of the uri are used as the key to store and lookup.
2347       * @return a statistics object
2348       */
2349      public static Statistics getStatistics(URI uri) {
2350        return AbstractFileSystem.getStatistics(uri);
2351      }
2352    
2353      /**
2354       * Clears all the statistics stored in AbstractFileSystem, for all the file
2355       * systems.
2356       */
2357      public static void clearStatistics() {
2358        AbstractFileSystem.clearStatistics();
2359      }
2360    
2361      /**
2362       * Prints the statistics to standard output. File System is identified by the
2363       * scheme and authority.
2364       */
2365      public static void printStatistics() {
2366        AbstractFileSystem.printStatistics();
2367      }
2368    
2369      /**
2370       * @return Map of uri and statistics for each filesystem instantiated. The uri
2371       *         consists of scheme and authority for the filesystem.
2372       */
2373      public static Map<URI, Statistics> getAllStatistics() {
2374        return AbstractFileSystem.getAllStatistics();
2375      }
2376      
2377      /**
2378       * Get delegation tokens for the file systems accessed for a given
2379       * path.
2380       * @param p Path for which delegations tokens are requested.
2381       * @param renewer the account name that is allowed to renew the token.
2382       * @return List of delegation tokens.
2383       * @throws IOException
2384       */
2385      @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
2386      public List<Token<?>> getDelegationTokens(
2387          Path p, String renewer) throws IOException {
2388        Set<AbstractFileSystem> afsSet = resolveAbstractFileSystems(p);
2389        List<Token<?>> tokenList = 
2390            new ArrayList<Token<?>>();
2391        for (AbstractFileSystem afs : afsSet) {
2392          List<Token<?>> afsTokens = afs.getDelegationTokens(renewer);
2393          tokenList.addAll(afsTokens);
2394        }
2395        return tokenList;
2396      }
2397    }