001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs;
019    
020    import java.io.Closeable;
021    import java.io.FileNotFoundException;
022    import java.io.IOException;
023    import java.net.URI;
024    import java.security.PrivilegedExceptionAction;
025    import java.util.ArrayList;
026    import java.util.Arrays;
027    import java.util.EnumSet;
028    import java.util.HashMap;
029    import java.util.HashSet;
030    import java.util.IdentityHashMap;
031    import java.util.Iterator;
032    import java.util.List;
033    import java.util.Map;
034    import java.util.NoSuchElementException;
035    import java.util.Set;
036    import java.util.Stack;
037    import java.util.TreeSet;
038    import java.util.concurrent.atomic.AtomicInteger;
039    import java.util.concurrent.atomic.AtomicLong;
040    
041    import org.apache.commons.logging.Log;
042    import org.apache.commons.logging.LogFactory;
043    import org.apache.hadoop.classification.InterfaceAudience;
044    import org.apache.hadoop.classification.InterfaceStability;
045    import org.apache.hadoop.conf.Configuration;
046    import org.apache.hadoop.conf.Configured;
047    import org.apache.hadoop.fs.Options.Rename;
048    import org.apache.hadoop.fs.permission.FsPermission;
049    import org.apache.hadoop.io.MultipleIOException;
050    import org.apache.hadoop.security.SecurityUtil;
051    import org.apache.hadoop.security.UserGroupInformation;
052    import org.apache.hadoop.security.token.Token;
053    import org.apache.hadoop.util.Progressable;
054    import org.apache.hadoop.util.ReflectionUtils;
055    
056    /****************************************************************
057     * An abstract base class for a fairly generic filesystem.  It
058     * may be implemented as a distributed filesystem, or as a "local"
059     * one that reflects the locally-connected disk.  The local version
060     * exists for small Hadoop instances and for testing.
061     *
062     * <p>
063     *
064     * All user code that may potentially use the Hadoop Distributed
065     * File System should be written to use a FileSystem object.  The
066     * Hadoop DFS is a multi-machine system that appears as a single
067     * disk.  It's useful because of its fault tolerance and potentially
068     * very large capacity.
069     * 
070     * <p>
071     * The local implementation is {@link LocalFileSystem} and distributed
072     * implementation is DistributedFileSystem.
073     *****************************************************************/
074    @InterfaceAudience.Public
075    @InterfaceStability.Stable
076    public abstract class FileSystem extends Configured implements Closeable {
077      public static final String FS_DEFAULT_NAME_KEY = 
078                       CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
079      public static final String DEFAULT_FS = 
080                       CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
081    
082      public static final Log LOG = LogFactory.getLog(FileSystem.class);
083    
084      /** FileSystem cache */
085      static final Cache CACHE = new Cache();
086    
087      /** The key this instance is stored under in the cache. */
088      private Cache.Key key;
089    
090      /** Recording statistics per a FileSystem class */
091      private static final Map<Class<? extends FileSystem>, Statistics> 
092        statisticsTable =
093          new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
094      
095      /**
096       * The statistics for this file system.
097       */
098      protected Statistics statistics;
099    
100      /**
101       * A cache of files that should be deleted when filsystem is closed
102       * or the JVM is exited.
103       */
104      private Set<Path> deleteOnExit = new TreeSet<Path>();
105      
106      /**
107       * This method adds a file system for testing so that we can find it later. It
108       * is only for testing.
109       * @param uri the uri to store it under
110       * @param conf the configuration to store it under
111       * @param fs the file system to store
112       * @throws IOException
113       */
114      static void addFileSystemForTesting(URI uri, Configuration conf,
115          FileSystem fs) throws IOException {
116        CACHE.map.put(new Cache.Key(uri, conf), fs);
117      }
118    
119      /**
120       * Get a filesystem instance based on the uri, the passed
121       * configuration and the user
122       * @param uri of the filesystem
123       * @param conf the configuration to use
124       * @param user to perform the get as
125       * @return the filesystem instance
126       * @throws IOException
127       * @throws InterruptedException
128       */
129      public static FileSystem get(final URI uri, final Configuration conf,
130            final String user) throws IOException, InterruptedException {
131        UserGroupInformation ugi;
132        if (user == null) {
133          ugi = UserGroupInformation.getCurrentUser();
134        } else {
135          ugi = UserGroupInformation.createRemoteUser(user);
136        }
137        return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
138          public FileSystem run() throws IOException {
139            return get(uri, conf);
140          }
141        });
142      }
143    
144      /**
145       * Returns the configured filesystem implementation.
146       * @param conf the configuration to use
147       */
148      public static FileSystem get(Configuration conf) throws IOException {
149        return get(getDefaultUri(conf), conf);
150      }
151      
152      /** Get the default filesystem URI from a configuration.
153       * @param conf the configuration to use
154       * @return the uri of the default filesystem
155       */
156      public static URI getDefaultUri(Configuration conf) {
157        return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
158      }
159    
160      /** Set the default filesystem URI in a configuration.
161       * @param conf the configuration to alter
162       * @param uri the new default filesystem uri
163       */
164      public static void setDefaultUri(Configuration conf, URI uri) {
165        conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
166      }
167    
168      /** Set the default filesystem URI in a configuration.
169       * @param conf the configuration to alter
170       * @param uri the new default filesystem uri
171       */
172      public static void setDefaultUri(Configuration conf, String uri) {
173        setDefaultUri(conf, URI.create(fixName(uri)));
174      }
175    
176      /** Called after a new FileSystem instance is constructed.
177       * @param name a uri whose authority section names the host, port, etc.
178       *   for this FileSystem
179       * @param conf the configuration
180       */
181      public void initialize(URI name, Configuration conf) throws IOException {
182        statistics = getStatistics(name.getScheme(), getClass());    
183      }
184    
185      /** Returns a URI whose scheme and authority identify this FileSystem.*/
186      public abstract URI getUri();
187      
188      /**
189       * Get the default port for this file system.
190       * @return the default port or 0 if there isn't one
191       */
192      protected int getDefaultPort() {
193        return 0;
194      }
195    
196      /**
197       * Get a canonical name for this file system.
198       * @return a URI string that uniquely identifies this file system
199       */
200      public String getCanonicalServiceName() {
201        return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort());
202      }
203    
204      /** @deprecated call #getUri() instead.*/
205      @Deprecated
206      public String getName() { return getUri().toString(); }
207    
208      /** @deprecated call #get(URI,Configuration) instead. */
209      @Deprecated
210      public static FileSystem getNamed(String name, Configuration conf)
211        throws IOException {
212        return get(URI.create(fixName(name)), conf);
213      }
214      
215      /** Update old-format filesystem names, for back-compatibility.  This should
216       * eventually be replaced with a checkName() method that throws an exception
217       * for old-format names. */ 
218      private static String fixName(String name) {
219        // convert old-format name to new-format name
220        if (name.equals("local")) {         // "local" is now "file:///".
221          LOG.warn("\"local\" is a deprecated filesystem name."
222                   +" Use \"file:///\" instead.");
223          name = "file:///";
224        } else if (name.indexOf('/')==-1) {   // unqualified is "hdfs://"
225          LOG.warn("\""+name+"\" is a deprecated filesystem name."
226                   +" Use \"hdfs://"+name+"/\" instead.");
227          name = "hdfs://"+name;
228        }
229        return name;
230      }
231    
232      /**
233       * Get the local file system.
234       * @param conf the configuration to configure the file system with
235       * @return a LocalFileSystem
236       */
237      public static LocalFileSystem getLocal(Configuration conf)
238        throws IOException {
239        return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
240      }
241    
242      /** Returns the FileSystem for this URI's scheme and authority.  The scheme
243       * of the URI determines a configuration property name,
244       * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
245       * The entire URI is passed to the FileSystem instance's initialize method.
246       */
247      public static FileSystem get(URI uri, Configuration conf) throws IOException {
248        String scheme = uri.getScheme();
249        String authority = uri.getAuthority();
250    
251        if (scheme == null) {                       // no scheme: use default FS
252          return get(conf);
253        }
254    
255        if (authority == null) {                       // no authority
256          URI defaultUri = getDefaultUri(conf);
257          if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
258              && defaultUri.getAuthority() != null) {  // & default has authority
259            return get(defaultUri, conf);              // return default
260          }
261        }
262        
263        String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
264        if (conf.getBoolean(disableCacheName, false)) {
265          return createFileSystem(uri, conf);
266        }
267    
268        return CACHE.get(uri, conf);
269      }
270    
271      /**
272       * Returns the FileSystem for this URI's scheme and authority and the 
273       * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
274       * @param uri of the filesystem
275       * @param conf the configuration to use
276       * @param user to perform the get as
277       * @return filesystem instance
278       * @throws IOException
279       * @throws InterruptedException
280       */
281      public static FileSystem newInstance(final URI uri, final Configuration conf,
282          final String user) throws IOException, InterruptedException {
283        UserGroupInformation ugi;
284        if (user == null) {
285          ugi = UserGroupInformation.getCurrentUser();
286        } else {
287          ugi = UserGroupInformation.createRemoteUser(user);
288        }
289        return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
290          public FileSystem run() throws IOException {
291            return newInstance(uri,conf); 
292          }
293        });
294      }
295      /** Returns the FileSystem for this URI's scheme and authority.  The scheme
296       * of the URI determines a configuration property name,
297       * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
298       * The entire URI is passed to the FileSystem instance's initialize method.
299       * This always returns a new FileSystem object.
300       */
301      public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
302        String scheme = uri.getScheme();
303        String authority = uri.getAuthority();
304    
305        if (scheme == null) {                       // no scheme: use default FS
306          return newInstance(conf);
307        }
308    
309        if (authority == null) {                       // no authority
310          URI defaultUri = getDefaultUri(conf);
311          if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
312              && defaultUri.getAuthority() != null) {  // & default has authority
313            return newInstance(defaultUri, conf);              // return default
314          }
315        }
316        return CACHE.getUnique(uri, conf);
317      }
318    
319      /** Returns a unique configured filesystem implementation.
320       * This always returns a new FileSystem object.
321       * @param conf the configuration to use
322       */
323      public static FileSystem newInstance(Configuration conf) throws IOException {
324        return newInstance(getDefaultUri(conf), conf);
325      }
326    
327      /**
328       * Get a unique local file system object
329       * @param conf the configuration to configure the file system with
330       * @return a LocalFileSystem
331       * This always returns a new FileSystem object.
332       */
333      public static LocalFileSystem newInstanceLocal(Configuration conf)
334        throws IOException {
335        return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
336      }
337    
338      /**
339       * Close all cached filesystems. Be sure those filesystems are not
340       * used anymore.
341       * 
342       * @throws IOException
343       */
344      public static void closeAll() throws IOException {
345        CACHE.closeAll();
346      }
347    
348      /**
349       * Close all cached filesystems for a given UGI. Be sure those filesystems 
350       * are not used anymore.
351       * @param ugi user group info to close
352       * @throws IOException
353       */
354      public static void closeAllForUGI(UserGroupInformation ugi) 
355      throws IOException {
356        CACHE.closeAll(ugi);
357      }
358    
359      /** 
360       * Make sure that a path specifies a FileSystem.
361       * @param path to use
362       */
363      public Path makeQualified(Path path) {
364        checkPath(path);
365        return path.makeQualified(this.getUri(), this.getWorkingDirectory());
366      }
367        
368      /**
369       * Deprecated  - use @link {@link #getDelegationTokens(String)}
370       * Get a new delegation token for this file system.
371       * @param renewer the account name that is allowed to renew the token.
372       * @return a new delegation token
373       * @throws IOException
374       */
375      @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
376      @Deprecated
377      public Token<?> getDelegationToken(String renewer) throws IOException {
378        return null;
379      }
380      
381      /**
382       * Get one or more delegation tokens associated with the filesystem. Normally
383       * a file system returns a single delegation token. A file system that manages
384       * multiple file systems underneath, could return set of delegation tokens for
385       * all the file systems it manages.
386       * 
387       * @param renewer the account name that is allowed to renew the token.
388       * @return list of new delegation tokens
389       *    If delegation tokens not supported then return a list of size zero.
390       * @throws IOException
391       */
392      @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
393      public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
394        return new ArrayList<Token<?>>(0);
395      }
396    
397      /** create a file with the provided permission
398       * The permission of the file is set to be the provided permission as in
399       * setPermission, not permission&~umask
400       * 
401       * It is implemented using two RPCs. It is understood that it is inefficient,
402       * but the implementation is thread-safe. The other option is to change the
403       * value of umask in configuration to be 0, but it is not thread-safe.
404       * 
405       * @param fs file system handle
406       * @param file the name of the file to be created
407       * @param permission the permission of the file
408       * @return an output stream
409       * @throws IOException
410       */
411      public static FSDataOutputStream create(FileSystem fs,
412          Path file, FsPermission permission) throws IOException {
413        // create the file with default permission
414        FSDataOutputStream out = fs.create(file);
415        // set its permission to the supplied one
416        fs.setPermission(file, permission);
417        return out;
418      }
419    
420      /** create a directory with the provided permission
421       * The permission of the directory is set to be the provided permission as in
422       * setPermission, not permission&~umask
423       * 
424       * @see #create(FileSystem, Path, FsPermission)
425       * 
426       * @param fs file system handle
427       * @param dir the name of the directory to be created
428       * @param permission the permission of the directory
429       * @return true if the directory creation succeeds; false otherwise
430       * @throws IOException
431       */
432      public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
433      throws IOException {
434        // create the directory using the default permission
435        boolean result = fs.mkdirs(dir);
436        // set its permission to be the supplied one
437        fs.setPermission(dir, permission);
438        return result;
439      }
440    
441      ///////////////////////////////////////////////////////////////
442      // FileSystem
443      ///////////////////////////////////////////////////////////////
444    
445      protected FileSystem() {
446        super(null);
447      }
448    
449      /** 
450       * Check that a Path belongs to this FileSystem.
451       * @param path to check
452       */
453      protected void checkPath(Path path) {
454        URI uri = path.toUri();
455        if (uri.getScheme() == null)                // fs is relative 
456          return;
457        String thisScheme = this.getUri().getScheme();
458        String thatScheme = uri.getScheme();
459        String thisAuthority = this.getUri().getAuthority();
460        String thatAuthority = uri.getAuthority();
461        //authority and scheme are not case sensitive
462        if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
463          if (thisAuthority == thatAuthority ||       // & authorities match
464              (thisAuthority != null && 
465               thisAuthority.equalsIgnoreCase(thatAuthority)))
466            return;
467    
468          if (thatAuthority == null &&                // path's authority is null
469              thisAuthority != null) {                // fs has an authority
470            URI defaultUri = getDefaultUri(getConf()); // & is the conf default 
471            if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) &&
472                thisAuthority.equalsIgnoreCase(defaultUri.getAuthority()))
473              return;
474            try {                                     // or the default fs's uri
475              defaultUri = get(getConf()).getUri();
476            } catch (IOException e) {
477              throw new RuntimeException(e);
478            }
479            if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) &&
480                thisAuthority.equalsIgnoreCase(defaultUri.getAuthority()))
481              return;
482          }
483        }
484        throw new IllegalArgumentException("Wrong FS: "+path+
485                                           ", expected: "+this.getUri());
486      }
487    
488      /**
489       * Return an array containing hostnames, offset and size of 
490       * portions of the given file.  For a nonexistent 
491       * file or regions, null will be returned.
492       *
493       * This call is most helpful with DFS, where it returns 
494       * hostnames of machines that contain the given file.
495       *
496       * The FileSystem will simply return an elt containing 'localhost'.
497       *
498       * @param file FilesStatus to get data from
499       * @param start offset into the given file
500       * @param len length for which to get locations for
501       */
502      public BlockLocation[] getFileBlockLocations(FileStatus file, 
503          long start, long len) throws IOException {
504        if (file == null) {
505          return null;
506        }
507    
508        if (start < 0 || len < 0) {
509          throw new IllegalArgumentException("Invalid start or len parameter");
510        }
511    
512        if (file.getLen() < start) {
513          return new BlockLocation[0];
514    
515        }
516        String[] name = { "localhost:50010" };
517        String[] host = { "localhost" };
518        return new BlockLocation[] {
519          new BlockLocation(name, host, 0, file.getLen()) };
520      }
521     
522    
523      /**
524       * Return an array containing hostnames, offset and size of 
525       * portions of the given file.  For a nonexistent 
526       * file or regions, null will be returned.
527       *
528       * This call is most helpful with DFS, where it returns 
529       * hostnames of machines that contain the given file.
530       *
531       * The FileSystem will simply return an elt containing 'localhost'.
532       *
533       * @param p path of file to get locations for
534       * @param start offset into the given file
535       * @param len length for which to get locations for
536       */
537      public BlockLocation[] getFileBlockLocations(Path p, 
538          long start, long len) throws IOException {
539        if (p == null) {
540          throw new NullPointerException();
541        }
542        FileStatus file = getFileStatus(p);
543        return getFileBlockLocations(file, start, len);
544      }
545      
546      /**
547       * Return a set of server default configuration values
548       * @return server default configuration values
549       * @throws IOException
550       */
551      public FsServerDefaults getServerDefaults() throws IOException {
552        Configuration conf = getConf();
553        return new FsServerDefaults(getDefaultBlockSize(), 
554            conf.getInt("io.bytes.per.checksum", 512), 
555            64 * 1024, 
556            getDefaultReplication(), 
557            conf.getInt("io.file.buffer.size", 4096));
558      }
559      
560      /**
561       * Return the fully-qualified path of path f resolving the path
562       * through any symlinks or mount point
563       * @param p path to be resolved
564       * @return fully qualified path 
565       * @throws FileNotFoundException
566       */
567       public Path resolvePath(final Path p) throws IOException {
568         checkPath(p);
569         return getFileStatus(p).getPath();
570       }
571    
572      /**
573       * Opens an FSDataInputStream at the indicated Path.
574       * @param f the file name to open
575       * @param bufferSize the size of the buffer to be used.
576       */
577      public abstract FSDataInputStream open(Path f, int bufferSize)
578        throws IOException;
579        
580      /**
581       * Opens an FSDataInputStream at the indicated Path.
582       * @param f the file to open
583       */
584      public FSDataInputStream open(Path f) throws IOException {
585        return open(f, getConf().getInt("io.file.buffer.size", 4096));
586      }
587    
588      /**
589       * Create an FSDataOutputStream at the indicated Path.
590       * Files are overwritten by default.
591       * @param f the file to create
592       */
593      public FSDataOutputStream create(Path f) throws IOException {
594        return create(f, true);
595      }
596    
597      /**
598       * Create an FSDataOutputStream at the indicated Path.
599       * @param f the file to create
600       * @param overwrite if a file with this name already exists, then if true,
601       *   the file will be overwritten, and if false an exception will be thrown.
602       */
603      public FSDataOutputStream create(Path f, boolean overwrite)
604          throws IOException {
605        return create(f, overwrite, 
606                      getConf().getInt("io.file.buffer.size", 4096),
607                      getDefaultReplication(),
608                      getDefaultBlockSize());
609      }
610    
611      /**
612       * Create an FSDataOutputStream at the indicated Path with write-progress
613       * reporting.
614       * Files are overwritten by default.
615       * @param f the file to create
616       * @param progress to report progress
617       */
618      public FSDataOutputStream create(Path f, Progressable progress) 
619          throws IOException {
620        return create(f, true, 
621                      getConf().getInt("io.file.buffer.size", 4096),
622                      getDefaultReplication(),
623                      getDefaultBlockSize(), progress);
624      }
625    
626      /**
627       * Create an FSDataOutputStream at the indicated Path.
628       * Files are overwritten by default.
629       * @param f the file to create
630       * @param replication the replication factor
631       */
632      public FSDataOutputStream create(Path f, short replication)
633          throws IOException {
634        return create(f, true, 
635                      getConf().getInt("io.file.buffer.size", 4096),
636                      replication,
637                      getDefaultBlockSize());
638      }
639    
640      /**
641       * Create an FSDataOutputStream at the indicated Path with write-progress
642       * reporting.
643       * Files are overwritten by default.
644       * @param f the file to create
645       * @param replication the replication factor
646       * @param progress to report progress
647       */
648      public FSDataOutputStream create(Path f, short replication, 
649          Progressable progress) throws IOException {
650        return create(f, true, 
651                      getConf().getInt("io.file.buffer.size", 4096),
652                      replication,
653                      getDefaultBlockSize(), progress);
654      }
655    
656        
657      /**
658       * Create an FSDataOutputStream at the indicated Path.
659       * @param f the file name to create
660       * @param overwrite if a file with this name already exists, then if true,
661       *   the file will be overwritten, and if false an error will be thrown.
662       * @param bufferSize the size of the buffer to be used.
663       */
664      public FSDataOutputStream create(Path f, 
665                                       boolean overwrite,
666                                       int bufferSize
667                                       ) throws IOException {
668        return create(f, overwrite, bufferSize, 
669                      getDefaultReplication(),
670                      getDefaultBlockSize());
671      }
672        
673      /**
674       * Create an FSDataOutputStream at the indicated Path with write-progress
675       * reporting.
676       * @param f the path of the file to open
677       * @param overwrite if a file with this name already exists, then if true,
678       *   the file will be overwritten, and if false an error will be thrown.
679       * @param bufferSize the size of the buffer to be used.
680       */
681      public FSDataOutputStream create(Path f, 
682                                       boolean overwrite,
683                                       int bufferSize,
684                                       Progressable progress
685                                       ) throws IOException {
686        return create(f, overwrite, bufferSize, 
687                      getDefaultReplication(),
688                      getDefaultBlockSize(), progress);
689      }
690        
691        
692      /**
693       * Create an FSDataOutputStream at the indicated Path.
694       * @param f the file name to open
695       * @param overwrite if a file with this name already exists, then if true,
696       *   the file will be overwritten, and if false an error will be thrown.
697       * @param bufferSize the size of the buffer to be used.
698       * @param replication required block replication for the file. 
699       */
700      public FSDataOutputStream create(Path f, 
701                                       boolean overwrite,
702                                       int bufferSize,
703                                       short replication,
704                                       long blockSize
705                                       ) throws IOException {
706        return create(f, overwrite, bufferSize, replication, blockSize, null);
707      }
708    
709      /**
710       * Create an FSDataOutputStream at the indicated Path with write-progress
711       * reporting.
712       * @param f the file name to open
713       * @param overwrite if a file with this name already exists, then if true,
714       *   the file will be overwritten, and if false an error will be thrown.
715       * @param bufferSize the size of the buffer to be used.
716       * @param replication required block replication for the file. 
717       */
718      public FSDataOutputStream create(Path f,
719                                                boolean overwrite,
720                                                int bufferSize,
721                                                short replication,
722                                                long blockSize,
723                                                Progressable progress
724                                                ) throws IOException {
725        return this.create(f, FsPermission.getDefault().applyUMask(
726            FsPermission.getUMask(getConf())), overwrite, bufferSize,
727            replication, blockSize, progress);
728      }
729    
730      /**
731       * Create an FSDataOutputStream at the indicated Path with write-progress
732       * reporting.
733       * @param f the file name to open
734       * @param permission
735       * @param overwrite if a file with this name already exists, then if true,
736       *   the file will be overwritten, and if false an error will be thrown.
737       * @param bufferSize the size of the buffer to be used.
738       * @param replication required block replication for the file.
739       * @param blockSize
740       * @param progress
741       * @throws IOException
742       * @see #setPermission(Path, FsPermission)
743       */
744      public abstract FSDataOutputStream create(Path f,
745          FsPermission permission,
746          boolean overwrite,
747          int bufferSize,
748          short replication,
749          long blockSize,
750          Progressable progress) throws IOException;
751      
752      
753      /*.
754       * This create has been added to support the FileContext that processes
755       * the permission
756       * with umask before calling this method.
757       * This a temporary method added to support the transition from FileSystem
758       * to FileContext for user applications.
759       */
760      @Deprecated
761      protected FSDataOutputStream primitiveCreate(Path f,
762         FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
763         short replication, long blockSize, Progressable progress,
764         int bytesPerChecksum) throws IOException {
765    
766        boolean pathExists = exists(f);
767        CreateFlag.validate(f, pathExists, flag);
768        
769        // Default impl  assumes that permissions do not matter and 
770        // nor does the bytesPerChecksum  hence
771        // calling the regular create is good enough.
772        // FSs that implement permissions should override this.
773    
774        if (pathExists && flag.contains(CreateFlag.APPEND)) {
775          return append(f, bufferSize, progress);
776        }
777        
778        return this.create(f, absolutePermission,
779            flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
780            blockSize, progress);
781      }
782      
783      /**
784       * This version of the mkdirs method assumes that the permission is absolute.
785       * It has been added to support the FileContext that processes the permission
786       * with umask before calling this method.
787       * This a temporary method added to support the transition from FileSystem
788       * to FileContext for user applications.
789       */
790      @Deprecated
791      protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
792        throws IOException {
793        // Default impl is to assume that permissions do not matter and hence
794        // calling the regular mkdirs is good enough.
795        // FSs that implement permissions should override this.
796       return this.mkdirs(f, absolutePermission);
797      }
798    
799    
800      /**
801       * This version of the mkdirs method assumes that the permission is absolute.
802       * It has been added to support the FileContext that processes the permission
803       * with umask before calling this method.
804       * This a temporary method added to support the transition from FileSystem
805       * to FileContext for user applications.
806       */
807      @Deprecated
808      protected void primitiveMkdir(Path f, FsPermission absolutePermission, 
809                        boolean createParent)
810        throws IOException {
811        
812        if (!createParent) { // parent must exist.
813          // since the this.mkdirs makes parent dirs automatically
814          // we must throw exception if parent does not exist.
815          final FileStatus stat = getFileStatus(f.getParent());
816          if (stat == null) {
817            throw new FileNotFoundException("Missing parent:" + f);
818          }
819          if (!stat.isDirectory()) {
820            throw new ParentNotDirectoryException("parent is not a dir");
821          }
822          // parent does exist - go ahead with mkdir of leaf
823        }
824        // Default impl is to assume that permissions do not matter and hence
825        // calling the regular mkdirs is good enough.
826        // FSs that implement permissions should override this.
827        if (!this.mkdirs(f, absolutePermission)) {
828          throw new IOException("mkdir of "+ f + " failed");
829        }
830      }
831    
832    
833      /**
834       * Creates the given Path as a brand-new zero-length file.  If
835       * create fails, or if it already existed, return false.
836       *
837       * @param f path to use for create
838       */
839      public boolean createNewFile(Path f) throws IOException {
840        if (exists(f)) {
841          return false;
842        } else {
843          create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
844          return true;
845        }
846      }
847    
848      /**
849       * Append to an existing file (optional operation).
850       * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
851       * @param f the existing file to be appended.
852       * @throws IOException
853       */
854      public FSDataOutputStream append(Path f) throws IOException {
855        return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
856      }
857      /**
858       * Append to an existing file (optional operation).
859       * Same as append(f, bufferSize, null).
860       * @param f the existing file to be appended.
861       * @param bufferSize the size of the buffer to be used.
862       * @throws IOException
863       */
864      public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
865        return append(f, bufferSize, null);
866      }
867    
868      /**
869       * Append to an existing file (optional operation).
870       * @param f the existing file to be appended.
871       * @param bufferSize the size of the buffer to be used.
872       * @param progress for reporting progress if it is not null.
873       * @throws IOException
874       */
875      public abstract FSDataOutputStream append(Path f, int bufferSize,
876          Progressable progress) throws IOException;
877    
878     /**
879       * Get replication.
880       * 
881       * @deprecated Use getFileStatus() instead
882       * @param src file name
883       * @return file replication
884       * @throws IOException
885       */ 
886      @Deprecated
887      public short getReplication(Path src) throws IOException {
888        return getFileStatus(src).getReplication();
889      }
890    
891      /**
892       * Set replication for an existing file.
893       * 
894       * @param src file name
895       * @param replication new replication
896       * @throws IOException
897       * @return true if successful;
898       *         false if file does not exist or is a directory
899       */
900      public boolean setReplication(Path src, short replication)
901        throws IOException {
902        return true;
903      }
904    
905      /**
906       * Renames Path src to Path dst.  Can take place on local fs
907       * or remote DFS.
908       * @param src path to be renamed
909       * @param dst new path after rename
910       * @throws IOException on failure
911       * @return true if rename is successful
912       */
913      public abstract boolean rename(Path src, Path dst) throws IOException;
914    
915      /**
916       * Renames Path src to Path dst
917       * <ul>
918       * <li
919       * <li>Fails if src is a file and dst is a directory.
920       * <li>Fails if src is a directory and dst is a file.
921       * <li>Fails if the parent of dst does not exist or is a file.
922       * </ul>
923       * <p>
924       * If OVERWRITE option is not passed as an argument, rename fails
925       * if the dst already exists.
926       * <p>
927       * If OVERWRITE option is passed as an argument, rename overwrites
928       * the dst if it is a file or an empty directory. Rename fails if dst is
929       * a non-empty directory.
930       * <p>
931       * Note that atomicity of rename is dependent on the file system
932       * implementation. Please refer to the file system documentation for
933       * details. This default implementation is non atomic.
934       * <p>
935       * This method is deprecated since it is a temporary method added to 
936       * support the transition from FileSystem to FileContext for user 
937       * applications.
938       * 
939       * @param src path to be renamed
940       * @param dst new path after rename
941       * @throws IOException on failure
942       */
943      @Deprecated
944      protected void rename(final Path src, final Path dst,
945          final Rename... options) throws IOException {
946        // Default implementation
947        final FileStatus srcStatus = getFileStatus(src);
948        if (srcStatus == null) {
949          throw new FileNotFoundException("rename source " + src + " not found.");
950        }
951    
952        boolean overwrite = false;
953        if (null != options) {
954          for (Rename option : options) {
955            if (option == Rename.OVERWRITE) {
956              overwrite = true;
957            }
958          }
959        }
960    
961        FileStatus dstStatus;
962        try {
963          dstStatus = getFileStatus(dst);
964        } catch (IOException e) {
965          dstStatus = null;
966        }
967        if (dstStatus != null) {
968          if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
969            throw new IOException("Source " + src + " Destination " + dst
970                + " both should be either file or directory");
971          }
972          if (!overwrite) {
973            throw new FileAlreadyExistsException("rename destination " + dst
974                + " already exists.");
975          }
976          // Delete the destination that is a file or an empty directory
977          if (dstStatus.isDirectory()) {
978            FileStatus[] list = listStatus(dst);
979            if (list != null && list.length != 0) {
980              throw new IOException(
981                  "rename cannot overwrite non empty destination directory " + dst);
982            }
983          }
984          delete(dst, false);
985        } else {
986          final Path parent = dst.getParent();
987          final FileStatus parentStatus = getFileStatus(parent);
988          if (parentStatus == null) {
989            throw new FileNotFoundException("rename destination parent " + parent
990                + " not found.");
991          }
992          if (!parentStatus.isDirectory()) {
993            throw new ParentNotDirectoryException("rename destination parent " + parent
994                + " is a file.");
995          }
996        }
997        if (!rename(src, dst)) {
998          throw new IOException("rename from " + src + " to " + dst + " failed.");
999        }
1000      }
1001      
1002      /**
1003       * Delete a file 
1004       * @deprecated Use {@link #delete(Path, boolean)} instead.
1005       */
1006      @Deprecated
1007      public boolean delete(Path f) throws IOException {
1008        return delete(f, true);
1009      }
1010      
1011      /** Delete a file.
1012       *
1013       * @param f the path to delete.
1014       * @param recursive if path is a directory and set to 
1015       * true, the directory is deleted else throws an exception. In
1016       * case of a file the recursive can be set to either true or false. 
1017       * @return  true if delete is successful else false. 
1018       * @throws IOException
1019       */
1020      public abstract boolean delete(Path f, boolean recursive) throws IOException;
1021    
1022      /**
1023       * Mark a path to be deleted when FileSystem is closed.
1024       * When the JVM shuts down,
1025       * all FileSystem objects will be closed automatically.
1026       * Then,
1027       * the marked path will be deleted as a result of closing the FileSystem.
1028       *
1029       * The path has to exist in the file system.
1030       * 
1031       * @param f the path to delete.
1032       * @return  true if deleteOnExit is successful, otherwise false.
1033       * @throws IOException
1034       */
1035      public boolean deleteOnExit(Path f) throws IOException {
1036        if (!exists(f)) {
1037          return false;
1038        }
1039        synchronized (deleteOnExit) {
1040          deleteOnExit.add(f);
1041        }
1042        return true;
1043      }
1044    
1045      /**
1046       * Delete all files that were marked as delete-on-exit. This recursively
1047       * deletes all files in the specified paths.
1048       */
1049      protected void processDeleteOnExit() {
1050        synchronized (deleteOnExit) {
1051          for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
1052            Path path = iter.next();
1053            try {
1054              delete(path, true);
1055            }
1056            catch (IOException e) {
1057              LOG.info("Ignoring failure to deleteOnExit for path " + path);
1058            }
1059            iter.remove();
1060          }
1061        }
1062      }
1063      
1064      /** Check if exists.
1065       * @param f source file
1066       */
1067      public boolean exists(Path f) throws IOException {
1068        try {
1069          return getFileStatus(f) != null;
1070        } catch (FileNotFoundException e) {
1071          return false;
1072        }
1073      }
1074    
1075      /** True iff the named path is a directory.
1076       * Note: Avoid using this method. Instead reuse the FileStatus 
1077       * returned by getFileStatus() or listStatus() methods.
1078       * @param f path to check
1079       */
1080      public boolean isDirectory(Path f) throws IOException {
1081        try {
1082          return getFileStatus(f).isDirectory();
1083        } catch (FileNotFoundException e) {
1084          return false;               // f does not exist
1085        }
1086      }
1087    
1088      /** True iff the named path is a regular file.
1089       * Note: Avoid using this method. Instead reuse the FileStatus 
1090       * returned by getFileStatus() or listStatus() methods.
1091       * @param f path to check
1092       */
1093      public boolean isFile(Path f) throws IOException {
1094        try {
1095          return getFileStatus(f).isFile();
1096        } catch (FileNotFoundException e) {
1097          return false;               // f does not exist
1098        }
1099      }
1100      
1101      /** The number of bytes in a file. */
1102      /** @deprecated Use getFileStatus() instead */
1103      @Deprecated
1104      public long getLength(Path f) throws IOException {
1105        return getFileStatus(f).getLen();
1106      }
1107        
1108      /** Return the {@link ContentSummary} of a given {@link Path}.
1109      * @param f path to use
1110      */
1111      public ContentSummary getContentSummary(Path f) throws IOException {
1112        FileStatus status = getFileStatus(f);
1113        if (status.isFile()) {
1114          // f is a file
1115          return new ContentSummary(status.getLen(), 1, 0);
1116        }
1117        // f is a directory
1118        long[] summary = {0, 0, 1};
1119        for(FileStatus s : listStatus(f)) {
1120          ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1121                                         new ContentSummary(s.getLen(), 1, 0);
1122          summary[0] += c.getLength();
1123          summary[1] += c.getFileCount();
1124          summary[2] += c.getDirectoryCount();
1125        }
1126        return new ContentSummary(summary[0], summary[1], summary[2]);
1127      }
1128    
1129      final private static PathFilter DEFAULT_FILTER = new PathFilter() {
1130          public boolean accept(Path file) {
1131            return true;
1132          }     
1133        };
1134        
1135      /**
1136       * List the statuses of the files/directories in the given path if the path is
1137       * a directory.
1138       * 
1139       * @param f given path
1140       * @return the statuses of the files/directories in the given patch
1141       * @throws FileNotFoundException when the path does not exist;
1142       *         IOException see specific implementation
1143       */
1144      public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, 
1145                                                             IOException;
1146        
1147      /*
1148       * Filter files/directories in the given path using the user-supplied path
1149       * filter. Results are added to the given array <code>results</code>.
1150       */
1151      private void listStatus(ArrayList<FileStatus> results, Path f,
1152          PathFilter filter) throws FileNotFoundException, IOException {
1153        FileStatus listing[] = listStatus(f);
1154        if (listing == null) {
1155          throw new IOException("Error accessing " + f);
1156        }
1157    
1158        for (int i = 0; i < listing.length; i++) {
1159          if (filter.accept(listing[i].getPath())) {
1160            results.add(listing[i]);
1161          }
1162        }
1163      }
1164    
1165      /**
1166       * @return an iterator over the corrupt files under the given path
1167       * (may contain duplicates if a file has more than one corrupt block)
1168       * @throws IOException
1169       */
1170      public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1171        throws IOException {
1172        throw new UnsupportedOperationException(getClass().getCanonicalName() +
1173                                                " does not support" +
1174                                                " listCorruptFileBlocks");
1175      }
1176    
1177      /**
1178       * Filter files/directories in the given path using the user-supplied path
1179       * filter.
1180       * 
1181       * @param f
1182       *          a path name
1183       * @param filter
1184       *          the user-supplied path filter
1185       * @return an array of FileStatus objects for the files under the given path
1186       *         after applying the filter
1187       * @throws FileNotFoundException when the path does not exist;
1188       *         IOException see specific implementation   
1189       */
1190      public FileStatus[] listStatus(Path f, PathFilter filter) 
1191                                       throws FileNotFoundException, IOException {
1192        ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1193        listStatus(results, f, filter);
1194        return results.toArray(new FileStatus[results.size()]);
1195      }
1196    
1197      /**
1198       * Filter files/directories in the given list of paths using default
1199       * path filter.
1200       * 
1201       * @param files
1202       *          a list of paths
1203       * @return a list of statuses for the files under the given paths after
1204       *         applying the filter default Path filter
1205       * @throws FileNotFoundException when the path does not exist;
1206       *         IOException see specific implementation
1207       */
1208      public FileStatus[] listStatus(Path[] files)
1209          throws FileNotFoundException, IOException {
1210        return listStatus(files, DEFAULT_FILTER);
1211      }
1212    
1213      /**
1214       * Filter files/directories in the given list of paths using user-supplied
1215       * path filter.
1216       * 
1217       * @param files
1218       *          a list of paths
1219       * @param filter
1220       *          the user-supplied path filter
1221       * @return a list of statuses for the files under the given paths after
1222       *         applying the filter
1223       * @throws FileNotFoundException when the path does not exist;
1224       *         IOException see specific implementation
1225       */
1226      public FileStatus[] listStatus(Path[] files, PathFilter filter)
1227          throws FileNotFoundException, IOException {
1228        ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1229        for (int i = 0; i < files.length; i++) {
1230          listStatus(results, files[i], filter);
1231        }
1232        return results.toArray(new FileStatus[results.size()]);
1233      }
1234    
1235      /**
1236       * <p>Return all the files that match filePattern and are not checksum
1237       * files. Results are sorted by their names.
1238       * 
1239       * <p>
1240       * A filename pattern is composed of <i>regular</i> characters and
1241       * <i>special pattern matching</i> characters, which are:
1242       *
1243       * <dl>
1244       *  <dd>
1245       *   <dl>
1246       *    <p>
1247       *    <dt> <tt> ? </tt>
1248       *    <dd> Matches any single character.
1249       *
1250       *    <p>
1251       *    <dt> <tt> * </tt>
1252       *    <dd> Matches zero or more characters.
1253       *
1254       *    <p>
1255       *    <dt> <tt> [<i>abc</i>] </tt>
1256       *    <dd> Matches a single character from character set
1257       *     <tt>{<i>a,b,c</i>}</tt>.
1258       *
1259       *    <p>
1260       *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1261       *    <dd> Matches a single character from the character range
1262       *     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
1263       *     lexicographically less than or equal to character <tt><i>b</i></tt>.
1264       *
1265       *    <p>
1266       *    <dt> <tt> [^<i>a</i>] </tt>
1267       *    <dd> Matches a single character that is not from character set or range
1268       *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
1269       *     immediately to the right of the opening bracket.
1270       *
1271       *    <p>
1272       *    <dt> <tt> \<i>c</i> </tt>
1273       *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
1274       *
1275       *    <p>
1276       *    <dt> <tt> {ab,cd} </tt>
1277       *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1278       *    
1279       *    <p>
1280       *    <dt> <tt> {ab,c{de,fh}} </tt>
1281       *    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
1282       *
1283       *   </dl>
1284       *  </dd>
1285       * </dl>
1286       *
1287       * @param pathPattern a regular expression specifying a pth pattern
1288    
1289       * @return an array of paths that match the path pattern
1290       * @throws IOException
1291       */
1292      public FileStatus[] globStatus(Path pathPattern) throws IOException {
1293        return globStatus(pathPattern, DEFAULT_FILTER);
1294      }
1295      
1296      /**
1297       * Return an array of FileStatus objects whose path names match pathPattern
1298       * and is accepted by the user-supplied path filter. Results are sorted by
1299       * their path names.
1300       * Return null if pathPattern has no glob and the path does not exist.
1301       * Return an empty array if pathPattern has a glob and no path matches it. 
1302       * 
1303       * @param pathPattern
1304       *          a regular expression specifying the path pattern
1305       * @param filter
1306       *          a user-supplied path filter
1307       * @return an array of FileStatus objects
1308       * @throws IOException if any I/O error occurs when fetching file status
1309       */
1310      public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
1311          throws IOException {
1312        String filename = pathPattern.toUri().getPath();
1313        List<String> filePatterns = GlobExpander.expand(filename);
1314        if (filePatterns.size() == 1) {
1315          return globStatusInternal(pathPattern, filter);
1316        } else {
1317          List<FileStatus> results = new ArrayList<FileStatus>();
1318          for (String filePattern : filePatterns) {
1319            FileStatus[] files = globStatusInternal(new Path(filePattern), filter);
1320            for (FileStatus file : files) {
1321              results.add(file);
1322            }
1323          }
1324          return results.toArray(new FileStatus[results.size()]);
1325        }
1326      }
1327    
1328      private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter)
1329          throws IOException {
1330        Path[] parents = new Path[1];
1331        int level = 0;
1332        String filename = pathPattern.toUri().getPath();
1333        
1334        // path has only zero component
1335        if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
1336          return getFileStatus(new Path[]{pathPattern});
1337        }
1338    
1339        // path has at least one component
1340        String[] components = filename.split(Path.SEPARATOR);
1341        // get the first component
1342        if (pathPattern.isAbsolute()) {
1343          parents[0] = new Path(Path.SEPARATOR);
1344          level = 1;
1345        } else {
1346          parents[0] = new Path(Path.CUR_DIR);
1347        }
1348    
1349        // glob the paths that match the parent path, i.e., [0, components.length-1]
1350        boolean[] hasGlob = new boolean[]{false};
1351        Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob);
1352        FileStatus[] results;
1353        if (parentPaths == null || parentPaths.length == 0) {
1354          results = null;
1355        } else {
1356          // Now work on the last component of the path
1357          GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
1358          if (fp.hasPattern()) { // last component has a pattern
1359            // list parent directories and then glob the results
1360            results = listStatus(parentPaths, fp);
1361            hasGlob[0] = true;
1362          } else { // last component does not have a pattern
1363            // get all the path names
1364            ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
1365            for (int i = 0; i < parentPaths.length; i++) {
1366              parentPaths[i] = new Path(parentPaths[i],
1367                components[components.length - 1]);
1368              if (fp.accept(parentPaths[i])) {
1369                filteredPaths.add(parentPaths[i]);
1370              }
1371            }
1372            // get all their statuses
1373            results = getFileStatus(
1374                filteredPaths.toArray(new Path[filteredPaths.size()]));
1375          }
1376        }
1377    
1378        // Decide if the pathPattern contains a glob or not
1379        if (results == null) {
1380          if (hasGlob[0]) {
1381            results = new FileStatus[0];
1382          }
1383        } else {
1384          if (results.length == 0 ) {
1385            if (!hasGlob[0]) {
1386              results = null;
1387            }
1388          } else {
1389            Arrays.sort(results);
1390          }
1391        }
1392        return results;
1393      }
1394    
1395      /*
1396       * For a path of N components, return a list of paths that match the
1397       * components [<code>level</code>, <code>N-1</code>].
1398       */
1399      private Path[] globPathsLevel(Path[] parents, String[] filePattern,
1400          int level, boolean[] hasGlob) throws IOException {
1401        if (level == filePattern.length - 1)
1402          return parents;
1403        if (parents == null || parents.length == 0) {
1404          return null;
1405        }
1406        GlobFilter fp = new GlobFilter(filePattern[level]);
1407        if (fp.hasPattern()) {
1408          parents = FileUtil.stat2Paths(listStatus(parents, fp));
1409          hasGlob[0] = true;
1410        } else {
1411          for (int i = 0; i < parents.length; i++) {
1412            parents[i] = new Path(parents[i], filePattern[level]);
1413          }
1414        }
1415        return globPathsLevel(parents, filePattern, level + 1, hasGlob);
1416      }
1417    
1418      /**
1419       * List the statuses of the files/directories in the given path if the path is
1420       * a directory. 
1421       * Return the file's status and block locations If the path is a file.
1422       * 
1423       * If a returned status is a file, it contains the file's block locations.
1424       * 
1425       * @param f is the path
1426       *
1427       * @return an iterator that traverses statuses of the files/directories 
1428       *         in the given path
1429       *
1430       * @throws FileNotFoundException If <code>f</code> does not exist
1431       * @throws IOException If an I/O error occurred
1432       */
1433      public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
1434      throws FileNotFoundException, IOException {
1435        return listLocatedStatus(f, DEFAULT_FILTER);
1436      }
1437    
1438      /**
1439       * Listing a directory
1440       * The returned results include its block location if it is a file
1441       * The results are filtered by the given path filter
1442       * @param f a path
1443       * @param filter a path filter
1444       * @return an iterator that traverses statuses of the files/directories 
1445       *         in the given path
1446       * @throws FileNotFoundException if <code>f</code> does not exist
1447       * @throws IOException if any I/O error occurred
1448       */
1449      protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
1450          final PathFilter filter)
1451      throws FileNotFoundException, IOException {
1452        return new RemoteIterator<LocatedFileStatus>() {
1453          private final FileStatus[] stats = listStatus(f, filter);
1454          private int i = 0;
1455    
1456          @Override
1457          public boolean hasNext() {
1458            return i<stats.length;
1459          }
1460    
1461          @Override
1462          public LocatedFileStatus next() throws IOException {
1463            if (!hasNext()) {
1464              throw new NoSuchElementException("No more entry in " + f);
1465            }
1466            FileStatus result = stats[i++];
1467            BlockLocation[] locs = result.isFile() ?
1468                getFileBlockLocations(result.getPath(), 0, result.getLen()) :
1469                null;
1470            return new LocatedFileStatus(result, locs);
1471          }
1472        };
1473      }
1474    
1475      /**
1476       * List the statuses and block locations of the files in the given path.
1477       * 
1478       * If the path is a directory, 
1479       *   if recursive is false, returns files in the directory;
1480       *   if recursive is true, return files in the subtree rooted at the path.
1481       * If the path is a file, return the file's status and block locations.
1482       * 
1483       * @param f is the path
1484       * @param recursive if the subdirectories need to be traversed recursively
1485       *
1486       * @return an iterator that traverses statuses of the files
1487       *
1488       * @throws FileNotFoundException when the path does not exist;
1489       *         IOException see specific implementation
1490       */
1491      public RemoteIterator<LocatedFileStatus> listFiles(
1492          final Path f, final boolean recursive)
1493      throws FileNotFoundException, IOException {
1494        return new RemoteIterator<LocatedFileStatus>() {
1495          private Stack<RemoteIterator<LocatedFileStatus>> itors = 
1496            new Stack<RemoteIterator<LocatedFileStatus>>();
1497          private RemoteIterator<LocatedFileStatus> curItor =
1498            listLocatedStatus(f);
1499          private LocatedFileStatus curFile;
1500         
1501          @Override
1502          public boolean hasNext() throws IOException {
1503            while (curFile == null) {
1504              if (curItor.hasNext()) {
1505                handleFileStat(curItor.next());
1506              } else if (!itors.empty()) {
1507                curItor = itors.pop();
1508              } else {
1509                return false;
1510              }
1511            }
1512            return true;
1513          }
1514    
1515          /**
1516           * Process the input stat.
1517           * If it is a file, return the file stat.
1518           * If it is a directory, traverse the directory if recursive is true;
1519           * ignore it if recursive is false.
1520           * @param stat input status
1521           * @throws IOException if any IO error occurs
1522           */
1523          private void handleFileStat(LocatedFileStatus stat) throws IOException {
1524            if (stat.isFile()) { // file
1525              curFile = stat;
1526            } else if (recursive) { // directory
1527              itors.push(curItor);
1528              curItor = listLocatedStatus(stat.getPath());
1529            }
1530          }
1531    
1532          @Override
1533          public LocatedFileStatus next() throws IOException {
1534            if (hasNext()) {
1535              LocatedFileStatus result = curFile;
1536              curFile = null;
1537              return result;
1538            } 
1539            throw new java.util.NoSuchElementException("No more entry in " + f);
1540          }
1541        };
1542      }
1543      
1544      /** Return the current user's home directory in this filesystem.
1545       * The default implementation returns "/user/$USER/".
1546       */
1547      public Path getHomeDirectory() {
1548        return this.makeQualified(
1549            new Path("/user/"+System.getProperty("user.name")));
1550      }
1551    
1552    
1553      /**
1554       * Set the current working directory for the given file system. All relative
1555       * paths will be resolved relative to it.
1556       * 
1557       * @param new_dir
1558       */
1559      public abstract void setWorkingDirectory(Path new_dir);
1560        
1561      /**
1562       * Get the current working directory for the given file system
1563       * @return the directory pathname
1564       */
1565      public abstract Path getWorkingDirectory();
1566      
1567      
1568      /**
1569       * Note: with the new FilesContext class, getWorkingDirectory()
1570       * will be removed. 
1571       * The working directory is implemented in FilesContext.
1572       * 
1573       * Some file systems like LocalFileSystem have an initial workingDir
1574       * that we use as the starting workingDir. For other file systems
1575       * like HDFS there is no built in notion of an inital workingDir.
1576       * 
1577       * @return if there is built in notion of workingDir then it
1578       * is returned; else a null is returned.
1579       */
1580      protected Path getInitialWorkingDirectory() {
1581        return null;
1582      }
1583    
1584      /**
1585       * Call {@link #mkdirs(Path, FsPermission)} with default permission.
1586       */
1587      public boolean mkdirs(Path f) throws IOException {
1588        return mkdirs(f, FsPermission.getDefault());
1589      }
1590    
1591      /**
1592       * Make the given file and all non-existent parents into
1593       * directories. Has the semantics of Unix 'mkdir -p'.
1594       * Existence of the directory hierarchy is not an error.
1595       * @param f path to create
1596       * @param permission to apply to f
1597       */
1598      public abstract boolean mkdirs(Path f, FsPermission permission
1599          ) throws IOException;
1600    
1601      /**
1602       * The src file is on the local disk.  Add it to FS at
1603       * the given dst name and the source is kept intact afterwards
1604       * @param src path
1605       * @param dst path
1606       */
1607      public void copyFromLocalFile(Path src, Path dst)
1608        throws IOException {
1609        copyFromLocalFile(false, src, dst);
1610      }
1611    
1612      /**
1613       * The src files is on the local disk.  Add it to FS at
1614       * the given dst name, removing the source afterwards.
1615       * @param srcs path
1616       * @param dst path
1617       */
1618      public void moveFromLocalFile(Path[] srcs, Path dst)
1619        throws IOException {
1620        copyFromLocalFile(true, true, srcs, dst);
1621      }
1622    
1623      /**
1624       * The src file is on the local disk.  Add it to FS at
1625       * the given dst name, removing the source afterwards.
1626       * @param src path
1627       * @param dst path
1628       */
1629      public void moveFromLocalFile(Path src, Path dst)
1630        throws IOException {
1631        copyFromLocalFile(true, src, dst);
1632      }
1633    
1634      /**
1635       * The src file is on the local disk.  Add it to FS at
1636       * the given dst name.
1637       * delSrc indicates if the source should be removed
1638       * @param delSrc whether to delete the src
1639       * @param src path
1640       * @param dst path
1641       */
1642      public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
1643        throws IOException {
1644        copyFromLocalFile(delSrc, true, src, dst);
1645      }
1646      
1647      /**
1648       * The src files are on the local disk.  Add it to FS at
1649       * the given dst name.
1650       * delSrc indicates if the source should be removed
1651       * @param delSrc whether to delete the src
1652       * @param overwrite whether to overwrite an existing file
1653       * @param srcs array of paths which are source
1654       * @param dst path
1655       */
1656      public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
1657                                    Path[] srcs, Path dst)
1658        throws IOException {
1659        Configuration conf = getConf();
1660        FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
1661      }
1662      
1663      /**
1664       * The src file is on the local disk.  Add it to FS at
1665       * the given dst name.
1666       * delSrc indicates if the source should be removed
1667       * @param delSrc whether to delete the src
1668       * @param overwrite whether to overwrite an existing file
1669       * @param src path
1670       * @param dst path
1671       */
1672      public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
1673                                    Path src, Path dst)
1674        throws IOException {
1675        Configuration conf = getConf();
1676        FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
1677      }
1678        
1679      /**
1680       * The src file is under FS, and the dst is on the local disk.
1681       * Copy it from FS control to the local dst name.
1682       * @param src path
1683       * @param dst path
1684       */
1685      public void copyToLocalFile(Path src, Path dst) throws IOException {
1686        copyToLocalFile(false, src, dst);
1687      }
1688        
1689      /**
1690       * The src file is under FS, and the dst is on the local disk.
1691       * Copy it from FS control to the local dst name.
1692       * Remove the source afterwards
1693       * @param src path
1694       * @param dst path
1695       */
1696      public void moveToLocalFile(Path src, Path dst) throws IOException {
1697        copyToLocalFile(true, src, dst);
1698      }
1699    
1700      /**
1701       * The src file is under FS, and the dst is on the local disk.
1702       * Copy it from FS control to the local dst name.
1703       * delSrc indicates if the src will be removed or not.
1704       * @param delSrc whether to delete the src
1705       * @param src path
1706       * @param dst path
1707       */   
1708      public void copyToLocalFile(boolean delSrc, Path src, Path dst)
1709        throws IOException {
1710        copyToLocalFile(delSrc, src, dst, false);
1711      }
1712      
1713        /**
1714       * The src file is under FS, and the dst is on the local disk. Copy it from FS
1715       * control to the local dst name. delSrc indicates if the src will be removed
1716       * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
1717       * as local file system or not. RawLocalFileSystem is non crc file system.So,
1718       * It will not create any crc files at local.
1719       * 
1720       * @param delSrc
1721       *          whether to delete the src
1722       * @param src
1723       *          path
1724       * @param dst
1725       *          path
1726       * @param useRawLocalFileSystem
1727       *          whether to use RawLocalFileSystem as local file system or not.
1728       * 
1729       * @throws IOException
1730       *           - if any IO error
1731       */
1732      public void copyToLocalFile(boolean delSrc, Path src, Path dst,
1733          boolean useRawLocalFileSystem) throws IOException {
1734        Configuration conf = getConf();
1735        FileSystem local = null;
1736        if (useRawLocalFileSystem) {
1737          local = getLocal(conf).getRawFileSystem();
1738        } else {
1739          local = getLocal(conf);
1740        }
1741        FileUtil.copy(this, src, local, dst, delSrc, conf);
1742      }
1743    
1744      /**
1745       * Returns a local File that the user can write output to.  The caller
1746       * provides both the eventual FS target name and the local working
1747       * file.  If the FS is local, we write directly into the target.  If
1748       * the FS is remote, we write into the tmp local area.
1749       * @param fsOutputFile path of output file
1750       * @param tmpLocalFile path of local tmp file
1751       */
1752      public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1753        throws IOException {
1754        return tmpLocalFile;
1755      }
1756    
1757      /**
1758       * Called when we're all done writing to the target.  A local FS will
1759       * do nothing, because we've written to exactly the right place.  A remote
1760       * FS will copy the contents of tmpLocalFile to the correct target at
1761       * fsOutputFile.
1762       * @param fsOutputFile path of output file
1763       * @param tmpLocalFile path to local tmp file
1764       */
1765      public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
1766        throws IOException {
1767        moveFromLocalFile(tmpLocalFile, fsOutputFile);
1768      }
1769    
1770      /**
1771       * No more filesystem operations are needed.  Will
1772       * release any held locks.
1773       */
1774      public void close() throws IOException {
1775        // delete all files that were marked as delete-on-exit.
1776        processDeleteOnExit();
1777        CACHE.remove(this.key, this);
1778      }
1779    
1780      /** Return the total size of all files in the filesystem.*/
1781      public long getUsed() throws IOException{
1782        long used = 0;
1783        FileStatus[] files = listStatus(new Path("/"));
1784        for(FileStatus file:files){
1785          used += file.getLen();
1786        }
1787        return used;
1788      }
1789      
1790      /**
1791       * Get the block size for a particular file.
1792       * @param f the filename
1793       * @return the number of bytes in a block
1794       */
1795      /** @deprecated Use getFileStatus() instead */
1796      @Deprecated
1797      public long getBlockSize(Path f) throws IOException {
1798        return getFileStatus(f).getBlockSize();
1799      }
1800    
1801      /** Return the number of bytes that large input files should be optimally
1802       * be split into to minimize i/o time. */
1803      public long getDefaultBlockSize() {
1804        // default to 32MB: large enough to minimize the impact of seeks
1805        return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
1806      }
1807        
1808      /**
1809       * Get the default replication.
1810       */
1811      public short getDefaultReplication() { return 1; }
1812    
1813      /**
1814       * Return a file status object that represents the path.
1815       * @param f The path we want information from
1816       * @return a FileStatus object
1817       * @throws FileNotFoundException when the path does not exist;
1818       *         IOException see specific implementation
1819       */
1820      public abstract FileStatus getFileStatus(Path f) throws IOException;
1821    
1822      /**
1823       * Get the checksum of a file.
1824       *
1825       * @param f The file path
1826       * @return The file checksum.  The default return value is null,
1827       *  which indicates that no checksum algorithm is implemented
1828       *  in the corresponding FileSystem.
1829       */
1830      public FileChecksum getFileChecksum(Path f) throws IOException {
1831        return null;
1832      }
1833      
1834      /**
1835       * Set the verify checksum flag. This is only applicable if the 
1836       * corresponding FileSystem supports checksum. By default doesn't do anything.
1837       * @param verifyChecksum
1838       */
1839      public void setVerifyChecksum(boolean verifyChecksum) {
1840        //doesn't do anything
1841      }
1842    
1843      /**
1844       * Return a list of file status objects that corresponds to the list of paths
1845       * excluding those non-existent paths.
1846       * 
1847       * @param paths
1848       *          the list of paths we want information from
1849       * @return a list of FileStatus objects
1850       * @throws IOException
1851       *           see specific implementation
1852       */
1853      private FileStatus[] getFileStatus(Path[] paths) throws IOException {
1854        if (paths == null) {
1855          return null;
1856        }
1857        ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
1858        for (int i = 0; i < paths.length; i++) {
1859          try {
1860            results.add(getFileStatus(paths[i]));
1861          } catch (FileNotFoundException e) { // do nothing
1862          }
1863        }
1864        return results.toArray(new FileStatus[results.size()]);
1865      }
1866      
1867      /**
1868       * Returns a status object describing the use and capacity of the
1869       * file system. If the file system has multiple partitions, the
1870       * use and capacity of the root partition is reflected.
1871       * 
1872       * @return a FsStatus object
1873       * @throws IOException
1874       *           see specific implementation
1875       */
1876      public FsStatus getStatus() throws IOException {
1877        return getStatus(null);
1878      }
1879    
1880      /**
1881       * Returns a status object describing the use and capacity of the
1882       * file system. If the file system has multiple partitions, the
1883       * use and capacity of the partition pointed to by the specified
1884       * path is reflected.
1885       * @param p Path for which status should be obtained. null means
1886       * the default partition. 
1887       * @return a FsStatus object
1888       * @throws IOException
1889       *           see specific implementation
1890       */
1891      public FsStatus getStatus(Path p) throws IOException {
1892        return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE);
1893      }
1894    
1895      /**
1896       * Set permission of a path.
1897       * @param p
1898       * @param permission
1899       */
1900      public void setPermission(Path p, FsPermission permission
1901          ) throws IOException {
1902      }
1903    
1904      /**
1905       * Set owner of a path (i.e. a file or a directory).
1906       * The parameters username and groupname cannot both be null.
1907       * @param p The path
1908       * @param username If it is null, the original username remains unchanged.
1909       * @param groupname If it is null, the original groupname remains unchanged.
1910       */
1911      public void setOwner(Path p, String username, String groupname
1912          ) throws IOException {
1913      }
1914    
1915      /**
1916       * Set access time of a file
1917       * @param p The path
1918       * @param mtime Set the modification time of this file.
1919       *              The number of milliseconds since Jan 1, 1970. 
1920       *              A value of -1 means that this call should not set modification time.
1921       * @param atime Set the access time of this file.
1922       *              The number of milliseconds since Jan 1, 1970. 
1923       *              A value of -1 means that this call should not set access time.
1924       */
1925      public void setTimes(Path p, long mtime, long atime
1926          ) throws IOException {
1927      }
1928    
1929      private static FileSystem createFileSystem(URI uri, Configuration conf
1930          ) throws IOException {
1931        Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
1932        if (clazz == null) {
1933          throw new IOException("No FileSystem for scheme: " + uri.getScheme());
1934        }
1935        FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
1936        fs.initialize(uri, conf);
1937        return fs;
1938      }
1939    
1940      /** Caching FileSystem objects */
1941      static class Cache {
1942        private final ClientFinalizer clientFinalizer = new ClientFinalizer();
1943    
1944        private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
1945        private final Set<Key> toAutoClose = new HashSet<Key>();
1946    
1947        /** A variable that makes all objects in the cache unique */
1948        private static AtomicLong unique = new AtomicLong(1);
1949    
1950        FileSystem get(URI uri, Configuration conf) throws IOException{
1951          Key key = new Key(uri, conf);
1952          return getInternal(uri, conf, key);
1953        }
1954    
1955        /** The objects inserted into the cache using this method are all unique */
1956        FileSystem getUnique(URI uri, Configuration conf) throws IOException{
1957          Key key = new Key(uri, conf, unique.getAndIncrement());
1958          return getInternal(uri, conf, key);
1959        }
1960    
1961        private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
1962          FileSystem fs;
1963          synchronized (this) {
1964            fs = map.get(key);
1965          }
1966          if (fs != null) {
1967            return fs;
1968          }
1969    
1970          fs = createFileSystem(uri, conf);
1971          synchronized (this) { // refetch the lock again
1972            FileSystem oldfs = map.get(key);
1973            if (oldfs != null) { // a file system is created while lock is releasing
1974              fs.close(); // close the new file system
1975              return oldfs;  // return the old file system
1976            }
1977            
1978            // now insert the new file system into the map
1979            if (map.isEmpty() && !clientFinalizer.isAlive()) {
1980              Runtime.getRuntime().addShutdownHook(clientFinalizer);
1981            }
1982            fs.key = key;
1983            map.put(key, fs);
1984            if (conf.getBoolean("fs.automatic.close", true)) {
1985              toAutoClose.add(key);
1986            }
1987            return fs;
1988          }
1989        }
1990    
1991        synchronized void remove(Key key, FileSystem fs) {
1992          if (map.containsKey(key) && fs == map.get(key)) {
1993            map.remove(key);
1994            toAutoClose.remove(key);
1995            if (map.isEmpty() && !clientFinalizer.isAlive()) {
1996              if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
1997                LOG.info("Could not cancel cleanup thread, though no " +
1998                         "FileSystems are open");
1999              }
2000            }
2001          }
2002        }
2003    
2004        synchronized void closeAll() throws IOException {
2005          closeAll(false);
2006        }
2007    
2008        /**
2009         * Close all FileSystem instances in the Cache.
2010         * @param onlyAutomatic only close those that are marked for automatic closing
2011         */
2012        synchronized void closeAll(boolean onlyAutomatic) throws IOException {
2013          List<IOException> exceptions = new ArrayList<IOException>();
2014    
2015          // Make a copy of the keys in the map since we'll be modifying
2016          // the map while iterating over it, which isn't safe.
2017          List<Key> keys = new ArrayList<Key>();
2018          keys.addAll(map.keySet());
2019    
2020          for (Key key : keys) {
2021            final FileSystem fs = map.get(key);
2022    
2023            if (onlyAutomatic && !toAutoClose.contains(key)) {
2024              continue;
2025            }
2026    
2027            //remove from cache
2028            remove(key, fs);
2029    
2030            if (fs != null) {
2031              try {
2032                fs.close();
2033              }
2034              catch(IOException ioe) {
2035                exceptions.add(ioe);
2036              }
2037            }
2038          }
2039    
2040          if (!exceptions.isEmpty()) {
2041            throw MultipleIOException.createIOException(exceptions);
2042          }
2043        }
2044    
2045        private class ClientFinalizer extends Thread {
2046          public synchronized void run() {
2047            try {
2048              closeAll(true);
2049            } catch (IOException e) {
2050              LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e);
2051            }
2052          }
2053        }
2054    
2055        synchronized void closeAll(UserGroupInformation ugi) throws IOException {
2056          List<FileSystem> targetFSList = new ArrayList<FileSystem>();
2057          //Make a pass over the list and collect the filesystems to close
2058          //we cannot close inline since close() removes the entry from the Map
2059          for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
2060            final Key key = entry.getKey();
2061            final FileSystem fs = entry.getValue();
2062            if (ugi.equals(key.ugi) && fs != null) {
2063              targetFSList.add(fs);   
2064            }
2065          }
2066          List<IOException> exceptions = new ArrayList<IOException>();
2067          //now make a pass over the target list and close each
2068          for (FileSystem fs : targetFSList) {
2069            try {
2070              fs.close();
2071            }
2072            catch(IOException ioe) {
2073              exceptions.add(ioe);
2074            }
2075          }
2076          if (!exceptions.isEmpty()) {
2077            throw MultipleIOException.createIOException(exceptions);
2078          }
2079        }
2080    
2081        /** FileSystem.Cache.Key */
2082        static class Key {
2083          final String scheme;
2084          final String authority;
2085          final UserGroupInformation ugi;
2086          final long unique;   // an artificial way to make a key unique
2087    
2088          Key(URI uri, Configuration conf) throws IOException {
2089            this(uri, conf, 0);
2090          }
2091    
2092          Key(URI uri, Configuration conf, long unique) throws IOException {
2093            scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
2094            authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
2095            this.unique = unique;
2096            
2097            this.ugi = UserGroupInformation.getCurrentUser();
2098          }
2099    
2100          /** {@inheritDoc} */
2101          public int hashCode() {
2102            return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique;
2103          }
2104    
2105          static boolean isEqual(Object a, Object b) {
2106            return a == b || (a != null && a.equals(b));        
2107          }
2108    
2109          /** {@inheritDoc} */
2110          public boolean equals(Object obj) {
2111            if (obj == this) {
2112              return true;
2113            }
2114            if (obj != null && obj instanceof Key) {
2115              Key that = (Key)obj;
2116              return isEqual(this.scheme, that.scheme)
2117                     && isEqual(this.authority, that.authority)
2118                     && isEqual(this.ugi, that.ugi)
2119                     && (this.unique == that.unique);
2120            }
2121            return false;        
2122          }
2123    
2124          /** {@inheritDoc} */
2125          public String toString() {
2126            return "("+ugi.toString() + ")@" + scheme + "://" + authority;        
2127          }
2128        }
2129      }
2130      
2131      public static final class Statistics {
2132        private final String scheme;
2133        private AtomicLong bytesRead = new AtomicLong();
2134        private AtomicLong bytesWritten = new AtomicLong();
2135        private AtomicInteger readOps = new AtomicInteger();
2136        private AtomicInteger largeReadOps = new AtomicInteger();
2137        private AtomicInteger writeOps = new AtomicInteger();
2138        
2139        public Statistics(String scheme) {
2140          this.scheme = scheme;
2141        }
2142    
2143        /**
2144         * Copy constructor.
2145         * 
2146         * @param st
2147         *          The input Statistics object which is cloned.
2148         */
2149        public Statistics(Statistics st) {
2150          this.scheme = st.scheme;
2151          this.bytesRead = new AtomicLong(st.bytesRead.longValue());
2152          this.bytesWritten = new AtomicLong(st.bytesWritten.longValue());
2153        }
2154    
2155        /**
2156         * Increment the bytes read in the statistics
2157         * @param newBytes the additional bytes read
2158         */
2159        public void incrementBytesRead(long newBytes) {
2160          bytesRead.getAndAdd(newBytes);
2161        }
2162        
2163        /**
2164         * Increment the bytes written in the statistics
2165         * @param newBytes the additional bytes written
2166         */
2167        public void incrementBytesWritten(long newBytes) {
2168          bytesWritten.getAndAdd(newBytes);
2169        }
2170        
2171        /**
2172         * Increment the number of read operations
2173         * @param count number of read operations
2174         */
2175        public void incrementReadOps(int count) {
2176          readOps.getAndAdd(count);
2177        }
2178    
2179        /**
2180         * Increment the number of large read operations
2181         * @param count number of large read operations
2182         */
2183        public void incrementLargeReadOps(int count) {
2184          largeReadOps.getAndAdd(count);
2185        }
2186    
2187        /**
2188         * Increment the number of write operations
2189         * @param count number of write operations
2190         */
2191        public void incrementWriteOps(int count) {
2192          writeOps.getAndAdd(count);
2193        }
2194    
2195        /**
2196         * Get the total number of bytes read
2197         * @return the number of bytes
2198         */
2199        public long getBytesRead() {
2200          return bytesRead.get();
2201        }
2202        
2203        /**
2204         * Get the total number of bytes written
2205         * @return the number of bytes
2206         */
2207        public long getBytesWritten() {
2208          return bytesWritten.get();
2209        }
2210        
2211        /**
2212         * Get the number of file system read operations such as list files
2213         * @return number of read operations
2214         */
2215        public int getReadOps() {
2216          return readOps.get() + largeReadOps.get();
2217        }
2218    
2219        /**
2220         * Get the number of large file system read operations such as list files
2221         * under a large directory
2222         * @return number of large read operations
2223         */
2224        public int getLargeReadOps() {
2225          return largeReadOps.get();
2226        }
2227    
2228        /**
2229         * Get the number of file system write operations such as create, append 
2230         * rename etc.
2231         * @return number of write operations
2232         */
2233        public int getWriteOps() {
2234          return writeOps.get();
2235        }
2236    
2237        public String toString() {
2238          return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
2239              + readOps + " read ops, " + largeReadOps + " large read ops, "
2240              + writeOps + " write ops";
2241        }
2242        
2243        /**
2244         * Reset the counts of bytes to 0.
2245         */
2246        public void reset() {
2247          bytesWritten.set(0);
2248          bytesRead.set(0);
2249        }
2250        
2251        /**
2252         * Get the uri scheme associated with this statistics object.
2253         * @return the schema associated with this set of statistics
2254         */
2255        public String getScheme() {
2256          return scheme;
2257        }
2258      }
2259      
2260      /**
2261       * Get the Map of Statistics object indexed by URI Scheme.
2262       * @return a Map having a key as URI scheme and value as Statistics object
2263       * @deprecated use {@link #getAllStatistics} instead
2264       */
2265      @Deprecated
2266      public static synchronized Map<String, Statistics> getStatistics() {
2267        Map<String, Statistics> result = new HashMap<String, Statistics>();
2268        for(Statistics stat: statisticsTable.values()) {
2269          result.put(stat.getScheme(), stat);
2270        }
2271        return result;
2272      }
2273    
2274      /**
2275       * Return the FileSystem classes that have Statistics
2276       */
2277      public static synchronized List<Statistics> getAllStatistics() {
2278        return new ArrayList<Statistics>(statisticsTable.values());
2279      }
2280      
2281      /**
2282       * Get the statistics for a particular file system
2283       * @param cls the class to lookup
2284       * @return a statistics object
2285       */
2286      public static synchronized 
2287      Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
2288        Statistics result = statisticsTable.get(cls);
2289        if (result == null) {
2290          result = new Statistics(scheme);
2291          statisticsTable.put(cls, result);
2292        }
2293        return result;
2294      }
2295      
2296      /**
2297       * Reset all statistics for all file systems
2298       */
2299      public static synchronized void clearStatistics() {
2300        for(Statistics stat: statisticsTable.values()) {
2301          stat.reset();
2302        }
2303      }
2304    
2305      /**
2306       * Print all statistics for all file systems
2307       */
2308      public static synchronized
2309      void printStatistics() throws IOException {
2310        for (Map.Entry<Class<? extends FileSystem>, Statistics> pair: 
2311                statisticsTable.entrySet()) {
2312          System.out.println("  FileSystem " + pair.getKey().getName() + 
2313                             ": " + pair.getValue());
2314        }
2315      }
2316    }