001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs; 019 020 import java.io.Closeable; 021 import java.io.FileNotFoundException; 022 import java.io.IOException; 023 import java.net.URI; 024 import java.security.PrivilegedExceptionAction; 025 import java.util.ArrayList; 026 import java.util.Arrays; 027 import java.util.EnumSet; 028 import java.util.HashMap; 029 import java.util.HashSet; 030 import java.util.IdentityHashMap; 031 import java.util.Iterator; 032 import java.util.List; 033 import java.util.Map; 034 import java.util.NoSuchElementException; 035 import java.util.Set; 036 import java.util.Stack; 037 import java.util.TreeSet; 038 import java.util.concurrent.atomic.AtomicInteger; 039 import java.util.concurrent.atomic.AtomicLong; 040 041 import org.apache.commons.logging.Log; 042 import org.apache.commons.logging.LogFactory; 043 import org.apache.hadoop.classification.InterfaceAudience; 044 import org.apache.hadoop.classification.InterfaceStability; 045 import org.apache.hadoop.conf.Configuration; 046 import org.apache.hadoop.conf.Configured; 047 import org.apache.hadoop.fs.Options.Rename; 048 import org.apache.hadoop.fs.permission.FsPermission; 049 import org.apache.hadoop.io.MultipleIOException; 050 import org.apache.hadoop.security.SecurityUtil; 051 import org.apache.hadoop.security.UserGroupInformation; 052 import org.apache.hadoop.security.token.Token; 053 import org.apache.hadoop.util.Progressable; 054 import org.apache.hadoop.util.ReflectionUtils; 055 056 /**************************************************************** 057 * An abstract base class for a fairly generic filesystem. It 058 * may be implemented as a distributed filesystem, or as a "local" 059 * one that reflects the locally-connected disk. The local version 060 * exists for small Hadoop instances and for testing. 061 * 062 * <p> 063 * 064 * All user code that may potentially use the Hadoop Distributed 065 * File System should be written to use a FileSystem object. The 066 * Hadoop DFS is a multi-machine system that appears as a single 067 * disk. It's useful because of its fault tolerance and potentially 068 * very large capacity. 069 * 070 * <p> 071 * The local implementation is {@link LocalFileSystem} and distributed 072 * implementation is DistributedFileSystem. 073 *****************************************************************/ 074 @InterfaceAudience.Public 075 @InterfaceStability.Stable 076 public abstract class FileSystem extends Configured implements Closeable { 077 public static final String FS_DEFAULT_NAME_KEY = 078 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY; 079 public static final String DEFAULT_FS = 080 CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT; 081 082 public static final Log LOG = LogFactory.getLog(FileSystem.class); 083 084 /** FileSystem cache */ 085 static final Cache CACHE = new Cache(); 086 087 /** The key this instance is stored under in the cache. */ 088 private Cache.Key key; 089 090 /** Recording statistics per a FileSystem class */ 091 private static final Map<Class<? extends FileSystem>, Statistics> 092 statisticsTable = 093 new IdentityHashMap<Class<? extends FileSystem>, Statistics>(); 094 095 /** 096 * The statistics for this file system. 097 */ 098 protected Statistics statistics; 099 100 /** 101 * A cache of files that should be deleted when filsystem is closed 102 * or the JVM is exited. 103 */ 104 private Set<Path> deleteOnExit = new TreeSet<Path>(); 105 106 /** 107 * This method adds a file system for testing so that we can find it later. It 108 * is only for testing. 109 * @param uri the uri to store it under 110 * @param conf the configuration to store it under 111 * @param fs the file system to store 112 * @throws IOException 113 */ 114 static void addFileSystemForTesting(URI uri, Configuration conf, 115 FileSystem fs) throws IOException { 116 CACHE.map.put(new Cache.Key(uri, conf), fs); 117 } 118 119 /** 120 * Get a filesystem instance based on the uri, the passed 121 * configuration and the user 122 * @param uri of the filesystem 123 * @param conf the configuration to use 124 * @param user to perform the get as 125 * @return the filesystem instance 126 * @throws IOException 127 * @throws InterruptedException 128 */ 129 public static FileSystem get(final URI uri, final Configuration conf, 130 final String user) throws IOException, InterruptedException { 131 UserGroupInformation ugi; 132 if (user == null) { 133 ugi = UserGroupInformation.getCurrentUser(); 134 } else { 135 ugi = UserGroupInformation.createRemoteUser(user); 136 } 137 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 138 public FileSystem run() throws IOException { 139 return get(uri, conf); 140 } 141 }); 142 } 143 144 /** 145 * Returns the configured filesystem implementation. 146 * @param conf the configuration to use 147 */ 148 public static FileSystem get(Configuration conf) throws IOException { 149 return get(getDefaultUri(conf), conf); 150 } 151 152 /** Get the default filesystem URI from a configuration. 153 * @param conf the configuration to use 154 * @return the uri of the default filesystem 155 */ 156 public static URI getDefaultUri(Configuration conf) { 157 return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS))); 158 } 159 160 /** Set the default filesystem URI in a configuration. 161 * @param conf the configuration to alter 162 * @param uri the new default filesystem uri 163 */ 164 public static void setDefaultUri(Configuration conf, URI uri) { 165 conf.set(FS_DEFAULT_NAME_KEY, uri.toString()); 166 } 167 168 /** Set the default filesystem URI in a configuration. 169 * @param conf the configuration to alter 170 * @param uri the new default filesystem uri 171 */ 172 public static void setDefaultUri(Configuration conf, String uri) { 173 setDefaultUri(conf, URI.create(fixName(uri))); 174 } 175 176 /** Called after a new FileSystem instance is constructed. 177 * @param name a uri whose authority section names the host, port, etc. 178 * for this FileSystem 179 * @param conf the configuration 180 */ 181 public void initialize(URI name, Configuration conf) throws IOException { 182 statistics = getStatistics(name.getScheme(), getClass()); 183 } 184 185 /** Returns a URI whose scheme and authority identify this FileSystem.*/ 186 public abstract URI getUri(); 187 188 /** 189 * Get the default port for this file system. 190 * @return the default port or 0 if there isn't one 191 */ 192 protected int getDefaultPort() { 193 return 0; 194 } 195 196 /** 197 * Get a canonical name for this file system. 198 * @return a URI string that uniquely identifies this file system 199 */ 200 public String getCanonicalServiceName() { 201 return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()); 202 } 203 204 /** @deprecated call #getUri() instead.*/ 205 @Deprecated 206 public String getName() { return getUri().toString(); } 207 208 /** @deprecated call #get(URI,Configuration) instead. */ 209 @Deprecated 210 public static FileSystem getNamed(String name, Configuration conf) 211 throws IOException { 212 return get(URI.create(fixName(name)), conf); 213 } 214 215 /** Update old-format filesystem names, for back-compatibility. This should 216 * eventually be replaced with a checkName() method that throws an exception 217 * for old-format names. */ 218 private static String fixName(String name) { 219 // convert old-format name to new-format name 220 if (name.equals("local")) { // "local" is now "file:///". 221 LOG.warn("\"local\" is a deprecated filesystem name." 222 +" Use \"file:///\" instead."); 223 name = "file:///"; 224 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://" 225 LOG.warn("\""+name+"\" is a deprecated filesystem name." 226 +" Use \"hdfs://"+name+"/\" instead."); 227 name = "hdfs://"+name; 228 } 229 return name; 230 } 231 232 /** 233 * Get the local file system. 234 * @param conf the configuration to configure the file system with 235 * @return a LocalFileSystem 236 */ 237 public static LocalFileSystem getLocal(Configuration conf) 238 throws IOException { 239 return (LocalFileSystem)get(LocalFileSystem.NAME, conf); 240 } 241 242 /** Returns the FileSystem for this URI's scheme and authority. The scheme 243 * of the URI determines a configuration property name, 244 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class. 245 * The entire URI is passed to the FileSystem instance's initialize method. 246 */ 247 public static FileSystem get(URI uri, Configuration conf) throws IOException { 248 String scheme = uri.getScheme(); 249 String authority = uri.getAuthority(); 250 251 if (scheme == null) { // no scheme: use default FS 252 return get(conf); 253 } 254 255 if (authority == null) { // no authority 256 URI defaultUri = getDefaultUri(conf); 257 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default 258 && defaultUri.getAuthority() != null) { // & default has authority 259 return get(defaultUri, conf); // return default 260 } 261 } 262 263 String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); 264 if (conf.getBoolean(disableCacheName, false)) { 265 return createFileSystem(uri, conf); 266 } 267 268 return CACHE.get(uri, conf); 269 } 270 271 /** 272 * Returns the FileSystem for this URI's scheme and authority and the 273 * passed user. Internally invokes {@link #newInstance(URI, Configuration)} 274 * @param uri of the filesystem 275 * @param conf the configuration to use 276 * @param user to perform the get as 277 * @return filesystem instance 278 * @throws IOException 279 * @throws InterruptedException 280 */ 281 public static FileSystem newInstance(final URI uri, final Configuration conf, 282 final String user) throws IOException, InterruptedException { 283 UserGroupInformation ugi; 284 if (user == null) { 285 ugi = UserGroupInformation.getCurrentUser(); 286 } else { 287 ugi = UserGroupInformation.createRemoteUser(user); 288 } 289 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 290 public FileSystem run() throws IOException { 291 return newInstance(uri,conf); 292 } 293 }); 294 } 295 /** Returns the FileSystem for this URI's scheme and authority. The scheme 296 * of the URI determines a configuration property name, 297 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class. 298 * The entire URI is passed to the FileSystem instance's initialize method. 299 * This always returns a new FileSystem object. 300 */ 301 public static FileSystem newInstance(URI uri, Configuration conf) throws IOException { 302 String scheme = uri.getScheme(); 303 String authority = uri.getAuthority(); 304 305 if (scheme == null) { // no scheme: use default FS 306 return newInstance(conf); 307 } 308 309 if (authority == null) { // no authority 310 URI defaultUri = getDefaultUri(conf); 311 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default 312 && defaultUri.getAuthority() != null) { // & default has authority 313 return newInstance(defaultUri, conf); // return default 314 } 315 } 316 return CACHE.getUnique(uri, conf); 317 } 318 319 /** Returns a unique configured filesystem implementation. 320 * This always returns a new FileSystem object. 321 * @param conf the configuration to use 322 */ 323 public static FileSystem newInstance(Configuration conf) throws IOException { 324 return newInstance(getDefaultUri(conf), conf); 325 } 326 327 /** 328 * Get a unique local file system object 329 * @param conf the configuration to configure the file system with 330 * @return a LocalFileSystem 331 * This always returns a new FileSystem object. 332 */ 333 public static LocalFileSystem newInstanceLocal(Configuration conf) 334 throws IOException { 335 return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf); 336 } 337 338 /** 339 * Close all cached filesystems. Be sure those filesystems are not 340 * used anymore. 341 * 342 * @throws IOException 343 */ 344 public static void closeAll() throws IOException { 345 CACHE.closeAll(); 346 } 347 348 /** 349 * Close all cached filesystems for a given UGI. Be sure those filesystems 350 * are not used anymore. 351 * @param ugi user group info to close 352 * @throws IOException 353 */ 354 public static void closeAllForUGI(UserGroupInformation ugi) 355 throws IOException { 356 CACHE.closeAll(ugi); 357 } 358 359 /** 360 * Make sure that a path specifies a FileSystem. 361 * @param path to use 362 */ 363 public Path makeQualified(Path path) { 364 checkPath(path); 365 return path.makeQualified(this.getUri(), this.getWorkingDirectory()); 366 } 367 368 /** 369 * Deprecated - use @link {@link #getDelegationTokens(String)} 370 * Get a new delegation token for this file system. 371 * @param renewer the account name that is allowed to renew the token. 372 * @return a new delegation token 373 * @throws IOException 374 */ 375 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) 376 @Deprecated 377 public Token<?> getDelegationToken(String renewer) throws IOException { 378 return null; 379 } 380 381 /** 382 * Get one or more delegation tokens associated with the filesystem. Normally 383 * a file system returns a single delegation token. A file system that manages 384 * multiple file systems underneath, could return set of delegation tokens for 385 * all the file systems it manages. 386 * 387 * @param renewer the account name that is allowed to renew the token. 388 * @return list of new delegation tokens 389 * If delegation tokens not supported then return a list of size zero. 390 * @throws IOException 391 */ 392 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" }) 393 public List<Token<?>> getDelegationTokens(String renewer) throws IOException { 394 return new ArrayList<Token<?>>(0); 395 } 396 397 /** create a file with the provided permission 398 * The permission of the file is set to be the provided permission as in 399 * setPermission, not permission&~umask 400 * 401 * It is implemented using two RPCs. It is understood that it is inefficient, 402 * but the implementation is thread-safe. The other option is to change the 403 * value of umask in configuration to be 0, but it is not thread-safe. 404 * 405 * @param fs file system handle 406 * @param file the name of the file to be created 407 * @param permission the permission of the file 408 * @return an output stream 409 * @throws IOException 410 */ 411 public static FSDataOutputStream create(FileSystem fs, 412 Path file, FsPermission permission) throws IOException { 413 // create the file with default permission 414 FSDataOutputStream out = fs.create(file); 415 // set its permission to the supplied one 416 fs.setPermission(file, permission); 417 return out; 418 } 419 420 /** create a directory with the provided permission 421 * The permission of the directory is set to be the provided permission as in 422 * setPermission, not permission&~umask 423 * 424 * @see #create(FileSystem, Path, FsPermission) 425 * 426 * @param fs file system handle 427 * @param dir the name of the directory to be created 428 * @param permission the permission of the directory 429 * @return true if the directory creation succeeds; false otherwise 430 * @throws IOException 431 */ 432 public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) 433 throws IOException { 434 // create the directory using the default permission 435 boolean result = fs.mkdirs(dir); 436 // set its permission to be the supplied one 437 fs.setPermission(dir, permission); 438 return result; 439 } 440 441 /////////////////////////////////////////////////////////////// 442 // FileSystem 443 /////////////////////////////////////////////////////////////// 444 445 protected FileSystem() { 446 super(null); 447 } 448 449 /** 450 * Check that a Path belongs to this FileSystem. 451 * @param path to check 452 */ 453 protected void checkPath(Path path) { 454 URI uri = path.toUri(); 455 if (uri.getScheme() == null) // fs is relative 456 return; 457 String thisScheme = this.getUri().getScheme(); 458 String thatScheme = uri.getScheme(); 459 String thisAuthority = this.getUri().getAuthority(); 460 String thatAuthority = uri.getAuthority(); 461 //authority and scheme are not case sensitive 462 if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match 463 if (thisAuthority == thatAuthority || // & authorities match 464 (thisAuthority != null && 465 thisAuthority.equalsIgnoreCase(thatAuthority))) 466 return; 467 468 if (thatAuthority == null && // path's authority is null 469 thisAuthority != null) { // fs has an authority 470 URI defaultUri = getDefaultUri(getConf()); // & is the conf default 471 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) && 472 thisAuthority.equalsIgnoreCase(defaultUri.getAuthority())) 473 return; 474 try { // or the default fs's uri 475 defaultUri = get(getConf()).getUri(); 476 } catch (IOException e) { 477 throw new RuntimeException(e); 478 } 479 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) && 480 thisAuthority.equalsIgnoreCase(defaultUri.getAuthority())) 481 return; 482 } 483 } 484 throw new IllegalArgumentException("Wrong FS: "+path+ 485 ", expected: "+this.getUri()); 486 } 487 488 /** 489 * Return an array containing hostnames, offset and size of 490 * portions of the given file. For a nonexistent 491 * file or regions, null will be returned. 492 * 493 * This call is most helpful with DFS, where it returns 494 * hostnames of machines that contain the given file. 495 * 496 * The FileSystem will simply return an elt containing 'localhost'. 497 * 498 * @param file FilesStatus to get data from 499 * @param start offset into the given file 500 * @param len length for which to get locations for 501 */ 502 public BlockLocation[] getFileBlockLocations(FileStatus file, 503 long start, long len) throws IOException { 504 if (file == null) { 505 return null; 506 } 507 508 if (start < 0 || len < 0) { 509 throw new IllegalArgumentException("Invalid start or len parameter"); 510 } 511 512 if (file.getLen() < start) { 513 return new BlockLocation[0]; 514 515 } 516 String[] name = { "localhost:50010" }; 517 String[] host = { "localhost" }; 518 return new BlockLocation[] { 519 new BlockLocation(name, host, 0, file.getLen()) }; 520 } 521 522 523 /** 524 * Return an array containing hostnames, offset and size of 525 * portions of the given file. For a nonexistent 526 * file or regions, null will be returned. 527 * 528 * This call is most helpful with DFS, where it returns 529 * hostnames of machines that contain the given file. 530 * 531 * The FileSystem will simply return an elt containing 'localhost'. 532 * 533 * @param p path of file to get locations for 534 * @param start offset into the given file 535 * @param len length for which to get locations for 536 */ 537 public BlockLocation[] getFileBlockLocations(Path p, 538 long start, long len) throws IOException { 539 if (p == null) { 540 throw new NullPointerException(); 541 } 542 FileStatus file = getFileStatus(p); 543 return getFileBlockLocations(file, start, len); 544 } 545 546 /** 547 * Return a set of server default configuration values 548 * @return server default configuration values 549 * @throws IOException 550 */ 551 public FsServerDefaults getServerDefaults() throws IOException { 552 Configuration conf = getConf(); 553 return new FsServerDefaults(getDefaultBlockSize(), 554 conf.getInt("io.bytes.per.checksum", 512), 555 64 * 1024, 556 getDefaultReplication(), 557 conf.getInt("io.file.buffer.size", 4096)); 558 } 559 560 /** 561 * Return the fully-qualified path of path f resolving the path 562 * through any symlinks or mount point 563 * @param p path to be resolved 564 * @return fully qualified path 565 * @throws FileNotFoundException 566 */ 567 public Path resolvePath(final Path p) throws IOException { 568 checkPath(p); 569 return getFileStatus(p).getPath(); 570 } 571 572 /** 573 * Opens an FSDataInputStream at the indicated Path. 574 * @param f the file name to open 575 * @param bufferSize the size of the buffer to be used. 576 */ 577 public abstract FSDataInputStream open(Path f, int bufferSize) 578 throws IOException; 579 580 /** 581 * Opens an FSDataInputStream at the indicated Path. 582 * @param f the file to open 583 */ 584 public FSDataInputStream open(Path f) throws IOException { 585 return open(f, getConf().getInt("io.file.buffer.size", 4096)); 586 } 587 588 /** 589 * Create an FSDataOutputStream at the indicated Path. 590 * Files are overwritten by default. 591 * @param f the file to create 592 */ 593 public FSDataOutputStream create(Path f) throws IOException { 594 return create(f, true); 595 } 596 597 /** 598 * Create an FSDataOutputStream at the indicated Path. 599 * @param f the file to create 600 * @param overwrite if a file with this name already exists, then if true, 601 * the file will be overwritten, and if false an exception will be thrown. 602 */ 603 public FSDataOutputStream create(Path f, boolean overwrite) 604 throws IOException { 605 return create(f, overwrite, 606 getConf().getInt("io.file.buffer.size", 4096), 607 getDefaultReplication(), 608 getDefaultBlockSize()); 609 } 610 611 /** 612 * Create an FSDataOutputStream at the indicated Path with write-progress 613 * reporting. 614 * Files are overwritten by default. 615 * @param f the file to create 616 * @param progress to report progress 617 */ 618 public FSDataOutputStream create(Path f, Progressable progress) 619 throws IOException { 620 return create(f, true, 621 getConf().getInt("io.file.buffer.size", 4096), 622 getDefaultReplication(), 623 getDefaultBlockSize(), progress); 624 } 625 626 /** 627 * Create an FSDataOutputStream at the indicated Path. 628 * Files are overwritten by default. 629 * @param f the file to create 630 * @param replication the replication factor 631 */ 632 public FSDataOutputStream create(Path f, short replication) 633 throws IOException { 634 return create(f, true, 635 getConf().getInt("io.file.buffer.size", 4096), 636 replication, 637 getDefaultBlockSize()); 638 } 639 640 /** 641 * Create an FSDataOutputStream at the indicated Path with write-progress 642 * reporting. 643 * Files are overwritten by default. 644 * @param f the file to create 645 * @param replication the replication factor 646 * @param progress to report progress 647 */ 648 public FSDataOutputStream create(Path f, short replication, 649 Progressable progress) throws IOException { 650 return create(f, true, 651 getConf().getInt("io.file.buffer.size", 4096), 652 replication, 653 getDefaultBlockSize(), progress); 654 } 655 656 657 /** 658 * Create an FSDataOutputStream at the indicated Path. 659 * @param f the file name to create 660 * @param overwrite if a file with this name already exists, then if true, 661 * the file will be overwritten, and if false an error will be thrown. 662 * @param bufferSize the size of the buffer to be used. 663 */ 664 public FSDataOutputStream create(Path f, 665 boolean overwrite, 666 int bufferSize 667 ) throws IOException { 668 return create(f, overwrite, bufferSize, 669 getDefaultReplication(), 670 getDefaultBlockSize()); 671 } 672 673 /** 674 * Create an FSDataOutputStream at the indicated Path with write-progress 675 * reporting. 676 * @param f the path of the file to open 677 * @param overwrite if a file with this name already exists, then if true, 678 * the file will be overwritten, and if false an error will be thrown. 679 * @param bufferSize the size of the buffer to be used. 680 */ 681 public FSDataOutputStream create(Path f, 682 boolean overwrite, 683 int bufferSize, 684 Progressable progress 685 ) throws IOException { 686 return create(f, overwrite, bufferSize, 687 getDefaultReplication(), 688 getDefaultBlockSize(), progress); 689 } 690 691 692 /** 693 * Create an FSDataOutputStream at the indicated Path. 694 * @param f the file name to open 695 * @param overwrite if a file with this name already exists, then if true, 696 * the file will be overwritten, and if false an error will be thrown. 697 * @param bufferSize the size of the buffer to be used. 698 * @param replication required block replication for the file. 699 */ 700 public FSDataOutputStream create(Path f, 701 boolean overwrite, 702 int bufferSize, 703 short replication, 704 long blockSize 705 ) throws IOException { 706 return create(f, overwrite, bufferSize, replication, blockSize, null); 707 } 708 709 /** 710 * Create an FSDataOutputStream at the indicated Path with write-progress 711 * reporting. 712 * @param f the file name to open 713 * @param overwrite if a file with this name already exists, then if true, 714 * the file will be overwritten, and if false an error will be thrown. 715 * @param bufferSize the size of the buffer to be used. 716 * @param replication required block replication for the file. 717 */ 718 public FSDataOutputStream create(Path f, 719 boolean overwrite, 720 int bufferSize, 721 short replication, 722 long blockSize, 723 Progressable progress 724 ) throws IOException { 725 return this.create(f, FsPermission.getDefault().applyUMask( 726 FsPermission.getUMask(getConf())), overwrite, bufferSize, 727 replication, blockSize, progress); 728 } 729 730 /** 731 * Create an FSDataOutputStream at the indicated Path with write-progress 732 * reporting. 733 * @param f the file name to open 734 * @param permission 735 * @param overwrite if a file with this name already exists, then if true, 736 * the file will be overwritten, and if false an error will be thrown. 737 * @param bufferSize the size of the buffer to be used. 738 * @param replication required block replication for the file. 739 * @param blockSize 740 * @param progress 741 * @throws IOException 742 * @see #setPermission(Path, FsPermission) 743 */ 744 public abstract FSDataOutputStream create(Path f, 745 FsPermission permission, 746 boolean overwrite, 747 int bufferSize, 748 short replication, 749 long blockSize, 750 Progressable progress) throws IOException; 751 752 753 /*. 754 * This create has been added to support the FileContext that processes 755 * the permission 756 * with umask before calling this method. 757 * This a temporary method added to support the transition from FileSystem 758 * to FileContext for user applications. 759 */ 760 @Deprecated 761 protected FSDataOutputStream primitiveCreate(Path f, 762 FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize, 763 short replication, long blockSize, Progressable progress, 764 int bytesPerChecksum) throws IOException { 765 766 boolean pathExists = exists(f); 767 CreateFlag.validate(f, pathExists, flag); 768 769 // Default impl assumes that permissions do not matter and 770 // nor does the bytesPerChecksum hence 771 // calling the regular create is good enough. 772 // FSs that implement permissions should override this. 773 774 if (pathExists && flag.contains(CreateFlag.APPEND)) { 775 return append(f, bufferSize, progress); 776 } 777 778 return this.create(f, absolutePermission, 779 flag.contains(CreateFlag.OVERWRITE), bufferSize, replication, 780 blockSize, progress); 781 } 782 783 /** 784 * This version of the mkdirs method assumes that the permission is absolute. 785 * It has been added to support the FileContext that processes the permission 786 * with umask before calling this method. 787 * This a temporary method added to support the transition from FileSystem 788 * to FileContext for user applications. 789 */ 790 @Deprecated 791 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) 792 throws IOException { 793 // Default impl is to assume that permissions do not matter and hence 794 // calling the regular mkdirs is good enough. 795 // FSs that implement permissions should override this. 796 return this.mkdirs(f, absolutePermission); 797 } 798 799 800 /** 801 * This version of the mkdirs method assumes that the permission is absolute. 802 * It has been added to support the FileContext that processes the permission 803 * with umask before calling this method. 804 * This a temporary method added to support the transition from FileSystem 805 * to FileContext for user applications. 806 */ 807 @Deprecated 808 protected void primitiveMkdir(Path f, FsPermission absolutePermission, 809 boolean createParent) 810 throws IOException { 811 812 if (!createParent) { // parent must exist. 813 // since the this.mkdirs makes parent dirs automatically 814 // we must throw exception if parent does not exist. 815 final FileStatus stat = getFileStatus(f.getParent()); 816 if (stat == null) { 817 throw new FileNotFoundException("Missing parent:" + f); 818 } 819 if (!stat.isDirectory()) { 820 throw new ParentNotDirectoryException("parent is not a dir"); 821 } 822 // parent does exist - go ahead with mkdir of leaf 823 } 824 // Default impl is to assume that permissions do not matter and hence 825 // calling the regular mkdirs is good enough. 826 // FSs that implement permissions should override this. 827 if (!this.mkdirs(f, absolutePermission)) { 828 throw new IOException("mkdir of "+ f + " failed"); 829 } 830 } 831 832 833 /** 834 * Creates the given Path as a brand-new zero-length file. If 835 * create fails, or if it already existed, return false. 836 * 837 * @param f path to use for create 838 */ 839 public boolean createNewFile(Path f) throws IOException { 840 if (exists(f)) { 841 return false; 842 } else { 843 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close(); 844 return true; 845 } 846 } 847 848 /** 849 * Append to an existing file (optional operation). 850 * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) 851 * @param f the existing file to be appended. 852 * @throws IOException 853 */ 854 public FSDataOutputStream append(Path f) throws IOException { 855 return append(f, getConf().getInt("io.file.buffer.size", 4096), null); 856 } 857 /** 858 * Append to an existing file (optional operation). 859 * Same as append(f, bufferSize, null). 860 * @param f the existing file to be appended. 861 * @param bufferSize the size of the buffer to be used. 862 * @throws IOException 863 */ 864 public FSDataOutputStream append(Path f, int bufferSize) throws IOException { 865 return append(f, bufferSize, null); 866 } 867 868 /** 869 * Append to an existing file (optional operation). 870 * @param f the existing file to be appended. 871 * @param bufferSize the size of the buffer to be used. 872 * @param progress for reporting progress if it is not null. 873 * @throws IOException 874 */ 875 public abstract FSDataOutputStream append(Path f, int bufferSize, 876 Progressable progress) throws IOException; 877 878 /** 879 * Get replication. 880 * 881 * @deprecated Use getFileStatus() instead 882 * @param src file name 883 * @return file replication 884 * @throws IOException 885 */ 886 @Deprecated 887 public short getReplication(Path src) throws IOException { 888 return getFileStatus(src).getReplication(); 889 } 890 891 /** 892 * Set replication for an existing file. 893 * 894 * @param src file name 895 * @param replication new replication 896 * @throws IOException 897 * @return true if successful; 898 * false if file does not exist or is a directory 899 */ 900 public boolean setReplication(Path src, short replication) 901 throws IOException { 902 return true; 903 } 904 905 /** 906 * Renames Path src to Path dst. Can take place on local fs 907 * or remote DFS. 908 * @param src path to be renamed 909 * @param dst new path after rename 910 * @throws IOException on failure 911 * @return true if rename is successful 912 */ 913 public abstract boolean rename(Path src, Path dst) throws IOException; 914 915 /** 916 * Renames Path src to Path dst 917 * <ul> 918 * <li 919 * <li>Fails if src is a file and dst is a directory. 920 * <li>Fails if src is a directory and dst is a file. 921 * <li>Fails if the parent of dst does not exist or is a file. 922 * </ul> 923 * <p> 924 * If OVERWRITE option is not passed as an argument, rename fails 925 * if the dst already exists. 926 * <p> 927 * If OVERWRITE option is passed as an argument, rename overwrites 928 * the dst if it is a file or an empty directory. Rename fails if dst is 929 * a non-empty directory. 930 * <p> 931 * Note that atomicity of rename is dependent on the file system 932 * implementation. Please refer to the file system documentation for 933 * details. This default implementation is non atomic. 934 * <p> 935 * This method is deprecated since it is a temporary method added to 936 * support the transition from FileSystem to FileContext for user 937 * applications. 938 * 939 * @param src path to be renamed 940 * @param dst new path after rename 941 * @throws IOException on failure 942 */ 943 @Deprecated 944 protected void rename(final Path src, final Path dst, 945 final Rename... options) throws IOException { 946 // Default implementation 947 final FileStatus srcStatus = getFileStatus(src); 948 if (srcStatus == null) { 949 throw new FileNotFoundException("rename source " + src + " not found."); 950 } 951 952 boolean overwrite = false; 953 if (null != options) { 954 for (Rename option : options) { 955 if (option == Rename.OVERWRITE) { 956 overwrite = true; 957 } 958 } 959 } 960 961 FileStatus dstStatus; 962 try { 963 dstStatus = getFileStatus(dst); 964 } catch (IOException e) { 965 dstStatus = null; 966 } 967 if (dstStatus != null) { 968 if (srcStatus.isDirectory() != dstStatus.isDirectory()) { 969 throw new IOException("Source " + src + " Destination " + dst 970 + " both should be either file or directory"); 971 } 972 if (!overwrite) { 973 throw new FileAlreadyExistsException("rename destination " + dst 974 + " already exists."); 975 } 976 // Delete the destination that is a file or an empty directory 977 if (dstStatus.isDirectory()) { 978 FileStatus[] list = listStatus(dst); 979 if (list != null && list.length != 0) { 980 throw new IOException( 981 "rename cannot overwrite non empty destination directory " + dst); 982 } 983 } 984 delete(dst, false); 985 } else { 986 final Path parent = dst.getParent(); 987 final FileStatus parentStatus = getFileStatus(parent); 988 if (parentStatus == null) { 989 throw new FileNotFoundException("rename destination parent " + parent 990 + " not found."); 991 } 992 if (!parentStatus.isDirectory()) { 993 throw new ParentNotDirectoryException("rename destination parent " + parent 994 + " is a file."); 995 } 996 } 997 if (!rename(src, dst)) { 998 throw new IOException("rename from " + src + " to " + dst + " failed."); 999 } 1000 } 1001 1002 /** 1003 * Delete a file 1004 * @deprecated Use {@link #delete(Path, boolean)} instead. 1005 */ 1006 @Deprecated 1007 public boolean delete(Path f) throws IOException { 1008 return delete(f, true); 1009 } 1010 1011 /** Delete a file. 1012 * 1013 * @param f the path to delete. 1014 * @param recursive if path is a directory and set to 1015 * true, the directory is deleted else throws an exception. In 1016 * case of a file the recursive can be set to either true or false. 1017 * @return true if delete is successful else false. 1018 * @throws IOException 1019 */ 1020 public abstract boolean delete(Path f, boolean recursive) throws IOException; 1021 1022 /** 1023 * Mark a path to be deleted when FileSystem is closed. 1024 * When the JVM shuts down, 1025 * all FileSystem objects will be closed automatically. 1026 * Then, 1027 * the marked path will be deleted as a result of closing the FileSystem. 1028 * 1029 * The path has to exist in the file system. 1030 * 1031 * @param f the path to delete. 1032 * @return true if deleteOnExit is successful, otherwise false. 1033 * @throws IOException 1034 */ 1035 public boolean deleteOnExit(Path f) throws IOException { 1036 if (!exists(f)) { 1037 return false; 1038 } 1039 synchronized (deleteOnExit) { 1040 deleteOnExit.add(f); 1041 } 1042 return true; 1043 } 1044 1045 /** 1046 * Delete all files that were marked as delete-on-exit. This recursively 1047 * deletes all files in the specified paths. 1048 */ 1049 protected void processDeleteOnExit() { 1050 synchronized (deleteOnExit) { 1051 for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) { 1052 Path path = iter.next(); 1053 try { 1054 delete(path, true); 1055 } 1056 catch (IOException e) { 1057 LOG.info("Ignoring failure to deleteOnExit for path " + path); 1058 } 1059 iter.remove(); 1060 } 1061 } 1062 } 1063 1064 /** Check if exists. 1065 * @param f source file 1066 */ 1067 public boolean exists(Path f) throws IOException { 1068 try { 1069 return getFileStatus(f) != null; 1070 } catch (FileNotFoundException e) { 1071 return false; 1072 } 1073 } 1074 1075 /** True iff the named path is a directory. 1076 * Note: Avoid using this method. Instead reuse the FileStatus 1077 * returned by getFileStatus() or listStatus() methods. 1078 * @param f path to check 1079 */ 1080 public boolean isDirectory(Path f) throws IOException { 1081 try { 1082 return getFileStatus(f).isDirectory(); 1083 } catch (FileNotFoundException e) { 1084 return false; // f does not exist 1085 } 1086 } 1087 1088 /** True iff the named path is a regular file. 1089 * Note: Avoid using this method. Instead reuse the FileStatus 1090 * returned by getFileStatus() or listStatus() methods. 1091 * @param f path to check 1092 */ 1093 public boolean isFile(Path f) throws IOException { 1094 try { 1095 return getFileStatus(f).isFile(); 1096 } catch (FileNotFoundException e) { 1097 return false; // f does not exist 1098 } 1099 } 1100 1101 /** The number of bytes in a file. */ 1102 /** @deprecated Use getFileStatus() instead */ 1103 @Deprecated 1104 public long getLength(Path f) throws IOException { 1105 return getFileStatus(f).getLen(); 1106 } 1107 1108 /** Return the {@link ContentSummary} of a given {@link Path}. 1109 * @param f path to use 1110 */ 1111 public ContentSummary getContentSummary(Path f) throws IOException { 1112 FileStatus status = getFileStatus(f); 1113 if (status.isFile()) { 1114 // f is a file 1115 return new ContentSummary(status.getLen(), 1, 0); 1116 } 1117 // f is a directory 1118 long[] summary = {0, 0, 1}; 1119 for(FileStatus s : listStatus(f)) { 1120 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : 1121 new ContentSummary(s.getLen(), 1, 0); 1122 summary[0] += c.getLength(); 1123 summary[1] += c.getFileCount(); 1124 summary[2] += c.getDirectoryCount(); 1125 } 1126 return new ContentSummary(summary[0], summary[1], summary[2]); 1127 } 1128 1129 final private static PathFilter DEFAULT_FILTER = new PathFilter() { 1130 public boolean accept(Path file) { 1131 return true; 1132 } 1133 }; 1134 1135 /** 1136 * List the statuses of the files/directories in the given path if the path is 1137 * a directory. 1138 * 1139 * @param f given path 1140 * @return the statuses of the files/directories in the given patch 1141 * @throws FileNotFoundException when the path does not exist; 1142 * IOException see specific implementation 1143 */ 1144 public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, 1145 IOException; 1146 1147 /* 1148 * Filter files/directories in the given path using the user-supplied path 1149 * filter. Results are added to the given array <code>results</code>. 1150 */ 1151 private void listStatus(ArrayList<FileStatus> results, Path f, 1152 PathFilter filter) throws FileNotFoundException, IOException { 1153 FileStatus listing[] = listStatus(f); 1154 if (listing == null) { 1155 throw new IOException("Error accessing " + f); 1156 } 1157 1158 for (int i = 0; i < listing.length; i++) { 1159 if (filter.accept(listing[i].getPath())) { 1160 results.add(listing[i]); 1161 } 1162 } 1163 } 1164 1165 /** 1166 * @return an iterator over the corrupt files under the given path 1167 * (may contain duplicates if a file has more than one corrupt block) 1168 * @throws IOException 1169 */ 1170 public RemoteIterator<Path> listCorruptFileBlocks(Path path) 1171 throws IOException { 1172 throw new UnsupportedOperationException(getClass().getCanonicalName() + 1173 " does not support" + 1174 " listCorruptFileBlocks"); 1175 } 1176 1177 /** 1178 * Filter files/directories in the given path using the user-supplied path 1179 * filter. 1180 * 1181 * @param f 1182 * a path name 1183 * @param filter 1184 * the user-supplied path filter 1185 * @return an array of FileStatus objects for the files under the given path 1186 * after applying the filter 1187 * @throws FileNotFoundException when the path does not exist; 1188 * IOException see specific implementation 1189 */ 1190 public FileStatus[] listStatus(Path f, PathFilter filter) 1191 throws FileNotFoundException, IOException { 1192 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1193 listStatus(results, f, filter); 1194 return results.toArray(new FileStatus[results.size()]); 1195 } 1196 1197 /** 1198 * Filter files/directories in the given list of paths using default 1199 * path filter. 1200 * 1201 * @param files 1202 * a list of paths 1203 * @return a list of statuses for the files under the given paths after 1204 * applying the filter default Path filter 1205 * @throws FileNotFoundException when the path does not exist; 1206 * IOException see specific implementation 1207 */ 1208 public FileStatus[] listStatus(Path[] files) 1209 throws FileNotFoundException, IOException { 1210 return listStatus(files, DEFAULT_FILTER); 1211 } 1212 1213 /** 1214 * Filter files/directories in the given list of paths using user-supplied 1215 * path filter. 1216 * 1217 * @param files 1218 * a list of paths 1219 * @param filter 1220 * the user-supplied path filter 1221 * @return a list of statuses for the files under the given paths after 1222 * applying the filter 1223 * @throws FileNotFoundException when the path does not exist; 1224 * IOException see specific implementation 1225 */ 1226 public FileStatus[] listStatus(Path[] files, PathFilter filter) 1227 throws FileNotFoundException, IOException { 1228 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1229 for (int i = 0; i < files.length; i++) { 1230 listStatus(results, files[i], filter); 1231 } 1232 return results.toArray(new FileStatus[results.size()]); 1233 } 1234 1235 /** 1236 * <p>Return all the files that match filePattern and are not checksum 1237 * files. Results are sorted by their names. 1238 * 1239 * <p> 1240 * A filename pattern is composed of <i>regular</i> characters and 1241 * <i>special pattern matching</i> characters, which are: 1242 * 1243 * <dl> 1244 * <dd> 1245 * <dl> 1246 * <p> 1247 * <dt> <tt> ? </tt> 1248 * <dd> Matches any single character. 1249 * 1250 * <p> 1251 * <dt> <tt> * </tt> 1252 * <dd> Matches zero or more characters. 1253 * 1254 * <p> 1255 * <dt> <tt> [<i>abc</i>] </tt> 1256 * <dd> Matches a single character from character set 1257 * <tt>{<i>a,b,c</i>}</tt>. 1258 * 1259 * <p> 1260 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> 1261 * <dd> Matches a single character from the character range 1262 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be 1263 * lexicographically less than or equal to character <tt><i>b</i></tt>. 1264 * 1265 * <p> 1266 * <dt> <tt> [^<i>a</i>] </tt> 1267 * <dd> Matches a single character that is not from character set or range 1268 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur 1269 * immediately to the right of the opening bracket. 1270 * 1271 * <p> 1272 * <dt> <tt> \<i>c</i> </tt> 1273 * <dd> Removes (escapes) any special meaning of character <i>c</i>. 1274 * 1275 * <p> 1276 * <dt> <tt> {ab,cd} </tt> 1277 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> 1278 * 1279 * <p> 1280 * <dt> <tt> {ab,c{de,fh}} </tt> 1281 * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt> 1282 * 1283 * </dl> 1284 * </dd> 1285 * </dl> 1286 * 1287 * @param pathPattern a regular expression specifying a pth pattern 1288 1289 * @return an array of paths that match the path pattern 1290 * @throws IOException 1291 */ 1292 public FileStatus[] globStatus(Path pathPattern) throws IOException { 1293 return globStatus(pathPattern, DEFAULT_FILTER); 1294 } 1295 1296 /** 1297 * Return an array of FileStatus objects whose path names match pathPattern 1298 * and is accepted by the user-supplied path filter. Results are sorted by 1299 * their path names. 1300 * Return null if pathPattern has no glob and the path does not exist. 1301 * Return an empty array if pathPattern has a glob and no path matches it. 1302 * 1303 * @param pathPattern 1304 * a regular expression specifying the path pattern 1305 * @param filter 1306 * a user-supplied path filter 1307 * @return an array of FileStatus objects 1308 * @throws IOException if any I/O error occurs when fetching file status 1309 */ 1310 public FileStatus[] globStatus(Path pathPattern, PathFilter filter) 1311 throws IOException { 1312 String filename = pathPattern.toUri().getPath(); 1313 List<String> filePatterns = GlobExpander.expand(filename); 1314 if (filePatterns.size() == 1) { 1315 return globStatusInternal(pathPattern, filter); 1316 } else { 1317 List<FileStatus> results = new ArrayList<FileStatus>(); 1318 for (String filePattern : filePatterns) { 1319 FileStatus[] files = globStatusInternal(new Path(filePattern), filter); 1320 for (FileStatus file : files) { 1321 results.add(file); 1322 } 1323 } 1324 return results.toArray(new FileStatus[results.size()]); 1325 } 1326 } 1327 1328 private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter) 1329 throws IOException { 1330 Path[] parents = new Path[1]; 1331 int level = 0; 1332 String filename = pathPattern.toUri().getPath(); 1333 1334 // path has only zero component 1335 if ("".equals(filename) || Path.SEPARATOR.equals(filename)) { 1336 return getFileStatus(new Path[]{pathPattern}); 1337 } 1338 1339 // path has at least one component 1340 String[] components = filename.split(Path.SEPARATOR); 1341 // get the first component 1342 if (pathPattern.isAbsolute()) { 1343 parents[0] = new Path(Path.SEPARATOR); 1344 level = 1; 1345 } else { 1346 parents[0] = new Path(Path.CUR_DIR); 1347 } 1348 1349 // glob the paths that match the parent path, i.e., [0, components.length-1] 1350 boolean[] hasGlob = new boolean[]{false}; 1351 Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob); 1352 FileStatus[] results; 1353 if (parentPaths == null || parentPaths.length == 0) { 1354 results = null; 1355 } else { 1356 // Now work on the last component of the path 1357 GlobFilter fp = new GlobFilter(components[components.length - 1], filter); 1358 if (fp.hasPattern()) { // last component has a pattern 1359 // list parent directories and then glob the results 1360 results = listStatus(parentPaths, fp); 1361 hasGlob[0] = true; 1362 } else { // last component does not have a pattern 1363 // get all the path names 1364 ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length); 1365 for (int i = 0; i < parentPaths.length; i++) { 1366 parentPaths[i] = new Path(parentPaths[i], 1367 components[components.length - 1]); 1368 if (fp.accept(parentPaths[i])) { 1369 filteredPaths.add(parentPaths[i]); 1370 } 1371 } 1372 // get all their statuses 1373 results = getFileStatus( 1374 filteredPaths.toArray(new Path[filteredPaths.size()])); 1375 } 1376 } 1377 1378 // Decide if the pathPattern contains a glob or not 1379 if (results == null) { 1380 if (hasGlob[0]) { 1381 results = new FileStatus[0]; 1382 } 1383 } else { 1384 if (results.length == 0 ) { 1385 if (!hasGlob[0]) { 1386 results = null; 1387 } 1388 } else { 1389 Arrays.sort(results); 1390 } 1391 } 1392 return results; 1393 } 1394 1395 /* 1396 * For a path of N components, return a list of paths that match the 1397 * components [<code>level</code>, <code>N-1</code>]. 1398 */ 1399 private Path[] globPathsLevel(Path[] parents, String[] filePattern, 1400 int level, boolean[] hasGlob) throws IOException { 1401 if (level == filePattern.length - 1) 1402 return parents; 1403 if (parents == null || parents.length == 0) { 1404 return null; 1405 } 1406 GlobFilter fp = new GlobFilter(filePattern[level]); 1407 if (fp.hasPattern()) { 1408 parents = FileUtil.stat2Paths(listStatus(parents, fp)); 1409 hasGlob[0] = true; 1410 } else { 1411 for (int i = 0; i < parents.length; i++) { 1412 parents[i] = new Path(parents[i], filePattern[level]); 1413 } 1414 } 1415 return globPathsLevel(parents, filePattern, level + 1, hasGlob); 1416 } 1417 1418 /** 1419 * List the statuses of the files/directories in the given path if the path is 1420 * a directory. 1421 * Return the file's status and block locations If the path is a file. 1422 * 1423 * If a returned status is a file, it contains the file's block locations. 1424 * 1425 * @param f is the path 1426 * 1427 * @return an iterator that traverses statuses of the files/directories 1428 * in the given path 1429 * 1430 * @throws FileNotFoundException If <code>f</code> does not exist 1431 * @throws IOException If an I/O error occurred 1432 */ 1433 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f) 1434 throws FileNotFoundException, IOException { 1435 return listLocatedStatus(f, DEFAULT_FILTER); 1436 } 1437 1438 /** 1439 * Listing a directory 1440 * The returned results include its block location if it is a file 1441 * The results are filtered by the given path filter 1442 * @param f a path 1443 * @param filter a path filter 1444 * @return an iterator that traverses statuses of the files/directories 1445 * in the given path 1446 * @throws FileNotFoundException if <code>f</code> does not exist 1447 * @throws IOException if any I/O error occurred 1448 */ 1449 protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f, 1450 final PathFilter filter) 1451 throws FileNotFoundException, IOException { 1452 return new RemoteIterator<LocatedFileStatus>() { 1453 private final FileStatus[] stats = listStatus(f, filter); 1454 private int i = 0; 1455 1456 @Override 1457 public boolean hasNext() { 1458 return i<stats.length; 1459 } 1460 1461 @Override 1462 public LocatedFileStatus next() throws IOException { 1463 if (!hasNext()) { 1464 throw new NoSuchElementException("No more entry in " + f); 1465 } 1466 FileStatus result = stats[i++]; 1467 BlockLocation[] locs = result.isFile() ? 1468 getFileBlockLocations(result.getPath(), 0, result.getLen()) : 1469 null; 1470 return new LocatedFileStatus(result, locs); 1471 } 1472 }; 1473 } 1474 1475 /** 1476 * List the statuses and block locations of the files in the given path. 1477 * 1478 * If the path is a directory, 1479 * if recursive is false, returns files in the directory; 1480 * if recursive is true, return files in the subtree rooted at the path. 1481 * If the path is a file, return the file's status and block locations. 1482 * 1483 * @param f is the path 1484 * @param recursive if the subdirectories need to be traversed recursively 1485 * 1486 * @return an iterator that traverses statuses of the files 1487 * 1488 * @throws FileNotFoundException when the path does not exist; 1489 * IOException see specific implementation 1490 */ 1491 public RemoteIterator<LocatedFileStatus> listFiles( 1492 final Path f, final boolean recursive) 1493 throws FileNotFoundException, IOException { 1494 return new RemoteIterator<LocatedFileStatus>() { 1495 private Stack<RemoteIterator<LocatedFileStatus>> itors = 1496 new Stack<RemoteIterator<LocatedFileStatus>>(); 1497 private RemoteIterator<LocatedFileStatus> curItor = 1498 listLocatedStatus(f); 1499 private LocatedFileStatus curFile; 1500 1501 @Override 1502 public boolean hasNext() throws IOException { 1503 while (curFile == null) { 1504 if (curItor.hasNext()) { 1505 handleFileStat(curItor.next()); 1506 } else if (!itors.empty()) { 1507 curItor = itors.pop(); 1508 } else { 1509 return false; 1510 } 1511 } 1512 return true; 1513 } 1514 1515 /** 1516 * Process the input stat. 1517 * If it is a file, return the file stat. 1518 * If it is a directory, traverse the directory if recursive is true; 1519 * ignore it if recursive is false. 1520 * @param stat input status 1521 * @throws IOException if any IO error occurs 1522 */ 1523 private void handleFileStat(LocatedFileStatus stat) throws IOException { 1524 if (stat.isFile()) { // file 1525 curFile = stat; 1526 } else if (recursive) { // directory 1527 itors.push(curItor); 1528 curItor = listLocatedStatus(stat.getPath()); 1529 } 1530 } 1531 1532 @Override 1533 public LocatedFileStatus next() throws IOException { 1534 if (hasNext()) { 1535 LocatedFileStatus result = curFile; 1536 curFile = null; 1537 return result; 1538 } 1539 throw new java.util.NoSuchElementException("No more entry in " + f); 1540 } 1541 }; 1542 } 1543 1544 /** Return the current user's home directory in this filesystem. 1545 * The default implementation returns "/user/$USER/". 1546 */ 1547 public Path getHomeDirectory() { 1548 return this.makeQualified( 1549 new Path("/user/"+System.getProperty("user.name"))); 1550 } 1551 1552 1553 /** 1554 * Set the current working directory for the given file system. All relative 1555 * paths will be resolved relative to it. 1556 * 1557 * @param new_dir 1558 */ 1559 public abstract void setWorkingDirectory(Path new_dir); 1560 1561 /** 1562 * Get the current working directory for the given file system 1563 * @return the directory pathname 1564 */ 1565 public abstract Path getWorkingDirectory(); 1566 1567 1568 /** 1569 * Note: with the new FilesContext class, getWorkingDirectory() 1570 * will be removed. 1571 * The working directory is implemented in FilesContext. 1572 * 1573 * Some file systems like LocalFileSystem have an initial workingDir 1574 * that we use as the starting workingDir. For other file systems 1575 * like HDFS there is no built in notion of an inital workingDir. 1576 * 1577 * @return if there is built in notion of workingDir then it 1578 * is returned; else a null is returned. 1579 */ 1580 protected Path getInitialWorkingDirectory() { 1581 return null; 1582 } 1583 1584 /** 1585 * Call {@link #mkdirs(Path, FsPermission)} with default permission. 1586 */ 1587 public boolean mkdirs(Path f) throws IOException { 1588 return mkdirs(f, FsPermission.getDefault()); 1589 } 1590 1591 /** 1592 * Make the given file and all non-existent parents into 1593 * directories. Has the semantics of Unix 'mkdir -p'. 1594 * Existence of the directory hierarchy is not an error. 1595 * @param f path to create 1596 * @param permission to apply to f 1597 */ 1598 public abstract boolean mkdirs(Path f, FsPermission permission 1599 ) throws IOException; 1600 1601 /** 1602 * The src file is on the local disk. Add it to FS at 1603 * the given dst name and the source is kept intact afterwards 1604 * @param src path 1605 * @param dst path 1606 */ 1607 public void copyFromLocalFile(Path src, Path dst) 1608 throws IOException { 1609 copyFromLocalFile(false, src, dst); 1610 } 1611 1612 /** 1613 * The src files is on the local disk. Add it to FS at 1614 * the given dst name, removing the source afterwards. 1615 * @param srcs path 1616 * @param dst path 1617 */ 1618 public void moveFromLocalFile(Path[] srcs, Path dst) 1619 throws IOException { 1620 copyFromLocalFile(true, true, srcs, dst); 1621 } 1622 1623 /** 1624 * The src file is on the local disk. Add it to FS at 1625 * the given dst name, removing the source afterwards. 1626 * @param src path 1627 * @param dst path 1628 */ 1629 public void moveFromLocalFile(Path src, Path dst) 1630 throws IOException { 1631 copyFromLocalFile(true, src, dst); 1632 } 1633 1634 /** 1635 * The src file is on the local disk. Add it to FS at 1636 * the given dst name. 1637 * delSrc indicates if the source should be removed 1638 * @param delSrc whether to delete the src 1639 * @param src path 1640 * @param dst path 1641 */ 1642 public void copyFromLocalFile(boolean delSrc, Path src, Path dst) 1643 throws IOException { 1644 copyFromLocalFile(delSrc, true, src, dst); 1645 } 1646 1647 /** 1648 * The src files are on the local disk. Add it to FS at 1649 * the given dst name. 1650 * delSrc indicates if the source should be removed 1651 * @param delSrc whether to delete the src 1652 * @param overwrite whether to overwrite an existing file 1653 * @param srcs array of paths which are source 1654 * @param dst path 1655 */ 1656 public void copyFromLocalFile(boolean delSrc, boolean overwrite, 1657 Path[] srcs, Path dst) 1658 throws IOException { 1659 Configuration conf = getConf(); 1660 FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf); 1661 } 1662 1663 /** 1664 * The src file is on the local disk. Add it to FS at 1665 * the given dst name. 1666 * delSrc indicates if the source should be removed 1667 * @param delSrc whether to delete the src 1668 * @param overwrite whether to overwrite an existing file 1669 * @param src path 1670 * @param dst path 1671 */ 1672 public void copyFromLocalFile(boolean delSrc, boolean overwrite, 1673 Path src, Path dst) 1674 throws IOException { 1675 Configuration conf = getConf(); 1676 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf); 1677 } 1678 1679 /** 1680 * The src file is under FS, and the dst is on the local disk. 1681 * Copy it from FS control to the local dst name. 1682 * @param src path 1683 * @param dst path 1684 */ 1685 public void copyToLocalFile(Path src, Path dst) throws IOException { 1686 copyToLocalFile(false, src, dst); 1687 } 1688 1689 /** 1690 * The src file is under FS, and the dst is on the local disk. 1691 * Copy it from FS control to the local dst name. 1692 * Remove the source afterwards 1693 * @param src path 1694 * @param dst path 1695 */ 1696 public void moveToLocalFile(Path src, Path dst) throws IOException { 1697 copyToLocalFile(true, src, dst); 1698 } 1699 1700 /** 1701 * The src file is under FS, and the dst is on the local disk. 1702 * Copy it from FS control to the local dst name. 1703 * delSrc indicates if the src will be removed or not. 1704 * @param delSrc whether to delete the src 1705 * @param src path 1706 * @param dst path 1707 */ 1708 public void copyToLocalFile(boolean delSrc, Path src, Path dst) 1709 throws IOException { 1710 copyToLocalFile(delSrc, src, dst, false); 1711 } 1712 1713 /** 1714 * The src file is under FS, and the dst is on the local disk. Copy it from FS 1715 * control to the local dst name. delSrc indicates if the src will be removed 1716 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem 1717 * as local file system or not. RawLocalFileSystem is non crc file system.So, 1718 * It will not create any crc files at local. 1719 * 1720 * @param delSrc 1721 * whether to delete the src 1722 * @param src 1723 * path 1724 * @param dst 1725 * path 1726 * @param useRawLocalFileSystem 1727 * whether to use RawLocalFileSystem as local file system or not. 1728 * 1729 * @throws IOException 1730 * - if any IO error 1731 */ 1732 public void copyToLocalFile(boolean delSrc, Path src, Path dst, 1733 boolean useRawLocalFileSystem) throws IOException { 1734 Configuration conf = getConf(); 1735 FileSystem local = null; 1736 if (useRawLocalFileSystem) { 1737 local = getLocal(conf).getRawFileSystem(); 1738 } else { 1739 local = getLocal(conf); 1740 } 1741 FileUtil.copy(this, src, local, dst, delSrc, conf); 1742 } 1743 1744 /** 1745 * Returns a local File that the user can write output to. The caller 1746 * provides both the eventual FS target name and the local working 1747 * file. If the FS is local, we write directly into the target. If 1748 * the FS is remote, we write into the tmp local area. 1749 * @param fsOutputFile path of output file 1750 * @param tmpLocalFile path of local tmp file 1751 */ 1752 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) 1753 throws IOException { 1754 return tmpLocalFile; 1755 } 1756 1757 /** 1758 * Called when we're all done writing to the target. A local FS will 1759 * do nothing, because we've written to exactly the right place. A remote 1760 * FS will copy the contents of tmpLocalFile to the correct target at 1761 * fsOutputFile. 1762 * @param fsOutputFile path of output file 1763 * @param tmpLocalFile path to local tmp file 1764 */ 1765 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) 1766 throws IOException { 1767 moveFromLocalFile(tmpLocalFile, fsOutputFile); 1768 } 1769 1770 /** 1771 * No more filesystem operations are needed. Will 1772 * release any held locks. 1773 */ 1774 public void close() throws IOException { 1775 // delete all files that were marked as delete-on-exit. 1776 processDeleteOnExit(); 1777 CACHE.remove(this.key, this); 1778 } 1779 1780 /** Return the total size of all files in the filesystem.*/ 1781 public long getUsed() throws IOException{ 1782 long used = 0; 1783 FileStatus[] files = listStatus(new Path("/")); 1784 for(FileStatus file:files){ 1785 used += file.getLen(); 1786 } 1787 return used; 1788 } 1789 1790 /** 1791 * Get the block size for a particular file. 1792 * @param f the filename 1793 * @return the number of bytes in a block 1794 */ 1795 /** @deprecated Use getFileStatus() instead */ 1796 @Deprecated 1797 public long getBlockSize(Path f) throws IOException { 1798 return getFileStatus(f).getBlockSize(); 1799 } 1800 1801 /** Return the number of bytes that large input files should be optimally 1802 * be split into to minimize i/o time. */ 1803 public long getDefaultBlockSize() { 1804 // default to 32MB: large enough to minimize the impact of seeks 1805 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); 1806 } 1807 1808 /** 1809 * Get the default replication. 1810 */ 1811 public short getDefaultReplication() { return 1; } 1812 1813 /** 1814 * Return a file status object that represents the path. 1815 * @param f The path we want information from 1816 * @return a FileStatus object 1817 * @throws FileNotFoundException when the path does not exist; 1818 * IOException see specific implementation 1819 */ 1820 public abstract FileStatus getFileStatus(Path f) throws IOException; 1821 1822 /** 1823 * Get the checksum of a file. 1824 * 1825 * @param f The file path 1826 * @return The file checksum. The default return value is null, 1827 * which indicates that no checksum algorithm is implemented 1828 * in the corresponding FileSystem. 1829 */ 1830 public FileChecksum getFileChecksum(Path f) throws IOException { 1831 return null; 1832 } 1833 1834 /** 1835 * Set the verify checksum flag. This is only applicable if the 1836 * corresponding FileSystem supports checksum. By default doesn't do anything. 1837 * @param verifyChecksum 1838 */ 1839 public void setVerifyChecksum(boolean verifyChecksum) { 1840 //doesn't do anything 1841 } 1842 1843 /** 1844 * Return a list of file status objects that corresponds to the list of paths 1845 * excluding those non-existent paths. 1846 * 1847 * @param paths 1848 * the list of paths we want information from 1849 * @return a list of FileStatus objects 1850 * @throws IOException 1851 * see specific implementation 1852 */ 1853 private FileStatus[] getFileStatus(Path[] paths) throws IOException { 1854 if (paths == null) { 1855 return null; 1856 } 1857 ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length); 1858 for (int i = 0; i < paths.length; i++) { 1859 try { 1860 results.add(getFileStatus(paths[i])); 1861 } catch (FileNotFoundException e) { // do nothing 1862 } 1863 } 1864 return results.toArray(new FileStatus[results.size()]); 1865 } 1866 1867 /** 1868 * Returns a status object describing the use and capacity of the 1869 * file system. If the file system has multiple partitions, the 1870 * use and capacity of the root partition is reflected. 1871 * 1872 * @return a FsStatus object 1873 * @throws IOException 1874 * see specific implementation 1875 */ 1876 public FsStatus getStatus() throws IOException { 1877 return getStatus(null); 1878 } 1879 1880 /** 1881 * Returns a status object describing the use and capacity of the 1882 * file system. If the file system has multiple partitions, the 1883 * use and capacity of the partition pointed to by the specified 1884 * path is reflected. 1885 * @param p Path for which status should be obtained. null means 1886 * the default partition. 1887 * @return a FsStatus object 1888 * @throws IOException 1889 * see specific implementation 1890 */ 1891 public FsStatus getStatus(Path p) throws IOException { 1892 return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE); 1893 } 1894 1895 /** 1896 * Set permission of a path. 1897 * @param p 1898 * @param permission 1899 */ 1900 public void setPermission(Path p, FsPermission permission 1901 ) throws IOException { 1902 } 1903 1904 /** 1905 * Set owner of a path (i.e. a file or a directory). 1906 * The parameters username and groupname cannot both be null. 1907 * @param p The path 1908 * @param username If it is null, the original username remains unchanged. 1909 * @param groupname If it is null, the original groupname remains unchanged. 1910 */ 1911 public void setOwner(Path p, String username, String groupname 1912 ) throws IOException { 1913 } 1914 1915 /** 1916 * Set access time of a file 1917 * @param p The path 1918 * @param mtime Set the modification time of this file. 1919 * The number of milliseconds since Jan 1, 1970. 1920 * A value of -1 means that this call should not set modification time. 1921 * @param atime Set the access time of this file. 1922 * The number of milliseconds since Jan 1, 1970. 1923 * A value of -1 means that this call should not set access time. 1924 */ 1925 public void setTimes(Path p, long mtime, long atime 1926 ) throws IOException { 1927 } 1928 1929 private static FileSystem createFileSystem(URI uri, Configuration conf 1930 ) throws IOException { 1931 Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null); 1932 if (clazz == null) { 1933 throw new IOException("No FileSystem for scheme: " + uri.getScheme()); 1934 } 1935 FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf); 1936 fs.initialize(uri, conf); 1937 return fs; 1938 } 1939 1940 /** Caching FileSystem objects */ 1941 static class Cache { 1942 private final ClientFinalizer clientFinalizer = new ClientFinalizer(); 1943 1944 private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>(); 1945 private final Set<Key> toAutoClose = new HashSet<Key>(); 1946 1947 /** A variable that makes all objects in the cache unique */ 1948 private static AtomicLong unique = new AtomicLong(1); 1949 1950 FileSystem get(URI uri, Configuration conf) throws IOException{ 1951 Key key = new Key(uri, conf); 1952 return getInternal(uri, conf, key); 1953 } 1954 1955 /** The objects inserted into the cache using this method are all unique */ 1956 FileSystem getUnique(URI uri, Configuration conf) throws IOException{ 1957 Key key = new Key(uri, conf, unique.getAndIncrement()); 1958 return getInternal(uri, conf, key); 1959 } 1960 1961 private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{ 1962 FileSystem fs; 1963 synchronized (this) { 1964 fs = map.get(key); 1965 } 1966 if (fs != null) { 1967 return fs; 1968 } 1969 1970 fs = createFileSystem(uri, conf); 1971 synchronized (this) { // refetch the lock again 1972 FileSystem oldfs = map.get(key); 1973 if (oldfs != null) { // a file system is created while lock is releasing 1974 fs.close(); // close the new file system 1975 return oldfs; // return the old file system 1976 } 1977 1978 // now insert the new file system into the map 1979 if (map.isEmpty() && !clientFinalizer.isAlive()) { 1980 Runtime.getRuntime().addShutdownHook(clientFinalizer); 1981 } 1982 fs.key = key; 1983 map.put(key, fs); 1984 if (conf.getBoolean("fs.automatic.close", true)) { 1985 toAutoClose.add(key); 1986 } 1987 return fs; 1988 } 1989 } 1990 1991 synchronized void remove(Key key, FileSystem fs) { 1992 if (map.containsKey(key) && fs == map.get(key)) { 1993 map.remove(key); 1994 toAutoClose.remove(key); 1995 if (map.isEmpty() && !clientFinalizer.isAlive()) { 1996 if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) { 1997 LOG.info("Could not cancel cleanup thread, though no " + 1998 "FileSystems are open"); 1999 } 2000 } 2001 } 2002 } 2003 2004 synchronized void closeAll() throws IOException { 2005 closeAll(false); 2006 } 2007 2008 /** 2009 * Close all FileSystem instances in the Cache. 2010 * @param onlyAutomatic only close those that are marked for automatic closing 2011 */ 2012 synchronized void closeAll(boolean onlyAutomatic) throws IOException { 2013 List<IOException> exceptions = new ArrayList<IOException>(); 2014 2015 // Make a copy of the keys in the map since we'll be modifying 2016 // the map while iterating over it, which isn't safe. 2017 List<Key> keys = new ArrayList<Key>(); 2018 keys.addAll(map.keySet()); 2019 2020 for (Key key : keys) { 2021 final FileSystem fs = map.get(key); 2022 2023 if (onlyAutomatic && !toAutoClose.contains(key)) { 2024 continue; 2025 } 2026 2027 //remove from cache 2028 remove(key, fs); 2029 2030 if (fs != null) { 2031 try { 2032 fs.close(); 2033 } 2034 catch(IOException ioe) { 2035 exceptions.add(ioe); 2036 } 2037 } 2038 } 2039 2040 if (!exceptions.isEmpty()) { 2041 throw MultipleIOException.createIOException(exceptions); 2042 } 2043 } 2044 2045 private class ClientFinalizer extends Thread { 2046 public synchronized void run() { 2047 try { 2048 closeAll(true); 2049 } catch (IOException e) { 2050 LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e); 2051 } 2052 } 2053 } 2054 2055 synchronized void closeAll(UserGroupInformation ugi) throws IOException { 2056 List<FileSystem> targetFSList = new ArrayList<FileSystem>(); 2057 //Make a pass over the list and collect the filesystems to close 2058 //we cannot close inline since close() removes the entry from the Map 2059 for (Map.Entry<Key, FileSystem> entry : map.entrySet()) { 2060 final Key key = entry.getKey(); 2061 final FileSystem fs = entry.getValue(); 2062 if (ugi.equals(key.ugi) && fs != null) { 2063 targetFSList.add(fs); 2064 } 2065 } 2066 List<IOException> exceptions = new ArrayList<IOException>(); 2067 //now make a pass over the target list and close each 2068 for (FileSystem fs : targetFSList) { 2069 try { 2070 fs.close(); 2071 } 2072 catch(IOException ioe) { 2073 exceptions.add(ioe); 2074 } 2075 } 2076 if (!exceptions.isEmpty()) { 2077 throw MultipleIOException.createIOException(exceptions); 2078 } 2079 } 2080 2081 /** FileSystem.Cache.Key */ 2082 static class Key { 2083 final String scheme; 2084 final String authority; 2085 final UserGroupInformation ugi; 2086 final long unique; // an artificial way to make a key unique 2087 2088 Key(URI uri, Configuration conf) throws IOException { 2089 this(uri, conf, 0); 2090 } 2091 2092 Key(URI uri, Configuration conf, long unique) throws IOException { 2093 scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase(); 2094 authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase(); 2095 this.unique = unique; 2096 2097 this.ugi = UserGroupInformation.getCurrentUser(); 2098 } 2099 2100 /** {@inheritDoc} */ 2101 public int hashCode() { 2102 return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique; 2103 } 2104 2105 static boolean isEqual(Object a, Object b) { 2106 return a == b || (a != null && a.equals(b)); 2107 } 2108 2109 /** {@inheritDoc} */ 2110 public boolean equals(Object obj) { 2111 if (obj == this) { 2112 return true; 2113 } 2114 if (obj != null && obj instanceof Key) { 2115 Key that = (Key)obj; 2116 return isEqual(this.scheme, that.scheme) 2117 && isEqual(this.authority, that.authority) 2118 && isEqual(this.ugi, that.ugi) 2119 && (this.unique == that.unique); 2120 } 2121 return false; 2122 } 2123 2124 /** {@inheritDoc} */ 2125 public String toString() { 2126 return "("+ugi.toString() + ")@" + scheme + "://" + authority; 2127 } 2128 } 2129 } 2130 2131 public static final class Statistics { 2132 private final String scheme; 2133 private AtomicLong bytesRead = new AtomicLong(); 2134 private AtomicLong bytesWritten = new AtomicLong(); 2135 private AtomicInteger readOps = new AtomicInteger(); 2136 private AtomicInteger largeReadOps = new AtomicInteger(); 2137 private AtomicInteger writeOps = new AtomicInteger(); 2138 2139 public Statistics(String scheme) { 2140 this.scheme = scheme; 2141 } 2142 2143 /** 2144 * Copy constructor. 2145 * 2146 * @param st 2147 * The input Statistics object which is cloned. 2148 */ 2149 public Statistics(Statistics st) { 2150 this.scheme = st.scheme; 2151 this.bytesRead = new AtomicLong(st.bytesRead.longValue()); 2152 this.bytesWritten = new AtomicLong(st.bytesWritten.longValue()); 2153 } 2154 2155 /** 2156 * Increment the bytes read in the statistics 2157 * @param newBytes the additional bytes read 2158 */ 2159 public void incrementBytesRead(long newBytes) { 2160 bytesRead.getAndAdd(newBytes); 2161 } 2162 2163 /** 2164 * Increment the bytes written in the statistics 2165 * @param newBytes the additional bytes written 2166 */ 2167 public void incrementBytesWritten(long newBytes) { 2168 bytesWritten.getAndAdd(newBytes); 2169 } 2170 2171 /** 2172 * Increment the number of read operations 2173 * @param count number of read operations 2174 */ 2175 public void incrementReadOps(int count) { 2176 readOps.getAndAdd(count); 2177 } 2178 2179 /** 2180 * Increment the number of large read operations 2181 * @param count number of large read operations 2182 */ 2183 public void incrementLargeReadOps(int count) { 2184 largeReadOps.getAndAdd(count); 2185 } 2186 2187 /** 2188 * Increment the number of write operations 2189 * @param count number of write operations 2190 */ 2191 public void incrementWriteOps(int count) { 2192 writeOps.getAndAdd(count); 2193 } 2194 2195 /** 2196 * Get the total number of bytes read 2197 * @return the number of bytes 2198 */ 2199 public long getBytesRead() { 2200 return bytesRead.get(); 2201 } 2202 2203 /** 2204 * Get the total number of bytes written 2205 * @return the number of bytes 2206 */ 2207 public long getBytesWritten() { 2208 return bytesWritten.get(); 2209 } 2210 2211 /** 2212 * Get the number of file system read operations such as list files 2213 * @return number of read operations 2214 */ 2215 public int getReadOps() { 2216 return readOps.get() + largeReadOps.get(); 2217 } 2218 2219 /** 2220 * Get the number of large file system read operations such as list files 2221 * under a large directory 2222 * @return number of large read operations 2223 */ 2224 public int getLargeReadOps() { 2225 return largeReadOps.get(); 2226 } 2227 2228 /** 2229 * Get the number of file system write operations such as create, append 2230 * rename etc. 2231 * @return number of write operations 2232 */ 2233 public int getWriteOps() { 2234 return writeOps.get(); 2235 } 2236 2237 public String toString() { 2238 return bytesRead + " bytes read, " + bytesWritten + " bytes written, " 2239 + readOps + " read ops, " + largeReadOps + " large read ops, " 2240 + writeOps + " write ops"; 2241 } 2242 2243 /** 2244 * Reset the counts of bytes to 0. 2245 */ 2246 public void reset() { 2247 bytesWritten.set(0); 2248 bytesRead.set(0); 2249 } 2250 2251 /** 2252 * Get the uri scheme associated with this statistics object. 2253 * @return the schema associated with this set of statistics 2254 */ 2255 public String getScheme() { 2256 return scheme; 2257 } 2258 } 2259 2260 /** 2261 * Get the Map of Statistics object indexed by URI Scheme. 2262 * @return a Map having a key as URI scheme and value as Statistics object 2263 * @deprecated use {@link #getAllStatistics} instead 2264 */ 2265 @Deprecated 2266 public static synchronized Map<String, Statistics> getStatistics() { 2267 Map<String, Statistics> result = new HashMap<String, Statistics>(); 2268 for(Statistics stat: statisticsTable.values()) { 2269 result.put(stat.getScheme(), stat); 2270 } 2271 return result; 2272 } 2273 2274 /** 2275 * Return the FileSystem classes that have Statistics 2276 */ 2277 public static synchronized List<Statistics> getAllStatistics() { 2278 return new ArrayList<Statistics>(statisticsTable.values()); 2279 } 2280 2281 /** 2282 * Get the statistics for a particular file system 2283 * @param cls the class to lookup 2284 * @return a statistics object 2285 */ 2286 public static synchronized 2287 Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { 2288 Statistics result = statisticsTable.get(cls); 2289 if (result == null) { 2290 result = new Statistics(scheme); 2291 statisticsTable.put(cls, result); 2292 } 2293 return result; 2294 } 2295 2296 /** 2297 * Reset all statistics for all file systems 2298 */ 2299 public static synchronized void clearStatistics() { 2300 for(Statistics stat: statisticsTable.values()) { 2301 stat.reset(); 2302 } 2303 } 2304 2305 /** 2306 * Print all statistics for all file systems 2307 */ 2308 public static synchronized 2309 void printStatistics() throws IOException { 2310 for (Map.Entry<Class<? extends FileSystem>, Statistics> pair: 2311 statisticsTable.entrySet()) { 2312 System.out.println(" FileSystem " + pair.getKey().getName() + 2313 ": " + pair.getValue()); 2314 } 2315 } 2316 }