001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.fs; 020 021 import java.io.BufferedOutputStream; 022 import java.io.DataOutput; 023 import java.io.File; 024 import java.io.FileInputStream; 025 import java.io.FileNotFoundException; 026 import java.io.FileOutputStream; 027 import java.io.IOException; 028 import java.io.OutputStream; 029 import java.net.URI; 030 import java.nio.ByteBuffer; 031 import java.util.Arrays; 032 import java.util.EnumSet; 033 import java.util.StringTokenizer; 034 035 import org.apache.hadoop.classification.InterfaceAudience; 036 import org.apache.hadoop.classification.InterfaceStability; 037 import org.apache.hadoop.conf.Configuration; 038 import org.apache.hadoop.fs.permission.FsPermission; 039 import org.apache.hadoop.io.nativeio.NativeIO; 040 import org.apache.hadoop.util.Progressable; 041 import org.apache.hadoop.util.Shell; 042 import org.apache.hadoop.util.StringUtils; 043 044 /**************************************************************** 045 * Implement the FileSystem API for the raw local filesystem. 046 * 047 *****************************************************************/ 048 @InterfaceAudience.Public 049 @InterfaceStability.Stable 050 public class RawLocalFileSystem extends FileSystem { 051 static final URI NAME = URI.create("file:///"); 052 private Path workingDir; 053 054 public RawLocalFileSystem() { 055 workingDir = getInitialWorkingDirectory(); 056 } 057 058 private Path makeAbsolute(Path f) { 059 if (f.isAbsolute()) { 060 return f; 061 } else { 062 return new Path(workingDir, f); 063 } 064 } 065 066 /** Convert a path to a File. */ 067 public File pathToFile(Path path) { 068 checkPath(path); 069 if (!path.isAbsolute()) { 070 path = new Path(getWorkingDirectory(), path); 071 } 072 return new File(path.toUri().getPath()); 073 } 074 075 public URI getUri() { return NAME; } 076 077 public void initialize(URI uri, Configuration conf) throws IOException { 078 super.initialize(uri, conf); 079 setConf(conf); 080 } 081 082 class TrackingFileInputStream extends FileInputStream { 083 public TrackingFileInputStream(File f) throws IOException { 084 super(f); 085 } 086 087 public int read() throws IOException { 088 int result = super.read(); 089 if (result != -1) { 090 statistics.incrementBytesRead(1); 091 } 092 return result; 093 } 094 095 public int read(byte[] data) throws IOException { 096 int result = super.read(data); 097 if (result != -1) { 098 statistics.incrementBytesRead(result); 099 } 100 return result; 101 } 102 103 public int read(byte[] data, int offset, int length) throws IOException { 104 int result = super.read(data, offset, length); 105 if (result != -1) { 106 statistics.incrementBytesRead(result); 107 } 108 return result; 109 } 110 } 111 112 /******************************************************* 113 * For open()'s FSInputStream. 114 *******************************************************/ 115 class LocalFSFileInputStream extends FSInputStream { 116 private FileInputStream fis; 117 private long position; 118 119 public LocalFSFileInputStream(Path f) throws IOException { 120 this.fis = new TrackingFileInputStream(pathToFile(f)); 121 } 122 123 public void seek(long pos) throws IOException { 124 fis.getChannel().position(pos); 125 this.position = pos; 126 } 127 128 public long getPos() throws IOException { 129 return this.position; 130 } 131 132 public boolean seekToNewSource(long targetPos) throws IOException { 133 return false; 134 } 135 136 /* 137 * Just forward to the fis 138 */ 139 public int available() throws IOException { return fis.available(); } 140 public void close() throws IOException { fis.close(); } 141 @Override 142 public boolean markSupported() { return false; } 143 144 public int read() throws IOException { 145 try { 146 int value = fis.read(); 147 if (value >= 0) { 148 this.position++; 149 } 150 return value; 151 } catch (IOException e) { // unexpected exception 152 throw new FSError(e); // assume native fs error 153 } 154 } 155 156 public int read(byte[] b, int off, int len) throws IOException { 157 try { 158 int value = fis.read(b, off, len); 159 if (value > 0) { 160 this.position += value; 161 } 162 return value; 163 } catch (IOException e) { // unexpected exception 164 throw new FSError(e); // assume native fs error 165 } 166 } 167 168 public int read(long position, byte[] b, int off, int len) 169 throws IOException { 170 ByteBuffer bb = ByteBuffer.wrap(b, off, len); 171 try { 172 return fis.getChannel().read(bb, position); 173 } catch (IOException e) { 174 throw new FSError(e); 175 } 176 } 177 178 public long skip(long n) throws IOException { 179 long value = fis.skip(n); 180 if (value > 0) { 181 this.position += value; 182 } 183 return value; 184 } 185 } 186 187 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 188 if (!exists(f)) { 189 throw new FileNotFoundException(f.toString()); 190 } 191 return new FSDataInputStream(new BufferedFSInputStream( 192 new LocalFSFileInputStream(f), bufferSize)); 193 } 194 195 /********************************************************* 196 * For create()'s FSOutputStream. 197 *********************************************************/ 198 class LocalFSFileOutputStream extends OutputStream { 199 private FileOutputStream fos; 200 201 private LocalFSFileOutputStream(Path f, boolean append) throws IOException { 202 this.fos = new FileOutputStream(pathToFile(f), append); 203 } 204 205 /* 206 * Just forward to the fos 207 */ 208 public void close() throws IOException { fos.close(); } 209 public void flush() throws IOException { fos.flush(); } 210 public void write(byte[] b, int off, int len) throws IOException { 211 try { 212 fos.write(b, off, len); 213 } catch (IOException e) { // unexpected exception 214 throw new FSError(e); // assume native fs error 215 } 216 } 217 218 public void write(int b) throws IOException { 219 try { 220 fos.write(b); 221 } catch (IOException e) { // unexpected exception 222 throw new FSError(e); // assume native fs error 223 } 224 } 225 } 226 227 /** {@inheritDoc} */ 228 public FSDataOutputStream append(Path f, int bufferSize, 229 Progressable progress) throws IOException { 230 if (!exists(f)) { 231 throw new FileNotFoundException("File " + f + " not found"); 232 } 233 if (getFileStatus(f).isDirectory()) { 234 throw new IOException("Cannot append to a diretory (=" + f + " )"); 235 } 236 return new FSDataOutputStream(new BufferedOutputStream( 237 new LocalFSFileOutputStream(f, true), bufferSize), statistics); 238 } 239 240 /** {@inheritDoc} */ 241 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, 242 short replication, long blockSize, Progressable progress) 243 throws IOException { 244 if (exists(f) && !overwrite) { 245 throw new IOException("File already exists: "+f); 246 } 247 Path parent = f.getParent(); 248 if (parent != null && !mkdirs(parent)) { 249 throw new IOException("Mkdirs failed to create " + parent.toString()); 250 } 251 return new FSDataOutputStream(new BufferedOutputStream( 252 new LocalFSFileOutputStream(f, false), bufferSize), statistics); 253 } 254 255 /** {@inheritDoc} */ 256 @Override 257 public FSDataOutputStream create(Path f, FsPermission permission, 258 boolean overwrite, int bufferSize, short replication, long blockSize, 259 Progressable progress) throws IOException { 260 261 FSDataOutputStream out = create(f, 262 overwrite, bufferSize, replication, blockSize, progress); 263 setPermission(f, permission); 264 return out; 265 } 266 267 public boolean rename(Path src, Path dst) throws IOException { 268 if (pathToFile(src).renameTo(pathToFile(dst))) { 269 return true; 270 } 271 return FileUtil.copy(this, src, this, dst, true, getConf()); 272 } 273 274 /** 275 * Delete the given path to a file or directory. 276 * @param p the path to delete 277 * @param recursive to delete sub-directories 278 * @return true if the file or directory and all its contents were deleted 279 * @throws IOException if p is non-empty and recursive is false 280 */ 281 public boolean delete(Path p, boolean recursive) throws IOException { 282 File f = pathToFile(p); 283 if (f.isFile()) { 284 return f.delete(); 285 } else if (!recursive && f.isDirectory() && 286 (FileUtil.listFiles(f).length != 0)) { 287 throw new IOException("Directory " + f.toString() + " is not empty"); 288 } 289 return FileUtil.fullyDelete(f); 290 } 291 292 public FileStatus[] listStatus(Path f) throws IOException { 293 File localf = pathToFile(f); 294 FileStatus[] results; 295 296 if (!localf.exists()) { 297 throw new FileNotFoundException("File " + f + " does not exist"); 298 } 299 if (localf.isFile()) { 300 return new FileStatus[] { 301 new RawLocalFileStatus(localf, getDefaultBlockSize(), this) }; 302 } 303 304 String[] names = localf.list(); 305 if (names == null) { 306 return null; 307 } 308 results = new FileStatus[names.length]; 309 int j = 0; 310 for (int i = 0; i < names.length; i++) { 311 try { 312 results[j] = getFileStatus(new Path(f, names[i])); 313 j++; 314 } catch (FileNotFoundException e) { 315 // ignore the files not found since the dir list may have have changed 316 // since the names[] list was generated. 317 } 318 } 319 if (j == names.length) { 320 return results; 321 } 322 return Arrays.copyOf(results, j); 323 } 324 325 /** 326 * Creates the specified directory hierarchy. Does not 327 * treat existence as an error. 328 */ 329 public boolean mkdirs(Path f) throws IOException { 330 if(f == null) { 331 throw new IllegalArgumentException("mkdirs path arg is null"); 332 } 333 Path parent = f.getParent(); 334 File p2f = pathToFile(f); 335 if(parent != null) { 336 File parent2f = pathToFile(parent); 337 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) { 338 throw new FileAlreadyExistsException("Parent path is not a directory: " 339 + parent); 340 } 341 } 342 return (parent == null || mkdirs(parent)) && 343 (p2f.mkdir() || p2f.isDirectory()); 344 } 345 346 /** {@inheritDoc} */ 347 @Override 348 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 349 boolean b = mkdirs(f); 350 if(b) { 351 setPermission(f, permission); 352 } 353 return b; 354 } 355 356 357 @Override 358 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) 359 throws IOException { 360 boolean b = mkdirs(f); 361 setPermission(f, absolutePermission); 362 return b; 363 } 364 365 366 @Override 367 public Path getHomeDirectory() { 368 return this.makeQualified(new Path(System.getProperty("user.home"))); 369 } 370 371 /** 372 * Set the working directory to the given directory. 373 */ 374 @Override 375 public void setWorkingDirectory(Path newDir) { 376 workingDir = makeAbsolute(newDir); 377 checkPath(workingDir); 378 379 } 380 381 @Override 382 public Path getWorkingDirectory() { 383 return workingDir; 384 } 385 386 @Override 387 protected Path getInitialWorkingDirectory() { 388 return this.makeQualified(new Path(System.getProperty("user.dir"))); 389 } 390 391 /** {@inheritDoc} */ 392 @Override 393 public FsStatus getStatus(Path p) throws IOException { 394 File partition = pathToFile(p == null ? new Path("/") : p); 395 //File provides getUsableSpace() and getFreeSpace() 396 //File provides no API to obtain used space, assume used = total - free 397 return new FsStatus(partition.getTotalSpace(), 398 partition.getTotalSpace() - partition.getFreeSpace(), 399 partition.getFreeSpace()); 400 } 401 402 // In the case of the local filesystem, we can just rename the file. 403 public void moveFromLocalFile(Path src, Path dst) throws IOException { 404 rename(src, dst); 405 } 406 407 // We can write output directly to the final location 408 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) 409 throws IOException { 410 return fsOutputFile; 411 } 412 413 // It's in the right place - nothing to do. 414 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile) 415 throws IOException { 416 } 417 418 public void close() throws IOException { 419 super.close(); 420 } 421 422 public String toString() { 423 return "LocalFS"; 424 } 425 426 public FileStatus getFileStatus(Path f) throws IOException { 427 File path = pathToFile(f); 428 if (path.exists()) { 429 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this); 430 } else { 431 throw new FileNotFoundException("File " + f + " does not exist"); 432 } 433 } 434 435 static class RawLocalFileStatus extends FileStatus { 436 /* We can add extra fields here. It breaks at least CopyFiles.FilePair(). 437 * We recognize if the information is already loaded by check if 438 * onwer.equals(""). 439 */ 440 private boolean isPermissionLoaded() { 441 return !super.getOwner().equals(""); 442 } 443 444 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 445 super(f.length(), f.isDirectory(), 1, defaultBlockSize, 446 f.lastModified(), fs.makeQualified(new Path(f.getPath()))); 447 } 448 449 @Override 450 public FsPermission getPermission() { 451 if (!isPermissionLoaded()) { 452 loadPermissionInfo(); 453 } 454 return super.getPermission(); 455 } 456 457 @Override 458 public String getOwner() { 459 if (!isPermissionLoaded()) { 460 loadPermissionInfo(); 461 } 462 return super.getOwner(); 463 } 464 465 @Override 466 public String getGroup() { 467 if (!isPermissionLoaded()) { 468 loadPermissionInfo(); 469 } 470 return super.getGroup(); 471 } 472 473 /// loads permissions, owner, and group from `ls -ld` 474 private void loadPermissionInfo() { 475 IOException e = null; 476 try { 477 StringTokenizer t = new StringTokenizer( 478 execCommand(new File(getPath().toUri()), 479 Shell.getGET_PERMISSION_COMMAND())); 480 //expected format 481 //-rw------- 1 username groupname ... 482 String permission = t.nextToken(); 483 if (permission.length() > 10) { //files with ACLs might have a '+' 484 permission = permission.substring(0, 10); 485 } 486 setPermission(FsPermission.valueOf(permission)); 487 t.nextToken(); 488 setOwner(t.nextToken()); 489 setGroup(t.nextToken()); 490 } catch (Shell.ExitCodeException ioe) { 491 if (ioe.getExitCode() != 1) { 492 e = ioe; 493 } else { 494 setPermission(null); 495 setOwner(null); 496 setGroup(null); 497 } 498 } catch (IOException ioe) { 499 e = ioe; 500 } finally { 501 if (e != null) { 502 throw new RuntimeException("Error while running command to get " + 503 "file permissions : " + 504 StringUtils.stringifyException(e)); 505 } 506 } 507 } 508 509 @Override 510 public void write(DataOutput out) throws IOException { 511 if (!isPermissionLoaded()) { 512 loadPermissionInfo(); 513 } 514 super.write(out); 515 } 516 } 517 518 /** 519 * Use the command chown to set owner. 520 */ 521 @Override 522 public void setOwner(Path p, String username, String groupname) 523 throws IOException { 524 if (username == null && groupname == null) { 525 throw new IOException("username == null && groupname == null"); 526 } 527 528 if (username == null) { 529 execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 530 } else { 531 //OWNER[:[GROUP]] 532 String s = username + (groupname == null? "": ":" + groupname); 533 execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s); 534 } 535 } 536 537 /** 538 * Use the command chmod to set permission. 539 */ 540 @Override 541 public void setPermission(Path p, FsPermission permission) 542 throws IOException { 543 if (NativeIO.isAvailable()) { 544 NativeIO.chmod(pathToFile(p).getCanonicalPath(), 545 permission.toShort()); 546 } else { 547 execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND, 548 String.format("%05o", permission.toShort())); 549 } 550 } 551 552 private static String execCommand(File f, String... cmd) throws IOException { 553 String[] args = new String[cmd.length + 1]; 554 System.arraycopy(cmd, 0, args, 0, cmd.length); 555 args[cmd.length] = FileUtil.makeShellPath(f, true); 556 String output = Shell.execCommand(args); 557 return output; 558 } 559 560 }