001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.fs;
020    
021    import java.io.BufferedOutputStream;
022    import java.io.DataOutput;
023    import java.io.File;
024    import java.io.FileInputStream;
025    import java.io.FileNotFoundException;
026    import java.io.FileOutputStream;
027    import java.io.IOException;
028    import java.io.OutputStream;
029    import java.net.URI;
030    import java.nio.ByteBuffer;
031    import java.util.Arrays;
032    import java.util.EnumSet;
033    import java.util.StringTokenizer;
034    
035    import org.apache.hadoop.classification.InterfaceAudience;
036    import org.apache.hadoop.classification.InterfaceStability;
037    import org.apache.hadoop.conf.Configuration;
038    import org.apache.hadoop.fs.permission.FsPermission;
039    import org.apache.hadoop.io.nativeio.NativeIO;
040    import org.apache.hadoop.util.Progressable;
041    import org.apache.hadoop.util.Shell;
042    import org.apache.hadoop.util.StringUtils;
043    
044    /****************************************************************
045     * Implement the FileSystem API for the raw local filesystem.
046     *
047     *****************************************************************/
048    @InterfaceAudience.Public
049    @InterfaceStability.Stable
050    public class RawLocalFileSystem extends FileSystem {
051      static final URI NAME = URI.create("file:///");
052      private Path workingDir;
053      
054      public RawLocalFileSystem() {
055        workingDir = getInitialWorkingDirectory();
056      }
057      
058      private Path makeAbsolute(Path f) {
059        if (f.isAbsolute()) {
060          return f;
061        } else {
062          return new Path(workingDir, f);
063        }
064      }
065      
066      /** Convert a path to a File. */
067      public File pathToFile(Path path) {
068        checkPath(path);
069        if (!path.isAbsolute()) {
070          path = new Path(getWorkingDirectory(), path);
071        }
072        return new File(path.toUri().getPath());
073      }
074    
075      public URI getUri() { return NAME; }
076      
077      public void initialize(URI uri, Configuration conf) throws IOException {
078        super.initialize(uri, conf);
079        setConf(conf);
080      }
081      
082      class TrackingFileInputStream extends FileInputStream {
083        public TrackingFileInputStream(File f) throws IOException {
084          super(f);
085        }
086        
087        public int read() throws IOException {
088          int result = super.read();
089          if (result != -1) {
090            statistics.incrementBytesRead(1);
091          }
092          return result;
093        }
094        
095        public int read(byte[] data) throws IOException {
096          int result = super.read(data);
097          if (result != -1) {
098            statistics.incrementBytesRead(result);
099          }
100          return result;
101        }
102        
103        public int read(byte[] data, int offset, int length) throws IOException {
104          int result = super.read(data, offset, length);
105          if (result != -1) {
106            statistics.incrementBytesRead(result);
107          }
108          return result;
109        }
110      }
111    
112      /*******************************************************
113       * For open()'s FSInputStream.
114       *******************************************************/
115      class LocalFSFileInputStream extends FSInputStream {
116        private FileInputStream fis;
117        private long position;
118    
119        public LocalFSFileInputStream(Path f) throws IOException {
120          this.fis = new TrackingFileInputStream(pathToFile(f));
121        }
122        
123        public void seek(long pos) throws IOException {
124          fis.getChannel().position(pos);
125          this.position = pos;
126        }
127        
128        public long getPos() throws IOException {
129          return this.position;
130        }
131        
132        public boolean seekToNewSource(long targetPos) throws IOException {
133          return false;
134        }
135        
136        /*
137         * Just forward to the fis
138         */
139        public int available() throws IOException { return fis.available(); }
140        public void close() throws IOException { fis.close(); }
141        @Override
142        public boolean markSupported() { return false; }
143        
144        public int read() throws IOException {
145          try {
146            int value = fis.read();
147            if (value >= 0) {
148              this.position++;
149            }
150            return value;
151          } catch (IOException e) {                 // unexpected exception
152            throw new FSError(e);                   // assume native fs error
153          }
154        }
155        
156        public int read(byte[] b, int off, int len) throws IOException {
157          try {
158            int value = fis.read(b, off, len);
159            if (value > 0) {
160              this.position += value;
161            }
162            return value;
163          } catch (IOException e) {                 // unexpected exception
164            throw new FSError(e);                   // assume native fs error
165          }
166        }
167        
168        public int read(long position, byte[] b, int off, int len)
169          throws IOException {
170          ByteBuffer bb = ByteBuffer.wrap(b, off, len);
171          try {
172            return fis.getChannel().read(bb, position);
173          } catch (IOException e) {
174            throw new FSError(e);
175          }
176        }
177        
178        public long skip(long n) throws IOException {
179          long value = fis.skip(n);
180          if (value > 0) {
181            this.position += value;
182          }
183          return value;
184        }
185      }
186      
187      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
188        if (!exists(f)) {
189          throw new FileNotFoundException(f.toString());
190        }
191        return new FSDataInputStream(new BufferedFSInputStream(
192            new LocalFSFileInputStream(f), bufferSize));
193      }
194      
195      /*********************************************************
196       * For create()'s FSOutputStream.
197       *********************************************************/
198      class LocalFSFileOutputStream extends OutputStream {
199        private FileOutputStream fos;
200        
201        private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
202          this.fos = new FileOutputStream(pathToFile(f), append);
203        }
204        
205        /*
206         * Just forward to the fos
207         */
208        public void close() throws IOException { fos.close(); }
209        public void flush() throws IOException { fos.flush(); }
210        public void write(byte[] b, int off, int len) throws IOException {
211          try {
212            fos.write(b, off, len);
213          } catch (IOException e) {                // unexpected exception
214            throw new FSError(e);                  // assume native fs error
215          }
216        }
217        
218        public void write(int b) throws IOException {
219          try {
220            fos.write(b);
221          } catch (IOException e) {              // unexpected exception
222            throw new FSError(e);                // assume native fs error
223          }
224        }
225      }
226    
227      /** {@inheritDoc} */
228      public FSDataOutputStream append(Path f, int bufferSize,
229          Progressable progress) throws IOException {
230        if (!exists(f)) {
231          throw new FileNotFoundException("File " + f + " not found");
232        }
233        if (getFileStatus(f).isDirectory()) {
234          throw new IOException("Cannot append to a diretory (=" + f + " )");
235        }
236        return new FSDataOutputStream(new BufferedOutputStream(
237            new LocalFSFileOutputStream(f, true), bufferSize), statistics);
238      }
239    
240      /** {@inheritDoc} */
241      public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
242        short replication, long blockSize, Progressable progress)
243        throws IOException {
244        if (exists(f) && !overwrite) {
245          throw new IOException("File already exists: "+f);
246        }
247        Path parent = f.getParent();
248        if (parent != null && !mkdirs(parent)) {
249          throw new IOException("Mkdirs failed to create " + parent.toString());
250        }
251        return new FSDataOutputStream(new BufferedOutputStream(
252            new LocalFSFileOutputStream(f, false), bufferSize), statistics);
253      }
254    
255      /** {@inheritDoc} */
256      @Override
257      public FSDataOutputStream create(Path f, FsPermission permission,
258        boolean overwrite, int bufferSize, short replication, long blockSize,
259        Progressable progress) throws IOException {
260    
261        FSDataOutputStream out = create(f,
262            overwrite, bufferSize, replication, blockSize, progress);
263        setPermission(f, permission);
264        return out;
265      }
266      
267      public boolean rename(Path src, Path dst) throws IOException {
268        if (pathToFile(src).renameTo(pathToFile(dst))) {
269          return true;
270        }
271        return FileUtil.copy(this, src, this, dst, true, getConf());
272      }
273      
274      /**
275       * Delete the given path to a file or directory.
276       * @param p the path to delete
277       * @param recursive to delete sub-directories
278       * @return true if the file or directory and all its contents were deleted
279       * @throws IOException if p is non-empty and recursive is false 
280       */
281      public boolean delete(Path p, boolean recursive) throws IOException {
282        File f = pathToFile(p);
283        if (f.isFile()) {
284          return f.delete();
285        } else if (!recursive && f.isDirectory() && 
286            (FileUtil.listFiles(f).length != 0)) {
287          throw new IOException("Directory " + f.toString() + " is not empty");
288        }
289        return FileUtil.fullyDelete(f);
290      }
291     
292      public FileStatus[] listStatus(Path f) throws IOException {
293        File localf = pathToFile(f);
294        FileStatus[] results;
295    
296        if (!localf.exists()) {
297          throw new FileNotFoundException("File " + f + " does not exist");
298        }
299        if (localf.isFile()) {
300          return new FileStatus[] {
301            new RawLocalFileStatus(localf, getDefaultBlockSize(), this) };
302        }
303    
304        String[] names = localf.list();
305        if (names == null) {
306          return null;
307        }
308        results = new FileStatus[names.length];
309        int j = 0;
310        for (int i = 0; i < names.length; i++) {
311          try {
312            results[j] = getFileStatus(new Path(f, names[i]));
313            j++;
314          } catch (FileNotFoundException e) {
315            // ignore the files not found since the dir list may have have changed
316            // since the names[] list was generated.
317          }
318        }
319        if (j == names.length) {
320          return results;
321        }
322        return Arrays.copyOf(results, j);
323      }
324    
325      /**
326       * Creates the specified directory hierarchy. Does not
327       * treat existence as an error.
328       */
329      public boolean mkdirs(Path f) throws IOException {
330        if(f == null) {
331          throw new IllegalArgumentException("mkdirs path arg is null");
332        }
333        Path parent = f.getParent();
334        File p2f = pathToFile(f);
335        if(parent != null) {
336          File parent2f = pathToFile(parent);
337          if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
338            throw new FileAlreadyExistsException("Parent path is not a directory: " 
339                + parent);
340          }
341        }
342        return (parent == null || mkdirs(parent)) &&
343          (p2f.mkdir() || p2f.isDirectory());
344      }
345    
346      /** {@inheritDoc} */
347      @Override
348      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
349        boolean b = mkdirs(f);
350        if(b) {
351          setPermission(f, permission);
352        }
353        return b;
354      }
355      
356    
357      @Override
358      protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
359        throws IOException {
360        boolean b = mkdirs(f);
361        setPermission(f, absolutePermission);
362        return b;
363      }
364      
365      
366      @Override
367      public Path getHomeDirectory() {
368        return this.makeQualified(new Path(System.getProperty("user.home")));
369      }
370    
371      /**
372       * Set the working directory to the given directory.
373       */
374      @Override
375      public void setWorkingDirectory(Path newDir) {
376        workingDir = makeAbsolute(newDir);
377        checkPath(workingDir);
378        
379      }
380      
381      @Override
382      public Path getWorkingDirectory() {
383        return workingDir;
384      }
385      
386      @Override
387      protected Path getInitialWorkingDirectory() {
388        return this.makeQualified(new Path(System.getProperty("user.dir")));
389      }
390    
391      /** {@inheritDoc} */
392      @Override
393      public FsStatus getStatus(Path p) throws IOException {
394        File partition = pathToFile(p == null ? new Path("/") : p);
395        //File provides getUsableSpace() and getFreeSpace()
396        //File provides no API to obtain used space, assume used = total - free
397        return new FsStatus(partition.getTotalSpace(), 
398          partition.getTotalSpace() - partition.getFreeSpace(),
399          partition.getFreeSpace());
400      }
401      
402      // In the case of the local filesystem, we can just rename the file.
403      public void moveFromLocalFile(Path src, Path dst) throws IOException {
404        rename(src, dst);
405      }
406      
407      // We can write output directly to the final location
408      public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
409        throws IOException {
410        return fsOutputFile;
411      }
412      
413      // It's in the right place - nothing to do.
414      public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
415        throws IOException {
416      }
417      
418      public void close() throws IOException {
419        super.close();
420      }
421      
422      public String toString() {
423        return "LocalFS";
424      }
425      
426      public FileStatus getFileStatus(Path f) throws IOException {
427        File path = pathToFile(f);
428        if (path.exists()) {
429          return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
430        } else {
431          throw new FileNotFoundException("File " + f + " does not exist");
432        }
433      }
434    
435      static class RawLocalFileStatus extends FileStatus {
436        /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
437         * We recognize if the information is already loaded by check if
438         * onwer.equals("").
439         */
440        private boolean isPermissionLoaded() {
441          return !super.getOwner().equals(""); 
442        }
443        
444        RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
445          super(f.length(), f.isDirectory(), 1, defaultBlockSize,
446                f.lastModified(), fs.makeQualified(new Path(f.getPath())));
447        }
448        
449        @Override
450        public FsPermission getPermission() {
451          if (!isPermissionLoaded()) {
452            loadPermissionInfo();
453          }
454          return super.getPermission();
455        }
456    
457        @Override
458        public String getOwner() {
459          if (!isPermissionLoaded()) {
460            loadPermissionInfo();
461          }
462          return super.getOwner();
463        }
464    
465        @Override
466        public String getGroup() {
467          if (!isPermissionLoaded()) {
468            loadPermissionInfo();
469          }
470          return super.getGroup();
471        }
472    
473        /// loads permissions, owner, and group from `ls -ld`
474        private void loadPermissionInfo() {
475          IOException e = null;
476          try {
477            StringTokenizer t = new StringTokenizer(
478                execCommand(new File(getPath().toUri()), 
479                            Shell.getGET_PERMISSION_COMMAND()));
480            //expected format
481            //-rw-------    1 username groupname ...
482            String permission = t.nextToken();
483            if (permission.length() > 10) { //files with ACLs might have a '+'
484              permission = permission.substring(0, 10);
485            }
486            setPermission(FsPermission.valueOf(permission));
487            t.nextToken();
488            setOwner(t.nextToken());
489            setGroup(t.nextToken());
490          } catch (Shell.ExitCodeException ioe) {
491            if (ioe.getExitCode() != 1) {
492              e = ioe;
493            } else {
494              setPermission(null);
495              setOwner(null);
496              setGroup(null);
497            }
498          } catch (IOException ioe) {
499            e = ioe;
500          } finally {
501            if (e != null) {
502              throw new RuntimeException("Error while running command to get " +
503                                         "file permissions : " + 
504                                         StringUtils.stringifyException(e));
505            }
506          }
507        }
508    
509        @Override
510        public void write(DataOutput out) throws IOException {
511          if (!isPermissionLoaded()) {
512            loadPermissionInfo();
513          }
514          super.write(out);
515        }
516      }
517    
518      /**
519       * Use the command chown to set owner.
520       */
521      @Override
522      public void setOwner(Path p, String username, String groupname)
523        throws IOException {
524        if (username == null && groupname == null) {
525          throw new IOException("username == null && groupname == null");
526        }
527    
528        if (username == null) {
529          execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
530        } else {
531          //OWNER[:[GROUP]]
532          String s = username + (groupname == null? "": ":" + groupname);
533          execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
534        }
535      }
536    
537      /**
538       * Use the command chmod to set permission.
539       */
540      @Override
541      public void setPermission(Path p, FsPermission permission)
542        throws IOException {
543        if (NativeIO.isAvailable()) {
544          NativeIO.chmod(pathToFile(p).getCanonicalPath(),
545                         permission.toShort());
546        } else {
547          execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
548              String.format("%05o", permission.toShort()));
549        }
550      }
551    
552      private static String execCommand(File f, String... cmd) throws IOException {
553        String[] args = new String[cmd.length + 1];
554        System.arraycopy(cmd, 0, args, 0, cmd.length);
555        args[cmd.length] = FileUtil.makeShellPath(f, true);
556        String output = Shell.execCommand(args);
557        return output;
558      }
559    
560    }