001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.fs;
020
021import java.io.*;
022import java.net.InetAddress;
023import java.net.URI;
024import java.net.UnknownHostException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Enumeration;
028import java.util.List;
029import java.util.Map;
030import java.util.jar.Attributes;
031import java.util.jar.JarOutputStream;
032import java.util.jar.Manifest;
033import java.util.zip.GZIPInputStream;
034import java.util.zip.ZipEntry;
035import java.util.zip.ZipFile;
036
037import org.apache.commons.collections.map.CaseInsensitiveMap;
038import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
039import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
040import org.apache.hadoop.classification.InterfaceAudience;
041import org.apache.hadoop.classification.InterfaceStability;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.fs.permission.FsAction;
044import org.apache.hadoop.fs.permission.FsPermission;
045import org.apache.hadoop.io.IOUtils;
046import org.apache.hadoop.io.nativeio.NativeIO;
047import org.apache.hadoop.util.StringUtils;
048import org.apache.hadoop.util.Shell;
049import org.apache.hadoop.util.Shell.ShellCommandExecutor;
050import org.apache.commons.logging.Log;
051import org.apache.commons.logging.LogFactory;
052
053/**
054 * A collection of file-processing util methods
055 */
056@InterfaceAudience.Public
057@InterfaceStability.Evolving
058public class FileUtil {
059
060  private static final Log LOG = LogFactory.getLog(FileUtil.class);
061
062  /* The error code is defined in winutils to indicate insufficient
063   * privilege to create symbolic links. This value need to keep in
064   * sync with the constant of the same name in:
065   * "src\winutils\common.h"
066   * */
067  public static final int SYMLINK_NO_PRIVILEGE = 2;
068
069  /**
070   * convert an array of FileStatus to an array of Path
071   * 
072   * @param stats
073   *          an array of FileStatus objects
074   * @return an array of paths corresponding to the input
075   */
076  public static Path[] stat2Paths(FileStatus[] stats) {
077    if (stats == null)
078      return null;
079    Path[] ret = new Path[stats.length];
080    for (int i = 0; i < stats.length; ++i) {
081      ret[i] = stats[i].getPath();
082    }
083    return ret;
084  }
085
086  /**
087   * convert an array of FileStatus to an array of Path.
088   * If stats if null, return path
089   * @param stats
090   *          an array of FileStatus objects
091   * @param path
092   *          default path to return in stats is null
093   * @return an array of paths corresponding to the input
094   */
095  public static Path[] stat2Paths(FileStatus[] stats, Path path) {
096    if (stats == null)
097      return new Path[]{path};
098    else
099      return stat2Paths(stats);
100  }
101  
102  /**
103   * Delete a directory and all its contents.  If
104   * we return false, the directory may be partially-deleted.
105   * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
106   *     to by the symlink is not deleted.
107   * (2) If dir is symlink to a directory, symlink is deleted. The directory
108   *     pointed to by symlink is not deleted.
109   * (3) If dir is a normal file, it is deleted.
110   * (4) If dir is a normal directory, then dir and all its contents recursively
111   *     are deleted.
112   */
113  public static boolean fullyDelete(final File dir) {
114    return fullyDelete(dir, false);
115  }
116  
117  /**
118   * Delete a directory and all its contents.  If
119   * we return false, the directory may be partially-deleted.
120   * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
121   *     to by the symlink is not deleted.
122   * (2) If dir is symlink to a directory, symlink is deleted. The directory
123   *     pointed to by symlink is not deleted.
124   * (3) If dir is a normal file, it is deleted.
125   * (4) If dir is a normal directory, then dir and all its contents recursively
126   *     are deleted.
127   * @param dir the file or directory to be deleted
128   * @param tryGrantPermissions true if permissions should be modified to delete a file.
129   * @return true on success false on failure.
130   */
131  public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
132    if (tryGrantPermissions) {
133      // try to chmod +rwx the parent folder of the 'dir': 
134      File parent = dir.getParentFile();
135      grantPermissions(parent);
136    }
137    if (deleteImpl(dir, false)) {
138      // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
139      // (d) symlink to a directory
140      return true;
141    }
142    // handle nonempty directory deletion
143    if (!fullyDeleteContents(dir, tryGrantPermissions)) {
144      return false;
145    }
146    return deleteImpl(dir, true);
147  }
148
149  /**
150   * Returns the target of the given symlink. Returns the empty string if
151   * the given path does not refer to a symlink or there is an error
152   * accessing the symlink.
153   * @param f File representing the symbolic link.
154   * @return The target of the symbolic link, empty string on error or if not
155   *         a symlink.
156   */
157  public static String readLink(File f) {
158    /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
159     * use getCanonicalPath in File to get the target of the symlink but that
160     * does not indicate if the given path refers to a symlink.
161     */
162    try {
163      return Shell.execCommand(
164          Shell.getReadlinkCommand(f.toString())).trim();
165    } catch (IOException x) {
166      return "";
167    }
168  }
169
170  /*
171   * Pure-Java implementation of "chmod +rwx f".
172   */
173  private static void grantPermissions(final File f) {
174      FileUtil.setExecutable(f, true);
175      FileUtil.setReadable(f, true);
176      FileUtil.setWritable(f, true);
177  }
178
179  private static boolean deleteImpl(final File f, final boolean doLog) {
180    if (f == null) {
181      LOG.warn("null file argument.");
182      return false;
183    }
184    final boolean wasDeleted = f.delete();
185    if (wasDeleted) {
186      return true;
187    }
188    final boolean ex = f.exists();
189    if (doLog && ex) {
190      LOG.warn("Failed to delete file or dir ["
191          + f.getAbsolutePath() + "]: it still exists.");
192    }
193    return !ex;
194  }
195  
196  /**
197   * Delete the contents of a directory, not the directory itself.  If
198   * we return false, the directory may be partially-deleted.
199   * If dir is a symlink to a directory, all the contents of the actual
200   * directory pointed to by dir will be deleted.
201   */
202  public static boolean fullyDeleteContents(final File dir) {
203    return fullyDeleteContents(dir, false);
204  }
205  
206  /**
207   * Delete the contents of a directory, not the directory itself.  If
208   * we return false, the directory may be partially-deleted.
209   * If dir is a symlink to a directory, all the contents of the actual
210   * directory pointed to by dir will be deleted.
211   * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 
212   * and all the underlying directories before trying to delete their contents.
213   */
214  public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
215    if (tryGrantPermissions) {
216      // to be able to list the dir and delete files from it
217      // we must grant the dir rwx permissions: 
218      grantPermissions(dir);
219    }
220    boolean deletionSucceeded = true;
221    final File[] contents = dir.listFiles();
222    if (contents != null) {
223      for (int i = 0; i < contents.length; i++) {
224        if (contents[i].isFile()) {
225          if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
226            deletionSucceeded = false;
227            continue; // continue deletion of other files/dirs under dir
228          }
229        } else {
230          // Either directory or symlink to another directory.
231          // Try deleting the directory as this might be a symlink
232          boolean b = false;
233          b = deleteImpl(contents[i], false);
234          if (b){
235            //this was indeed a symlink or an empty directory
236            continue;
237          }
238          // if not an empty directory or symlink let
239          // fullydelete handle it.
240          if (!fullyDelete(contents[i], tryGrantPermissions)) {
241            deletionSucceeded = false;
242            // continue deletion of other files/dirs under dir
243          }
244        }
245      }
246    }
247    return deletionSucceeded;
248  }
249
250  /**
251   * Recursively delete a directory.
252   * 
253   * @param fs {@link FileSystem} on which the path is present
254   * @param dir directory to recursively delete 
255   * @throws IOException
256   * @deprecated Use {@link FileSystem#delete(Path, boolean)}
257   */
258  @Deprecated
259  public static void fullyDelete(FileSystem fs, Path dir) 
260  throws IOException {
261    fs.delete(dir, true);
262  }
263
264  //
265  // If the destination is a subdirectory of the source, then
266  // generate exception
267  //
268  private static void checkDependencies(FileSystem srcFS, 
269                                        Path src, 
270                                        FileSystem dstFS, 
271                                        Path dst)
272                                        throws IOException {
273    if (srcFS == dstFS) {
274      String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
275      String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
276      if (dstq.startsWith(srcq)) {
277        if (srcq.length() == dstq.length()) {
278          throw new IOException("Cannot copy " + src + " to itself.");
279        } else {
280          throw new IOException("Cannot copy " + src + " to its subdirectory " +
281                                dst);
282        }
283      }
284    }
285  }
286
287  /** Copy files between FileSystems. */
288  public static boolean copy(FileSystem srcFS, Path src, 
289                             FileSystem dstFS, Path dst, 
290                             boolean deleteSource,
291                             Configuration conf) throws IOException {
292    return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
293  }
294
295  public static boolean copy(FileSystem srcFS, Path[] srcs, 
296                             FileSystem dstFS, Path dst,
297                             boolean deleteSource, 
298                             boolean overwrite, Configuration conf)
299                             throws IOException {
300    boolean gotException = false;
301    boolean returnVal = true;
302    StringBuilder exceptions = new StringBuilder();
303
304    if (srcs.length == 1)
305      return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
306
307    // Check if dest is directory
308    if (!dstFS.exists(dst)) {
309      throw new IOException("`" + dst +"': specified destination directory " +
310                            "does not exist");
311    } else {
312      FileStatus sdst = dstFS.getFileStatus(dst);
313      if (!sdst.isDirectory()) 
314        throw new IOException("copying multiple files, but last argument `" +
315                              dst + "' is not a directory");
316    }
317
318    for (Path src : srcs) {
319      try {
320        if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
321          returnVal = false;
322      } catch (IOException e) {
323        gotException = true;
324        exceptions.append(e.getMessage());
325        exceptions.append("\n");
326      }
327    }
328    if (gotException) {
329      throw new IOException(exceptions.toString());
330    }
331    return returnVal;
332  }
333
334  /** Copy files between FileSystems. */
335  public static boolean copy(FileSystem srcFS, Path src, 
336                             FileSystem dstFS, Path dst, 
337                             boolean deleteSource,
338                             boolean overwrite,
339                             Configuration conf) throws IOException {
340    FileStatus fileStatus = srcFS.getFileStatus(src);
341    return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
342  }
343
344  /** Copy files between FileSystems. */
345  public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
346                             FileSystem dstFS, Path dst,
347                             boolean deleteSource,
348                             boolean overwrite,
349                             Configuration conf) throws IOException {
350    Path src = srcStatus.getPath();
351    dst = checkDest(src.getName(), dstFS, dst, overwrite);
352    if (srcStatus.isDirectory()) {
353      checkDependencies(srcFS, src, dstFS, dst);
354      if (!dstFS.mkdirs(dst)) {
355        return false;
356      }
357      FileStatus contents[] = srcFS.listStatus(src);
358      for (int i = 0; i < contents.length; i++) {
359        copy(srcFS, contents[i], dstFS,
360             new Path(dst, contents[i].getPath().getName()),
361             deleteSource, overwrite, conf);
362      }
363    } else {
364      InputStream in=null;
365      OutputStream out = null;
366      try {
367        in = srcFS.open(src);
368        out = dstFS.create(dst, overwrite);
369        IOUtils.copyBytes(in, out, conf, true);
370      } catch (IOException e) {
371        IOUtils.closeStream(out);
372        IOUtils.closeStream(in);
373        throw e;
374      }
375    }
376    if (deleteSource) {
377      return srcFS.delete(src, true);
378    } else {
379      return true;
380    }
381  
382  }
383
384  /** Copy all files in a directory to one output file (merge). */
385  public static boolean copyMerge(FileSystem srcFS, Path srcDir, 
386                                  FileSystem dstFS, Path dstFile, 
387                                  boolean deleteSource,
388                                  Configuration conf, String addString) throws IOException {
389    dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
390
391    if (!srcFS.getFileStatus(srcDir).isDirectory())
392      return false;
393   
394    OutputStream out = dstFS.create(dstFile);
395    
396    try {
397      FileStatus contents[] = srcFS.listStatus(srcDir);
398      Arrays.sort(contents);
399      for (int i = 0; i < contents.length; i++) {
400        if (contents[i].isFile()) {
401          InputStream in = srcFS.open(contents[i].getPath());
402          try {
403            IOUtils.copyBytes(in, out, conf, false);
404            if (addString!=null)
405              out.write(addString.getBytes("UTF-8"));
406                
407          } finally {
408            in.close();
409          } 
410        }
411      }
412    } finally {
413      out.close();
414    }
415    
416
417    if (deleteSource) {
418      return srcFS.delete(srcDir, true);
419    } else {
420      return true;
421    }
422  }  
423  
424  /** Copy local files to a FileSystem. */
425  public static boolean copy(File src,
426                             FileSystem dstFS, Path dst,
427                             boolean deleteSource,
428                             Configuration conf) throws IOException {
429    dst = checkDest(src.getName(), dstFS, dst, false);
430
431    if (src.isDirectory()) {
432      if (!dstFS.mkdirs(dst)) {
433        return false;
434      }
435      File contents[] = listFiles(src);
436      for (int i = 0; i < contents.length; i++) {
437        copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
438             deleteSource, conf);
439      }
440    } else if (src.isFile()) {
441      InputStream in = null;
442      OutputStream out =null;
443      try {
444        in = new FileInputStream(src);
445        out = dstFS.create(dst);
446        IOUtils.copyBytes(in, out, conf);
447      } catch (IOException e) {
448        IOUtils.closeStream( out );
449        IOUtils.closeStream( in );
450        throw e;
451      }
452    } else {
453      throw new IOException(src.toString() + 
454                            ": No such file or directory");
455    }
456    if (deleteSource) {
457      return FileUtil.fullyDelete(src);
458    } else {
459      return true;
460    }
461  }
462
463  /** Copy FileSystem files to local files. */
464  public static boolean copy(FileSystem srcFS, Path src, 
465                             File dst, boolean deleteSource,
466                             Configuration conf) throws IOException {
467    FileStatus filestatus = srcFS.getFileStatus(src);
468    return copy(srcFS, filestatus, dst, deleteSource, conf);
469  }
470
471  /** Copy FileSystem files to local files. */
472  private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
473                              File dst, boolean deleteSource,
474                              Configuration conf) throws IOException {
475    Path src = srcStatus.getPath();
476    if (srcStatus.isDirectory()) {
477      if (!dst.mkdirs()) {
478        return false;
479      }
480      FileStatus contents[] = srcFS.listStatus(src);
481      for (int i = 0; i < contents.length; i++) {
482        copy(srcFS, contents[i],
483             new File(dst, contents[i].getPath().getName()),
484             deleteSource, conf);
485      }
486    } else {
487      InputStream in = srcFS.open(src);
488      IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
489    }
490    if (deleteSource) {
491      return srcFS.delete(src, true);
492    } else {
493      return true;
494    }
495  }
496
497  private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
498      boolean overwrite) throws IOException {
499    if (dstFS.exists(dst)) {
500      FileStatus sdst = dstFS.getFileStatus(dst);
501      if (sdst.isDirectory()) {
502        if (null == srcName) {
503          throw new IOException("Target " + dst + " is a directory");
504        }
505        return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
506      } else if (!overwrite) {
507        throw new IOException("Target " + dst + " already exists");
508      }
509    }
510    return dst;
511  }
512
513  /**
514   * Convert a os-native filename to a path that works for the shell.
515   * @param filename The filename to convert
516   * @return The unix pathname
517   * @throws IOException on windows, there can be problems with the subprocess
518   */
519  public static String makeShellPath(String filename) throws IOException {
520    return filename;
521  }
522  
523  /**
524   * Convert a os-native filename to a path that works for the shell.
525   * @param file The filename to convert
526   * @return The unix pathname
527   * @throws IOException on windows, there can be problems with the subprocess
528   */
529  public static String makeShellPath(File file) throws IOException {
530    return makeShellPath(file, false);
531  }
532
533  /**
534   * Convert a os-native filename to a path that works for the shell.
535   * @param file The filename to convert
536   * @param makeCanonicalPath 
537   *          Whether to make canonical path for the file passed
538   * @return The unix pathname
539   * @throws IOException on windows, there can be problems with the subprocess
540   */
541  public static String makeShellPath(File file, boolean makeCanonicalPath) 
542  throws IOException {
543    if (makeCanonicalPath) {
544      return makeShellPath(file.getCanonicalPath());
545    } else {
546      return makeShellPath(file.toString());
547    }
548  }
549
550  /**
551   * Takes an input dir and returns the du on that local directory. Very basic
552   * implementation.
553   * 
554   * @param dir
555   *          The input dir to get the disk space of this local dir
556   * @return The total disk space of the input local directory
557   */
558  public static long getDU(File dir) {
559    long size = 0;
560    if (!dir.exists())
561      return 0;
562    if (!dir.isDirectory()) {
563      return dir.length();
564    } else {
565      File[] allFiles = dir.listFiles();
566      if(allFiles != null) {
567         for (int i = 0; i < allFiles.length; i++) {
568           boolean isSymLink;
569           try {
570             isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
571           } catch(IOException ioe) {
572             isSymLink = true;
573           }
574           if(!isSymLink) {
575             size += getDU(allFiles[i]);
576           }
577         }
578      }
579      return size;
580    }
581  }
582    
583  /**
584   * Given a File input it will unzip the file in a the unzip directory
585   * passed as the second parameter
586   * @param inFile The zip file as input
587   * @param unzipDir The unzip directory where to unzip the zip file.
588   * @throws IOException
589   */
590  public static void unZip(File inFile, File unzipDir) throws IOException {
591    Enumeration<? extends ZipEntry> entries;
592    ZipFile zipFile = new ZipFile(inFile);
593
594    try {
595      entries = zipFile.entries();
596      while (entries.hasMoreElements()) {
597        ZipEntry entry = entries.nextElement();
598        if (!entry.isDirectory()) {
599          InputStream in = zipFile.getInputStream(entry);
600          try {
601            File file = new File(unzipDir, entry.getName());
602            if (!file.getParentFile().mkdirs()) {           
603              if (!file.getParentFile().isDirectory()) {
604                throw new IOException("Mkdirs failed to create " + 
605                                      file.getParentFile().toString());
606              }
607            }
608            OutputStream out = new FileOutputStream(file);
609            try {
610              byte[] buffer = new byte[8192];
611              int i;
612              while ((i = in.read(buffer)) != -1) {
613                out.write(buffer, 0, i);
614              }
615            } finally {
616              out.close();
617            }
618          } finally {
619            in.close();
620          }
621        }
622      }
623    } finally {
624      zipFile.close();
625    }
626  }
627
628  /**
629   * Given a Tar File as input it will untar the file in a the untar directory
630   * passed as the second parameter
631   * 
632   * This utility will untar ".tar" files and ".tar.gz","tgz" files.
633   *  
634   * @param inFile The tar file as input. 
635   * @param untarDir The untar directory where to untar the tar file.
636   * @throws IOException
637   */
638  public static void unTar(File inFile, File untarDir) throws IOException {
639    if (!untarDir.mkdirs()) {
640      if (!untarDir.isDirectory()) {
641        throw new IOException("Mkdirs failed to create " + untarDir);
642      }
643    }
644
645    boolean gzipped = inFile.toString().endsWith("gz");
646    if(Shell.WINDOWS) {
647      // Tar is not native to Windows. Use simple Java based implementation for 
648      // tests and simple tar archives
649      unTarUsingJava(inFile, untarDir, gzipped);
650    }
651    else {
652      // spawn tar utility to untar archive for full fledged unix behavior such 
653      // as resolving symlinks in tar archives
654      unTarUsingTar(inFile, untarDir, gzipped);
655    }
656  }
657  
658  private static void unTarUsingTar(File inFile, File untarDir,
659      boolean gzipped) throws IOException {
660    StringBuffer untarCommand = new StringBuffer();
661    if (gzipped) {
662      untarCommand.append(" gzip -dc '");
663      untarCommand.append(FileUtil.makeShellPath(inFile));
664      untarCommand.append("' | (");
665    } 
666    untarCommand.append("cd '");
667    untarCommand.append(FileUtil.makeShellPath(untarDir)); 
668    untarCommand.append("' ; ");
669    untarCommand.append("tar -xf ");
670
671    if (gzipped) {
672      untarCommand.append(" -)");
673    } else {
674      untarCommand.append(FileUtil.makeShellPath(inFile));
675    }
676    String[] shellCmd = { "bash", "-c", untarCommand.toString() };
677    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
678    shexec.execute();
679    int exitcode = shexec.getExitCode();
680    if (exitcode != 0) {
681      throw new IOException("Error untarring file " + inFile + 
682                  ". Tar process exited with exit code " + exitcode);
683    }
684  }
685  
686  private static void unTarUsingJava(File inFile, File untarDir,
687      boolean gzipped) throws IOException {
688    InputStream inputStream = null;
689    TarArchiveInputStream tis = null;
690    try {
691      if (gzipped) {
692        inputStream = new BufferedInputStream(new GZIPInputStream(
693            new FileInputStream(inFile)));
694      } else {
695        inputStream = new BufferedInputStream(new FileInputStream(inFile));
696      }
697
698      tis = new TarArchiveInputStream(inputStream);
699
700      for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
701        unpackEntries(tis, entry, untarDir);
702        entry = tis.getNextTarEntry();
703      }
704    } finally {
705      IOUtils.cleanup(LOG, tis, inputStream);
706    }
707  }
708  
709  private static void unpackEntries(TarArchiveInputStream tis,
710      TarArchiveEntry entry, File outputDir) throws IOException {
711    if (entry.isDirectory()) {
712      File subDir = new File(outputDir, entry.getName());
713      if (!subDir.mkdirs() && !subDir.isDirectory()) {
714        throw new IOException("Mkdirs failed to create tar internal dir "
715            + outputDir);
716      }
717
718      for (TarArchiveEntry e : entry.getDirectoryEntries()) {
719        unpackEntries(tis, e, subDir);
720      }
721
722      return;
723    }
724
725    File outputFile = new File(outputDir, entry.getName());
726    if (!outputFile.getParentFile().exists()) {
727      if (!outputFile.getParentFile().mkdirs()) {
728        throw new IOException("Mkdirs failed to create tar internal dir "
729            + outputDir);
730      }
731    }
732
733    int count;
734    byte data[] = new byte[2048];
735    try (BufferedOutputStream outputStream = new BufferedOutputStream(
736        new FileOutputStream(outputFile));) {
737
738      while ((count = tis.read(data)) != -1) {
739        outputStream.write(data, 0, count);
740      }
741
742      outputStream.flush();
743    }
744  }
745  
746  /**
747   * Class for creating hardlinks.
748   * Supports Unix, WindXP.
749   * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
750   */
751  @Deprecated
752  public static class HardLink extends org.apache.hadoop.fs.HardLink { 
753    // This is a stub to assist with coordinated change between
754    // COMMON and HDFS projects.  It will be removed after the
755    // corresponding change is committed to HDFS.
756  }
757
758  /**
759   * Create a soft link between a src and destination
760   * only on a local disk. HDFS does not support this.
761   * On Windows, when symlink creation fails due to security
762   * setting, we will log a warning. The return code in this
763   * case is 2.
764   *
765   * @param target the target for symlink 
766   * @param linkname the symlink
767   * @return 0 on success
768   */
769  public static int symLink(String target, String linkname) throws IOException{
770    // Run the input paths through Java's File so that they are converted to the
771    // native OS form
772    File targetFile = new File(
773        Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
774    File linkFile = new File(
775        Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
776
777    // If not on Java7+, copy a file instead of creating a symlink since
778    // Java6 has close to no support for symlinks on Windows. Specifically
779    // File#length and File#renameTo do not work as expected.
780    // (see HADOOP-9061 for additional details)
781    // We still create symlinks for directories, since the scenario in this
782    // case is different. The directory content could change in which
783    // case the symlink loses its purpose (for example task attempt log folder
784    // is symlinked under userlogs and userlogs are generated afterwards).
785    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
786      try {
787        LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
788            "of creating a symlink. Copying " + target + " -> " + linkname);
789
790        if (!linkFile.getParentFile().exists()) {
791          LOG.warn("Parent directory " + linkFile.getParent() +
792              " does not exist.");
793          return 1;
794        } else {
795          org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
796        }
797      } catch (IOException ex) {
798        LOG.warn("FileUtil#symlink failed to copy the file with error: "
799            + ex.getMessage());
800        // Exit with non-zero exit code
801        return 1;
802      }
803      return 0;
804    }
805
806    String[] cmd = Shell.getSymlinkCommand(
807        targetFile.toString(),
808        linkFile.toString());
809
810    ShellCommandExecutor shExec;
811    try {
812      if (Shell.WINDOWS &&
813          linkFile.getParentFile() != null &&
814          !new Path(target).isAbsolute()) {
815        // Relative links on Windows must be resolvable at the time of
816        // creation. To ensure this we run the shell command in the directory
817        // of the link.
818        //
819        shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
820      } else {
821        shExec = new ShellCommandExecutor(cmd);
822      }
823      shExec.execute();
824    } catch (Shell.ExitCodeException ec) {
825      int returnVal = ec.getExitCode();
826      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
827        LOG.warn("Fail to create symbolic links on Windows. "
828            + "The default security settings in Windows disallow non-elevated "
829            + "administrators and all non-administrators from creating symbolic links. "
830            + "This behavior can be changed in the Local Security Policy management console");
831      } else if (returnVal != 0) {
832        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
833            + returnVal + " with: " + ec.getMessage());
834      }
835      return returnVal;
836    } catch (IOException e) {
837      if (LOG.isDebugEnabled()) {
838        LOG.debug("Error while create symlink " + linkname + " to " + target
839            + "." + " Exception: " + StringUtils.stringifyException(e));
840      }
841      throw e;
842    }
843    return shExec.getExitCode();
844  }
845
846  /**
847   * Change the permissions on a filename.
848   * @param filename the name of the file to change
849   * @param perm the permission string
850   * @return the exit code from the command
851   * @throws IOException
852   * @throws InterruptedException
853   */
854  public static int chmod(String filename, String perm
855                          ) throws IOException, InterruptedException {
856    return chmod(filename, perm, false);
857  }
858
859  /**
860   * Change the permissions on a file / directory, recursively, if
861   * needed.
862   * @param filename name of the file whose permissions are to change
863   * @param perm permission string
864   * @param recursive true, if permissions should be changed recursively
865   * @return the exit code from the command.
866   * @throws IOException
867   */
868  public static int chmod(String filename, String perm, boolean recursive)
869                            throws IOException {
870    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
871    String[] args = new String[cmd.length + 1];
872    System.arraycopy(cmd, 0, args, 0, cmd.length);
873    args[cmd.length] = new File(filename).getPath();
874    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
875    try {
876      shExec.execute();
877    }catch(IOException e) {
878      if(LOG.isDebugEnabled()) {
879        LOG.debug("Error while changing permission : " + filename 
880                  +" Exception: " + StringUtils.stringifyException(e));
881      }
882    }
883    return shExec.getExitCode();
884  }
885
886  /**
887   * Set the ownership on a file / directory. User name and group name
888   * cannot both be null.
889   * @param file the file to change
890   * @param username the new user owner name
891   * @param groupname the new group owner name
892   * @throws IOException
893   */
894  public static void setOwner(File file, String username,
895      String groupname) throws IOException {
896    if (username == null && groupname == null) {
897      throw new IOException("username == null && groupname == null");
898    }
899    String arg = (username == null ? "" : username)
900        + (groupname == null ? "" : ":" + groupname);
901    String [] cmd = Shell.getSetOwnerCommand(arg);
902    execCommand(file, cmd);
903  }
904
905  /**
906   * Platform independent implementation for {@link File#setReadable(boolean)}
907   * File#setReadable does not work as expected on Windows.
908   * @param f input file
909   * @param readable
910   * @return true on success, false otherwise
911   */
912  public static boolean setReadable(File f, boolean readable) {
913    if (Shell.WINDOWS) {
914      try {
915        String permission = readable ? "u+r" : "u-r";
916        FileUtil.chmod(f.getCanonicalPath(), permission, false);
917        return true;
918      } catch (IOException ex) {
919        return false;
920      }
921    } else {
922      return f.setReadable(readable);
923    }
924  }
925
926  /**
927   * Platform independent implementation for {@link File#setWritable(boolean)}
928   * File#setWritable does not work as expected on Windows.
929   * @param f input file
930   * @param writable
931   * @return true on success, false otherwise
932   */
933  public static boolean setWritable(File f, boolean writable) {
934    if (Shell.WINDOWS) {
935      try {
936        String permission = writable ? "u+w" : "u-w";
937        FileUtil.chmod(f.getCanonicalPath(), permission, false);
938        return true;
939      } catch (IOException ex) {
940        return false;
941      }
942    } else {
943      return f.setWritable(writable);
944    }
945  }
946
947  /**
948   * Platform independent implementation for {@link File#setExecutable(boolean)}
949   * File#setExecutable does not work as expected on Windows.
950   * Note: revoking execute permission on folders does not have the same
951   * behavior on Windows as on Unix platforms. Creating, deleting or renaming
952   * a file within that folder will still succeed on Windows.
953   * @param f input file
954   * @param executable
955   * @return true on success, false otherwise
956   */
957  public static boolean setExecutable(File f, boolean executable) {
958    if (Shell.WINDOWS) {
959      try {
960        String permission = executable ? "u+x" : "u-x";
961        FileUtil.chmod(f.getCanonicalPath(), permission, false);
962        return true;
963      } catch (IOException ex) {
964        return false;
965      }
966    } else {
967      return f.setExecutable(executable);
968    }
969  }
970
971  /**
972   * Platform independent implementation for {@link File#canRead()}
973   * @param f input file
974   * @return On Unix, same as {@link File#canRead()}
975   *         On Windows, true if process has read access on the path
976   */
977  public static boolean canRead(File f) {
978    if (Shell.WINDOWS) {
979      try {
980        return NativeIO.Windows.access(f.getCanonicalPath(),
981            NativeIO.Windows.AccessRight.ACCESS_READ);
982      } catch (IOException e) {
983        return false;
984      }
985    } else {
986      return f.canRead();
987    }
988  }
989
990  /**
991   * Platform independent implementation for {@link File#canWrite()}
992   * @param f input file
993   * @return On Unix, same as {@link File#canWrite()}
994   *         On Windows, true if process has write access on the path
995   */
996  public static boolean canWrite(File f) {
997    if (Shell.WINDOWS) {
998      try {
999        return NativeIO.Windows.access(f.getCanonicalPath(),
1000            NativeIO.Windows.AccessRight.ACCESS_WRITE);
1001      } catch (IOException e) {
1002        return false;
1003      }
1004    } else {
1005      return f.canWrite();
1006    }
1007  }
1008
1009  /**
1010   * Platform independent implementation for {@link File#canExecute()}
1011   * @param f input file
1012   * @return On Unix, same as {@link File#canExecute()}
1013   *         On Windows, true if process has execute access on the path
1014   */
1015  public static boolean canExecute(File f) {
1016    if (Shell.WINDOWS) {
1017      try {
1018        return NativeIO.Windows.access(f.getCanonicalPath(),
1019            NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
1020      } catch (IOException e) {
1021        return false;
1022      }
1023    } else {
1024      return f.canExecute();
1025    }
1026  }
1027
1028  /**
1029   * Set permissions to the required value. Uses the java primitives instead
1030   * of forking if group == other.
1031   * @param f the file to change
1032   * @param permission the new permissions
1033   * @throws IOException
1034   */
1035  public static void setPermission(File f, FsPermission permission
1036                                   ) throws IOException {
1037    FsAction user = permission.getUserAction();
1038    FsAction group = permission.getGroupAction();
1039    FsAction other = permission.getOtherAction();
1040
1041    // use the native/fork if the group/other permissions are different
1042    // or if the native is available or on Windows
1043    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
1044      execSetPermission(f, permission);
1045      return;
1046    }
1047    
1048    boolean rv = true;
1049    
1050    // read perms
1051    rv = f.setReadable(group.implies(FsAction.READ), false);
1052    checkReturnValue(rv, f, permission);
1053    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1054      rv = f.setReadable(user.implies(FsAction.READ), true);
1055      checkReturnValue(rv, f, permission);
1056    }
1057
1058    // write perms
1059    rv = f.setWritable(group.implies(FsAction.WRITE), false);
1060    checkReturnValue(rv, f, permission);
1061    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1062      rv = f.setWritable(user.implies(FsAction.WRITE), true);
1063      checkReturnValue(rv, f, permission);
1064    }
1065
1066    // exec perms
1067    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1068    checkReturnValue(rv, f, permission);
1069    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1070      rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1071      checkReturnValue(rv, f, permission);
1072    }
1073  }
1074
1075  private static void checkReturnValue(boolean rv, File p, 
1076                                       FsPermission permission
1077                                       ) throws IOException {
1078    if (!rv) {
1079      throw new IOException("Failed to set permissions of path: " + p + 
1080                            " to " + 
1081                            String.format("%04o", permission.toShort()));
1082    }
1083  }
1084  
1085  private static void execSetPermission(File f, 
1086                                        FsPermission permission
1087                                       )  throws IOException {
1088    if (NativeIO.isAvailable()) {
1089      NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1090    } else {
1091      execCommand(f, Shell.getSetPermissionCommand(
1092                  String.format("%04o", permission.toShort()), false));
1093    }
1094  }
1095  
1096  static String execCommand(File f, String... cmd) throws IOException {
1097    String[] args = new String[cmd.length + 1];
1098    System.arraycopy(cmd, 0, args, 0, cmd.length);
1099    args[cmd.length] = f.getCanonicalPath();
1100    String output = Shell.execCommand(args);
1101    return output;
1102  }
1103
1104  /**
1105   * Create a tmp file for a base file.
1106   * @param basefile the base file of the tmp
1107   * @param prefix file name prefix of tmp
1108   * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1109   * @return a newly created tmp file
1110   * @exception IOException If a tmp file cannot created
1111   * @see java.io.File#createTempFile(String, String, File)
1112   * @see java.io.File#deleteOnExit()
1113   */
1114  public static final File createLocalTempFile(final File basefile,
1115                                               final String prefix,
1116                                               final boolean isDeleteOnExit)
1117    throws IOException {
1118    File tmp = File.createTempFile(prefix + basefile.getName(),
1119                                   "", basefile.getParentFile());
1120    if (isDeleteOnExit) {
1121      tmp.deleteOnExit();
1122    }
1123    return tmp;
1124  }
1125
1126  /**
1127   * Move the src file to the name specified by target.
1128   * @param src the source file
1129   * @param target the target file
1130   * @exception IOException If this operation fails
1131   */
1132  public static void replaceFile(File src, File target) throws IOException {
1133    /* renameTo() has two limitations on Windows platform.
1134     * src.renameTo(target) fails if
1135     * 1) If target already exists OR
1136     * 2) If target is already open for reading/writing.
1137     */
1138    if (!src.renameTo(target)) {
1139      int retries = 5;
1140      while (target.exists() && !target.delete() && retries-- >= 0) {
1141        try {
1142          Thread.sleep(1000);
1143        } catch (InterruptedException e) {
1144          throw new IOException("replaceFile interrupted.");
1145        }
1146      }
1147      if (!src.renameTo(target)) {
1148        throw new IOException("Unable to rename " + src +
1149                              " to " + target);
1150      }
1151    }
1152  }
1153  
1154  /**
1155   * A wrapper for {@link File#listFiles()}. This java.io API returns null 
1156   * when a dir is not a directory or for any I/O error. Instead of having
1157   * null check everywhere File#listFiles() is used, we will add utility API
1158   * to get around this problem. For the majority of cases where we prefer 
1159   * an IOException to be thrown.
1160   * @param dir directory for which listing should be performed
1161   * @return list of files or empty list
1162   * @exception IOException for invalid directory or for a bad disk.
1163   */
1164  public static File[] listFiles(File dir) throws IOException {
1165    File[] files = dir.listFiles();
1166    if(files == null) {
1167      throw new IOException("Invalid directory or I/O error occurred for dir: "
1168                + dir.toString());
1169    }
1170    return files;
1171  }  
1172  
1173  /**
1174   * A wrapper for {@link File#list()}. This java.io API returns null 
1175   * when a dir is not a directory or for any I/O error. Instead of having
1176   * null check everywhere File#list() is used, we will add utility API
1177   * to get around this problem. For the majority of cases where we prefer 
1178   * an IOException to be thrown.
1179   * @param dir directory for which listing should be performed
1180   * @return list of file names or empty string list
1181   * @exception IOException for invalid directory or for a bad disk.
1182   */
1183  public static String[] list(File dir) throws IOException {
1184    String[] fileNames = dir.list();
1185    if(fileNames == null) {
1186      throw new IOException("Invalid directory or I/O error occurred for dir: "
1187                + dir.toString());
1188    }
1189    return fileNames;
1190  }  
1191  
1192  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1193      Map<String, String> callerEnv) throws IOException {
1194    return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
1195  }
1196  
1197  /**
1198   * Create a jar file at the given path, containing a manifest with a classpath
1199   * that references all specified entries.
1200   * 
1201   * Some platforms may have an upper limit on command line length.  For example,
1202   * the maximum command line length on Windows is 8191 characters, but the
1203   * length of the classpath may exceed this.  To work around this limitation,
1204   * use this method to create a small intermediate jar with a manifest that
1205   * contains the full classpath.  It returns the absolute path to the new jar,
1206   * which the caller may set as the classpath for a new process.
1207   * 
1208   * Environment variable evaluation is not supported within a jar manifest, so
1209   * this method expands environment variables before inserting classpath entries
1210   * to the manifest.  The method parses environment variables according to
1211   * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
1212   * environment variables are case-insensitive.  For example, %VAR% and %var%
1213   * evaluate to the same value.
1214   * 
1215   * Specifying the classpath in a jar manifest does not support wildcards, so
1216   * this method expands wildcards internally.  Any classpath entry that ends
1217   * with * is translated to all files at that path with extension .jar or .JAR.
1218   * 
1219   * @param inputClassPath String input classpath to bundle into the jar manifest
1220   * @param pwd Path to working directory to save jar
1221   * @param targetDir path to where the jar execution will have its working dir
1222   * @param callerEnv Map<String, String> caller's environment variables to use
1223   *   for expansion
1224   * @return String[] with absolute path to new jar in position 0 and
1225   *   unexpanded wild card entry path in position 1
1226   * @throws IOException if there is an I/O error while writing the jar file
1227   */
1228  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1229      Path targetDir,
1230      Map<String, String> callerEnv) throws IOException {
1231    // Replace environment variables, case-insensitive on Windows
1232    @SuppressWarnings("unchecked")
1233    Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1234      callerEnv;
1235    String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1236    for (int i = 0; i < classPathEntries.length; ++i) {
1237      classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1238        StringUtils.ENV_VAR_PATTERN, env);
1239    }
1240    File workingDir = new File(pwd.toString());
1241    if (!workingDir.mkdirs()) {
1242      // If mkdirs returns false because the working directory already exists,
1243      // then this is acceptable.  If it returns false due to some other I/O
1244      // error, then this method will fail later with an IOException while saving
1245      // the jar.
1246      LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1247    }
1248
1249    StringBuilder unexpandedWildcardClasspath = new StringBuilder();
1250    // Append all entries
1251    List<String> classPathEntryList = new ArrayList<String>(
1252      classPathEntries.length);
1253    for (String classPathEntry: classPathEntries) {
1254      if (classPathEntry.length() == 0) {
1255        continue;
1256      }
1257      if (classPathEntry.endsWith("*")) {
1258        boolean foundWildCardJar = false;
1259        // Append all jars that match the wildcard
1260        Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1261        FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1262          .globStatus(globPath);
1263        if (wildcardJars != null) {
1264          for (FileStatus wildcardJar: wildcardJars) {
1265            foundWildCardJar = true;
1266            classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1267              .toExternalForm());
1268          }
1269        }
1270        if (!foundWildCardJar) {
1271          unexpandedWildcardClasspath.append(File.pathSeparator);
1272          unexpandedWildcardClasspath.append(classPathEntry);
1273        }
1274      } else {
1275        // Append just this entry
1276        File fileCpEntry = null;
1277        if(!new Path(classPathEntry).isAbsolute()) {
1278          fileCpEntry = new File(targetDir.toString(), classPathEntry);
1279        }
1280        else {
1281          fileCpEntry = new File(classPathEntry);
1282        }
1283        String classPathEntryUrl = fileCpEntry.toURI().toURL()
1284          .toExternalForm();
1285
1286        // File.toURI only appends trailing '/' if it can determine that it is a
1287        // directory that already exists.  (See JavaDocs.)  If this entry had a
1288        // trailing '/' specified by the caller, then guarantee that the
1289        // classpath entry in the manifest has a trailing '/', and thus refers to
1290        // a directory instead of a file.  This can happen if the caller is
1291        // creating a classpath jar referencing a directory that hasn't been
1292        // created yet, but will definitely be created before running.
1293        if (classPathEntry.endsWith(Path.SEPARATOR) &&
1294            !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1295          classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1296        }
1297        classPathEntryList.add(classPathEntryUrl);
1298      }
1299    }
1300    String jarClassPath = StringUtils.join(" ", classPathEntryList);
1301
1302    // Create the manifest
1303    Manifest jarManifest = new Manifest();
1304    jarManifest.getMainAttributes().putValue(
1305        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1306    jarManifest.getMainAttributes().putValue(
1307        Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1308
1309    // Write the manifest to output JAR file
1310    File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1311    FileOutputStream fos = null;
1312    BufferedOutputStream bos = null;
1313    JarOutputStream jos = null;
1314    try {
1315      fos = new FileOutputStream(classPathJar);
1316      bos = new BufferedOutputStream(fos);
1317      jos = new JarOutputStream(bos, jarManifest);
1318    } finally {
1319      IOUtils.cleanup(LOG, jos, bos, fos);
1320    }
1321    String[] jarCp = {classPathJar.getCanonicalPath(),
1322                        unexpandedWildcardClasspath.toString()};
1323    return jarCp;
1324  }
1325
1326  public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
1327    if (srcFs==null || destFs==null) {
1328      return false;
1329    }
1330    URI srcUri = srcFs.getUri();
1331    URI dstUri = destFs.getUri();
1332    if (srcUri.getScheme()==null) {
1333      return false;
1334    }
1335    if (!srcUri.getScheme().equals(dstUri.getScheme())) {
1336      return false;
1337    }
1338    String srcHost = srcUri.getHost();
1339    String dstHost = dstUri.getHost();
1340    if ((srcHost!=null) && (dstHost!=null)) {
1341      if (srcHost.equals(dstHost)) {
1342        return srcUri.getPort()==dstUri.getPort();
1343      }
1344      try {
1345        srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
1346        dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
1347      } catch (UnknownHostException ue) {
1348        if (LOG.isDebugEnabled()) {
1349          LOG.debug("Could not compare file-systems. Unknown host: ", ue);
1350        }
1351        return false;
1352      }
1353      if (!srcHost.equals(dstHost)) {
1354        return false;
1355      }
1356    } else if (srcHost==null && dstHost!=null) {
1357      return false;
1358    } else if (srcHost!=null) {
1359      return false;
1360    }
1361    // check for ports
1362    return srcUri.getPort()==dstUri.getPort();
1363  }
1364}