View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.FSDataOutputStream;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.PathFilter;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.RemoteExceptionHandler;
34  import org.apache.hadoop.hbase.master.HMaster;
35  import org.apache.hadoop.hbase.regionserver.HRegion;
36  import org.apache.hadoop.hdfs.DistributedFileSystem;
37  import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
38  import org.apache.hadoop.hdfs.protocol.FSConstants;
39  import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
40  import org.apache.hadoop.io.SequenceFile;
41  
42  import java.io.DataInputStream;
43  import java.io.EOFException;
44  import java.io.FileNotFoundException;
45  import java.io.IOException;
46  import java.net.URI;
47  import java.net.URISyntaxException;
48  import java.util.HashMap;
49  import java.util.Map;
50  
51  /**
52   * Utility methods for interacting with the underlying file system.
53   */
54  public class FSUtils {
55    private static final Log LOG = LogFactory.getLog(FSUtils.class);
56  
57    /**
58     * Not instantiable
59     */
60    private FSUtils() {
61      super();
62    }
63  
64    /**
65     * Delete if exists.
66     * @param fs filesystem object
67     * @param dir directory to delete
68     * @return True if deleted <code>dir</code>
69     * @throws IOException e
70     */
71    public static boolean deleteDirectory(final FileSystem fs, final Path dir)
72    throws IOException {
73      return fs.exists(dir) && fs.delete(dir, true);
74    }
75  
76    /**
77     * Check if directory exists.  If it does not, create it.
78     * @param fs filesystem object
79     * @param dir path to check
80     * @return Path
81     * @throws IOException e
82     */
83    public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
84      if (!fs.exists(dir)) {
85        fs.mkdirs(dir);
86      }
87      return dir;
88    }
89  
90    /**
91     * Create file.
92     * @param fs filesystem object
93     * @param p path to create
94     * @return Path
95     * @throws IOException e
96     */
97    public static Path create(final FileSystem fs, final Path p)
98    throws IOException {
99      if (fs.exists(p)) {
100       throw new IOException("File already exists " + p.toString());
101     }
102     if (!fs.createNewFile(p)) {
103       throw new IOException("Failed create of " + p);
104     }
105     return p;
106   }
107 
108   /**
109    * Checks to see if the specified file system is available
110    *
111    * @param fs filesystem
112    * @throws IOException e
113    */
114   public static void checkFileSystemAvailable(final FileSystem fs)
115   throws IOException {
116     if (!(fs instanceof DistributedFileSystem)) {
117       return;
118     }
119     IOException exception = null;
120     DistributedFileSystem dfs = (DistributedFileSystem) fs;
121     try {
122       if (dfs.exists(new Path("/"))) {
123         return;
124       }
125     } catch (IOException e) {
126       exception = RemoteExceptionHandler.checkIOException(e);
127     }
128     try {
129       fs.close();
130     } catch (Exception e) {
131         LOG.error("file system close failed: ", e);
132     }
133     IOException io = new IOException("File system is not available");
134     io.initCause(exception);
135     throw io;
136   }
137 
138   /**
139    * Verifies current version of file system
140    *
141    * @param fs filesystem object
142    * @param rootdir root hbase directory
143    * @return null if no version file exists, version string otherwise.
144    * @throws IOException e
145    */
146   public static String getVersion(FileSystem fs, Path rootdir)
147   throws IOException {
148     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
149     String version = null;
150     if (fs.exists(versionFile)) {
151       FSDataInputStream s =
152         fs.open(versionFile);
153       try {
154         version = DataInputStream.readUTF(s);
155       } catch (EOFException eof) {
156         LOG.warn("Version file was empty, odd, will try to set it.");
157       } finally {
158         s.close();
159       }
160     }
161     return version;
162   }
163 
164   /**
165    * Verifies current version of file system
166    *
167    * @param fs file system
168    * @param rootdir root directory of HBase installation
169    * @param message if true, issues a message on System.out
170    *
171    * @throws IOException e
172    */
173   public static void checkVersion(FileSystem fs, Path rootdir,
174       boolean message) throws IOException {
175     String version = getVersion(fs, rootdir);
176 
177     if (version == null) {
178       if (!rootRegionExists(fs, rootdir)) {
179         // rootDir is empty (no version file and no root region)
180         // just create new version file (HBASE-1195)
181         FSUtils.setVersion(fs, rootdir);
182         return;
183       }
184     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0)
185         return;
186 
187     // version is deprecated require migration
188     // Output on stdout so user sees it in terminal.
189     String msg = "File system needs to be upgraded."
190       + "  You have version " + version
191       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
192       + ".  Run the '${HBASE_HOME}/bin/hbase migrate' script.";
193     if (message) {
194       System.out.println("WARNING! " + msg);
195     }
196     throw new FileSystemVersionException(msg);
197   }
198 
199   /**
200    * Sets version of file system
201    *
202    * @param fs filesystem object
203    * @param rootdir hbase root
204    * @throws IOException e
205    */
206   public static void setVersion(FileSystem fs, Path rootdir)
207   throws IOException {
208     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION);
209   }
210 
211   /**
212    * Sets version of file system
213    *
214    * @param fs filesystem object
215    * @param rootdir hbase root directory
216    * @param version version to set
217    * @throws IOException e
218    */
219   public static void setVersion(FileSystem fs, Path rootdir, String version)
220   throws IOException {
221     FSDataOutputStream s =
222       fs.create(new Path(rootdir, HConstants.VERSION_FILE_NAME));
223     s.writeUTF(version);
224     s.close();
225     LOG.debug("Created version file at " + rootdir.toString() + " set its version at:" + version);
226   }
227 
228   /**
229    * Verifies root directory path is a valid URI with a scheme
230    *
231    * @param root root directory path
232    * @return Passed <code>root</code> argument.
233    * @throws IOException if not a valid URI with a scheme
234    */
235   public static Path validateRootPath(Path root) throws IOException {
236     try {
237       URI rootURI = new URI(root.toString());
238       String scheme = rootURI.getScheme();
239       if (scheme == null) {
240         throw new IOException("Root directory does not have a scheme");
241       }
242       return root;
243     } catch (URISyntaxException e) {
244       IOException io = new IOException("Root directory path is not a valid " +
245         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
246       io.initCause(e);
247       throw io;
248     }
249   }
250 
251   /**
252    * If DFS, check safe mode and if so, wait until we clear it.
253    * @param conf configuration
254    * @param wait Sleep between retries
255    * @throws IOException e
256    */
257   public static void waitOnSafeMode(final Configuration conf,
258     final long wait)
259   throws IOException {
260     FileSystem fs = FileSystem.get(conf);
261     if (!(fs instanceof DistributedFileSystem)) return;
262     DistributedFileSystem dfs = (DistributedFileSystem)fs;
263     // Are there any data nodes up yet?
264     // Currently the safe mode check falls through if the namenode is up but no
265     // datanodes have reported in yet.
266     try {
267       while (dfs.getDataNodeStats().length == 0) {
268         LOG.info("Waiting for dfs to come up...");
269         try {
270           Thread.sleep(wait);
271         } catch (InterruptedException e) {
272           //continue
273         }
274       }
275     } catch (IOException e) {
276       // getDataNodeStats can fail if superuser privilege is required to run
277       // the datanode report, just ignore it
278     }
279     // Make sure dfs is not in safe mode
280     while (dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET)) {
281       LOG.info("Waiting for dfs to exit safe mode...");
282       try {
283         Thread.sleep(wait);
284       } catch (InterruptedException e) {
285         //continue
286       }
287     }
288   }
289 
290   /**
291    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
292    * method returns the 'path' component of a Path's URI: e.g. If a Path is
293    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
294    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
295    * This method is useful if you want to print out a Path without qualifying
296    * Filesystem instance.
297    * @param p Filesystem Path whose 'path' component we are to return.
298    * @return Path portion of the Filesystem
299    */
300   public static String getPath(Path p) {
301     return p.toUri().getPath();
302   }
303 
304   /**
305    * @param c configuration
306    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
307    * configuration as a Path.
308    * @throws IOException e
309    */
310   public static Path getRootDir(final Configuration c) throws IOException {
311     return new Path(c.get(HConstants.HBASE_DIR));
312   }
313 
314   /**
315    * Checks if root region exists
316    *
317    * @param fs file system
318    * @param rootdir root directory of HBase installation
319    * @return true if exists
320    * @throws IOException e
321    */
322   public static boolean rootRegionExists(FileSystem fs, Path rootdir)
323   throws IOException {
324     Path rootRegionDir =
325       HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO);
326     return fs.exists(rootRegionDir);
327   }
328 
329   /**
330    * Runs through the hbase rootdir and checks all stores have only
331    * one file in them -- that is, they've been major compacted.  Looks
332    * at root and meta tables too.
333    * @param fs filesystem
334    * @param hbaseRootDir hbase root directory
335    * @return True if this hbase install is major compacted.
336    * @throws IOException e
337    */
338   public static boolean isMajorCompacted(final FileSystem fs,
339       final Path hbaseRootDir)
340   throws IOException {
341     // Presumes any directory under hbase.rootdir is a table.
342     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
343     for (FileStatus tableDir : tableDirs) {
344       // Skip the .log directory.  All others should be tables.  Inside a table,
345       // there are compaction.dir directories to skip.  Otherwise, all else
346       // should be regions.  Then in each region, should only be family
347       // directories.  Under each of these, should be one file only.
348       Path d = tableDir.getPath();
349       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
350         continue;
351       }
352       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
353       for (FileStatus regionDir : regionDirs) {
354         Path dd = regionDir.getPath();
355         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
356           continue;
357         }
358         // Else its a region name.  Now look in region for families.
359         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
360         for (FileStatus familyDir : familyDirs) {
361           Path family = familyDir.getPath();
362           // Now in family make sure only one file.
363           FileStatus[] familyStatus = fs.listStatus(family);
364           if (familyStatus.length > 1) {
365             LOG.debug(family.toString() + " has " + familyStatus.length +
366                 " files.");
367             return false;
368           }
369         }
370       }
371     }
372     return true;
373   }
374 
375   // TODO move this method OUT of FSUtils. No dependencies to HMaster
376   /**
377    * Returns the total overall fragmentation percentage. Includes .META. and
378    * -ROOT- as well.
379    *
380    * @param master  The master defining the HBase root and file system.
381    * @return A map for each table and its percentage.
382    * @throws IOException When scanning the directory fails.
383    */
384   public static int getTotalTableFragmentation(final HMaster master)
385   throws IOException {
386     Map<String, Integer> map = getTableFragmentation(master);
387     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
388   }
389 
390   /**
391    * Runs through the HBase rootdir and checks how many stores for each table
392    * have more than one file in them. Checks -ROOT- and .META. too. The total
393    * percentage across all tables is stored under the special key "-TOTAL-".
394    *
395    * @param master  The master defining the HBase root and file system.
396    * @return A map for each table and its percentage.
397    * @throws IOException When scanning the directory fails.
398    */
399   public static Map<String, Integer> getTableFragmentation(
400     final HMaster master)
401   throws IOException {
402     Path path = getRootDir(master.getConfiguration());
403     // since HMaster.getFileSystem() is package private
404     FileSystem fs = path.getFileSystem(master.getConfiguration());
405     return getTableFragmentation(fs, path);
406   }
407 
408   /**
409    * Runs through the HBase rootdir and checks how many stores for each table
410    * have more than one file in them. Checks -ROOT- and .META. too. The total
411    * percentage across all tables is stored under the special key "-TOTAL-".
412    *
413    * @param fs  The file system to use.
414    * @param hbaseRootDir  The root directory to scan.
415    * @return A map for each table and its percentage.
416    * @throws IOException When scanning the directory fails.
417    */
418   public static Map<String, Integer> getTableFragmentation(
419     final FileSystem fs, final Path hbaseRootDir)
420   throws IOException {
421     Map<String, Integer> frags = new HashMap<String, Integer>();
422     int cfCountTotal = 0;
423     int cfFragTotal = 0;
424     DirFilter df = new DirFilter(fs);
425     // presumes any directory under hbase.rootdir is a table
426     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
427     for (FileStatus tableDir : tableDirs) {
428       // Skip the .log directory.  All others should be tables.  Inside a table,
429       // there are compaction.dir directories to skip.  Otherwise, all else
430       // should be regions.  Then in each region, should only be family
431       // directories.  Under each of these, should be one file only.
432       Path d = tableDir.getPath();
433       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
434         continue;
435       }
436       int cfCount = 0;
437       int cfFrag = 0;
438       FileStatus[] regionDirs = fs.listStatus(d, df);
439       for (FileStatus regionDir : regionDirs) {
440         Path dd = regionDir.getPath();
441         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
442           continue;
443         }
444         // else its a region name, now look in region for families
445         FileStatus[] familyDirs = fs.listStatus(dd, df);
446         for (FileStatus familyDir : familyDirs) {
447           cfCount++;
448           cfCountTotal++;
449           Path family = familyDir.getPath();
450           // now in family make sure only one file
451           FileStatus[] familyStatus = fs.listStatus(family);
452           if (familyStatus.length > 1) {
453             cfFrag++;
454             cfFragTotal++;
455           }
456         }
457       }
458       // compute percentage per table and store in result list
459       frags.put(d.getName(), Math.round((float) cfFrag / cfCount * 100));
460     }
461     // set overall percentage for all tables
462     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
463     return frags;
464   }
465 
466   /**
467    * Expects to find -ROOT- directory.
468    * @param fs filesystem
469    * @param hbaseRootDir hbase root directory
470    * @return True if this a pre020 layout.
471    * @throws IOException e
472    */
473   public static boolean isPre020FileLayout(final FileSystem fs,
474     final Path hbaseRootDir)
475   throws IOException {
476     Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
477       "70236052"), "info"), "mapfiles");
478     return fs.exists(mapfiles);
479   }
480 
481   /**
482    * Runs through the hbase rootdir and checks all stores have only
483    * one file in them -- that is, they've been major compacted.  Looks
484    * at root and meta tables too.  This version differs from
485    * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
486    * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
487    * @param fs filesystem
488    * @param hbaseRootDir hbase root directory
489    * @return True if this hbase install is major compacted.
490    * @throws IOException e
491    */
492   public static boolean isMajorCompactedPre020(final FileSystem fs,
493       final Path hbaseRootDir)
494   throws IOException {
495     // Presumes any directory under hbase.rootdir is a table.
496     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
497     for (FileStatus tableDir : tableDirs) {
498       // Inside a table, there are compaction.dir directories to skip.
499       // Otherwise, all else should be regions.  Then in each region, should
500       // only be family directories.  Under each of these, should be a mapfile
501       // and info directory and in these only one file.
502       Path d = tableDir.getPath();
503       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
504         continue;
505       }
506       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
507       for (FileStatus regionDir : regionDirs) {
508         Path dd = regionDir.getPath();
509         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
510           continue;
511         }
512         // Else its a region name.  Now look in region for families.
513         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
514         for (FileStatus familyDir : familyDirs) {
515           Path family = familyDir.getPath();
516           FileStatus[] infoAndMapfile = fs.listStatus(family);
517           // Assert that only info and mapfile in family dir.
518           if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
519             LOG.debug(family.toString() +
520                 " has more than just info and mapfile: " + infoAndMapfile.length);
521             return false;
522           }
523           // Make sure directory named info or mapfile.
524           for (int ll = 0; ll < 2; ll++) {
525             if (infoAndMapfile[ll].getPath().getName().equals("info") ||
526                 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
527               continue;
528             LOG.debug("Unexpected directory name: " +
529                 infoAndMapfile[ll].getPath());
530             return false;
531           }
532           // Now in family, there are 'mapfile' and 'info' subdirs.  Just
533           // look in the 'mapfile' subdir.
534           FileStatus[] familyStatus =
535               fs.listStatus(new Path(family, "mapfiles"));
536           if (familyStatus.length > 1) {
537             LOG.debug(family.toString() + " has " + familyStatus.length +
538                 " files.");
539             return false;
540           }
541         }
542       }
543     }
544     return true;
545   }
546 
547   /**
548    * A {@link PathFilter} that returns directories.
549    */
550   public static class DirFilter implements PathFilter {
551     private final FileSystem fs;
552 
553     public DirFilter(final FileSystem fs) {
554       this.fs = fs;
555     }
556 
557     public boolean accept(Path p) {
558       boolean isdir = false;
559       try {
560         isdir = this.fs.getFileStatus(p).isDir();
561       } catch (IOException e) {
562         e.printStackTrace();
563       }
564       return isdir;
565     }
566   }
567 
568   /**
569    * Heuristic to determine whether is safe or not to open a file for append
570    * Looks both for dfs.support.append and use reflection to search
571    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
572    * @param conf
573    * @return True if append support
574    */
575   public static boolean isAppendSupported(final Configuration conf) {
576     boolean append = conf.getBoolean("dfs.support.append", false);
577     if (append) {
578       try {
579         // TODO: The implementation that comes back when we do a createWriter
580         // may not be using SequenceFile so the below is not a definitive test.
581         // Will do for now (hdfs-200).
582         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
583         append = true;
584       } catch (SecurityException e) {
585       } catch (NoSuchMethodException e) {
586         append = false;
587       }
588     } else {
589       try {
590         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
591       } catch (NoSuchMethodException e) {
592         append = false;
593       }
594     }
595     return append;
596   }
597 
598   /**
599    * @param conf
600    * @return True if this filesystem whose scheme is 'hdfs'.
601    * @throws IOException
602    */
603   public static boolean isHDFS(final Configuration conf) throws IOException {
604     FileSystem fs = FileSystem.get(conf);
605     String scheme = fs.getUri().getScheme();
606     return scheme.equalsIgnoreCase("hdfs");
607   }
608 
609   /*
610    * Recover file lease. Used when a file might be suspect to be had been left open by another process. <code>p</code>
611    * @param fs
612    * @param p
613    * @param append True if append supported
614    * @throws IOException
615    */
616   public static void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
617   throws IOException{
618     if (!isAppendSupported(conf)) {
619       LOG.warn("Running on HDFS without append enabled may result in data loss");
620       return;
621     }
622     // lease recovery not needed for local file system case.
623     // currently, local file system doesn't implement append either.
624     if (!(fs instanceof DistributedFileSystem)) {
625       return;
626     }
627     LOG.info("Recovering file " + p);
628     long startWaiting = System.currentTimeMillis();
629 
630     // Trying recovery
631     boolean recovered = false;
632     while (!recovered) {
633       try {
634         FSDataOutputStream out = fs.append(p);
635         out.close();
636         recovered = true;
637       } catch (IOException e) {
638         e = RemoteExceptionHandler.checkIOException(e);
639         if (e instanceof AlreadyBeingCreatedException) {
640           // We expect that we'll get this message while the lease is still
641           // within its soft limit, but if we get it past that, it means
642           // that the RS is holding onto the file even though it lost its
643           // znode. We could potentially abort after some time here.
644           long waitedFor = System.currentTimeMillis() - startWaiting;
645           if (waitedFor > FSConstants.LEASE_SOFTLIMIT_PERIOD) {
646             LOG.warn("Waited " + waitedFor + "ms for lease recovery on " + p +
647               ":" + e.getMessage());
648           }
649           try {
650             Thread.sleep(1000);
651           } catch (InterruptedException ex) {
652             // ignore it and try again
653           }
654         } else if (e instanceof LeaseExpiredException &&
655             e.getMessage().contains("File does not exist")) {
656           // This exception comes out instead of FNFE, fix it
657           throw new FileNotFoundException(
658               "The given HLog wasn't found at " + p.toString());
659         } else {
660           throw new IOException("Failed to open " + p + " for append", e);
661         }
662       }
663     }
664     LOG.info("Finished lease recover attempt for " + p);
665   }
666 
667 }