org.apache.hadoop.hbase.util
Class FSUtils

java.lang.Object
  extended by org.apache.hadoop.hbase.util.FSUtils
Direct Known Subclasses:
FSHDFSUtils, FSMapRUtils

public abstract class FSUtils
extends Object

Utility methods for interacting with the underlying file system.


Nested Class Summary
static class FSUtils.DirFilter
          A PathFilter that returns directories.
 
Constructor Summary
protected FSUtils()
           
 
Method Summary
static boolean checkClusterIdExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, int wait)
          Checks that a cluster ID file exists in the HBase root directory
static void checkDfsSafeMode(org.apache.hadoop.conf.Configuration conf)
          Check whether dfs is in safemode.
 org.apache.hadoop.fs.Path checkdir(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir)
          Check if directory exists.
static void checkFileSystemAvailable(org.apache.hadoop.fs.FileSystem fs)
          Checks to see if the specified file system is available
static void checkVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, boolean message)
          Verifies current version of file system
static void checkVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, boolean message, int wait, int retries)
          Verifies current version of file system
static HDFSBlocksDistribution computeHDFSBlocksDistribution(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus status, long start, long length)
          Compute HDFS blocks distribution of a given file, or a portion of the file
static org.apache.hadoop.fs.Path create(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path p)
          Create file.
static boolean deleteDirectory(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir)
          Delete if exists.
static String getClusterId(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir)
          Returns the value of the unique cluster ID stored for this HBase instance.
static org.apache.hadoop.fs.FileSystem getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf)
           
static FSUtils getInstance(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.conf.Configuration conf)
           
static String getPath(org.apache.hadoop.fs.Path p)
          Return the 'path' component of a Path.
static org.apache.hadoop.fs.Path getRootDir(org.apache.hadoop.conf.Configuration c)
           
static List<org.apache.hadoop.fs.Path> getTableDirs(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir)
           
static Map<String,Integer> getTableFragmentation(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir)
          Runs through the HBase rootdir and checks how many stores for each table have more than one file in them.
static Map<String,Integer> getTableFragmentation(HMaster master)
          Runs through the HBase rootdir and checks how many stores for each table have more than one file in them.
static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir, byte[] tableName)
           
static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir, String tableName)
           
static Map<String,org.apache.hadoop.fs.Path> getTableStoreFilePathMap(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir)
          Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to the full Path.
static int getTotalTableFragmentation(HMaster master)
          Returns the total overall fragmentation percentage.
static String getVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir)
          Verifies current version of file system
static boolean isAppendSupported(org.apache.hadoop.conf.Configuration conf)
          Heuristic to determine whether is safe or not to open a file for append Looks both for dfs.support.append and use reflection to search for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
static boolean isHDFS(org.apache.hadoop.conf.Configuration conf)
           
static boolean isMajorCompacted(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir)
          Runs through the hbase rootdir and checks all stores have only one file in them -- that is, they've been major compacted.
static boolean isMajorCompactedPre020(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir)
          Runs through the hbase rootdir and checks all stores have only one file in them -- that is, they've been major compacted.
static boolean isPre020FileLayout(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path hbaseRootDir)
          Expects to find -ROOT- directory.
static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir, org.apache.hadoop.fs.PathFilter filter)
          Calls fs.listStatus() and treats FileNotFoundException as non-fatal This would accommodate difference in various hadoop versions
abstract  void recoverFileLease(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path p, org.apache.hadoop.conf.Configuration conf)
          Recover file lease.
static boolean rootRegionExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir)
          Checks if root region exists
static void setClusterId(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, String clusterId, int wait)
          Writes a new unique identifier for this cluster to the "hbase.id" file in the HBase root directory
static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir)
          Sets version of file system
static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, int wait, int retries)
          Sets version of file system
static void setVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path rootdir, String version, int wait, int retries)
          Sets version of file system
static org.apache.hadoop.fs.Path validateRootPath(org.apache.hadoop.fs.Path root)
          Verifies root directory path is a valid URI with a scheme
static void waitOnSafeMode(org.apache.hadoop.conf.Configuration conf, long wait)
          If DFS, check safe mode and if so, wait until we clear it.
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Constructor Detail

FSUtils

protected FSUtils()
Method Detail

getInstance

public static FSUtils getInstance(org.apache.hadoop.fs.FileSystem fs,
                                  org.apache.hadoop.conf.Configuration conf)

deleteDirectory

public static boolean deleteDirectory(org.apache.hadoop.fs.FileSystem fs,
                                      org.apache.hadoop.fs.Path dir)
                               throws IOException
Delete if exists.

Parameters:
fs - filesystem object
dir - directory to delete
Returns:
True if deleted dir
Throws:
IOException - e

checkdir

public org.apache.hadoop.fs.Path checkdir(org.apache.hadoop.fs.FileSystem fs,
                                          org.apache.hadoop.fs.Path dir)
                                   throws IOException
Check if directory exists. If it does not, create it.

Parameters:
fs - filesystem object
dir - path to check
Returns:
Path
Throws:
IOException - e

create

public static org.apache.hadoop.fs.Path create(org.apache.hadoop.fs.FileSystem fs,
                                               org.apache.hadoop.fs.Path p)
                                        throws IOException
Create file.

Parameters:
fs - filesystem object
p - path to create
Returns:
Path
Throws:
IOException - e

checkFileSystemAvailable

public static void checkFileSystemAvailable(org.apache.hadoop.fs.FileSystem fs)
                                     throws IOException
Checks to see if the specified file system is available

Parameters:
fs - filesystem
Throws:
IOException - e

checkDfsSafeMode

public static void checkDfsSafeMode(org.apache.hadoop.conf.Configuration conf)
                             throws IOException
Check whether dfs is in safemode.

Parameters:
conf -
Throws:
IOException

getVersion

public static String getVersion(org.apache.hadoop.fs.FileSystem fs,
                                org.apache.hadoop.fs.Path rootdir)
                         throws IOException
Verifies current version of file system

Parameters:
fs - filesystem object
rootdir - root hbase directory
Returns:
null if no version file exists, version string otherwise.
Throws:
IOException - e

checkVersion

public static void checkVersion(org.apache.hadoop.fs.FileSystem fs,
                                org.apache.hadoop.fs.Path rootdir,
                                boolean message)
                         throws IOException
Verifies current version of file system

Parameters:
fs - file system
rootdir - root directory of HBase installation
message - if true, issues a message on System.out
Throws:
IOException - e

checkVersion

public static void checkVersion(org.apache.hadoop.fs.FileSystem fs,
                                org.apache.hadoop.fs.Path rootdir,
                                boolean message,
                                int wait,
                                int retries)
                         throws IOException
Verifies current version of file system

Parameters:
fs - file system
rootdir - root directory of HBase installation
message - if true, issues a message on System.out
wait - wait interval
retries - number of times to retry
Throws:
IOException - e

setVersion

public static void setVersion(org.apache.hadoop.fs.FileSystem fs,
                              org.apache.hadoop.fs.Path rootdir)
                       throws IOException
Sets version of file system

Parameters:
fs - filesystem object
rootdir - hbase root
Throws:
IOException - e

setVersion

public static void setVersion(org.apache.hadoop.fs.FileSystem fs,
                              org.apache.hadoop.fs.Path rootdir,
                              int wait,
                              int retries)
                       throws IOException
Sets version of file system

Parameters:
fs - filesystem object
rootdir - hbase root
wait - time to wait for retry
retries - number of times to retry before failing
Throws:
IOException - e

setVersion

public static void setVersion(org.apache.hadoop.fs.FileSystem fs,
                              org.apache.hadoop.fs.Path rootdir,
                              String version,
                              int wait,
                              int retries)
                       throws IOException
Sets version of file system

Parameters:
fs - filesystem object
rootdir - hbase root directory
version - version to set
wait - time to wait for retry
retries - number of times to retry before throwing an IOException
Throws:
IOException - e

checkClusterIdExists

public static boolean checkClusterIdExists(org.apache.hadoop.fs.FileSystem fs,
                                           org.apache.hadoop.fs.Path rootdir,
                                           int wait)
                                    throws IOException
Checks that a cluster ID file exists in the HBase root directory

Parameters:
fs - the root directory FileSystem
rootdir - the HBase root directory in HDFS
wait - how long to wait between retries
Returns:
true if the file exists, otherwise false
Throws:
IOException - if checking the FileSystem fails

getClusterId

public static String getClusterId(org.apache.hadoop.fs.FileSystem fs,
                                  org.apache.hadoop.fs.Path rootdir)
                           throws IOException
Returns the value of the unique cluster ID stored for this HBase instance.

Parameters:
fs - the root directory FileSystem
rootdir - the path to the HBase root directory
Returns:
the unique cluster identifier
Throws:
IOException - if reading the cluster ID file fails

setClusterId

public static void setClusterId(org.apache.hadoop.fs.FileSystem fs,
                                org.apache.hadoop.fs.Path rootdir,
                                String clusterId,
                                int wait)
                         throws IOException
Writes a new unique identifier for this cluster to the "hbase.id" file in the HBase root directory

Parameters:
fs - the root directory FileSystem
rootdir - the path to the HBase root directory
clusterId - the unique identifier to store
wait - how long (in milliseconds) to wait between retries
Throws:
IOException - if writing to the FileSystem fails and no wait value

validateRootPath

public static org.apache.hadoop.fs.Path validateRootPath(org.apache.hadoop.fs.Path root)
                                                  throws IOException
Verifies root directory path is a valid URI with a scheme

Parameters:
root - root directory path
Returns:
Passed root argument.
Throws:
IOException - if not a valid URI with a scheme

waitOnSafeMode

public static void waitOnSafeMode(org.apache.hadoop.conf.Configuration conf,
                                  long wait)
                           throws IOException
If DFS, check safe mode and if so, wait until we clear it.

Parameters:
conf - configuration
wait - Sleep between retries
Throws:
IOException - e

getPath

public static String getPath(org.apache.hadoop.fs.Path p)
Return the 'path' component of a Path. In Hadoop, Path is an URI. This method returns the 'path' component of a Path's URI: e.g. If a Path is hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir, this method returns /hbase_trunk/TestTable/compaction.dir. This method is useful if you want to print out a Path without qualifying Filesystem instance.

Parameters:
p - Filesystem Path whose 'path' component we are to return.
Returns:
Path portion of the Filesystem

getRootDir

public static org.apache.hadoop.fs.Path getRootDir(org.apache.hadoop.conf.Configuration c)
                                            throws IOException
Parameters:
c - configuration
Returns:
Path to hbase root directory: i.e. hbase.rootdir from configuration as a qualified Path.
Throws:
IOException - e

rootRegionExists

public static boolean rootRegionExists(org.apache.hadoop.fs.FileSystem fs,
                                       org.apache.hadoop.fs.Path rootdir)
                                throws IOException
Checks if root region exists

Parameters:
fs - file system
rootdir - root directory of HBase installation
Returns:
true if exists
Throws:
IOException - e

computeHDFSBlocksDistribution

public static HDFSBlocksDistribution computeHDFSBlocksDistribution(org.apache.hadoop.fs.FileSystem fs,
                                                                   org.apache.hadoop.fs.FileStatus status,
                                                                   long start,
                                                                   long length)
                                                            throws IOException
Compute HDFS blocks distribution of a given file, or a portion of the file

Parameters:
fs - file system
status - file status of the file
start - start position of the portion
length - length of the portion
Returns:
The HDFS blocks distribution
Throws:
IOException

isMajorCompacted

public static boolean isMajorCompacted(org.apache.hadoop.fs.FileSystem fs,
                                       org.apache.hadoop.fs.Path hbaseRootDir)
                                throws IOException
Runs through the hbase rootdir and checks all stores have only one file in them -- that is, they've been major compacted. Looks at root and meta tables too.

Parameters:
fs - filesystem
hbaseRootDir - hbase root directory
Returns:
True if this hbase install is major compacted.
Throws:
IOException - e

getTotalTableFragmentation

public static int getTotalTableFragmentation(HMaster master)
                                      throws IOException
Returns the total overall fragmentation percentage. Includes .META. and -ROOT- as well.

Parameters:
master - The master defining the HBase root and file system.
Returns:
A map for each table and its percentage.
Throws:
IOException - When scanning the directory fails.

getTableFragmentation

public static Map<String,Integer> getTableFragmentation(HMaster master)
                                                 throws IOException
Runs through the HBase rootdir and checks how many stores for each table have more than one file in them. Checks -ROOT- and .META. too. The total percentage across all tables is stored under the special key "-TOTAL-".

Parameters:
master - The master defining the HBase root and file system.
Returns:
A map for each table and its percentage.
Throws:
IOException - When scanning the directory fails.

getTableFragmentation

public static Map<String,Integer> getTableFragmentation(org.apache.hadoop.fs.FileSystem fs,
                                                        org.apache.hadoop.fs.Path hbaseRootDir)
                                                 throws IOException
Runs through the HBase rootdir and checks how many stores for each table have more than one file in them. Checks -ROOT- and .META. too. The total percentage across all tables is stored under the special key "-TOTAL-".

Parameters:
fs - The file system to use.
hbaseRootDir - The root directory to scan.
Returns:
A map for each table and its percentage.
Throws:
IOException - When scanning the directory fails.

isPre020FileLayout

public static boolean isPre020FileLayout(org.apache.hadoop.fs.FileSystem fs,
                                         org.apache.hadoop.fs.Path hbaseRootDir)
                                  throws IOException
Expects to find -ROOT- directory.

Parameters:
fs - filesystem
hbaseRootDir - hbase root directory
Returns:
True if this a pre020 layout.
Throws:
IOException - e

isMajorCompactedPre020

public static boolean isMajorCompactedPre020(org.apache.hadoop.fs.FileSystem fs,
                                             org.apache.hadoop.fs.Path hbaseRootDir)
                                      throws IOException
Runs through the hbase rootdir and checks all stores have only one file in them -- that is, they've been major compacted. Looks at root and meta tables too. This version differs from isMajorCompacted(FileSystem, Path) in that it expects a pre-0.20.0 hbase layout on the filesystem. Used migrating.

Parameters:
fs - filesystem
hbaseRootDir - hbase root directory
Returns:
True if this hbase install is major compacted.
Throws:
IOException - e

isAppendSupported

public static boolean isAppendSupported(org.apache.hadoop.conf.Configuration conf)
Heuristic to determine whether is safe or not to open a file for append Looks both for dfs.support.append and use reflection to search for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()

Parameters:
conf -
Returns:
True if append support

isHDFS

public static boolean isHDFS(org.apache.hadoop.conf.Configuration conf)
                      throws IOException
Parameters:
conf -
Returns:
True if this filesystem whose scheme is 'hdfs'.
Throws:
IOException

recoverFileLease

public abstract void recoverFileLease(org.apache.hadoop.fs.FileSystem fs,
                                      org.apache.hadoop.fs.Path p,
                                      org.apache.hadoop.conf.Configuration conf)
                               throws IOException
Recover file lease. Used when a file might be suspect to be had been left open by another process.

Parameters:
fs - FileSystem handle
p - Path of file to recover lease
conf - Configuration handle
Throws:
IOException

getTableDirs

public static List<org.apache.hadoop.fs.Path> getTableDirs(org.apache.hadoop.fs.FileSystem fs,
                                                           org.apache.hadoop.fs.Path rootdir)
                                                    throws IOException
Parameters:
fs -
rootdir -
Returns:
All the table directories under rootdir. Ignore non table hbase folders such as .logs, .oldlogs, .corrupt, .META., and -ROOT- folders.
Throws:
IOException

getTablePath

public static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir,
                                                     byte[] tableName)

getTablePath

public static org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path rootdir,
                                                     String tableName)

getCurrentFileSystem

public static org.apache.hadoop.fs.FileSystem getCurrentFileSystem(org.apache.hadoop.conf.Configuration conf)
                                                            throws IOException
Parameters:
conf -
Returns:
Returns the filesystem of the hbase rootdir.
Throws:
IOException

getTableStoreFilePathMap

public static Map<String,org.apache.hadoop.fs.Path> getTableStoreFilePathMap(org.apache.hadoop.fs.FileSystem fs,
                                                                             org.apache.hadoop.fs.Path hbaseRootDir)
                                                                      throws IOException
Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to the full Path.
Example...
Key = 3944417774205889744
Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744

Parameters:
fs - The file system to use.
hbaseRootDir - The root directory to scan.
Returns:
Map keyed by StoreFile name with a value of the full Path.
Throws:
IOException - When scanning the directory fails.

listStatus

public static org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.FileSystem fs,
                                                           org.apache.hadoop.fs.Path dir,
                                                           org.apache.hadoop.fs.PathFilter filter)
                                                    throws IOException
Calls fs.listStatus() and treats FileNotFoundException as non-fatal This would accommodate difference in various hadoop versions

Parameters:
fs - file system
dir - directory
filter - path filter
Returns:
null if tabledir doesn't exist, otherwise FileStatus array
Throws:
IOException


Copyright © 2012 The Apache Software Foundation. All Rights Reserved.