1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.UUID;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReentrantLock;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
41 import org.apache.hadoop.hbase.RemoteExceptionHandler;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.hbase.ServerName;
44 import org.apache.hadoop.hbase.backup.HFileArchiver;
45 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
46 import org.apache.hadoop.hbase.regionserver.HRegion;
47 import org.apache.hadoop.hbase.regionserver.wal.HLog;
48 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
49 import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
52 import org.apache.hadoop.hbase.util.FSTableDescriptors;
53 import org.apache.hadoop.hbase.util.FSUtils;
54
55
56
57
58
59
60 public class MasterFileSystem {
61 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
62
63 Configuration conf;
64
65 Server master;
66
67 MasterMetrics metrics;
68
69 private String clusterId;
70
71 private final FileSystem fs;
72
73 private volatile boolean fsOk = true;
74
75 private final Path oldLogDir;
76
77 private final Path rootdir;
78
79 private final Path tempdir;
80
81 final Lock splitLogLock = new ReentrantLock();
82 final boolean distributedLogSplitting;
83 final SplitLogManager splitLogManager;
84 private final MasterServices services;
85
86 public MasterFileSystem(Server master, MasterServices services,
87 MasterMetrics metrics, boolean masterRecovery)
88 throws IOException {
89 this.conf = master.getConfiguration();
90 this.master = master;
91 this.services = services;
92 this.metrics = metrics;
93
94
95
96
97 this.rootdir = FSUtils.getRootDir(conf);
98 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
99
100
101 this.fs = this.rootdir.getFileSystem(conf);
102 String fsUri = this.fs.getUri().toString();
103 conf.set("fs.default.name", fsUri);
104 conf.set("fs.defaultFS", fsUri);
105
106 fs.setConf(conf);
107 this.distributedLogSplitting =
108 conf.getBoolean("hbase.master.distributed.log.splitting", true);
109 if (this.distributedLogSplitting) {
110 this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
111 master.getConfiguration(), master, master.getServerName().toString());
112 this.splitLogManager.finishInitialization(masterRecovery);
113 } else {
114 this.splitLogManager = null;
115 }
116
117
118 this.oldLogDir = createInitialFileSystemLayout();
119 }
120
121
122
123
124
125
126
127
128
129
130
131 private Path createInitialFileSystemLayout() throws IOException {
132
133 checkRootDir(this.rootdir, conf, this.fs);
134
135
136 checkTempDir(this.tempdir, conf, this.fs);
137
138 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
139
140
141 if(!this.fs.exists(oldLogDir)) {
142 this.fs.mkdirs(oldLogDir);
143 }
144
145 return oldLogDir;
146 }
147
148 public FileSystem getFileSystem() {
149 return this.fs;
150 }
151
152
153
154
155
156 public Path getOldLogDir() {
157 return this.oldLogDir;
158 }
159
160
161
162
163
164
165 public boolean checkFileSystem() {
166 if (this.fsOk) {
167 try {
168 FSUtils.checkFileSystemAvailable(this.fs);
169 FSUtils.checkDfsSafeMode(this.conf);
170 } catch (IOException e) {
171 master.abort("Shutting down HBase cluster: file system not available", e);
172 this.fsOk = false;
173 }
174 }
175 return this.fsOk;
176 }
177
178
179
180
181 public Path getRootDir() {
182 return this.rootdir;
183 }
184
185
186
187
188 public Path getTempDir() {
189 return this.tempdir;
190 }
191
192
193
194
195 public String getClusterId() {
196 return clusterId;
197 }
198
199
200
201
202
203 void splitLogAfterStartup() {
204 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
205 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
206 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
207 do {
208 if (master.isStopped()) {
209 LOG.warn("Master stopped while splitting logs");
210 break;
211 }
212 List<ServerName> serverNames = new ArrayList<ServerName>();
213 try {
214 if (!this.fs.exists(logsDirPath)) return;
215 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
216
217
218 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
219 .keySet();
220
221 if (logFolders == null || logFolders.length == 0) {
222 LOG.debug("No log files to split, proceeding...");
223 return;
224 }
225 for (FileStatus status : logFolders) {
226 String sn = status.getPath().getName();
227
228 if (sn.endsWith(HLog.SPLITTING_EXT)) {
229 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
230 }
231 ServerName serverName = ServerName.parseServerName(sn);
232 if (!onlineServers.contains(serverName)) {
233 LOG.info("Log folder " + status.getPath() + " doesn't belong "
234 + "to a known region server, splitting");
235 serverNames.add(serverName);
236 } else {
237 LOG.info("Log folder " + status.getPath()
238 + " belongs to an existing region server");
239 }
240 }
241 splitLog(serverNames);
242 retrySplitting = false;
243 } catch (IOException ioe) {
244 LOG.warn("Failed splitting of " + serverNames, ioe);
245 if (!checkFileSystem()) {
246 LOG.warn("Bad Filesystem, exiting");
247 Runtime.getRuntime().halt(1);
248 }
249 try {
250 if (retrySplitting) {
251 Thread.sleep(conf.getInt(
252 "hbase.hlog.split.failure.retry.interval", 30 * 1000));
253 }
254 } catch (InterruptedException e) {
255 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
256 Thread.currentThread().interrupt();
257 retrySplitting = false;
258 Runtime.getRuntime().halt(1);
259 }
260 }
261 } while (retrySplitting);
262 }
263
264 public void splitLog(final ServerName serverName) throws IOException {
265 List<ServerName> serverNames = new ArrayList<ServerName>();
266 serverNames.add(serverName);
267 splitLog(serverNames);
268 }
269
270 public void splitLog(final List<ServerName> serverNames) throws IOException {
271 long splitTime = 0, splitLogSize = 0;
272 List<Path> logDirs = new ArrayList<Path>();
273 for(ServerName serverName: serverNames){
274 Path logDir = new Path(this.rootdir,
275 HLog.getHLogDirectoryName(serverName.toString()));
276 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
277
278 if (fs.exists(logDir)) {
279 if (!this.fs.rename(logDir, splitDir)) {
280 throw new IOException("Failed fs.rename for log split: " + logDir);
281 }
282 logDir = splitDir;
283 LOG.debug("Renamed region directory: " + splitDir);
284 } else if (!fs.exists(splitDir)) {
285 LOG.info("Log dir for server " + serverName + " does not exist");
286 continue;
287 }
288 logDirs.add(splitDir);
289 }
290
291 if (logDirs.isEmpty()) {
292 LOG.info("No logs to split");
293 return;
294 }
295
296 if (distributedLogSplitting) {
297 splitLogManager.handleDeadWorkers(serverNames);
298 splitTime = EnvironmentEdgeManager.currentTimeMillis();
299 splitLogSize = splitLogManager.splitLogDistributed(logDirs);
300 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
301 } else {
302 for(Path logDir: logDirs){
303
304
305 this.splitLogLock.lock();
306 try {
307 HLogSplitter splitter = HLogSplitter.createLogSplitter(
308 conf, rootdir, logDir, oldLogDir, this.fs);
309 try {
310
311 FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
312 splitter.splitLog();
313 } catch (OrphanHLogAfterSplitException e) {
314 LOG.warn("Retrying splitting because of:", e);
315
316 splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
317 oldLogDir, this.fs);
318 splitter.splitLog();
319 }
320 splitTime = splitter.getTime();
321 splitLogSize = splitter.getSize();
322 } finally {
323 this.splitLogLock.unlock();
324 }
325 }
326 }
327
328 if (this.metrics != null) {
329 this.metrics.addSplit(splitTime, splitLogSize);
330 }
331 }
332
333
334
335
336
337
338
339
340
341
342 private Path checkRootDir(final Path rd, final Configuration c,
343 final FileSystem fs)
344 throws IOException {
345
346 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
347 10 * 1000));
348
349 try {
350 if (!fs.exists(rd)) {
351 fs.mkdirs(rd);
352
353
354
355
356
357
358
359 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
360 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
361 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
362 } else {
363 if (!fs.isDirectory(rd)) {
364 throw new IllegalArgumentException(rd.toString() + " is not a directory");
365 }
366
367 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
368 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
369 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
370 }
371 } catch (IllegalArgumentException iae) {
372 LOG.fatal("Please fix invalid configuration for "
373 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
374 throw iae;
375 }
376
377 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
378 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
379 FSUtils.setClusterId(fs, rd, UUID.randomUUID().toString(), c.getInt(
380 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
381 }
382 clusterId = FSUtils.getClusterId(fs, rd);
383
384
385 if (!FSUtils.rootRegionExists(fs, rd)) {
386 bootstrap(rd, c);
387 }
388 createRootTableInfo(rd);
389 return rd;
390 }
391
392 private void createRootTableInfo(Path rd) throws IOException {
393
394 if (!FSTableDescriptors.isTableInfoExists(fs, rd,
395 Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
396 FSTableDescriptors.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
397 }
398 }
399
400
401
402
403
404 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
405 throws IOException {
406
407 if (fs.exists(tmpdir)) {
408
409
410 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
411 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
412 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
413 }
414 }
415 if (!fs.delete(tmpdir, true)) {
416 throw new IOException("Unable to clean the temp directory: " + tmpdir);
417 }
418 }
419
420
421 if (!fs.mkdirs(tmpdir)) {
422 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
423 }
424 }
425
426 private static void bootstrap(final Path rd, final Configuration c)
427 throws IOException {
428 LOG.info("BOOTSTRAP: creating ROOT and first META regions");
429 try {
430
431
432
433
434 HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
435 setInfoFamilyCachingForRoot(false);
436 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
437 setInfoFamilyCachingForMeta(false);
438 HRegion root = HRegion.createHRegion(rootHRI, rd, c,
439 HTableDescriptor.ROOT_TABLEDESC);
440 HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
441 HTableDescriptor.META_TABLEDESC);
442 setInfoFamilyCachingForRoot(true);
443 setInfoFamilyCachingForMeta(true);
444
445 HRegion.addRegionToMETA(root, meta);
446 root.close();
447 root.getLog().closeAndDelete();
448 meta.close();
449 meta.getLog().closeAndDelete();
450 } catch (IOException e) {
451 e = RemoteExceptionHandler.checkIOException(e);
452 LOG.error("bootstrap", e);
453 throw e;
454 }
455 }
456
457
458
459
460 public static void setInfoFamilyCachingForRoot(final boolean b) {
461 for (HColumnDescriptor hcd:
462 HTableDescriptor.ROOT_TABLEDESC.getColumnFamilies()) {
463 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
464 hcd.setBlockCacheEnabled(b);
465 hcd.setInMemory(b);
466 }
467 }
468 }
469
470
471
472
473 public static void setInfoFamilyCachingForMeta(final boolean b) {
474 for (HColumnDescriptor hcd:
475 HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
476 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
477 hcd.setBlockCacheEnabled(b);
478 hcd.setInMemory(b);
479 }
480 }
481 }
482
483
484 public void deleteRegion(HRegionInfo region) throws IOException {
485 HFileArchiver.archiveRegion(conf, fs, region);
486 }
487
488 public void deleteTable(byte[] tableName) throws IOException {
489 fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
490 }
491
492
493
494
495
496
497
498 public Path moveToTemp(final Path path) throws IOException {
499 Path tempPath = new Path(this.tempdir, path.getName());
500
501
502 if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
503 throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
504 }
505
506 if (!fs.rename(path, tempPath)) {
507 throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
508 }
509
510 return tempPath;
511 }
512
513
514
515
516
517
518
519 public Path moveTableToTemp(byte[] tableName) throws IOException {
520 return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
521 }
522
523 public void updateRegionInfo(HRegionInfo region) {
524
525
526
527 }
528
529 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
530 throws IOException {
531
532 Path tableDir = new Path(rootdir, region.getTableNameAsString());
533 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
534
535
536 Path familyDir = new Path(tableDir,
537 new Path(region.getEncodedName(), Bytes.toString(familyName)));
538 if (fs.delete(familyDir, true) == false) {
539 throw new IOException("Could not delete family "
540 + Bytes.toString(familyName) + " from FileSystem for region "
541 + region.getRegionNameAsString() + "(" + region.getEncodedName()
542 + ")");
543 }
544 }
545
546 public void stop() {
547 if (splitLogManager != null) {
548 this.splitLogManager.stop();
549 }
550 }
551
552
553
554
555
556
557 public void createTableDescriptor(HTableDescriptor htableDescriptor)
558 throws IOException {
559 FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
560 }
561
562
563
564
565
566
567
568
569 public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
570 throws IOException {
571 LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
572 + " family = " + Bytes.toString(familyName));
573 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
574 htd.removeFamily(familyName);
575 this.services.getTableDescriptors().add(htd);
576 return htd;
577 }
578
579
580
581
582
583
584
585
586 public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
587 throws IOException {
588 LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
589 + " HCD = " + hcd.toString());
590
591 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
592 byte [] familyName = hcd.getName();
593 if(!htd.hasFamily(familyName)) {
594 throw new InvalidFamilyOperationException("Family '" +
595 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
596 }
597 htd.addFamily(hcd);
598 this.services.getTableDescriptors().add(htd);
599 return htd;
600 }
601
602
603
604
605
606
607
608
609 public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
610 throws IOException {
611 LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
612 hcd.toString());
613 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
614 if (htd == null) {
615 throw new InvalidFamilyOperationException("Family '" +
616 hcd.getNameAsString() + "' cannot be modified as HTD is null");
617 }
618 htd.addFamily(hcd);
619 this.services.getTableDescriptors().add(htd);
620 return htd;
621 }
622 }