1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.Arrays;
23 import java.util.Comparator;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.ConcurrentHashMap;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30
31 import org.apache.commons.lang.NotImplementedException;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.TableDescriptors;
44 import org.apache.hadoop.hbase.TableInfoMissingException;
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64 public class FSTableDescriptors implements TableDescriptors {
65 private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
66 private final FileSystem fs;
67 private final Path rootdir;
68 private final boolean fsreadonly;
69 long cachehits = 0;
70 long invocations = 0;
71
72
73 public static final String TABLEINFO_NAME = ".tableinfo";
74
75
76
77
78 private final Map<String, TableDescriptorModtime> cache =
79 new ConcurrentHashMap<String, TableDescriptorModtime>();
80
81
82
83
84 static class TableDescriptorModtime {
85 private final HTableDescriptor descriptor;
86 private final long modtime;
87
88 TableDescriptorModtime(final long modtime, final HTableDescriptor htd) {
89 this.descriptor = htd;
90 this.modtime = modtime;
91 }
92
93 long getModtime() {
94 return this.modtime;
95 }
96
97 HTableDescriptor getTableDescriptor() {
98 return this.descriptor;
99 }
100 }
101
102 public FSTableDescriptors(final FileSystem fs, final Path rootdir) {
103 this(fs, rootdir, false);
104 }
105
106
107
108
109
110
111
112 public FSTableDescriptors(final FileSystem fs, final Path rootdir,
113 final boolean fsreadOnly) {
114 super();
115 this.fs = fs;
116 this.rootdir = rootdir;
117 this.fsreadonly = fsreadOnly;
118 }
119
120
121
122
123 @Override
124 public HTableDescriptor get(final byte [] tablename)
125 throws IOException {
126 return get(Bytes.toString(tablename));
127 }
128
129
130
131
132 @Override
133 public HTableDescriptor get(final String tablename)
134 throws IOException {
135 invocations++;
136 if (HTableDescriptor.ROOT_TABLEDESC.getNameAsString().equals(tablename)) {
137 cachehits++;
138 return HTableDescriptor.ROOT_TABLEDESC;
139 }
140 if (HTableDescriptor.META_TABLEDESC.getNameAsString().equals(tablename)) {
141 cachehits++;
142 return HTableDescriptor.META_TABLEDESC;
143 }
144
145
146 if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename)) {
147 throw new IOException("No descriptor found for table = " + tablename);
148 }
149
150
151 TableDescriptorModtime cachedtdm = this.cache.get(tablename);
152
153 if (cachedtdm != null) {
154
155 if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) {
156 cachehits++;
157 return cachedtdm.getTableDescriptor();
158 }
159 }
160
161 TableDescriptorModtime tdmt = null;
162 try {
163 tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename);
164 } catch (NullPointerException e) {
165 LOG.debug("Exception during readTableDecriptor. Current table name = "
166 + tablename, e);
167 } catch (IOException ioe) {
168 LOG.debug("Exception during readTableDecriptor. Current table name = "
169 + tablename, ioe);
170 }
171
172 if (tdmt == null) {
173 LOG.warn("The following folder is in HBase's root directory and " +
174 "doesn't contain a table descriptor, " +
175 "do consider deleting it: " + tablename);
176 } else {
177 this.cache.put(tablename, tdmt);
178 }
179 return tdmt == null ? null : tdmt.getTableDescriptor();
180 }
181
182
183
184
185 @Override
186 public Map<String, HTableDescriptor> getAll()
187 throws IOException {
188 Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
189 List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
190 for (Path d: tableDirs) {
191 HTableDescriptor htd = null;
192 try {
193
194 htd = get(d.getName());
195 } catch (FileNotFoundException fnfe) {
196
197 LOG.warn("Trouble retrieving htd", fnfe);
198 }
199 if (htd == null) continue;
200 htds.put(d.getName(), htd);
201 }
202 return htds;
203 }
204
205 @Override
206 public void add(HTableDescriptor htd) throws IOException {
207 if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) {
208 throw new NotImplementedException();
209 }
210 if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) {
211 throw new NotImplementedException();
212 }
213 if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) {
214 throw new NotImplementedException();
215 }
216 if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd);
217 long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
218 this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
219 }
220
221 @Override
222 public HTableDescriptor remove(final String tablename)
223 throws IOException {
224 if (!this.fsreadonly) {
225 Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
226 if (this.fs.exists(tabledir)) {
227 if (!this.fs.delete(tabledir, true)) {
228 throw new IOException("Failed delete of " + tabledir.toString());
229 }
230 }
231 }
232 TableDescriptorModtime tdm = this.cache.remove(tablename);
233 return tdm == null ? null : tdm.getTableDescriptor();
234 }
235
236
237
238
239
240
241
242
243
244
245 public static boolean isTableInfoExists(FileSystem fs, Path rootdir,
246 String tableName) throws IOException {
247 FileStatus status = getTableInfoPath(fs, rootdir, tableName);
248 return status == null? false: fs.exists(status.getPath());
249 }
250
251 private static FileStatus getTableInfoPath(final FileSystem fs,
252 final Path rootdir, final String tableName)
253 throws IOException {
254 Path tabledir = FSUtils.getTablePath(rootdir, tableName);
255 return getTableInfoPath(fs, tabledir);
256 }
257
258
259
260
261
262
263
264
265
266 public static FileStatus getTableInfoPath(final FileSystem fs,
267 final Path tabledir)
268 throws IOException {
269 FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() {
270 @Override
271 public boolean accept(Path p) {
272
273 return p.getName().startsWith(TABLEINFO_NAME);
274 }
275 });
276 if (status == null || status.length < 1) return null;
277 Arrays.sort(status, new FileStatusFileNameComparator());
278 if (status.length > 1) {
279
280 for (int i = 1; i < status.length; i++) {
281 Path p = status[i].getPath();
282
283 if (!fs.delete(p, false)) {
284 LOG.warn("Failed cleanup of " + status);
285 } else {
286 LOG.debug("Cleaned up old tableinfo file " + p);
287 }
288 }
289 }
290 return status[0];
291 }
292
293
294
295
296
297 static class FileStatusFileNameComparator
298 implements Comparator<FileStatus> {
299 @Override
300 public int compare(FileStatus left, FileStatus right) {
301 return -left.compareTo(right);
302 }
303 }
304
305
306
307
308 static final int WIDTH_OF_SEQUENCE_ID = 10;
309
310
311
312
313
314
315 static String formatTableInfoSequenceId(final int number) {
316 byte [] b = new byte[WIDTH_OF_SEQUENCE_ID];
317 int d = Math.abs(number);
318 for (int i = b.length - 1; i >= 0; i--) {
319 b[i] = (byte)((d % 10) + '0');
320 d /= 10;
321 }
322 return Bytes.toString(b);
323 }
324
325
326
327
328
329
330 private static final Pattern SUFFIX =
331 Pattern.compile(TABLEINFO_NAME + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$");
332
333
334
335
336
337
338 static int getTableInfoSequenceid(final Path p) {
339 if (p == null) return 0;
340 Matcher m = SUFFIX.matcher(p.getName());
341 if (!m.matches()) throw new IllegalArgumentException(p.toString());
342 String suffix = m.group(2);
343 if (suffix == null || suffix.length() <= 0) return 0;
344 return Integer.parseInt(m.group(2));
345 }
346
347
348
349
350
351
352 static Path getTableInfoFileName(final Path tabledir, final int sequenceid) {
353 return new Path(tabledir,
354 TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid));
355 }
356
357
358
359
360
361
362
363
364
365 static long getTableInfoModtime(final FileSystem fs, final Path rootdir,
366 final String tableName)
367 throws IOException {
368 FileStatus status = getTableInfoPath(fs, rootdir, tableName);
369 return status == null? 0: status.getModificationTime();
370 }
371
372
373
374
375
376
377
378
379
380 public static HTableDescriptor getTableDescriptor(FileSystem fs,
381 Path hbaseRootDir, byte[] tableName)
382 throws IOException {
383 HTableDescriptor htd = null;
384 try {
385 TableDescriptorModtime tdmt =
386 getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName));
387 htd = tdmt == null ? null : tdmt.getTableDescriptor();
388 } catch (NullPointerException e) {
389 LOG.debug("Exception during readTableDecriptor. Current table name = "
390 + Bytes.toString(tableName), e);
391 }
392 return htd;
393 }
394
395 static HTableDescriptor getTableDescriptor(FileSystem fs,
396 Path hbaseRootDir, String tableName) throws NullPointerException, IOException {
397 TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName);
398 return tdmt == null ? null : tdmt.getTableDescriptor();
399 }
400
401 static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs,
402 Path hbaseRootDir, String tableName) throws NullPointerException, IOException{
403
404 if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
405 || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
406 return null;
407 }
408 return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
409 }
410
411 static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir)
412 throws NullPointerException, IOException {
413 if (tableDir == null) throw new NullPointerException();
414 FileStatus status = getTableInfoPath(fs, tableDir);
415 if (status == null) {
416 throw new TableInfoMissingException("No .tableinfo file under "
417 + tableDir.toUri());
418 }
419 FSDataInputStream fsDataInputStream = fs.open(status.getPath());
420 HTableDescriptor hTableDescriptor = null;
421 try {
422 hTableDescriptor = new HTableDescriptor();
423 hTableDescriptor.readFields(fsDataInputStream);
424 } finally {
425 fsDataInputStream.close();
426 }
427 return new TableDescriptorModtime(status.getModificationTime(), hTableDescriptor);
428 }
429
430 public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
431 throws IOException, NullPointerException {
432 TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir);
433 return tdmt == null? null: tdmt.getTableDescriptor();
434 }
435
436
437
438
439
440
441
442
443
444
445 static Path updateHTableDescriptor(FileSystem fs, Path rootdir,
446 HTableDescriptor hTableDescriptor)
447 throws IOException {
448 Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName());
449 Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir,
450 getTableInfoPath(fs, tableDir));
451 if (p == null) throw new IOException("Failed update");
452 LOG.info("Updated tableinfo=" + p);
453 return p;
454 }
455
456
457
458
459
460 public static void deleteTableDescriptorIfExists(String tableName,
461 Configuration conf) throws IOException {
462 FileSystem fs = FSUtils.getCurrentFileSystem(conf);
463 FileStatus status = getTableInfoPath(fs, FSUtils.getRootDir(conf), tableName);
464
465 if (status != null && fs.exists(status.getPath())) {
466 FSUtils.deleteDirectory(fs, status.getPath());
467 }
468 }
469
470
471
472
473
474
475
476
477
478 private static Path writeTableDescriptor(final FileSystem fs,
479 final HTableDescriptor hTableDescriptor, final Path tableDir,
480 final FileStatus status)
481 throws IOException {
482
483
484 Path tmpTableDir = new Path(tableDir, ".tmp");
485
486
487
488
489
490 int currentSequenceid =
491 status == null? 0: getTableInfoSequenceid(status.getPath());
492 int sequenceid = currentSequenceid;
493
494 int retries = 10;
495 int retrymax = currentSequenceid + retries;
496 Path tableInfoPath = null;
497 do {
498 sequenceid += 1;
499 Path p = getTableInfoFileName(tmpTableDir, sequenceid);
500 if (fs.exists(p)) {
501 LOG.debug(p + " exists; retrying up to " + retries + " times");
502 continue;
503 }
504 try {
505 writeHTD(fs, p, hTableDescriptor);
506 tableInfoPath = getTableInfoFileName(tableDir, sequenceid);
507 if (!fs.rename(p, tableInfoPath)) {
508 throw new IOException("Failed rename of " + p + " to " + tableInfoPath);
509 }
510 } catch (IOException ioe) {
511
512 LOG.debug("Failed write and/or rename; retrying", ioe);
513 if (!FSUtils.deleteDirectory(fs, p)) {
514 LOG.warn("Failed cleanup of " + p);
515 }
516 tableInfoPath = null;
517 continue;
518 }
519
520 if (status != null) {
521 if (!FSUtils.deleteDirectory(fs, status.getPath())) {
522 LOG.warn("Failed delete of " + status.getPath() + "; continuing");
523 }
524 }
525 break;
526 } while (sequenceid < retrymax);
527 return tableInfoPath;
528 }
529
530 private static void writeHTD(final FileSystem fs, final Path p,
531 final HTableDescriptor htd)
532 throws IOException {
533 FSDataOutputStream out = fs.create(p, false);
534 try {
535 htd.write(out);
536 out.write('\n');
537 out.write('\n');
538 out.write(Bytes.toBytes(htd.toString()));
539 } finally {
540 out.close();
541 }
542 }
543
544
545
546
547
548
549
550 public static boolean createTableDescriptor(final HTableDescriptor htableDescriptor,
551 Configuration conf)
552 throws IOException {
553 return createTableDescriptor(htableDescriptor, conf, false);
554 }
555
556
557
558
559
560
561
562
563
564
565 static boolean createTableDescriptor(final HTableDescriptor htableDescriptor,
566 final Configuration conf, boolean forceCreation)
567 throws IOException {
568 FileSystem fs = FSUtils.getCurrentFileSystem(conf);
569 return createTableDescriptor(fs, FSUtils.getRootDir(conf), htableDescriptor,
570 forceCreation);
571 }
572
573
574
575
576
577
578
579
580 public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
581 HTableDescriptor htableDescriptor)
582 throws IOException {
583 return createTableDescriptor(fs, rootdir, htableDescriptor, false);
584 }
585
586
587
588
589
590
591
592
593
594
595
596
597 public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
598 HTableDescriptor htableDescriptor, boolean forceCreation)
599 throws IOException {
600 Path tabledir = FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString());
601 return createTableDescriptorForTableDirectory(fs, tabledir, htableDescriptor, forceCreation);
602 }
603
604
605
606
607
608
609
610
611
612
613
614
615
616 public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tabledir,
617 HTableDescriptor htableDescriptor, boolean forceCreation) throws IOException {
618 FileStatus status = getTableInfoPath(fs, tabledir);
619 if (status != null) {
620 LOG.info("Current tableInfoPath = " + status.getPath());
621 if (!forceCreation) {
622 if (fs.exists(status.getPath()) && status.getLen() > 0) {
623 LOG.info("TableInfo already exists.. Skipping creation");
624 return false;
625 }
626 }
627 }
628 Path p = writeTableDescriptor(fs, htableDescriptor, tabledir, status);
629 return p != null;
630 }
631 }