1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.Closeable;
23 import java.io.DataInput;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.concurrent.ArrayBlockingQueue;
31 import java.util.concurrent.BlockingQueue;
32 import java.util.concurrent.atomic.AtomicInteger;
33 import java.util.concurrent.atomic.AtomicLong;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FSDataInputStream;
39 import org.apache.hadoop.fs.FSDataOutputStream;
40 import org.apache.hadoop.fs.FileStatus;
41 import org.apache.hadoop.fs.FileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.PathFilter;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.KeyValue;
47 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
48 import org.apache.hadoop.hbase.fs.HFileSystem;
49 import org.apache.hadoop.hbase.io.HbaseMapWritable;
50 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
51 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
52 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware;
53 import org.apache.hadoop.hbase.util.BloomFilterWriter;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.ChecksumType;
56 import org.apache.hadoop.hbase.util.FSUtils;
57 import org.apache.hadoop.io.RawComparator;
58 import org.apache.hadoop.io.Writable;
59
60 import com.google.common.base.Preconditions;
61 import com.google.common.collect.Lists;
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126 public class HFile {
127 static final Log LOG = LogFactory.getLog(HFile.class);
128
129
130
131
132 public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
133
134
135
136
137 public final static int DEFAULT_BLOCKSIZE = 64 * 1024;
138
139
140
141
142 public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
143 Compression.Algorithm.NONE;
144
145
146 public static final int MIN_FORMAT_VERSION = 1;
147
148
149 public static final int MAX_FORMAT_VERSION = 2;
150
151
152 public final static String DEFAULT_COMPRESSION =
153 DEFAULT_COMPRESSION_ALGORITHM.getName();
154
155
156
157
158
159
160
161 public final static int MIN_NUM_HFILE_PATH_LEVELS = 5;
162
163
164
165
166 public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
167 public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
168
169
170 private static final AtomicInteger readOps = new AtomicInteger();
171 private static final AtomicLong readTimeNano = new AtomicLong();
172 private static final AtomicInteger writeOps = new AtomicInteger();
173 private static final AtomicLong writeTimeNano = new AtomicLong();
174
175
176 private static final AtomicInteger preadOps = new AtomicInteger();
177 private static final AtomicLong preadTimeNano = new AtomicLong();
178
179
180 static final AtomicLong checksumFailures = new AtomicLong();
181
182
183
184
185
186
187 private static final int LATENCY_BUFFER_SIZE = 5000;
188 private static final BlockingQueue<Long> fsReadLatenciesNanos =
189 new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
190 private static final BlockingQueue<Long> fsWriteLatenciesNanos =
191 new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
192 private static final BlockingQueue<Long> fsPreadLatenciesNanos =
193 new ArrayBlockingQueue<Long>(LATENCY_BUFFER_SIZE);
194
195 public static final void offerReadLatency(long latencyNanos, boolean pread) {
196 if (pread) {
197 fsPreadLatenciesNanos.offer(latencyNanos);
198 preadOps.incrementAndGet();
199 preadTimeNano.addAndGet(latencyNanos);
200 } else {
201 fsReadLatenciesNanos.offer(latencyNanos);
202 readTimeNano.addAndGet(latencyNanos);
203 readOps.incrementAndGet();
204 }
205 }
206
207 public static final void offerWriteLatency(long latencyNanos) {
208 fsWriteLatenciesNanos.offer(latencyNanos);
209
210 writeTimeNano.addAndGet(latencyNanos);
211 writeOps.incrementAndGet();
212 }
213
214 public static final Collection<Long> getReadLatenciesNanos() {
215 final List<Long> latencies =
216 Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
217 fsReadLatenciesNanos.drainTo(latencies);
218 return latencies;
219 }
220
221 public static final Collection<Long> getPreadLatenciesNanos() {
222 final List<Long> latencies =
223 Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
224 fsPreadLatenciesNanos.drainTo(latencies);
225 return latencies;
226 }
227
228 public static final Collection<Long> getWriteLatenciesNanos() {
229 final List<Long> latencies =
230 Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
231 fsWriteLatenciesNanos.drainTo(latencies);
232 return latencies;
233 }
234
235
236 public static volatile AtomicLong dataBlockReadCnt = new AtomicLong(0);
237
238
239 public static final int getReadOps() {
240 return readOps.getAndSet(0);
241 }
242
243 public static final long getReadTimeMs() {
244 return readTimeNano.getAndSet(0) / 1000000;
245 }
246
247
248 public static final int getPreadOps() {
249 return preadOps.getAndSet(0);
250 }
251
252 public static final long getPreadTimeMs() {
253 return preadTimeNano.getAndSet(0) / 1000000;
254 }
255
256 public static final int getWriteOps() {
257 return writeOps.getAndSet(0);
258 }
259
260 public static final long getWriteTimeMs() {
261 return writeTimeNano.getAndSet(0) / 1000000;
262 }
263
264
265
266
267
268 public static final long getChecksumFailuresCount() {
269 return checksumFailures.getAndSet(0);
270 }
271
272
273 public interface Writer extends Closeable {
274
275
276 void appendFileInfo(byte[] key, byte[] value) throws IOException;
277
278 void append(KeyValue kv) throws IOException;
279
280 void append(byte[] key, byte[] value) throws IOException;
281
282
283 Path getPath();
284
285 String getColumnFamilyName();
286
287 void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
288
289
290
291
292
293 void addInlineBlockWriter(InlineBlockWriter bloomWriter);
294
295
296
297
298
299
300 void addGeneralBloomFilter(BloomFilterWriter bfw);
301
302
303
304
305
306 void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
307 }
308
309
310
311
312
313 public static abstract class WriterFactory {
314 protected final Configuration conf;
315 protected final CacheConfig cacheConf;
316 protected FileSystem fs;
317 protected Path path;
318 protected FSDataOutputStream ostream;
319 protected int blockSize = HColumnDescriptor.DEFAULT_BLOCKSIZE;
320 protected Compression.Algorithm compression =
321 HFile.DEFAULT_COMPRESSION_ALGORITHM;
322 protected HFileDataBlockEncoder encoder = NoOpDataBlockEncoder.INSTANCE;
323 protected KeyComparator comparator;
324 protected ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE;
325 protected int bytesPerChecksum = DEFAULT_BYTES_PER_CHECKSUM;
326
327 WriterFactory(Configuration conf, CacheConfig cacheConf) {
328 this.conf = conf;
329 this.cacheConf = cacheConf;
330 }
331
332 public WriterFactory withPath(FileSystem fs, Path path) {
333 Preconditions.checkNotNull(fs);
334 Preconditions.checkNotNull(path);
335 this.fs = fs;
336 this.path = path;
337 return this;
338 }
339
340 public WriterFactory withOutputStream(FSDataOutputStream ostream) {
341 Preconditions.checkNotNull(ostream);
342 this.ostream = ostream;
343 return this;
344 }
345
346 public WriterFactory withBlockSize(int blockSize) {
347 this.blockSize = blockSize;
348 return this;
349 }
350
351 public WriterFactory withCompression(Compression.Algorithm compression) {
352 Preconditions.checkNotNull(compression);
353 this.compression = compression;
354 return this;
355 }
356
357 public WriterFactory withCompression(String compressAlgo) {
358 Preconditions.checkNotNull(compression);
359 this.compression = AbstractHFileWriter.compressionByName(compressAlgo);
360 return this;
361 }
362
363 public WriterFactory withDataBlockEncoder(HFileDataBlockEncoder encoder) {
364 Preconditions.checkNotNull(encoder);
365 this.encoder = encoder;
366 return this;
367 }
368
369 public WriterFactory withComparator(KeyComparator comparator) {
370 Preconditions.checkNotNull(comparator);
371 this.comparator = comparator;
372 return this;
373 }
374
375 public WriterFactory withChecksumType(ChecksumType checksumType) {
376 Preconditions.checkNotNull(checksumType);
377 this.checksumType = checksumType;
378 return this;
379 }
380
381 public WriterFactory withBytesPerChecksum(int bytesPerChecksum) {
382 this.bytesPerChecksum = bytesPerChecksum;
383 return this;
384 }
385
386 public Writer create() throws IOException {
387 if ((path != null ? 1 : 0) + (ostream != null ? 1 : 0) != 1) {
388 throw new AssertionError("Please specify exactly one of " +
389 "filesystem/path or path");
390 }
391 if (path != null) {
392 ostream = AbstractHFileWriter.createOutputStream(conf, fs, path);
393 }
394 return createWriter(fs, path, ostream, blockSize,
395 compression, encoder, comparator, checksumType, bytesPerChecksum);
396 }
397
398 protected abstract Writer createWriter(FileSystem fs, Path path,
399 FSDataOutputStream ostream, int blockSize,
400 Compression.Algorithm compress,
401 HFileDataBlockEncoder dataBlockEncoder,
402 KeyComparator comparator, ChecksumType checksumType,
403 int bytesPerChecksum) throws IOException;
404 }
405
406
407 public static final String FORMAT_VERSION_KEY = "hfile.format.version";
408
409 public static int getFormatVersion(Configuration conf) {
410 int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
411 checkFormatVersion(version);
412 return version;
413 }
414
415
416
417
418
419
420 public static final WriterFactory getWriterFactoryNoCache(Configuration
421 conf) {
422 Configuration tempConf = new Configuration(conf);
423 tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
424 return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
425 }
426
427
428
429
430 public static final WriterFactory getWriterFactory(Configuration conf,
431 CacheConfig cacheConf) {
432 SchemaMetrics.configureGlobally(conf);
433 int version = getFormatVersion(conf);
434 switch (version) {
435 case 1:
436 return new HFileWriterV1.WriterFactoryV1(conf, cacheConf);
437 case 2:
438 return new HFileWriterV2.WriterFactoryV2(conf, cacheConf);
439 default:
440 throw new IllegalArgumentException("Cannot create writer for HFile " +
441 "format version " + version);
442 }
443 }
444
445
446 public interface CachingBlockReader {
447 HFileBlock readBlock(long offset, long onDiskBlockSize,
448 boolean cacheBlock, final boolean pread, final boolean isCompaction,
449 BlockType expectedBlockType)
450 throws IOException;
451 }
452
453
454 public interface Reader extends Closeable, CachingBlockReader,
455 SchemaAware {
456
457
458
459
460
461 String getName();
462
463 String getColumnFamilyName();
464
465 RawComparator<byte []> getComparator();
466
467 HFileScanner getScanner(boolean cacheBlocks,
468 final boolean pread, final boolean isCompaction);
469
470 ByteBuffer getMetaBlock(String metaBlockName,
471 boolean cacheBlock) throws IOException;
472
473 Map<byte[], byte[]> loadFileInfo() throws IOException;
474
475 byte[] getLastKey();
476
477 byte[] midkey() throws IOException;
478
479 long length();
480
481 long getEntries();
482
483 byte[] getFirstKey();
484
485 long indexSize();
486
487 byte[] getFirstRowKey();
488
489 byte[] getLastRowKey();
490
491 FixedFileTrailer getTrailer();
492
493 HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
494
495 HFileScanner getScanner(boolean cacheBlocks, boolean pread);
496
497 Compression.Algorithm getCompressionAlgorithm();
498
499
500
501
502
503
504 DataInput getGeneralBloomFilterMetadata() throws IOException;
505
506
507
508
509
510
511 DataInput getDeleteBloomFilterMetadata() throws IOException;
512
513 Path getPath();
514
515
516 void close(boolean evictOnClose) throws IOException;
517
518 DataBlockEncoding getEncodingOnDisk();
519 }
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536 private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
537 FSDataInputStream fsdisNoFsChecksum,
538 long size, boolean closeIStream, CacheConfig cacheConf,
539 DataBlockEncoding preferredEncodingInCache, HFileSystem hfs)
540 throws IOException {
541 FixedFileTrailer trailer = null;
542 try {
543 trailer = FixedFileTrailer.readFromStream(fsdis, size);
544 } catch (IllegalArgumentException iae) {
545 throw new CorruptHFileException("Problem reading HFile Trailer from file " + path, iae);
546 }
547 switch (trailer.getMajorVersion()) {
548 case 1:
549 return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
550 cacheConf);
551 case 2:
552 return new HFileReaderV2(path, trailer, fsdis, fsdisNoFsChecksum,
553 size, closeIStream,
554 cacheConf, preferredEncodingInCache, hfs);
555 default:
556 throw new CorruptHFileException("Invalid HFile version " + trailer.getMajorVersion());
557 }
558 }
559
560
561
562
563
564
565
566
567
568 public static Reader createReaderWithEncoding(
569 FileSystem fs, Path path, CacheConfig cacheConf,
570 DataBlockEncoding preferredEncodingInCache) throws IOException {
571 final boolean closeIStream = true;
572 HFileSystem hfs = null;
573 FSDataInputStream fsdis = fs.open(path);
574 FSDataInputStream fsdisNoFsChecksum = fsdis;
575
576
577
578
579 if (!(fs instanceof HFileSystem)) {
580 hfs = new HFileSystem(fs);
581 } else {
582 hfs = (HFileSystem)fs;
583
584
585 if (hfs != null) {
586 fsdisNoFsChecksum = hfs.getNoChecksumFs().open(path);
587 }
588 }
589 return pickReaderVersion(path, fsdis, fsdisNoFsChecksum,
590 fs.getFileStatus(path).getLen(), closeIStream, cacheConf,
591 preferredEncodingInCache, hfs);
592 }
593
594
595
596
597
598
599
600
601
602
603
604
605
606 public static Reader createReaderWithEncoding(
607 FileSystem fs, Path path, FSDataInputStream fsdis,
608 FSDataInputStream fsdisNoFsChecksum, long size, CacheConfig cacheConf,
609 DataBlockEncoding preferredEncodingInCache, boolean closeIStream)
610 throws IOException {
611 HFileSystem hfs = null;
612
613
614
615
616
617 if (!(fs instanceof HFileSystem)) {
618 hfs = new HFileSystem(fs);
619 } else {
620 hfs = (HFileSystem)fs;
621 }
622 return pickReaderVersion(path, fsdis, fsdisNoFsChecksum, size,
623 closeIStream, cacheConf,
624 preferredEncodingInCache, hfs);
625 }
626
627
628
629
630
631
632
633
634 public static Reader createReader(
635 FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
636 return createReaderWithEncoding(fs, path, cacheConf,
637 DataBlockEncoding.NONE);
638 }
639
640
641
642
643 static Reader createReaderFromStream(Path path,
644 FSDataInputStream fsdis, long size, CacheConfig cacheConf)
645 throws IOException {
646 final boolean closeIStream = false;
647 return pickReaderVersion(path, fsdis, fsdis, size, closeIStream, cacheConf,
648 DataBlockEncoding.NONE, null);
649 }
650
651
652
653
654 static class FileInfo extends HbaseMapWritable<byte [], byte []> {
655 static final String RESERVED_PREFIX = "hfile.";
656 static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
657 static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
658 static final byte [] AVG_KEY_LEN =
659 Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
660 static final byte [] AVG_VALUE_LEN =
661 Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
662 static final byte [] COMPARATOR =
663 Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
664
665
666
667
668
669
670
671
672
673
674
675
676 public FileInfo append(final byte[] k, final byte[] v,
677 final boolean checkPrefix) throws IOException {
678 if (k == null || v == null) {
679 throw new NullPointerException("Key nor value may be null");
680 }
681 if (checkPrefix && isReservedFileInfoKey(k)) {
682 throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX
683 + " are reserved");
684 }
685 put(k, v);
686 return this;
687 }
688
689 }
690
691
692 public static boolean isReservedFileInfoKey(byte[] key) {
693 return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
694 }
695
696
697
698
699
700
701
702
703
704
705
706
707
708 public static String[] getSupportedCompressionAlgorithms() {
709 return Compression.getSupportedAlgorithms();
710 }
711
712
713
714
715
716
717 static int longToInt(final long l) {
718
719
720 return (int)(l & 0x00000000ffffffffL);
721 }
722
723
724
725
726
727
728
729
730
731
732 static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
733 throws IOException {
734 List<Path> res = new ArrayList<Path>();
735 PathFilter dirFilter = new FSUtils.DirFilter(fs);
736 FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
737 for(FileStatus dir : familyDirs) {
738 FileStatus[] files = fs.listStatus(dir.getPath());
739 for (FileStatus file : files) {
740 if (!file.isDir()) {
741 res.add(file.getPath());
742 }
743 }
744 }
745 return res;
746 }
747
748 public static void main(String[] args) throws IOException {
749 HFilePrettyPrinter prettyPrinter = new HFilePrettyPrinter();
750 System.exit(prettyPrinter.run(args));
751 }
752
753
754
755
756
757
758
759
760
761
762 public static void checkFormatVersion(int version)
763 throws IllegalArgumentException {
764 if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
765 throw new IllegalArgumentException("Invalid HFile version: " + version
766 + " (expected to be " + "between " + MIN_FORMAT_VERSION + " and "
767 + MAX_FORMAT_VERSION + ")");
768 }
769 }
770
771 }