1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.apache.hadoop.hbase.io.hfile.BlockType.MAGIC_LENGTH;
21 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.NONE;
22
23 import java.io.BufferedInputStream;
24 import java.io.ByteArrayInputStream;
25 import java.io.ByteArrayOutputStream;
26 import java.io.DataInputStream;
27 import java.io.DataOutput;
28 import java.io.DataOutputStream;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.nio.ByteBuffer;
32 import java.util.concurrent.locks.Lock;
33 import java.util.concurrent.locks.ReentrantLock;
34
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.fs.HFileSystem;
40 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
41 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
42 import org.apache.hadoop.hbase.regionserver.MemStore;
43 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.ChecksumType;
46 import org.apache.hadoop.hbase.util.ClassSize;
47 import org.apache.hadoop.hbase.util.CompoundBloomFilter;
48 import org.apache.hadoop.hbase.util.Pair;
49 import org.apache.hadoop.hbase.util.Writables;
50 import org.apache.hadoop.io.IOUtils;
51 import org.apache.hadoop.io.Writable;
52 import org.apache.hadoop.io.compress.CompressionOutputStream;
53 import org.apache.hadoop.io.compress.Compressor;
54 import org.apache.hadoop.io.compress.Decompressor;
55
56 import com.google.common.base.Preconditions;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89 public class HFileBlock extends SchemaConfigured implements Cacheable {
90
91
92 static final int MINOR_VERSION_WITH_CHECKSUM = 1;
93
94
95 static final int MINOR_VERSION_NO_CHECKSUM = 0;
96
97
98
99
100
101
102 static final int CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD = 3;
103
104
105 static final int HEADER_SIZE_NO_CHECKSUM = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
106 + Bytes.SIZEOF_LONG;
107
108 public static final boolean FILL_HEADER = true;
109 public static final boolean DONT_FILL_HEADER = false;
110
111
112
113
114
115 static final int HEADER_SIZE_WITH_CHECKSUMS = HEADER_SIZE_NO_CHECKSUM + Bytes.SIZEOF_BYTE +
116 2 * Bytes.SIZEOF_INT;
117
118
119
120
121
122 public static final int ENCODED_HEADER_SIZE = HEADER_SIZE_WITH_CHECKSUMS
123 + DataBlockEncoding.ID_SIZE;
124
125
126 static final byte[] DUMMY_HEADER_WITH_CHECKSUM = new byte[HEADER_SIZE_WITH_CHECKSUMS];
127 static final byte[] DUMMY_HEADER_NO_CHECKSUM =
128 new byte[HEADER_SIZE_NO_CHECKSUM];
129
130 public static final int BYTE_BUFFER_HEAP_SIZE = (int) ClassSize.estimateBase(
131 ByteBuffer.wrap(new byte[0], 0, 0).getClass(), false);
132
133 static final int EXTRA_SERIALIZATION_SPACE = Bytes.SIZEOF_LONG +
134 Bytes.SIZEOF_INT;
135
136
137
138
139 static final int CHECKSUM_SIZE = Bytes.SIZEOF_INT;
140
141 private static final CacheableDeserializer<Cacheable> blockDeserializer =
142 new CacheableDeserializer<Cacheable>() {
143 public HFileBlock deserialize(ByteBuffer buf) throws IOException{
144 ByteBuffer newByteBuffer = ByteBuffer.allocate(buf.limit()
145 - HFileBlock.EXTRA_SERIALIZATION_SPACE);
146 buf.limit(buf.limit()
147 - HFileBlock.EXTRA_SERIALIZATION_SPACE).rewind();
148 newByteBuffer.put(buf);
149 HFileBlock ourBuffer = new HFileBlock(newByteBuffer,
150 MINOR_VERSION_NO_CHECKSUM);
151
152 buf.position(buf.limit());
153 buf.limit(buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE);
154 ourBuffer.offset = buf.getLong();
155 ourBuffer.nextBlockOnDiskSizeWithHeader = buf.getInt();
156 return ourBuffer;
157 }
158 };
159
160 private BlockType blockType;
161
162
163 private int onDiskSizeWithoutHeader;
164
165
166 private final int uncompressedSizeWithoutHeader;
167
168
169 private final long prevBlockOffset;
170
171
172 private final byte checksumType;
173
174
175 private final int bytesPerChecksum;
176
177
178 private final int onDiskDataSizeWithHeader;
179
180
181 private final int minorVersion;
182
183
184 private ByteBuffer buf;
185
186
187 private boolean includesMemstoreTS;
188
189
190
191
192
193 private long offset = -1;
194
195
196
197
198
199
200 private int nextBlockOnDiskSizeWithHeader = -1;
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226 HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
227 int uncompressedSizeWithoutHeader, long prevBlockOffset, ByteBuffer buf,
228 boolean fillHeader, long offset, boolean includesMemstoreTS,
229 int minorVersion, int bytesPerChecksum, byte checksumType,
230 int onDiskDataSizeWithHeader) {
231 this.blockType = blockType;
232 this.onDiskSizeWithoutHeader = onDiskSizeWithoutHeader;
233 this.uncompressedSizeWithoutHeader = uncompressedSizeWithoutHeader;
234 this.prevBlockOffset = prevBlockOffset;
235 this.buf = buf;
236 if (fillHeader)
237 overwriteHeader();
238 this.offset = offset;
239 this.includesMemstoreTS = includesMemstoreTS;
240 this.minorVersion = minorVersion;
241 this.bytesPerChecksum = bytesPerChecksum;
242 this.checksumType = checksumType;
243 this.onDiskDataSizeWithHeader = onDiskDataSizeWithHeader;
244 }
245
246
247
248
249
250
251
252
253
254 HFileBlock(ByteBuffer b, int minorVersion) throws IOException {
255 b.rewind();
256 blockType = BlockType.read(b);
257 onDiskSizeWithoutHeader = b.getInt();
258 uncompressedSizeWithoutHeader = b.getInt();
259 prevBlockOffset = b.getLong();
260 this.minorVersion = minorVersion;
261 if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
262 this.checksumType = b.get();
263 this.bytesPerChecksum = b.getInt();
264 this.onDiskDataSizeWithHeader = b.getInt();
265 } else {
266 this.checksumType = ChecksumType.NULL.getCode();
267 this.bytesPerChecksum = 0;
268 this.onDiskDataSizeWithHeader = onDiskSizeWithoutHeader +
269 HEADER_SIZE_NO_CHECKSUM;
270 }
271 buf = b;
272 buf.rewind();
273 }
274
275 public BlockType getBlockType() {
276 return blockType;
277 }
278
279
280 public short getDataBlockEncodingId() {
281 if (blockType != BlockType.ENCODED_DATA) {
282 throw new IllegalArgumentException("Querying encoder ID of a block " +
283 "of type other than " + BlockType.ENCODED_DATA + ": " + blockType);
284 }
285 return buf.getShort(headerSize());
286 }
287
288
289
290
291
292 public int getOnDiskSizeWithHeader() {
293 return onDiskSizeWithoutHeader + headerSize();
294 }
295
296
297
298
299
300
301
302
303
304 int getOnDiskSizeWithoutHeader() {
305 return onDiskSizeWithoutHeader;
306 }
307
308
309
310
311
312 public int getUncompressedSizeWithoutHeader() {
313 return uncompressedSizeWithoutHeader;
314 }
315
316
317
318
319
320 public long getPrevBlockOffset() {
321 return prevBlockOffset;
322 }
323
324
325
326
327
328 private void overwriteHeader() {
329 buf.rewind();
330 blockType.write(buf);
331 buf.putInt(onDiskSizeWithoutHeader);
332 buf.putInt(uncompressedSizeWithoutHeader);
333 buf.putLong(prevBlockOffset);
334 }
335
336
337
338
339
340
341
342
343 ByteBuffer getBufferWithoutHeader() {
344 return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + headerSize(),
345 buf.limit() - headerSize() - totalChecksumBytes()).slice();
346 }
347
348
349
350
351
352
353
354
355
356
357 public ByteBuffer getBufferReadOnly() {
358 return ByteBuffer.wrap(buf.array(), buf.arrayOffset(),
359 buf.limit() - totalChecksumBytes()).slice();
360 }
361
362
363
364
365
366
367
368 ByteBuffer getBufferWithHeader() {
369 ByteBuffer dupBuf = buf.duplicate();
370 dupBuf.rewind();
371 return dupBuf;
372 }
373
374
375
376
377
378 void readInto(Writable w) throws IOException {
379 Preconditions.checkNotNull(w);
380
381 if (Writables.getWritable(buf.array(), buf.arrayOffset() + headerSize(),
382 buf.limit() - headerSize(), w) == null) {
383 throw new IOException("Failed to deserialize block " + this + " into a "
384 + w.getClass().getSimpleName());
385 }
386 }
387
388 private void sanityCheckAssertion(long valueFromBuf, long valueFromField,
389 String fieldName) throws IOException {
390 if (valueFromBuf != valueFromField) {
391 throw new AssertionError(fieldName + " in the buffer (" + valueFromBuf
392 + ") is different from that in the field (" + valueFromField + ")");
393 }
394 }
395
396
397
398
399
400
401
402 void sanityCheck() throws IOException {
403 buf.rewind();
404
405 {
406 BlockType blockTypeFromBuf = BlockType.read(buf);
407 if (blockTypeFromBuf != blockType) {
408 throw new IOException("Block type stored in the buffer: " +
409 blockTypeFromBuf + ", block type field: " + blockType);
410 }
411 }
412
413 sanityCheckAssertion(buf.getInt(), onDiskSizeWithoutHeader,
414 "onDiskSizeWithoutHeader");
415
416 sanityCheckAssertion(buf.getInt(), uncompressedSizeWithoutHeader,
417 "uncompressedSizeWithoutHeader");
418
419 sanityCheckAssertion(buf.getLong(), prevBlockOffset, "prevBlocKOffset");
420 if (minorVersion >= MINOR_VERSION_WITH_CHECKSUM) {
421 sanityCheckAssertion(buf.get(), checksumType, "checksumType");
422 sanityCheckAssertion(buf.getInt(), bytesPerChecksum, "bytesPerChecksum");
423 sanityCheckAssertion(buf.getInt(), onDiskDataSizeWithHeader,
424 "onDiskDataSizeWithHeader");
425 }
426
427 int cksumBytes = totalChecksumBytes();
428 int hdrSize = headerSize();
429 int expectedBufLimit = uncompressedSizeWithoutHeader + headerSize() +
430 cksumBytes;
431 if (buf.limit() != expectedBufLimit) {
432 throw new AssertionError("Expected buffer limit " + expectedBufLimit
433 + ", got " + buf.limit());
434 }
435
436
437
438 int size = uncompressedSizeWithoutHeader + hdrSize + cksumBytes;
439 if (buf.capacity() != size &&
440 buf.capacity() != size + hdrSize) {
441 throw new AssertionError("Invalid buffer capacity: " + buf.capacity() +
442 ", expected " + size + " or " + (size + hdrSize));
443 }
444 }
445
446 @Override
447 public String toString() {
448 return "blockType="
449 + blockType
450 + ", onDiskSizeWithoutHeader="
451 + onDiskSizeWithoutHeader
452 + ", uncompressedSizeWithoutHeader="
453 + uncompressedSizeWithoutHeader
454 + ", prevBlockOffset="
455 + prevBlockOffset
456 + ", dataBeginsWith="
457 + Bytes.toStringBinary(buf.array(), buf.arrayOffset() + headerSize(),
458 Math.min(32, buf.limit() - buf.arrayOffset() - headerSize()))
459 + ", fileOffset=" + offset;
460 }
461
462 private void validateOnDiskSizeWithoutHeader(
463 int expectedOnDiskSizeWithoutHeader) throws IOException {
464 if (onDiskSizeWithoutHeader != expectedOnDiskSizeWithoutHeader) {
465 String blockInfoMsg =
466 "Block offset: " + offset + ", data starts with: "
467 + Bytes.toStringBinary(buf.array(), buf.arrayOffset(),
468 buf.arrayOffset() + Math.min(32, buf.limit()));
469 throw new IOException("On-disk size without header provided is "
470 + expectedOnDiskSizeWithoutHeader + ", but block "
471 + "header contains " + onDiskSizeWithoutHeader + ". " +
472 blockInfoMsg);
473 }
474 }
475
476
477
478
479
480
481
482
483
484 private void allocateBuffer(boolean extraBytes) {
485 int cksumBytes = totalChecksumBytes();
486 int capacityNeeded = headerSize() + uncompressedSizeWithoutHeader +
487 cksumBytes +
488 (extraBytes ? headerSize() : 0);
489
490 ByteBuffer newBuf = ByteBuffer.allocate(capacityNeeded);
491
492
493 System.arraycopy(buf.array(), buf.arrayOffset(), newBuf.array(),
494 newBuf.arrayOffset(), headerSize());
495
496 buf = newBuf;
497 buf.limit(headerSize() + uncompressedSizeWithoutHeader + cksumBytes);
498 }
499
500
501 public void assumeUncompressed() throws IOException {
502 if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader +
503 totalChecksumBytes()) {
504 throw new IOException("Using no compression but "
505 + "onDiskSizeWithoutHeader=" + onDiskSizeWithoutHeader + ", "
506 + "uncompressedSizeWithoutHeader=" + uncompressedSizeWithoutHeader
507 + ", numChecksumbytes=" + totalChecksumBytes());
508 }
509 }
510
511
512
513
514
515 public void expectType(BlockType expectedType) throws IOException {
516 if (blockType != expectedType) {
517 throw new IOException("Invalid block type: expected=" + expectedType
518 + ", actual=" + blockType);
519 }
520 }
521
522
523 public long getOffset() {
524 if (offset < 0) {
525 throw new IllegalStateException(
526 "HFile block offset not initialized properly");
527 }
528 return offset;
529 }
530
531
532
533
534 public DataInputStream getByteStream() {
535 return new DataInputStream(new ByteArrayInputStream(buf.array(),
536 buf.arrayOffset() + headerSize(), buf.limit() - headerSize()));
537 }
538
539 @Override
540 public long heapSize() {
541 long size = ClassSize.align(
542
543 SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
544
545 2 * ClassSize.REFERENCE +
546
547
548 6 * Bytes.SIZEOF_INT +
549
550 1 * Bytes.SIZEOF_BYTE +
551
552 2 * Bytes.SIZEOF_LONG +
553
554 Bytes.SIZEOF_BOOLEAN
555 );
556
557 if (buf != null) {
558
559 size += ClassSize.align(buf.capacity() + BYTE_BUFFER_HEAP_SIZE);
560 }
561
562 return ClassSize.align(size);
563 }
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580 public static boolean readWithExtra(InputStream in, byte buf[],
581 int bufOffset, int necessaryLen, int extraLen) throws IOException {
582 int bytesRemaining = necessaryLen + extraLen;
583 while (bytesRemaining > 0) {
584 int ret = in.read(buf, bufOffset, bytesRemaining);
585 if (ret == -1 && bytesRemaining <= extraLen) {
586
587 break;
588 }
589
590 if (ret < 0) {
591 throw new IOException("Premature EOF from inputStream (read "
592 + "returned " + ret + ", was trying to read " + necessaryLen
593 + " necessary bytes and " + extraLen + " extra bytes, "
594 + "successfully read "
595 + (necessaryLen + extraLen - bytesRemaining));
596 }
597 bufOffset += ret;
598 bytesRemaining -= ret;
599 }
600 return bytesRemaining <= 0;
601 }
602
603
604
605
606
607 public int getNextBlockOnDiskSizeWithHeader() {
608 return nextBlockOnDiskSizeWithHeader;
609 }
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628 public static class Writer {
629
630 private enum State {
631 INIT,
632 WRITING,
633 BLOCK_READY
634 };
635
636
637 private State state = State.INIT;
638
639
640 private final Compression.Algorithm compressAlgo;
641
642
643 private final HFileDataBlockEncoder dataBlockEncoder;
644
645
646
647
648
649
650
651 private ByteArrayOutputStream baosInMemory;
652
653
654 private Compressor compressor;
655
656
657 private CompressionOutputStream compressionStream;
658
659
660 private ByteArrayOutputStream compressedByteStream;
661
662
663
664
665
666
667 private BlockType blockType;
668
669
670
671
672
673 private DataOutputStream userDataStream;
674
675
676
677
678
679
680 private byte[] onDiskBytesWithHeader;
681
682
683
684
685
686 private int onDiskDataSizeWithHeader;
687
688
689
690
691
692
693
694 private byte[] onDiskChecksum;
695
696
697
698
699
700
701
702 private byte[] uncompressedBytesWithHeader;
703
704
705
706
707
708 private long startOffset;
709
710
711
712
713
714 private long[] prevOffsetByType;
715
716
717 private long prevOffset;
718
719
720 private boolean includesMemstoreTS;
721
722
723 private ChecksumType checksumType;
724 private int bytesPerChecksum;
725
726 private final int minorVersion;
727
728
729
730
731
732
733
734 public Writer(Compression.Algorithm compressionAlgorithm,
735 HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS,
736 int minorVersion,
737 ChecksumType checksumType, int bytesPerChecksum) {
738 this.minorVersion = minorVersion;
739 compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
740 this.dataBlockEncoder = dataBlockEncoder != null
741 ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
742
743 baosInMemory = new ByteArrayOutputStream();
744 if (compressAlgo != NONE) {
745 compressor = compressionAlgorithm.getCompressor();
746 compressedByteStream = new ByteArrayOutputStream();
747 try {
748 compressionStream =
749 compressionAlgorithm.createPlainCompressionStream(
750 compressedByteStream, compressor);
751 } catch (IOException e) {
752 throw new RuntimeException("Could not create compression stream " +
753 "for algorithm " + compressionAlgorithm, e);
754 }
755 }
756 if (minorVersion > MINOR_VERSION_NO_CHECKSUM
757 && bytesPerChecksum < HEADER_SIZE_WITH_CHECKSUMS) {
758 throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
759 " Minimum is " + HEADER_SIZE_WITH_CHECKSUMS + " but the configured value is " +
760 bytesPerChecksum);
761 }
762
763 prevOffsetByType = new long[BlockType.values().length];
764 for (int i = 0; i < prevOffsetByType.length; ++i)
765 prevOffsetByType[i] = -1;
766
767 this.includesMemstoreTS = includesMemstoreTS;
768 this.checksumType = checksumType;
769 this.bytesPerChecksum = bytesPerChecksum;
770 }
771
772
773
774
775
776
777
778 public DataOutputStream startWriting(BlockType newBlockType)
779 throws IOException {
780 if (state == State.BLOCK_READY && startOffset != -1) {
781
782
783 prevOffsetByType[blockType.getId()] = startOffset;
784 }
785
786 startOffset = -1;
787 blockType = newBlockType;
788
789 baosInMemory.reset();
790 baosInMemory.write(getDummyHeaderForVersion(this.minorVersion));
791
792 state = State.WRITING;
793
794
795 userDataStream = new DataOutputStream(baosInMemory);
796 return userDataStream;
797 }
798
799
800
801
802
803
804
805
806 DataOutputStream getUserDataStream() {
807 expectState(State.WRITING);
808 return userDataStream;
809 }
810
811
812
813
814
815 private void ensureBlockReady() throws IOException {
816 Preconditions.checkState(state != State.INIT,
817 "Unexpected state: " + state);
818
819 if (state == State.BLOCK_READY)
820 return;
821
822
823 finishBlock();
824 }
825
826
827
828
829
830
831
832 private void finishBlock() throws IOException {
833 userDataStream.flush();
834
835
836 uncompressedBytesWithHeader = baosInMemory.toByteArray();
837 prevOffset = prevOffsetByType[blockType.getId()];
838
839
840
841
842 state = State.BLOCK_READY;
843 encodeDataBlockForDisk();
844
845 doCompressionAndChecksumming();
846 }
847
848
849
850
851
852
853
854
855
856 private void doCompressionAndChecksumming() throws IOException {
857 if ( minorVersion <= MINOR_VERSION_NO_CHECKSUM) {
858 version20compression();
859 } else {
860 version21ChecksumAndCompression();
861 }
862 }
863
864 private void version20compression() throws IOException {
865 onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
866
867 if (compressAlgo != NONE) {
868 compressedByteStream.reset();
869 compressedByteStream.write(DUMMY_HEADER_NO_CHECKSUM);
870
871 compressionStream.resetState();
872
873 compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
874 uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
875
876
877 compressionStream.flush();
878 compressionStream.finish();
879 onDiskDataSizeWithHeader = compressedByteStream.size();
880 onDiskBytesWithHeader = compressedByteStream.toByteArray();
881
882 put20Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
883 uncompressedBytesWithHeader.length);
884
885
886
887 put20Header(uncompressedBytesWithHeader, 0,
888 onDiskBytesWithHeader.length + onDiskChecksum.length,
889 uncompressedBytesWithHeader.length);
890
891 } else {
892 onDiskBytesWithHeader = uncompressedBytesWithHeader;
893
894 onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
895
896
897 put20Header(uncompressedBytesWithHeader, 0,
898 onDiskBytesWithHeader.length,
899 uncompressedBytesWithHeader.length);
900 }
901 }
902
903 private void version21ChecksumAndCompression() throws IOException {
904
905 if (compressAlgo != NONE) {
906 compressedByteStream.reset();
907 compressedByteStream.write(DUMMY_HEADER_WITH_CHECKSUM);
908
909 compressionStream.resetState();
910
911 compressionStream.write(uncompressedBytesWithHeader, headerSize(this.minorVersion),
912 uncompressedBytesWithHeader.length - headerSize(this.minorVersion));
913
914 compressionStream.flush();
915 compressionStream.finish();
916
917
918 onDiskDataSizeWithHeader = compressedByteStream.size();
919
920
921 ChecksumUtil.reserveSpaceForChecksums(compressedByteStream,
922 onDiskDataSizeWithHeader, bytesPerChecksum);
923
924
925 onDiskBytesWithHeader = compressedByteStream.toByteArray();
926 put21Header(onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length,
927 uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
928
929
930
931 ChecksumUtil.generateChecksums(
932 onDiskBytesWithHeader, 0, onDiskDataSizeWithHeader,
933 onDiskBytesWithHeader, onDiskDataSizeWithHeader,
934 checksumType, bytesPerChecksum);
935
936
937 onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;
938
939
940 put21Header(uncompressedBytesWithHeader, 0,
941 onDiskBytesWithHeader.length + onDiskChecksum.length,
942 uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
943
944 } else {
945
946
947 onDiskBytesWithHeader = uncompressedBytesWithHeader;
948
949 onDiskDataSizeWithHeader = onDiskBytesWithHeader.length;
950 int numBytes = (int)ChecksumUtil.numBytes(
951 uncompressedBytesWithHeader.length,
952 bytesPerChecksum);
953 onDiskChecksum = new byte[numBytes];
954
955
956 put21Header(uncompressedBytesWithHeader, 0,
957 onDiskBytesWithHeader.length + onDiskChecksum.length,
958 uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader);
959
960 ChecksumUtil.generateChecksums(
961 uncompressedBytesWithHeader, 0, uncompressedBytesWithHeader.length,
962 onDiskChecksum, 0,
963 checksumType, bytesPerChecksum);
964 }
965 }
966
967
968
969
970
971 private void encodeDataBlockForDisk() throws IOException {
972 if (blockType != BlockType.DATA) {
973 return;
974 }
975
976
977 ByteBuffer rawKeyValues = ByteBuffer.wrap(uncompressedBytesWithHeader,
978 headerSize(this.minorVersion), uncompressedBytesWithHeader.length -
979 headerSize(this.minorVersion)).slice();
980 Pair<ByteBuffer, BlockType> encodingResult =
981 dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
982 includesMemstoreTS, getDummyHeaderForVersion(this.minorVersion));
983
984 BlockType encodedBlockType = encodingResult.getSecond();
985 if (encodedBlockType == BlockType.ENCODED_DATA) {
986 uncompressedBytesWithHeader = encodingResult.getFirst().array();
987 blockType = BlockType.ENCODED_DATA;
988 } else {
989
990 if (encodedBlockType != BlockType.DATA) {
991 throw new IOException("Unexpected block type coming out of data " +
992 "block encoder: " + encodedBlockType);
993 }
994 if (userDataStream.size() !=
995 uncompressedBytesWithHeader.length - headerSize(this.minorVersion)) {
996 throw new IOException("Uncompressed size mismatch: "
997 + userDataStream.size() + " vs. "
998 + (uncompressedBytesWithHeader.length - headerSize(this.minorVersion)));
999 }
1000 }
1001 }
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011 private void put21Header(byte[] dest, int offset, int onDiskSize,
1012 int uncompressedSize, int onDiskDataSize) {
1013 offset = blockType.put(dest, offset);
1014 offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_WITH_CHECKSUMS);
1015 offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_WITH_CHECKSUMS);
1016 offset = Bytes.putLong(dest, offset, prevOffset);
1017 offset = Bytes.putByte(dest, offset, checksumType.getCode());
1018 offset = Bytes.putInt(dest, offset, bytesPerChecksum);
1019 offset = Bytes.putInt(dest, offset, onDiskDataSizeWithHeader);
1020 }
1021
1022
1023 private void put20Header(byte[] dest, int offset, int onDiskSize,
1024 int uncompressedSize) {
1025 offset = blockType.put(dest, offset);
1026 offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_NO_CHECKSUM);
1027 offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_NO_CHECKSUM);
1028 Bytes.putLong(dest, offset, prevOffset);
1029 }
1030
1031
1032
1033
1034
1035
1036
1037
1038 public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
1039 long offset = out.getPos();
1040 if (startOffset != -1 && offset != startOffset) {
1041 throw new IOException("A " + blockType + " block written to a "
1042 + "stream twice, first at offset " + startOffset + ", then at "
1043 + offset);
1044 }
1045 startOffset = offset;
1046
1047 writeHeaderAndData((DataOutputStream) out);
1048 }
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059 private void writeHeaderAndData(DataOutputStream out) throws IOException {
1060 ensureBlockReady();
1061 out.write(onDiskBytesWithHeader);
1062 if (compressAlgo == NONE && minorVersion > MINOR_VERSION_NO_CHECKSUM) {
1063 if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
1064 throw new IOException("A " + blockType
1065 + " without compression should have checksums "
1066 + " stored separately.");
1067 }
1068 out.write(onDiskChecksum);
1069 }
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082 byte[] getHeaderAndDataForTest() throws IOException {
1083 ensureBlockReady();
1084 if (compressAlgo == NONE) {
1085 if (onDiskChecksum == HConstants.EMPTY_BYTE_ARRAY) {
1086 throw new IOException("A " + blockType
1087 + " without compression should have checksums "
1088 + " stored separately.");
1089 }
1090
1091
1092 byte[] output = new byte[onDiskBytesWithHeader.length +
1093 onDiskChecksum.length];
1094 System.arraycopy(onDiskBytesWithHeader, 0,
1095 output, 0, onDiskBytesWithHeader.length);
1096 System.arraycopy(onDiskChecksum, 0,
1097 output, onDiskBytesWithHeader.length,
1098 onDiskChecksum.length);
1099 return output;
1100 }
1101 return onDiskBytesWithHeader;
1102 }
1103
1104
1105
1106
1107
1108 public void releaseCompressor() {
1109 if (compressor != null) {
1110 compressAlgo.returnCompressor(compressor);
1111 compressor = null;
1112 }
1113 }
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123 int getOnDiskSizeWithoutHeader() {
1124 expectState(State.BLOCK_READY);
1125 return onDiskBytesWithHeader.length + onDiskChecksum.length - headerSize(this.minorVersion);
1126 }
1127
1128
1129
1130
1131
1132
1133
1134
1135 int getOnDiskSizeWithHeader() {
1136 expectState(State.BLOCK_READY);
1137 return onDiskBytesWithHeader.length + onDiskChecksum.length;
1138 }
1139
1140
1141
1142
1143 int getUncompressedSizeWithoutHeader() {
1144 expectState(State.BLOCK_READY);
1145 return uncompressedBytesWithHeader.length - headerSize(this.minorVersion);
1146 }
1147
1148
1149
1150
1151 int getUncompressedSizeWithHeader() {
1152 expectState(State.BLOCK_READY);
1153 return uncompressedBytesWithHeader.length;
1154 }
1155
1156
1157 public boolean isWriting() {
1158 return state == State.WRITING;
1159 }
1160
1161
1162
1163
1164
1165
1166
1167
1168 public int blockSizeWritten() {
1169 if (state != State.WRITING)
1170 return 0;
1171 return userDataStream.size();
1172 }
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182 ByteBuffer getUncompressedBufferWithHeader() {
1183 expectState(State.BLOCK_READY);
1184 return ByteBuffer.wrap(uncompressedBytesWithHeader);
1185 }
1186
1187 private void expectState(State expectedState) {
1188 if (state != expectedState) {
1189 throw new IllegalStateException("Expected state: " + expectedState +
1190 ", actual state: " + state);
1191 }
1192 }
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204 public void writeBlock(BlockWritable bw, FSDataOutputStream out)
1205 throws IOException {
1206 bw.writeToBlock(startWriting(bw.getBlockType()));
1207 writeHeaderAndData(out);
1208 }
1209
1210
1211
1212
1213
1214
1215
1216
1217 public HFileBlock getBlockForCaching() {
1218 return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
1219 getUncompressedSizeWithoutHeader(), prevOffset,
1220 getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
1221 includesMemstoreTS, this.minorVersion,
1222 0, ChecksumType.NULL.getCode(),
1223 onDiskBytesWithHeader.length + onDiskChecksum.length);
1224 }
1225 }
1226
1227
1228 public interface BlockWritable {
1229
1230
1231 BlockType getBlockType();
1232
1233
1234
1235
1236
1237
1238
1239 void writeToBlock(DataOutput out) throws IOException;
1240 }
1241
1242
1243
1244
1245 public interface BlockIterator {
1246
1247
1248
1249
1250 HFileBlock nextBlock() throws IOException;
1251
1252
1253
1254
1255
1256 HFileBlock nextBlockWithBlockType(BlockType blockType) throws IOException;
1257 }
1258
1259
1260 public interface FSReader {
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273 HFileBlock readBlockData(long offset, long onDiskSize,
1274 int uncompressedSize, boolean pread) throws IOException;
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285 BlockIterator blockRange(long startOffset, long endOffset);
1286 }
1287
1288
1289
1290
1291
1292 private abstract static class AbstractFSReader implements FSReader {
1293
1294
1295
1296 protected final FSDataInputStream istream;
1297
1298
1299
1300 protected final FSDataInputStream istreamNoFsChecksum;
1301
1302
1303 protected Compression.Algorithm compressAlgo;
1304
1305
1306 protected long fileSize;
1307
1308
1309 private int minorVersion;
1310
1311
1312 protected int hdrSize;
1313
1314
1315 protected HFileSystem hfs;
1316
1317
1318 protected Path path;
1319
1320 private final Lock streamLock = new ReentrantLock();
1321
1322
1323 public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
1324
1325 public AbstractFSReader(FSDataInputStream istream,
1326 FSDataInputStream istreamNoFsChecksum,
1327 Algorithm compressAlgo,
1328 long fileSize, int minorVersion, HFileSystem hfs, Path path)
1329 throws IOException {
1330 this.istream = istream;
1331 this.compressAlgo = compressAlgo;
1332 this.fileSize = fileSize;
1333 this.minorVersion = minorVersion;
1334 this.hfs = hfs;
1335 this.path = path;
1336 this.hdrSize = headerSize(minorVersion);
1337 this.istreamNoFsChecksum = istreamNoFsChecksum;
1338 }
1339
1340 @Override
1341 public BlockIterator blockRange(final long startOffset,
1342 final long endOffset) {
1343 return new BlockIterator() {
1344 private long offset = startOffset;
1345
1346 @Override
1347 public HFileBlock nextBlock() throws IOException {
1348 if (offset >= endOffset)
1349 return null;
1350 HFileBlock b = readBlockData(offset, -1, -1, false);
1351 offset += b.getOnDiskSizeWithHeader();
1352 return b;
1353 }
1354
1355 @Override
1356 public HFileBlock nextBlockWithBlockType(BlockType blockType)
1357 throws IOException {
1358 HFileBlock blk = nextBlock();
1359 if (blk.getBlockType() != blockType) {
1360 throw new IOException("Expected block of type " + blockType
1361 + " but found " + blk.getBlockType());
1362 }
1363 return blk;
1364 }
1365 };
1366 }
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383 protected int readAtOffset(FSDataInputStream istream,
1384 byte[] dest, int destOffset, int size,
1385 boolean peekIntoNextBlock, long fileOffset, boolean pread)
1386 throws IOException {
1387 if (peekIntoNextBlock &&
1388 destOffset + size + hdrSize > dest.length) {
1389
1390
1391 throw new IOException("Attempted to read " + size + " bytes and " +
1392 hdrSize + " bytes of next header into a " + dest.length +
1393 "-byte array at offset " + destOffset);
1394 }
1395
1396 if (!pread && streamLock.tryLock()) {
1397
1398 try {
1399 istream.seek(fileOffset);
1400
1401 long realOffset = istream.getPos();
1402 if (realOffset != fileOffset) {
1403 throw new IOException("Tried to seek to " + fileOffset + " to "
1404 + "read " + size + " bytes, but pos=" + realOffset
1405 + " after seek");
1406 }
1407
1408 if (!peekIntoNextBlock) {
1409 IOUtils.readFully(istream, dest, destOffset, size);
1410 return -1;
1411 }
1412
1413
1414 if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
1415 return -1;
1416 } finally {
1417 streamLock.unlock();
1418 }
1419 } else {
1420
1421 int extraSize = peekIntoNextBlock ? hdrSize : 0;
1422
1423 int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
1424 if (ret < size) {
1425 throw new IOException("Positional read of " + size + " bytes " +
1426 "failed at offset " + fileOffset + " (returned " + ret + ")");
1427 }
1428
1429 if (ret == size || ret < size + extraSize) {
1430
1431 return -1;
1432 }
1433 }
1434
1435 assert peekIntoNextBlock;
1436 return Bytes.toInt(dest, destOffset + size + BlockType.MAGIC_LENGTH) +
1437 hdrSize;
1438 }
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452 protected void decompress(byte[] dest, int destOffset,
1453 InputStream bufferedBoundedStream,
1454 int uncompressedSize) throws IOException {
1455 Decompressor decompressor = null;
1456 try {
1457 decompressor = compressAlgo.getDecompressor();
1458 InputStream is = compressAlgo.createDecompressionStream(
1459 bufferedBoundedStream, decompressor, 0);
1460
1461 IOUtils.readFully(is, dest, destOffset, uncompressedSize);
1462 is.close();
1463 } finally {
1464 if (decompressor != null) {
1465 compressAlgo.returnDecompressor(decompressor);
1466 }
1467 }
1468 }
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480 protected InputStream createBufferedBoundedStream(long offset,
1481 int size, boolean pread) {
1482 return new BufferedInputStream(new BoundedRangeFileInputStream(istream,
1483 offset, size, pread), Math.min(DEFAULT_BUFFER_SIZE, size));
1484 }
1485
1486
1487
1488
1489 protected int getMinorVersion() {
1490 return minorVersion;
1491 }
1492 }
1493
1494
1495
1496
1497
1498
1499
1500
1501 static class FSReaderV1 extends AbstractFSReader {
1502
1503
1504 private static final int HEADER_DELTA = HEADER_SIZE_NO_CHECKSUM -
1505 MAGIC_LENGTH;
1506
1507 public FSReaderV1(FSDataInputStream istream, Algorithm compressAlgo,
1508 long fileSize) throws IOException {
1509 super(istream, istream, compressAlgo, fileSize, 0, null, null);
1510 }
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530 @Override
1531 public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
1532 int uncompressedSizeWithMagic, boolean pread) throws IOException {
1533 if (uncompressedSizeWithMagic <= 0) {
1534 throw new IOException("Invalid uncompressedSize="
1535 + uncompressedSizeWithMagic + " for a version 1 block");
1536 }
1537
1538 if (onDiskSizeWithMagic <= 0 || onDiskSizeWithMagic >= Integer.MAX_VALUE)
1539 {
1540 throw new IOException("Invalid onDiskSize=" + onDiskSizeWithMagic
1541 + " (maximum allowed: " + Integer.MAX_VALUE + ")");
1542 }
1543
1544 int onDiskSize = (int) onDiskSizeWithMagic;
1545
1546 if (uncompressedSizeWithMagic < MAGIC_LENGTH) {
1547 throw new IOException("Uncompressed size for a version 1 block is "
1548 + uncompressedSizeWithMagic + " but must be at least "
1549 + MAGIC_LENGTH);
1550 }
1551
1552
1553
1554 ByteBuffer buf = ByteBuffer.allocate(uncompressedSizeWithMagic
1555 + HEADER_DELTA);
1556
1557 int onDiskSizeWithoutHeader;
1558 if (compressAlgo == Compression.Algorithm.NONE) {
1559
1560 if (onDiskSize != uncompressedSizeWithMagic) {
1561 throw new IOException("onDiskSize=" + onDiskSize
1562 + " and uncompressedSize=" + uncompressedSizeWithMagic
1563 + " must be equal for version 1 with no compression");
1564 }
1565
1566
1567
1568 readAtOffset(istream, buf.array(), buf.arrayOffset() + HEADER_DELTA,
1569 onDiskSize, false, offset, pread);
1570
1571 onDiskSizeWithoutHeader = uncompressedSizeWithMagic - MAGIC_LENGTH;
1572 } else {
1573 InputStream bufferedBoundedStream = createBufferedBoundedStream(
1574 offset, onDiskSize, pread);
1575 decompress(buf.array(), buf.arrayOffset() + HEADER_DELTA,
1576 bufferedBoundedStream, uncompressedSizeWithMagic);
1577
1578
1579
1580 onDiskSizeWithoutHeader = onDiskSize;
1581 }
1582
1583 BlockType newBlockType = BlockType.parse(buf.array(), buf.arrayOffset()
1584 + HEADER_DELTA, MAGIC_LENGTH);
1585
1586
1587
1588
1589 HFileBlock b = new HFileBlock(newBlockType, onDiskSizeWithoutHeader,
1590 uncompressedSizeWithMagic - MAGIC_LENGTH, -1L, buf, FILL_HEADER,
1591 offset, MemStore.NO_PERSISTENT_TS, 0, 0, ChecksumType.NULL.getCode(),
1592 onDiskSizeWithoutHeader + HEADER_SIZE_NO_CHECKSUM);
1593 return b;
1594 }
1595 }
1596
1597
1598
1599
1600
1601 private static class PrefetchedHeader {
1602 long offset = -1;
1603 byte[] header = new byte[HEADER_SIZE_WITH_CHECKSUMS];
1604 ByteBuffer buf = ByteBuffer.wrap(header, 0, HEADER_SIZE_WITH_CHECKSUMS);
1605 }
1606
1607
1608 static class FSReaderV2 extends AbstractFSReader {
1609
1610
1611 private final boolean useHBaseChecksumConfigured;
1612
1613
1614
1615
1616
1617 private volatile boolean useHBaseChecksum;
1618
1619
1620
1621 private volatile int checksumOffCount = -1;
1622
1623
1624 protected boolean includesMemstoreTS;
1625
1626
1627 protected HFileDataBlockEncoder dataBlockEncoder =
1628 NoOpDataBlockEncoder.INSTANCE;
1629
1630 private ThreadLocal<PrefetchedHeader> prefetchedHeaderForThread =
1631 new ThreadLocal<PrefetchedHeader>() {
1632 @Override
1633 public PrefetchedHeader initialValue() {
1634 return new PrefetchedHeader();
1635 }
1636 };
1637
1638 public FSReaderV2(FSDataInputStream istream,
1639 FSDataInputStream istreamNoFsChecksum, Algorithm compressAlgo,
1640 long fileSize, int minorVersion, HFileSystem hfs, Path path)
1641 throws IOException {
1642 super(istream, istreamNoFsChecksum, compressAlgo, fileSize,
1643 minorVersion, hfs, path);
1644
1645 if (hfs != null) {
1646
1647
1648 useHBaseChecksum = hfs.useHBaseChecksum();
1649 } else {
1650
1651
1652
1653
1654 useHBaseChecksum = true;
1655 }
1656
1657
1658 if (getMinorVersion() < MINOR_VERSION_WITH_CHECKSUM) {
1659 useHBaseChecksum = false;
1660 }
1661 this.useHBaseChecksumConfigured = useHBaseChecksum;
1662 }
1663
1664
1665
1666
1667
1668 FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
1669 long fileSize) throws IOException {
1670 this(istream, istream, compressAlgo, fileSize,
1671 HFileReaderV2.MAX_MINOR_VERSION, null, null);
1672 }
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685 @Override
1686 public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
1687 int uncompressedSize, boolean pread) throws IOException {
1688
1689
1690
1691 FSDataInputStream is = this.istreamNoFsChecksum;
1692
1693
1694
1695
1696
1697
1698 boolean doVerificationThruHBaseChecksum = this.useHBaseChecksum;
1699 if (!doVerificationThruHBaseChecksum) {
1700 is = this.istream;
1701 }
1702
1703 HFileBlock blk = readBlockDataInternal(is, offset,
1704 onDiskSizeWithHeaderL,
1705 uncompressedSize, pread,
1706 doVerificationThruHBaseChecksum);
1707 if (blk == null) {
1708 HFile.LOG.warn("HBase checksum verification failed for file " +
1709 path + " at offset " +
1710 offset + " filesize " + fileSize +
1711 ". Retrying read with HDFS checksums turned on...");
1712
1713 if (!doVerificationThruHBaseChecksum) {
1714 String msg = "HBase checksum verification failed for file " +
1715 path + " at offset " +
1716 offset + " filesize " + fileSize +
1717 " but this cannot happen because doVerify is " +
1718 doVerificationThruHBaseChecksum;
1719 HFile.LOG.warn(msg);
1720 throw new IOException(msg);
1721 }
1722 HFile.checksumFailures.incrementAndGet();
1723
1724
1725
1726
1727
1728
1729
1730 this.checksumOffCount = CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD;
1731 this.useHBaseChecksum = false;
1732 doVerificationThruHBaseChecksum = false;
1733 is = this.istream;
1734 blk = readBlockDataInternal(is, offset, onDiskSizeWithHeaderL,
1735 uncompressedSize, pread,
1736 doVerificationThruHBaseChecksum);
1737 if (blk != null) {
1738 HFile.LOG.warn("HDFS checksum verification suceeded for file " +
1739 path + " at offset " +
1740 offset + " filesize " + fileSize);
1741 }
1742 }
1743 if (blk == null && !doVerificationThruHBaseChecksum) {
1744 String msg = "readBlockData failed, possibly due to " +
1745 "checksum verification failed for file " + path +
1746 " at offset " + offset + " filesize " + fileSize;
1747 HFile.LOG.warn(msg);
1748 throw new IOException(msg);
1749 }
1750
1751
1752
1753
1754
1755
1756
1757
1758 if (!this.useHBaseChecksum && this.useHBaseChecksumConfigured) {
1759 if (this.checksumOffCount-- < 0) {
1760 this.useHBaseChecksum = true;
1761 }
1762 }
1763 return blk;
1764 }
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779 private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
1780 long onDiskSizeWithHeaderL,
1781 int uncompressedSize, boolean pread, boolean verifyChecksum)
1782 throws IOException {
1783 if (offset < 0) {
1784 throw new IOException("Invalid offset=" + offset + " trying to read "
1785 + "block (onDiskSize=" + onDiskSizeWithHeaderL
1786 + ", uncompressedSize=" + uncompressedSize + ")");
1787 }
1788 if (uncompressedSize != -1) {
1789 throw new IOException("Version 2 block reader API does not need " +
1790 "the uncompressed size parameter");
1791 }
1792
1793 if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1)
1794 || onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
1795 throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
1796 + ": expected to be at least " + hdrSize
1797 + " and at most " + Integer.MAX_VALUE + ", or -1 (offset="
1798 + offset + ", uncompressedSize=" + uncompressedSize + ")");
1799 }
1800
1801 int onDiskSizeWithHeader = (int) onDiskSizeWithHeaderL;
1802
1803 HFileBlock b;
1804 if (onDiskSizeWithHeader > 0) {
1805
1806
1807
1808
1809
1810
1811
1812
1813 int onDiskSizeWithoutHeader = onDiskSizeWithHeader - hdrSize;
1814 assert onDiskSizeWithoutHeader >= 0;
1815
1816
1817
1818
1819 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1820 byte[] header = prefetchedHeader.offset == offset
1821 ? prefetchedHeader.header : null;
1822
1823
1824 int preReadHeaderSize = header == null ? 0 : hdrSize;
1825
1826 if (compressAlgo == Compression.Algorithm.NONE) {
1827
1828
1829
1830 ByteBuffer headerAndData = ByteBuffer.allocate(onDiskSizeWithHeader
1831 + hdrSize);
1832 headerAndData.limit(onDiskSizeWithHeader);
1833
1834 if (header != null) {
1835 System.arraycopy(header, 0, headerAndData.array(), 0,
1836 hdrSize);
1837 }
1838
1839 int nextBlockOnDiskSizeWithHeader = readAtOffset(is,
1840 headerAndData.array(), headerAndData.arrayOffset()
1841 + preReadHeaderSize, onDiskSizeWithHeader
1842 - preReadHeaderSize, true, offset + preReadHeaderSize,
1843 pread);
1844
1845 b = new HFileBlock(headerAndData, getMinorVersion());
1846 b.assumeUncompressed();
1847 b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
1848 b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSizeWithHeader;
1849 if (verifyChecksum &&
1850 !validateBlockChecksum(b, headerAndData.array(), hdrSize)) {
1851 return null;
1852 }
1853 if (b.nextBlockOnDiskSizeWithHeader > 0)
1854 setNextBlockHeader(offset, b);
1855 } else {
1856
1857 byte[] onDiskBlock = new byte[onDiskSizeWithHeader + hdrSize];
1858
1859 int nextBlockOnDiskSize = readAtOffset(is, onDiskBlock,
1860 preReadHeaderSize, onDiskSizeWithHeader - preReadHeaderSize,
1861 true, offset + preReadHeaderSize, pread);
1862
1863 if (header == null)
1864 header = onDiskBlock;
1865
1866 try {
1867 b = new HFileBlock(ByteBuffer.wrap(header, 0, hdrSize),
1868 getMinorVersion());
1869 } catch (IOException ex) {
1870
1871 throw new IOException("Failed to read compressed block at "
1872 + offset + ", onDiskSizeWithoutHeader=" + onDiskSizeWithHeader
1873 + ", preReadHeaderSize=" + preReadHeaderSize
1874 + ", header.length=" + header.length + ", header bytes: "
1875 + Bytes.toStringBinary(header, 0, hdrSize), ex);
1876 }
1877 b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
1878 b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSize;
1879 if (verifyChecksum &&
1880 !validateBlockChecksum(b, onDiskBlock, hdrSize)) {
1881 return null;
1882 }
1883
1884 DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
1885 onDiskBlock, hdrSize, onDiskSizeWithoutHeader));
1886
1887
1888 b.allocateBuffer(b.nextBlockOnDiskSizeWithHeader > 0);
1889
1890 decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
1891 b.uncompressedSizeWithoutHeader);
1892
1893
1894 if (nextBlockOnDiskSize > 0) {
1895 System.arraycopy(onDiskBlock, onDiskSizeWithHeader, b.buf.array(),
1896 b.buf.arrayOffset() + hdrSize
1897 + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(),
1898 hdrSize);
1899
1900 setNextBlockHeader(offset, b);
1901 }
1902 }
1903
1904 } else {
1905
1906
1907
1908
1909
1910
1911
1912
1913 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1914 ByteBuffer headerBuf = prefetchedHeader.offset == offset ?
1915 prefetchedHeader.buf : null;
1916
1917 if (headerBuf == null) {
1918
1919
1920 headerBuf = ByteBuffer.allocate(hdrSize);
1921 readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize,
1922 false, offset, pread);
1923 }
1924
1925 b = new HFileBlock(headerBuf, getMinorVersion());
1926
1927
1928 b.allocateBuffer(true);
1929
1930 if (compressAlgo == Compression.Algorithm.NONE) {
1931
1932
1933
1934 b.assumeUncompressed();
1935 b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, b.buf.array(),
1936 b.buf.arrayOffset() + hdrSize,
1937 b.uncompressedSizeWithoutHeader + b.totalChecksumBytes(),
1938 true, offset + hdrSize,
1939 pread);
1940 if (verifyChecksum &&
1941 !validateBlockChecksum(b, b.buf.array(), hdrSize)) {
1942 return null;
1943 }
1944
1945 if (b.nextBlockOnDiskSizeWithHeader > 0) {
1946 setNextBlockHeader(offset, b);
1947 }
1948 } else {
1949
1950 byte[] compressedBytes = new byte[b.getOnDiskSizeWithHeader()
1951 + hdrSize];
1952
1953 b.nextBlockOnDiskSizeWithHeader = readAtOffset(is, compressedBytes,
1954 hdrSize, b.onDiskSizeWithoutHeader, true, offset
1955 + hdrSize, pread);
1956 if (verifyChecksum &&
1957 !validateBlockChecksum(b, compressedBytes, hdrSize)) {
1958 return null;
1959 }
1960 DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
1961 compressedBytes, hdrSize, b.onDiskSizeWithoutHeader));
1962
1963 decompress(b.buf.array(), b.buf.arrayOffset() + hdrSize, dis,
1964 b.uncompressedSizeWithoutHeader);
1965
1966 if (b.nextBlockOnDiskSizeWithHeader > 0) {
1967
1968 int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
1969 + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
1970 System.arraycopy(compressedBytes,
1971 compressedBytes.length - hdrSize,
1972 b.buf.array(),
1973 nextHeaderOffset,
1974 hdrSize);
1975
1976 setNextBlockHeader(offset, b);
1977 }
1978 }
1979 }
1980
1981 b.includesMemstoreTS = includesMemstoreTS;
1982 b.offset = offset;
1983 return b;
1984 }
1985
1986 private void setNextBlockHeader(long offset, HFileBlock b) {
1987 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1988 prefetchedHeader.offset = offset + b.getOnDiskSizeWithHeader();
1989 int nextHeaderOffset = b.buf.arrayOffset() + hdrSize
1990 + b.uncompressedSizeWithoutHeader + b.totalChecksumBytes();
1991 System.arraycopy(b.buf.array(), nextHeaderOffset,
1992 prefetchedHeader.header, 0, hdrSize);
1993 }
1994
1995 void setIncludesMemstoreTS(boolean enabled) {
1996 includesMemstoreTS = enabled;
1997 }
1998
1999 void setDataBlockEncoder(HFileDataBlockEncoder encoder) {
2000 this.dataBlockEncoder = encoder;
2001 }
2002
2003
2004
2005
2006
2007
2008
2009 protected boolean validateBlockChecksum(HFileBlock block,
2010 byte[] data, int hdrSize) throws IOException {
2011 return ChecksumUtil.validateBlockChecksum(path, block,
2012 data, hdrSize);
2013 }
2014 }
2015
2016 @Override
2017 public int getSerializedLength() {
2018 if (buf != null) {
2019 return this.buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE;
2020 }
2021 return 0;
2022 }
2023
2024 @Override
2025 public void serialize(ByteBuffer destination) {
2026 destination.put(this.buf.duplicate());
2027 destination.putLong(this.offset);
2028 destination.putInt(this.nextBlockOnDiskSizeWithHeader);
2029 destination.rewind();
2030 }
2031
2032 @Override
2033 public CacheableDeserializer<Cacheable> getDeserializer() {
2034 return HFileBlock.blockDeserializer;
2035 }
2036
2037 @Override
2038 public boolean equals(Object comparison) {
2039 if (this == comparison) {
2040 return true;
2041 }
2042 if (comparison == null) {
2043 return false;
2044 }
2045 if (comparison.getClass() != this.getClass()) {
2046 return false;
2047 }
2048
2049 HFileBlock castedComparison = (HFileBlock) comparison;
2050
2051 if (castedComparison.blockType != this.blockType) {
2052 return false;
2053 }
2054 if (castedComparison.nextBlockOnDiskSizeWithHeader != this.nextBlockOnDiskSizeWithHeader) {
2055 return false;
2056 }
2057 if (castedComparison.offset != this.offset) {
2058 return false;
2059 }
2060 if (castedComparison.onDiskSizeWithoutHeader != this.onDiskSizeWithoutHeader) {
2061 return false;
2062 }
2063 if (castedComparison.prevBlockOffset != this.prevBlockOffset) {
2064 return false;
2065 }
2066 if (castedComparison.uncompressedSizeWithoutHeader != this.uncompressedSizeWithoutHeader) {
2067 return false;
2068 }
2069 if (this.buf.compareTo(castedComparison.buf) != 0) {
2070 return false;
2071 }
2072 if (this.buf.position() != castedComparison.buf.position()){
2073 return false;
2074 }
2075 if (this.buf.limit() != castedComparison.buf.limit()){
2076 return false;
2077 }
2078 return true;
2079 }
2080
2081 public boolean doesIncludeMemstoreTS() {
2082 return includesMemstoreTS;
2083 }
2084
2085 public DataBlockEncoding getDataBlockEncoding() {
2086 if (blockType == BlockType.ENCODED_DATA) {
2087 return DataBlockEncoding.getEncodingById(getDataBlockEncodingId());
2088 }
2089 return DataBlockEncoding.NONE;
2090 }
2091
2092 byte getChecksumType() {
2093 return this.checksumType;
2094 }
2095
2096 int getBytesPerChecksum() {
2097 return this.bytesPerChecksum;
2098 }
2099
2100 int getOnDiskDataSizeWithHeader() {
2101 return this.onDiskDataSizeWithHeader;
2102 }
2103
2104 int getMinorVersion() {
2105 return this.minorVersion;
2106 }
2107
2108
2109
2110
2111
2112 int totalChecksumBytes() {
2113
2114
2115
2116
2117 if (minorVersion < MINOR_VERSION_WITH_CHECKSUM || this.bytesPerChecksum == 0) {
2118 return 0;
2119 }
2120 return (int)ChecksumUtil.numBytes(onDiskDataSizeWithHeader, bytesPerChecksum);
2121 }
2122
2123
2124
2125
2126 public int headerSize() {
2127 return headerSize(this.minorVersion);
2128 }
2129
2130
2131
2132
2133 static private int headerSize(int minorVersion) {
2134 if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
2135 return HEADER_SIZE_NO_CHECKSUM;
2136 }
2137 return HEADER_SIZE_WITH_CHECKSUMS;
2138 }
2139
2140
2141
2142
2143 public byte[] getDummyHeaderForVersion() {
2144 return getDummyHeaderForVersion(minorVersion);
2145 }
2146
2147
2148
2149
2150 static private byte[] getDummyHeaderForVersion(int minorVersion) {
2151 if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) {
2152 return DUMMY_HEADER_NO_CHECKSUM;
2153 }
2154 return DUMMY_HEADER_WITH_CHECKSUM;
2155 }
2156
2157
2158
2159
2160
2161
2162 static String toStringHeader(ByteBuffer buf) throws IOException {
2163 int offset = buf.arrayOffset();
2164 byte[] b = buf.array();
2165 long magic = Bytes.toLong(b, offset);
2166 BlockType bt = BlockType.read(buf);
2167 offset += Bytes.SIZEOF_LONG;
2168 int compressedBlockSizeNoHeader = Bytes.toInt(b, offset);
2169 offset += Bytes.SIZEOF_INT;
2170 int uncompressedBlockSizeNoHeader = Bytes.toInt(b, offset);
2171 offset += Bytes.SIZEOF_INT;
2172 long prevBlockOffset = Bytes.toLong(b, offset);
2173 offset += Bytes.SIZEOF_LONG;
2174 byte cksumtype = b[offset];
2175 offset += Bytes.SIZEOF_BYTE;
2176 long bytesPerChecksum = Bytes.toInt(b, offset);
2177 offset += Bytes.SIZEOF_INT;
2178 long onDiskDataSizeWithHeader = Bytes.toInt(b, offset);
2179 offset += Bytes.SIZEOF_INT;
2180 return " Header dump: magic: " + magic +
2181 " blockType " + bt +
2182 " compressedBlockSizeNoHeader " +
2183 compressedBlockSizeNoHeader +
2184 " uncompressedBlockSizeNoHeader " +
2185 uncompressedBlockSizeNoHeader +
2186 " prevBlockOffset " + prevBlockOffset +
2187 " checksumType " + ChecksumType.codeToType(cksumtype) +
2188 " bytesPerChecksum " + bytesPerChecksum +
2189 " onDiskDataSizeWithHeader " + onDiskDataSizeWithHeader;
2190 }
2191 }
2192