1 /**
2 *
3 * Licensed to the Apache Software Foundation (ASF) under one
4 * or more contributor license agreements. See the NOTICE file
5 * distributed with this work for additional information
6 * regarding copyright ownership. The ASF licenses this file
7 * to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance
9 * with the License. You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.apache.hadoop.hbase;
20
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.ByteArrayOutputStream;
28 import java.io.DataInputStream;
29 import java.io.DataOutputStream;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.NavigableSet;
33 import java.util.Set;
34
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.exceptions.DeserializationException;
38 import org.apache.hadoop.hbase.filter.BinaryComparator;
39 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
40 import org.apache.hadoop.hbase.filter.Filter;
41 import org.apache.hadoop.hbase.filter.PrefixFilter;
42 import org.apache.hadoop.hbase.filter.RowFilter;
43 import org.apache.hadoop.hbase.io.TimeRange;
44 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
45 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.io.DataInputBuffer;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 /**
53 * Test HBase Writables serializations
54 */
55 @Category(SmallTests.class)
56 public class TestSerialization {
57 @Test public void testKeyValue() throws Exception {
58 final String name = "testKeyValue2";
59 byte[] row = name.getBytes();
60 byte[] fam = "fam".getBytes();
61 byte[] qf = "qf".getBytes();
62 long ts = System.currentTimeMillis();
63 byte[] val = "val".getBytes();
64 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
65 ByteArrayOutputStream baos = new ByteArrayOutputStream();
66 DataOutputStream dos = new DataOutputStream(baos);
67 long l = KeyValue.write(kv, dos);
68 dos.close();
69 byte [] mb = baos.toByteArray();
70 ByteArrayInputStream bais = new ByteArrayInputStream(mb);
71 DataInputStream dis = new DataInputStream(bais);
72 KeyValue deserializedKv = KeyValue.create(dis);
73 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
74 assertEquals(kv.getOffset(), deserializedKv.getOffset());
75 assertEquals(kv.getLength(), deserializedKv.getLength());
76 }
77
78 @Test
79 public void testSplitLogTask() throws DeserializationException {
80 SplitLogTask slt = new SplitLogTask.Unassigned(new ServerName("mgr,1,1"));
81 byte [] bytes = slt.toByteArray();
82 SplitLogTask sltDeserialized = SplitLogTask.parseFrom(bytes);
83 assertTrue(slt.equals(sltDeserialized));
84 }
85
86 @Test public void testCompareFilter() throws Exception {
87 Filter f = new RowFilter(CompareOp.EQUAL,
88 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
89 byte [] bytes = f.toByteArray();
90 Filter ff = RowFilter.parseFrom(bytes);
91 assertNotNull(ff);
92 }
93
94 @Test public void testTableDescriptor() throws Exception {
95 final String name = "testTableDescriptor";
96 HTableDescriptor htd = createTableDescriptor(name);
97 byte [] mb = Writables.getBytes(htd);
98 HTableDescriptor deserializedHtd =
99 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
100 assertEquals(htd.getTableName(), deserializedHtd.getTableName());
101 }
102
103 /**
104 * Test RegionInfo serialization
105 * @throws Exception
106 */
107 @Test public void testRegionInfo() throws Exception {
108 HRegionInfo hri = createRandomRegion("testRegionInfo");
109
110 //test toByteArray()
111 byte [] hrib = hri.toByteArray();
112 HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib);
113 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
114 assertEquals(hri, deserializedHri);
115
116 //test toDelimitedByteArray()
117 hrib = hri.toDelimitedByteArray();
118 DataInputBuffer buf = new DataInputBuffer();
119 try {
120 buf.reset(hrib, hrib.length);
121 deserializedHri = HRegionInfo.parseFrom(buf);
122 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
123 assertEquals(hri, deserializedHri);
124 } finally {
125 buf.close();
126 }
127 }
128
129 @Test public void testRegionInfos() throws Exception {
130 HRegionInfo hri = createRandomRegion("testRegionInfos");
131 byte[] triple = HRegionInfo.toDelimitedByteArray(hri, hri, hri);
132 List<HRegionInfo> regions = HRegionInfo.parseDelimitedFrom(triple, 0, triple.length);
133 assertTrue(regions.size() == 3);
134 assertTrue(regions.get(0).equals(regions.get(1)));
135 assertTrue(regions.get(0).equals(regions.get(2)));
136 }
137
138 private HRegionInfo createRandomRegion(final String name) {
139 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
140 String [] families = new String [] {"info", "anchor"};
141 for (int i = 0; i < families.length; i++) {
142 htd.addFamily(new HColumnDescriptor(families[i]));
143 }
144 return new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW,
145 HConstants.EMPTY_END_ROW);
146 }
147
148 /*
149 * TODO
150 @Test public void testPut() throws Exception{
151 byte[] row = "row".getBytes();
152 byte[] fam = "fam".getBytes();
153 byte[] qf1 = "qf1".getBytes();
154 byte[] qf2 = "qf2".getBytes();
155 byte[] qf3 = "qf3".getBytes();
156 byte[] qf4 = "qf4".getBytes();
157 byte[] qf5 = "qf5".getBytes();
158 byte[] qf6 = "qf6".getBytes();
159 byte[] qf7 = "qf7".getBytes();
160 byte[] qf8 = "qf8".getBytes();
161
162 long ts = System.currentTimeMillis();
163 byte[] val = "val".getBytes();
164
165 Put put = new Put(row);
166 put.setWriteToWAL(false);
167 put.add(fam, qf1, ts, val);
168 put.add(fam, qf2, ts, val);
169 put.add(fam, qf3, ts, val);
170 put.add(fam, qf4, ts, val);
171 put.add(fam, qf5, ts, val);
172 put.add(fam, qf6, ts, val);
173 put.add(fam, qf7, ts, val);
174 put.add(fam, qf8, ts, val);
175
176 byte[] sb = Writables.getBytes(put);
177 Put desPut = (Put)Writables.getWritable(sb, new Put());
178
179 //Timing test
180 // long start = System.nanoTime();
181 // desPut = (Put)Writables.getWritable(sb, new Put());
182 // long stop = System.nanoTime();
183 // System.out.println("timer " +(stop-start));
184
185 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
186 List<KeyValue> list = null;
187 List<KeyValue> desList = null;
188 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
189 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
190 list = entry.getValue();
191 desList = desPut.getFamilyMap().get(entry.getKey());
192 for(int i=0; i<list.size(); i++){
193 assertTrue(list.get(i).equals(desList.get(i)));
194 }
195 }
196 }
197
198
199 @Test public void testPut2() throws Exception{
200 byte[] row = "testAbort,,1243116656250".getBytes();
201 byte[] fam = "historian".getBytes();
202 byte[] qf1 = "creation".getBytes();
203
204 long ts = 9223372036854775807L;
205 byte[] val = "dont-care".getBytes();
206
207 Put put = new Put(row);
208 put.add(fam, qf1, ts, val);
209
210 byte[] sb = Writables.getBytes(put);
211 Put desPut = (Put)Writables.getWritable(sb, new Put());
212
213 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
214 List<KeyValue> list = null;
215 List<KeyValue> desList = null;
216 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
217 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
218 list = entry.getValue();
219 desList = desPut.getFamilyMap().get(entry.getKey());
220 for(int i=0; i<list.size(); i++){
221 assertTrue(list.get(i).equals(desList.get(i)));
222 }
223 }
224 }
225
226
227 @Test public void testDelete() throws Exception{
228 byte[] row = "row".getBytes();
229 byte[] fam = "fam".getBytes();
230 byte[] qf1 = "qf1".getBytes();
231
232 long ts = System.currentTimeMillis();
233
234 Delete delete = new Delete(row);
235 delete.deleteColumn(fam, qf1, ts);
236
237 byte[] sb = Writables.getBytes(delete);
238 Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
239
240 assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
241 List<KeyValue> list = null;
242 List<KeyValue> desList = null;
243 for(Map.Entry<byte[], List<KeyValue>> entry :
244 delete.getFamilyMap().entrySet()){
245 assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
246 list = entry.getValue();
247 desList = desDelete.getFamilyMap().get(entry.getKey());
248 for(int i=0; i<list.size(); i++){
249 assertTrue(list.get(i).equals(desList.get(i)));
250 }
251 }
252 }
253 */
254
255 @Test public void testGet() throws Exception{
256 byte[] row = "row".getBytes();
257 byte[] fam = "fam".getBytes();
258 byte[] qf1 = "qf1".getBytes();
259
260 long ts = System.currentTimeMillis();
261 int maxVersions = 2;
262
263 Get get = new Get(row);
264 get.addColumn(fam, qf1);
265 get.setTimeRange(ts, ts+1);
266 get.setMaxVersions(maxVersions);
267
268 ClientProtos.Get getProto = ProtobufUtil.toGet(get);
269 Get desGet = ProtobufUtil.toGet(getProto);
270
271 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
272 Set<byte[]> set = null;
273 Set<byte[]> desSet = null;
274
275 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
276 get.getFamilyMap().entrySet()){
277 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
278 set = entry.getValue();
279 desSet = desGet.getFamilyMap().get(entry.getKey());
280 for(byte [] qualifier : set){
281 assertTrue(desSet.contains(qualifier));
282 }
283 }
284
285 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
286 TimeRange tr = get.getTimeRange();
287 TimeRange desTr = desGet.getTimeRange();
288 assertEquals(tr.getMax(), desTr.getMax());
289 assertEquals(tr.getMin(), desTr.getMin());
290 }
291
292
293 @Test public void testScan() throws Exception {
294
295 byte[] startRow = "startRow".getBytes();
296 byte[] stopRow = "stopRow".getBytes();
297 byte[] fam = "fam".getBytes();
298 byte[] qf1 = "qf1".getBytes();
299
300 long ts = System.currentTimeMillis();
301 int maxVersions = 2;
302
303 Scan scan = new Scan(startRow, stopRow);
304 scan.addColumn(fam, qf1);
305 scan.setTimeRange(ts, ts+1);
306 scan.setMaxVersions(maxVersions);
307
308 ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
309 Scan desScan = ProtobufUtil.toScan(scanProto);
310
311 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
312 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
313 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
314 Set<byte[]> set = null;
315 Set<byte[]> desSet = null;
316
317 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
318 scan.getFamilyMap().entrySet()){
319 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
320 set = entry.getValue();
321 desSet = desScan.getFamilyMap().get(entry.getKey());
322 for(byte[] column : set){
323 assertTrue(desSet.contains(column));
324 }
325
326 // Test filters are serialized properly.
327 scan = new Scan(startRow);
328 final String name = "testScan";
329 byte [] prefix = Bytes.toBytes(name);
330 scan.setFilter(new PrefixFilter(prefix));
331 scanProto = ProtobufUtil.toScan(scan);
332 desScan = ProtobufUtil.toScan(scanProto);
333 Filter f = desScan.getFilter();
334 assertTrue(f instanceof PrefixFilter);
335 }
336
337 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
338 TimeRange tr = scan.getTimeRange();
339 TimeRange desTr = desScan.getTimeRange();
340 assertEquals(tr.getMax(), desTr.getMax());
341 assertEquals(tr.getMin(), desTr.getMin());
342 }
343
344 /*
345 * TODO
346 @Test public void testResultEmpty() throws Exception {
347 List<KeyValue> keys = new ArrayList<KeyValue>();
348 Result r = new Result(keys);
349 assertTrue(r.isEmpty());
350 byte [] rb = Writables.getBytes(r);
351 Result deserializedR = (Result)Writables.getWritable(rb, new Result());
352 assertTrue(deserializedR.isEmpty());
353 }
354
355
356 @Test public void testResult() throws Exception {
357 byte [] rowA = Bytes.toBytes("rowA");
358 byte [] famA = Bytes.toBytes("famA");
359 byte [] qfA = Bytes.toBytes("qfA");
360 byte [] valueA = Bytes.toBytes("valueA");
361
362 byte [] rowB = Bytes.toBytes("rowB");
363 byte [] famB = Bytes.toBytes("famB");
364 byte [] qfB = Bytes.toBytes("qfB");
365 byte [] valueB = Bytes.toBytes("valueB");
366
367 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
368 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
369
370 Result result = new Result(new KeyValue[]{kvA, kvB});
371
372 byte [] rb = Writables.getBytes(result);
373 Result deResult = (Result)Writables.getWritable(rb, new Result());
374
375 assertTrue("results are not equivalent, first key mismatch",
376 result.raw()[0].equals(deResult.raw()[0]));
377
378 assertTrue("results are not equivalent, second key mismatch",
379 result.raw()[1].equals(deResult.raw()[1]));
380
381 // Test empty Result
382 Result r = new Result();
383 byte [] b = Writables.getBytes(r);
384 Result deserialized = (Result)Writables.getWritable(b, new Result());
385 assertEquals(r.size(), deserialized.size());
386 }
387
388 @Test public void testResultDynamicBuild() throws Exception {
389 byte [] rowA = Bytes.toBytes("rowA");
390 byte [] famA = Bytes.toBytes("famA");
391 byte [] qfA = Bytes.toBytes("qfA");
392 byte [] valueA = Bytes.toBytes("valueA");
393
394 byte [] rowB = Bytes.toBytes("rowB");
395 byte [] famB = Bytes.toBytes("famB");
396 byte [] qfB = Bytes.toBytes("qfB");
397 byte [] valueB = Bytes.toBytes("valueB");
398
399 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
400 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
401
402 Result result = new Result(new KeyValue[]{kvA, kvB});
403
404 byte [] rb = Writables.getBytes(result);
405
406
407 // Call getRow() first
408 Result deResult = (Result)Writables.getWritable(rb, new Result());
409 byte [] row = deResult.getRow();
410 assertTrue(Bytes.equals(row, rowA));
411
412 // Call sorted() first
413 deResult = (Result)Writables.getWritable(rb, new Result());
414 assertTrue("results are not equivalent, first key mismatch",
415 result.raw()[0].equals(deResult.raw()[0]));
416 assertTrue("results are not equivalent, second key mismatch",
417 result.raw()[1].equals(deResult.raw()[1]));
418
419 // Call raw() first
420 deResult = (Result)Writables.getWritable(rb, new Result());
421 assertTrue("results are not equivalent, first key mismatch",
422 result.raw()[0].equals(deResult.raw()[0]));
423 assertTrue("results are not equivalent, second key mismatch",
424 result.raw()[1].equals(deResult.raw()[1]));
425
426
427 }
428
429 @Test public void testResultArray() throws Exception {
430 byte [] rowA = Bytes.toBytes("rowA");
431 byte [] famA = Bytes.toBytes("famA");
432 byte [] qfA = Bytes.toBytes("qfA");
433 byte [] valueA = Bytes.toBytes("valueA");
434
435 byte [] rowB = Bytes.toBytes("rowB");
436 byte [] famB = Bytes.toBytes("famB");
437 byte [] qfB = Bytes.toBytes("qfB");
438 byte [] valueB = Bytes.toBytes("valueB");
439
440 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
441 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
442
443
444 Result result1 = new Result(new KeyValue[]{kvA, kvB});
445 Result result2 = new Result(new KeyValue[]{kvB});
446 Result result3 = new Result(new KeyValue[]{kvB});
447
448 Result [] results = new Result [] {result1, result2, result3};
449
450 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
451 DataOutputStream out = new DataOutputStream(byteStream);
452 Result.writeArray(out, results);
453
454 byte [] rb = byteStream.toByteArray();
455
456 DataInputBuffer in = new DataInputBuffer();
457 in.reset(rb, 0, rb.length);
458
459 Result [] deResults = Result.readArray(in);
460
461 assertTrue(results.length == deResults.length);
462
463 for(int i=0;i<results.length;i++) {
464 KeyValue [] keysA = results[i].raw();
465 KeyValue [] keysB = deResults[i].raw();
466 assertTrue(keysA.length == keysB.length);
467 for(int j=0;j<keysA.length;j++) {
468 assertTrue("Expected equivalent keys but found:\n" +
469 "KeyA : " + keysA[j].toString() + "\n" +
470 "KeyB : " + keysB[j].toString() + "\n" +
471 keysA.length + " total keys, " + i + "th so far"
472 ,keysA[j].equals(keysB[j]));
473 }
474 }
475
476 }
477
478 @Test public void testResultArrayEmpty() throws Exception {
479 List<KeyValue> keys = new ArrayList<KeyValue>();
480 Result r = new Result(keys);
481 Result [] results = new Result [] {r};
482
483 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
484 DataOutputStream out = new DataOutputStream(byteStream);
485
486 Result.writeArray(out, results);
487
488 results = null;
489
490 byteStream = new ByteArrayOutputStream();
491 out = new DataOutputStream(byteStream);
492 Result.writeArray(out, results);
493
494 byte [] rb = byteStream.toByteArray();
495
496 DataInputBuffer in = new DataInputBuffer();
497 in.reset(rb, 0, rb.length);
498
499 Result [] deResults = Result.readArray(in);
500
501 assertTrue(deResults.length == 0);
502
503 results = new Result[0];
504
505 byteStream = new ByteArrayOutputStream();
506 out = new DataOutputStream(byteStream);
507 Result.writeArray(out, results);
508
509 rb = byteStream.toByteArray();
510
511 in = new DataInputBuffer();
512 in.reset(rb, 0, rb.length);
513
514 deResults = Result.readArray(in);
515
516 assertTrue(deResults.length == 0);
517
518 }
519 */
520
521 protected static final int MAXVERSIONS = 3;
522 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
523 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
524 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
525 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
526
527 /**
528 * Create a table of name <code>name</code> with {@link COLUMNS} for
529 * families.
530 * @param name Name to give table.
531 * @return Column descriptor.
532 */
533 protected HTableDescriptor createTableDescriptor(final String name) {
534 return createTableDescriptor(name, MAXVERSIONS);
535 }
536
537 /**
538 * Create a table of name <code>name</code> with {@link COLUMNS} for
539 * families.
540 * @param name Name to give table.
541 * @param versions How many versions to allow per column.
542 * @return Column descriptor.
543 */
544 protected HTableDescriptor createTableDescriptor(final String name,
545 final int versions) {
546 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
547 htd.addFamily(new HColumnDescriptor(fam1)
548 .setMaxVersions(versions)
549 .setBlockCacheEnabled(false)
550 );
551 htd.addFamily(new HColumnDescriptor(fam2)
552 .setMaxVersions(versions)
553 .setBlockCacheEnabled(false)
554 );
555 htd.addFamily(new HColumnDescriptor(fam3)
556 .setMaxVersions(versions)
557 .setBlockCacheEnabled(false)
558 );
559 return htd;
560 }
561 }