1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.lang.reflect.Field;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.concurrent.ThreadPoolExecutor;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.*;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.JVMClusterUtil;
33  import org.junit.AfterClass;
34  import org.junit.Assert;
35  import org.junit.Before;
36  import org.junit.BeforeClass;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  import static org.junit.Assert.*;
41  
42  @Category(MediumTests.class)
43  public class TestMultiParallel {
44    private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
45    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
46    private static final byte[] VALUE = Bytes.toBytes("value");
47    private static final byte[] QUALIFIER = Bytes.toBytes("qual");
48    private static final String FAMILY = "family";
49    private static final String TEST_TABLE = "multi_test_table";
50    private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
51    private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
52    private static final byte [][] KEYS = makeKeys();
53  
54    private static final int slaves = 2; // also used for testing HTable pool size
55  
56    @BeforeClass public static void beforeClass() throws Exception {
57      UTIL.startMiniCluster(slaves);
58      HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
59      UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
60      UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000);
61      t.close();
62    }
63  
64    @AfterClass public static void afterClass() throws Exception {
65      UTIL.shutdownMiniCluster();
66    }
67  
68    @Before public void before() throws IOException {
69      LOG.info("before");
70      if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
71        // Distribute regions
72        UTIL.getMiniHBaseCluster().getMaster().balance();
73      }
74      LOG.info("before done");
75    }
76  
77    private static byte[][] makeKeys() {
78      byte [][] starterKeys = HBaseTestingUtility.KEYS;
79      // Create a "non-uniform" test set with the following characteristics:
80      // a) Unequal number of keys per region
81  
82      // Don't use integer as a multiple, so that we have a number of keys that is
83      // not a multiple of the number of regions
84      int numKeys = (int) ((float) starterKeys.length * 10.33F);
85  
86      List<byte[]> keys = new ArrayList<byte[]>();
87      for (int i = 0; i < numKeys; i++) {
88        int kIdx = i % starterKeys.length;
89        byte[] k = starterKeys[kIdx];
90        byte[] cp = new byte[k.length + 1];
91        System.arraycopy(k, 0, cp, 0, k.length);
92        cp[k.length] = new Integer(i % 256).byteValue();
93        keys.add(cp);
94      }
95  
96      // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which
97      // should work)
98      // c) keys are not in sorted order (within a region), to ensure that the
99      // sorting code and index mapping doesn't break the functionality
100     for (int i = 0; i < 100; i++) {
101       int kIdx = i % starterKeys.length;
102       byte[] k = starterKeys[kIdx];
103       byte[] cp = new byte[k.length + 1];
104       System.arraycopy(k, 0, cp, 0, k.length);
105       cp[k.length] = new Integer(i % 256).byteValue();
106       keys.add(cp);
107     }
108     return keys.toArray(new byte [][] {new byte [] {}});
109   }
110 
111 
112   /**
113    * This is for testing the active number of threads that were used while
114    * doing a batch operation. It inserts one row per region via the batch
115    * operation, and then checks the number of active threads.
116    * For HBASE-3553
117    * @throws IOException
118    * @throws InterruptedException
119    * @throws NoSuchFieldException
120    * @throws SecurityException
121    */
122   @Test(timeout=300000) 
123   public void testActiveThreadsCount() throws Exception{
124     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
125     List<Row> puts = constructPutRequests(); // creates a Put for every region
126     table.batch(puts);
127     Field poolField = table.getClass().getDeclaredField("pool");
128     poolField.setAccessible(true);
129     ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
130     assertEquals(slaves, tExecutor.getLargestPoolSize());
131     table.close();
132   }
133 
134   @Test(timeout=300000) 
135   public void testBatchWithGet() throws Exception {
136     LOG.info("test=testBatchWithGet");
137     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
138 
139     // load test data
140     List<Row> puts = constructPutRequests();
141     table.batch(puts);
142 
143     // create a list of gets and run it
144     List<Row> gets = new ArrayList<Row>();
145     for (byte[] k : KEYS) {
146       Get get = new Get(k);
147       get.addColumn(BYTES_FAMILY, QUALIFIER);
148       gets.add(get);
149     }
150     Result[] multiRes = new Result[gets.size()];
151     table.batch(gets, multiRes);
152 
153     // Same gets using individual call API
154     List<Result> singleRes = new ArrayList<Result>();
155     for (Row get : gets) {
156       singleRes.add(table.get((Get) get));
157     }
158 
159     // Compare results
160     Assert.assertEquals(singleRes.size(), multiRes.length);
161     for (int i = 0; i < singleRes.size(); i++) {
162       Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
163       KeyValue[] singleKvs = singleRes.get(i).raw();
164       KeyValue[] multiKvs = multiRes[i].raw();
165       for (int j = 0; j < singleKvs.length; j++) {
166         Assert.assertEquals(singleKvs[j], multiKvs[j]);
167         Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
168             .getValue()));
169       }
170     }
171     table.close();
172   }
173 
174   @Test
175   public void testBadFam() throws Exception {
176     LOG.info("test=testBadFam");
177     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
178 
179     List<Row> actions = new ArrayList<Row>();
180     Put p = new Put(Bytes.toBytes("row1"));
181     p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
182     actions.add(p);
183     p = new Put(Bytes.toBytes("row2"));
184     p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
185     actions.add(p);
186 
187     // row1 and row2 should be in the same region.
188 
189     Object [] r = new Object[actions.size()];
190     try {
191       table.batch(actions, r);
192       fail();
193     } catch (RetriesExhaustedWithDetailsException ex) {
194       LOG.debug(ex);
195       // good!
196       assertFalse(ex.mayHaveClusterIssues());
197     }
198     assertEquals(2, r.length);
199     assertTrue(r[0] instanceof Throwable);
200     assertTrue(r[1] instanceof Result);
201     table.close();
202   }
203 
204   /**
205    * Only run one Multi test with a forced RegionServer abort. Otherwise, the
206    * unit tests will take an unnecessarily long time to run.
207    *
208    * @throws Exception
209    */
210   @Test (timeout=300000) 
211   public void testFlushCommitsWithAbort() throws Exception {
212     LOG.info("test=testFlushCommitsWithAbort");
213     doTestFlushCommits(true);
214   }
215 
216   @Test (timeout=300000)
217   public void testFlushCommitsNoAbort() throws Exception {
218     LOG.info("test=testFlushCommitsNoAbort");
219     doTestFlushCommits(false);
220   }
221 
222   /**
223    * Set table auto flush to false and test flushing commits
224    * @param doAbort true if abort one regionserver in the testing
225    * @throws Exception
226    */
227   private void doTestFlushCommits(boolean doAbort) throws Exception {
228     // Load the data
229     LOG.info("get new table");
230     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
231     table.setAutoFlush(false);
232     table.setWriteBufferSize(10 * 1024 * 1024);
233 
234     LOG.info("constructPutRequests");
235     List<Row> puts = constructPutRequests();
236     for (Row put : puts) {
237       table.put((Put) put);
238     }
239     LOG.info("puts");
240     table.flushCommits();
241     int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
242         .size();
243     assert liveRScount > 0;
244     JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
245         .getLiveRegionServerThreads().get(0);
246     if (doAbort) {
247       LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
248 
249       // If we waiting for no regions being online after we abort the server, we
250       // could ensure the master has re-assigned the regions on killed server
251       // after putting keys successfully, it means the server we abort is dead
252       // and detected by matser
253       while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
254         Thread.sleep(100);
255       }
256       while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) {
257         Thread.sleep(100);
258       }
259 
260       // try putting more keys after the abort. same key/qual... just validating
261       // no exceptions thrown
262       puts = constructPutRequests();
263       for (Row put : puts) {
264         table.put((Put) put);
265       }
266 
267       table.flushCommits();
268     }
269 
270     LOG.info("validating loaded data");
271     validateLoadedData(table);
272 
273     // Validate server and region count
274     List<JVMClusterUtil.RegionServerThread> liveRSs =
275       UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
276     int count = 0;
277     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
278       count++;
279       LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
280     }
281     LOG.info("Count=" + count);
282     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
283         (doAbort ? (liveRScount - 1) : liveRScount), count);
284     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
285       int regions = t.getRegionServer().getOnlineRegions().size();
286       Assert.assertTrue("Count of regions=" + regions, regions > 10);
287     }
288     table.close();
289     LOG.info("done");
290   }
291 
292   @Test (timeout=300000)
293   public void testBatchWithPut() throws Exception {
294     LOG.info("test=testBatchWithPut");
295     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
296 
297     // put multiple rows using a batch
298     List<Row> puts = constructPutRequests();
299 
300     Object[] results = table.batch(puts);
301     validateSizeAndEmpty(results, KEYS.length);
302 
303     if (true) {
304       int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
305           .size();
306       assert liveRScount > 0;
307       JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
308           .getLiveRegionServerThreads().get(0);
309       liveRS.getRegionServer().abort("Aborting for tests",
310           new Exception("testBatchWithPut"));
311 
312       puts = constructPutRequests();
313       results = table.batch(puts);
314       validateSizeAndEmpty(results, KEYS.length);
315     }
316 
317     validateLoadedData(table);
318     table.close();
319   }
320 
321   @Test(timeout=300000) 
322   public void testBatchWithDelete() throws Exception {
323     LOG.info("test=testBatchWithDelete");
324     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
325 
326     // Load some data
327     List<Row> puts = constructPutRequests();
328     Object[] results = table.batch(puts);
329     validateSizeAndEmpty(results, KEYS.length);
330 
331     // Deletes
332     List<Row> deletes = new ArrayList<Row>();
333     for (int i = 0; i < KEYS.length; i++) {
334       Delete delete = new Delete(KEYS[i]);
335       delete.deleteFamily(BYTES_FAMILY);
336       deletes.add(delete);
337     }
338     results = table.batch(deletes);
339     validateSizeAndEmpty(results, KEYS.length);
340 
341     // Get to make sure ...
342     for (byte[] k : KEYS) {
343       Get get = new Get(k);
344       get.addColumn(BYTES_FAMILY, QUALIFIER);
345       Assert.assertFalse(table.exists(get));
346     }
347     table.close();
348   }
349 
350   @Test(timeout=300000)
351   public void testHTableDeleteWithList() throws Exception {
352     LOG.info("test=testHTableDeleteWithList");
353     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
354 
355     // Load some data
356     List<Row> puts = constructPutRequests();
357     Object[] results = table.batch(puts);
358     validateSizeAndEmpty(results, KEYS.length);
359 
360     // Deletes
361     ArrayList<Delete> deletes = new ArrayList<Delete>();
362     for (int i = 0; i < KEYS.length; i++) {
363       Delete delete = new Delete(KEYS[i]);
364       delete.deleteFamily(BYTES_FAMILY);
365       deletes.add(delete);
366     }
367     table.delete(deletes);
368     Assert.assertTrue(deletes.isEmpty());
369 
370     // Get to make sure ...
371     for (byte[] k : KEYS) {
372       Get get = new Get(k);
373       get.addColumn(BYTES_FAMILY, QUALIFIER);
374       Assert.assertFalse(table.exists(get));
375     }
376     table.close();
377   }
378 
379   @Test(timeout=300000)
380   public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
381     LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
382     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
383 
384     List<Row> puts = new ArrayList<Row>();
385     for (int i = 0; i < 100; i++) {
386       Put put = new Put(ONE_ROW);
387       byte[] qual = Bytes.toBytes("column" + i);
388       put.add(BYTES_FAMILY, qual, VALUE);
389       puts.add(put);
390     }
391     Object[] results = table.batch(puts);
392 
393     // validate
394     validateSizeAndEmpty(results, 100);
395 
396     // get the data back and validate that it is correct
397     List<Row> gets = new ArrayList<Row>();
398     for (int i = 0; i < 100; i++) {
399       Get get = new Get(ONE_ROW);
400       byte[] qual = Bytes.toBytes("column" + i);
401       get.addColumn(BYTES_FAMILY, qual);
402       gets.add(get);
403     }
404 
405     Object[] multiRes = table.batch(gets);
406 
407     int idx = 0;
408     for (Object r : multiRes) {
409       byte[] qual = Bytes.toBytes("column" + idx);
410       validateResult(r, qual, VALUE);
411       idx++;
412     }
413     table.close();
414   }
415 
416   @Test(timeout=300000)
417   public void testBatchWithIncrementAndAppend() throws Exception {
418     LOG.info("test=testBatchWithIncrementAndAppend");
419     final byte[] QUAL1 = Bytes.toBytes("qual1");
420     final byte[] QUAL2 = Bytes.toBytes("qual2");
421     final byte[] QUAL3 = Bytes.toBytes("qual3");
422     final byte[] QUAL4 = Bytes.toBytes("qual4");
423     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
424     Delete d = new Delete(ONE_ROW);
425     table.delete(d);
426     Put put = new Put(ONE_ROW);
427     put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
428     put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
429     table.put(put);
430 
431     Increment inc = new Increment(ONE_ROW);
432     inc.addColumn(BYTES_FAMILY, QUAL2, 1);
433     inc.addColumn(BYTES_FAMILY, QUAL3, 1);
434 
435     Append a = new Append(ONE_ROW);
436     a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
437     a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
438     List<Row> actions = new ArrayList<Row>();
439     actions.add(inc);
440     actions.add(a);
441 
442     Object[] multiRes = table.batch(actions);
443     validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
444     validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
445     validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
446     validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
447     table.close();
448   }
449 
450   @Test(timeout=300000)
451   public void testBatchWithMixedActions() throws Exception {
452     LOG.info("test=testBatchWithMixedActions");
453     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
454 
455     // Load some data to start
456     Object[] results = table.batch(constructPutRequests());
457     validateSizeAndEmpty(results, KEYS.length);
458 
459     // Batch: get, get, put(new col), delete, get, get of put, get of deleted,
460     // put
461     List<Row> actions = new ArrayList<Row>();
462 
463     byte[] qual2 = Bytes.toBytes("qual2");
464     byte[] val2 = Bytes.toBytes("putvalue2");
465 
466     // 0 get
467     Get get = new Get(KEYS[10]);
468     get.addColumn(BYTES_FAMILY, QUALIFIER);
469     actions.add(get);
470 
471     // 1 get
472     get = new Get(KEYS[11]);
473     get.addColumn(BYTES_FAMILY, QUALIFIER);
474     actions.add(get);
475 
476     // 2 put of new column
477     Put put = new Put(KEYS[10]);
478     put.add(BYTES_FAMILY, qual2, val2);
479     actions.add(put);
480 
481     // 3 delete
482     Delete delete = new Delete(KEYS[20]);
483     delete.deleteFamily(BYTES_FAMILY);
484     actions.add(delete);
485 
486     // 4 get
487     get = new Get(KEYS[30]);
488     get.addColumn(BYTES_FAMILY, QUALIFIER);
489     actions.add(get);
490 
491     // There used to be a 'get' of a previous put here, but removed
492     // since this API really cannot guarantee order in terms of mixed
493     // get/puts.
494 
495     // 5 put of new column
496     put = new Put(KEYS[40]);
497     put.add(BYTES_FAMILY, qual2, val2);
498     actions.add(put);
499 
500     results = table.batch(actions);
501 
502     // Validation
503 
504     validateResult(results[0]);
505     validateResult(results[1]);
506     validateEmpty(results[2]);
507     validateEmpty(results[3]);
508     validateResult(results[4]);
509     validateEmpty(results[5]);
510 
511     // validate last put, externally from the batch
512     get = new Get(KEYS[40]);
513     get.addColumn(BYTES_FAMILY, qual2);
514     Result r = table.get(get);
515     validateResult(r, qual2, val2);
516 
517     table.close();
518   }
519 
520   // // Helper methods ////
521 
522   private void validateResult(Object r) {
523     validateResult(r, QUALIFIER, VALUE);
524   }
525 
526   private void validateResult(Object r1, byte[] qual, byte[] val) {
527     // TODO provide nice assert here or something.
528     Result r = (Result)r1;
529     Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
530     Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
531   }
532 
533   private List<Row> constructPutRequests() {
534     List<Row> puts = new ArrayList<Row>();
535     for (byte[] k : KEYS) {
536       Put put = new Put(k);
537       put.add(BYTES_FAMILY, QUALIFIER, VALUE);
538       puts.add(put);
539     }
540     return puts;
541   }
542 
543   private void validateLoadedData(HTable table) throws IOException {
544     // get the data back and validate that it is correct
545     for (byte[] k : KEYS) {
546       Get get = new Get(k);
547       get.addColumn(BYTES_FAMILY, QUALIFIER);
548       Result r = table.get(get);
549       Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
550       Assert.assertEquals(0, Bytes.compareTo(VALUE, r
551           .getValue(BYTES_FAMILY, QUALIFIER)));
552     }
553   }
554 
555   private void validateEmpty(Object r1) {
556     Result result = (Result)r1;
557     Assert.assertTrue(result != null);
558     Assert.assertTrue(result.getRow() == null);
559     Assert.assertEquals(0, result.raw().length);
560   }
561 
562   private void validateSizeAndEmpty(Object[] results, int expectedSize) {
563     // Validate got back the same number of Result objects, all empty
564     Assert.assertEquals(expectedSize, results.length);
565     for (Object result : results) {
566       validateEmpty(result);
567     }
568   }
569 
570   @org.junit.Rule
571   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
572     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
573 }
574