1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.JVMClusterUtil;
32 import org.junit.AfterClass;
33 import org.junit.Assert;
34 import org.junit.Before;
35 import org.junit.BeforeClass;
36 import org.junit.Test;
37
38 import static org.junit.Assert.*;
39
40 public class TestMultiParallel {
41 private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
42 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
43 private static final byte[] VALUE = Bytes.toBytes("value");
44 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
45 private static final String FAMILY = "family";
46 private static final String TEST_TABLE = "multi_test_table";
47 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
48 private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
49 private static final byte [][] KEYS = makeKeys();
50
51 @BeforeClass public static void beforeClass() throws Exception {
52 UTIL.startMiniCluster(2);
53 HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
54 UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
55 }
56
57 @AfterClass public static void afterClass() throws IOException {
58 UTIL.getMiniHBaseCluster().shutdown();
59 }
60
61 @Before public void before() throws IOException {
62 LOG.info("before");
63 if (UTIL.ensureSomeRegionServersAvailable(2)) {
64
65 UTIL.getMiniHBaseCluster().getMaster().balance();
66 }
67 LOG.info("before done");
68 }
69
70 private static byte[][] makeKeys() {
71 byte [][] starterKeys = HBaseTestingUtility.KEYS;
72
73
74
75
76
77 int numKeys = (int) ((float) starterKeys.length * 10.33F);
78
79 List<byte[]> keys = new ArrayList<byte[]>();
80 for (int i = 0; i < numKeys; i++) {
81 int kIdx = i % starterKeys.length;
82 byte[] k = starterKeys[kIdx];
83 byte[] cp = new byte[k.length + 1];
84 System.arraycopy(k, 0, cp, 0, k.length);
85 cp[k.length] = new Integer(i % 256).byteValue();
86 keys.add(cp);
87 }
88
89
90
91
92
93 for (int i = 0; i < 100; i++) {
94 int kIdx = i % starterKeys.length;
95 byte[] k = starterKeys[kIdx];
96 byte[] cp = new byte[k.length + 1];
97 System.arraycopy(k, 0, cp, 0, k.length);
98 cp[k.length] = new Integer(i % 256).byteValue();
99 keys.add(cp);
100 }
101 return keys.toArray(new byte [][] {new byte [] {}});
102 }
103
104 @Test public void testBatchWithGet() throws Exception {
105 LOG.info("test=testBatchWithGet");
106 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
107
108
109 List<Row> puts = constructPutRequests();
110 table.batch(puts);
111
112
113 List<Row> gets = new ArrayList<Row>();
114 for (byte[] k : KEYS) {
115 Get get = new Get(k);
116 get.addColumn(BYTES_FAMILY, QUALIFIER);
117 gets.add(get);
118 }
119 Result[] multiRes = new Result[gets.size()];
120 table.batch(gets, multiRes);
121
122
123 List<Result> singleRes = new ArrayList<Result>();
124 for (Row get : gets) {
125 singleRes.add(table.get((Get) get));
126 }
127
128
129 Assert.assertEquals(singleRes.size(), multiRes.length);
130 for (int i = 0; i < singleRes.size(); i++) {
131 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
132 KeyValue[] singleKvs = singleRes.get(i).raw();
133 KeyValue[] multiKvs = multiRes[i].raw();
134 for (int j = 0; j < singleKvs.length; j++) {
135 Assert.assertEquals(singleKvs[j], multiKvs[j]);
136 Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
137 .getValue()));
138 }
139 }
140 }
141
142 @Test
143 public void testBadFam() throws Exception {
144 LOG.info("test=testBadFam");
145 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
146
147 List<Row> actions = new ArrayList<Row>();
148 Put p = new Put(Bytes.toBytes("row1"));
149 p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
150 actions.add(p);
151 p = new Put(Bytes.toBytes("row2"));
152 p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
153 actions.add(p);
154
155
156
157 Object [] r = new Object[actions.size()];
158 try {
159 table.batch(actions, r);
160 fail();
161 } catch (RetriesExhaustedWithDetailsException ex) {
162 LOG.debug(ex);
163
164 assertFalse(ex.mayHaveClusterIssues());
165 }
166 assertEquals(2, r.length);
167 assertTrue(r[0] instanceof Throwable);
168 assertTrue(r[1] instanceof Result);
169 }
170
171
172
173
174
175
176
177 @Test public void testFlushCommitsWithAbort() throws Exception {
178 LOG.info("test=testFlushCommitsWithAbort");
179 doTestFlushCommits(true);
180 }
181
182 @Test public void testFlushCommitsNoAbort() throws Exception {
183 LOG.info("test=testFlushCommitsNoAbort");
184 doTestFlushCommits(false);
185 }
186
187 private void doTestFlushCommits(boolean doAbort) throws Exception {
188
189 LOG.info("get new table");
190 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
191 table.setAutoFlush(false);
192 table.setWriteBufferSize(10 * 1024 * 1024);
193
194 LOG.info("constructPutRequests");
195 List<Row> puts = constructPutRequests();
196 for (Row put : puts) {
197 table.put((Put) put);
198 }
199 LOG.info("puts");
200 table.flushCommits();
201 if (doAbort) {
202 LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
203
204
205
206 puts = constructPutRequests();
207 for (Row put : puts) {
208 table.put((Put) put);
209 }
210
211 table.flushCommits();
212 }
213
214 LOG.info("validating loaded data");
215 validateLoadedData(table);
216
217
218 List<JVMClusterUtil.RegionServerThread> liveRSs =
219 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
220 int count = 0;
221 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
222 count++;
223 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
224 }
225 LOG.info("Count=" + count);
226 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
227 (doAbort ? 1 : 2), count);
228 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
229 int regions = t.getRegionServer().getOnlineRegions().size();
230 Assert.assertTrue("Count of regions=" + regions, regions > 10);
231 }
232 LOG.info("done");
233 }
234
235 @Test public void testBatchWithPut() throws Exception {
236 LOG.info("test=testBatchWithPut");
237 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
238
239
240 List<Row> puts = constructPutRequests();
241
242 Object[] results = table.batch(puts);
243 validateSizeAndEmpty(results, KEYS.length);
244
245 if (true) {
246 UTIL.getMiniHBaseCluster().abortRegionServer(0);
247
248 puts = constructPutRequests();
249 results = table.batch(puts);
250 validateSizeAndEmpty(results, KEYS.length);
251 }
252
253 validateLoadedData(table);
254 }
255
256 @Test public void testBatchWithDelete() throws Exception {
257 LOG.info("test=testBatchWithDelete");
258 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
259
260
261 List<Row> puts = constructPutRequests();
262 Object[] results = table.batch(puts);
263 validateSizeAndEmpty(results, KEYS.length);
264
265
266 List<Row> deletes = new ArrayList<Row>();
267 for (int i = 0; i < KEYS.length; i++) {
268 Delete delete = new Delete(KEYS[i]);
269 delete.deleteFamily(BYTES_FAMILY);
270 deletes.add(delete);
271 }
272 results = table.batch(deletes);
273 validateSizeAndEmpty(results, KEYS.length);
274
275
276 for (byte[] k : KEYS) {
277 Get get = new Get(k);
278 get.addColumn(BYTES_FAMILY, QUALIFIER);
279 Assert.assertFalse(table.exists(get));
280 }
281
282 }
283
284 @Test public void testHTableDeleteWithList() throws Exception {
285 LOG.info("test=testHTableDeleteWithList");
286 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
287
288
289 List<Row> puts = constructPutRequests();
290 Object[] results = table.batch(puts);
291 validateSizeAndEmpty(results, KEYS.length);
292
293
294 ArrayList<Delete> deletes = new ArrayList<Delete>();
295 for (int i = 0; i < KEYS.length; i++) {
296 Delete delete = new Delete(KEYS[i]);
297 delete.deleteFamily(BYTES_FAMILY);
298 deletes.add(delete);
299 }
300 table.delete(deletes);
301 Assert.assertTrue(deletes.isEmpty());
302
303
304 for (byte[] k : KEYS) {
305 Get get = new Get(k);
306 get.addColumn(BYTES_FAMILY, QUALIFIER);
307 Assert.assertFalse(table.exists(get));
308 }
309
310 }
311
312 @Test public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
313 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
314 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
315
316 List<Row> puts = new ArrayList<Row>();
317 for (int i = 0; i < 100; i++) {
318 Put put = new Put(ONE_ROW);
319 byte[] qual = Bytes.toBytes("column" + i);
320 put.add(BYTES_FAMILY, qual, VALUE);
321 puts.add(put);
322 }
323 Object[] results = table.batch(puts);
324
325
326 validateSizeAndEmpty(results, 100);
327
328
329 List<Row> gets = new ArrayList<Row>();
330 for (int i = 0; i < 100; i++) {
331 Get get = new Get(ONE_ROW);
332 byte[] qual = Bytes.toBytes("column" + i);
333 get.addColumn(BYTES_FAMILY, qual);
334 gets.add(get);
335 }
336
337 Object[] multiRes = table.batch(gets);
338
339 int idx = 0;
340 for (Object r : multiRes) {
341 byte[] qual = Bytes.toBytes("column" + idx);
342 validateResult(r, qual, VALUE);
343 idx++;
344 }
345
346 }
347
348 @Test public void testBatchWithMixedActions() throws Exception {
349 LOG.info("test=testBatchWithMixedActions");
350 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
351
352
353 Object[] results = table.batch(constructPutRequests());
354 validateSizeAndEmpty(results, KEYS.length);
355
356
357
358 List<Row> actions = new ArrayList<Row>();
359
360 byte[] qual2 = Bytes.toBytes("qual2");
361 byte[] val2 = Bytes.toBytes("putvalue2");
362
363
364 Get get = new Get(KEYS[10]);
365 get.addColumn(BYTES_FAMILY, QUALIFIER);
366 actions.add(get);
367
368
369 get = new Get(KEYS[11]);
370 get.addColumn(BYTES_FAMILY, QUALIFIER);
371 actions.add(get);
372
373
374 Put put = new Put(KEYS[10]);
375 put.add(BYTES_FAMILY, qual2, val2);
376 actions.add(put);
377
378
379 Delete delete = new Delete(KEYS[20]);
380 delete.deleteFamily(BYTES_FAMILY);
381 actions.add(delete);
382
383
384 get = new Get(KEYS[30]);
385 get.addColumn(BYTES_FAMILY, QUALIFIER);
386 actions.add(get);
387
388
389
390
391
392
393 put = new Put(KEYS[40]);
394 put.add(BYTES_FAMILY, qual2, val2);
395 actions.add(put);
396
397 results = table.batch(actions);
398
399
400
401 validateResult(results[0]);
402 validateResult(results[1]);
403 validateEmpty(results[2]);
404 validateEmpty(results[3]);
405 validateResult(results[4]);
406 validateEmpty(results[5]);
407
408
409 get = new Get(KEYS[40]);
410 get.addColumn(BYTES_FAMILY, qual2);
411 Result r = table.get(get);
412 validateResult(r, qual2, val2);
413 }
414
415
416
417 private void validateResult(Object r) {
418 validateResult(r, QUALIFIER, VALUE);
419 }
420
421 private void validateResult(Object r1, byte[] qual, byte[] val) {
422
423 Result r = (Result)r1;
424 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
425 Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
426 }
427
428 private List<Row> constructPutRequests() {
429 List<Row> puts = new ArrayList<Row>();
430 for (byte[] k : KEYS) {
431 Put put = new Put(k);
432 put.add(BYTES_FAMILY, QUALIFIER, VALUE);
433 puts.add(put);
434 }
435 return puts;
436 }
437
438 private void validateLoadedData(HTable table) throws IOException {
439
440 for (byte[] k : KEYS) {
441 LOG.info("Assert=" + Bytes.toString(k));
442 Get get = new Get(k);
443 get.addColumn(BYTES_FAMILY, QUALIFIER);
444 Result r = table.get(get);
445 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
446 Assert.assertEquals(0, Bytes.compareTo(VALUE, r
447 .getValue(BYTES_FAMILY, QUALIFIER)));
448 }
449 }
450
451 private void validateEmpty(Object r1) {
452 Result result = (Result)r1;
453 Assert.assertTrue(result != null);
454 Assert.assertTrue(result.getRow() == null);
455 Assert.assertEquals(0, result.raw().length);
456 }
457
458 private void validateSizeAndEmpty(Object[] results, int expectedSize) {
459
460 Assert.assertEquals(expectedSize, results.length);
461 for (Object result : results) {
462 validateEmpty(result);
463 }
464 }
465 }