1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.concurrent.atomic.AtomicBoolean;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.Put;
33 import org.apache.hadoop.hbase.client.Result;
34 import org.apache.hadoop.hbase.client.ResultScanner;
35 import org.apache.hadoop.hbase.client.Scan;
36 import org.apache.hadoop.hbase.executor.EventHandler;
37 import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
38 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
39 import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
40 import org.apache.hadoop.hbase.regionserver.HRegionServer;
41 import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.Threads;
44 import org.apache.hadoop.hbase.util.Writables;
45 import org.junit.AfterClass;
46 import org.junit.Assert;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51 import org.mockito.Mockito;
52 import org.mockito.internal.util.reflection.Whitebox;
53
54 import static org.junit.Assert.assertEquals;
55 import static org.junit.Assert.assertTrue;
56 import static org.junit.Assert.fail;
57 import static org.junit.Assert.assertFalse;
58
59
60
61
62 @Category(MediumTests.class)
63 public class TestZKBasedOpenCloseRegion {
64 private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
65 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
67 private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
68 Bytes.toBytes("b"), Bytes.toBytes("c")};
69 private static int countOfRegions;
70
71 @BeforeClass public static void beforeAllTests() throws Exception {
72 Configuration c = TEST_UTIL.getConfiguration();
73 c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class,
74 HRegionServer.class);
75 c.setBoolean("dfs.support.append", true);
76 c.setInt("hbase.regionserver.info.port", 0);
77 TEST_UTIL.startMiniCluster(2);
78 TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
79 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
80 countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
81 waitUntilAllRegionsAssigned();
82 addToEachStartKey(countOfRegions);
83 t.close();
84 }
85
86 @AfterClass public static void afterAllTests() throws Exception {
87 TEST_UTIL.shutdownMiniCluster();
88 }
89
90 @Before public void setup() throws IOException {
91 if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
92
93 LOG.info("Started new server=" +
94 TEST_UTIL.getHBaseCluster().startRegionServer());
95
96 }
97 waitUntilAllRegionsAssigned();
98 }
99
100
101
102
103
104 public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer {
105 public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
106 throws IOException, InterruptedException {
107 super(conf);
108 }
109 @Override
110 public boolean addRegionsInTransition(HRegionInfo region,
111 String currentAction) throws RegionAlreadyInTransitionException {
112 return super.addRegionsInTransition(region, currentAction);
113 }
114 }
115
116
117
118
119
120 @Test (timeout=300000) public void testReOpenRegion()
121 throws Exception {
122 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
123 LOG.info("Number of region servers = " +
124 cluster.getLiveRegionServerThreads().size());
125
126 int rsIdx = 0;
127 HRegionServer regionServer =
128 TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
129 HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
130 LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
131
132 AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
133 AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
134
135 EventHandlerListener closeListener =
136 new ReopenEventListener(hri.getRegionNameAsString(),
137 closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
138 cluster.getMaster().executorService.
139 registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
140
141 EventHandlerListener openListener =
142 new ReopenEventListener(hri.getRegionNameAsString(),
143 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
144 cluster.getMaster().executorService.
145 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
146
147 LOG.info("Unassign " + hri.getRegionNameAsString());
148 cluster.getMaster().assignmentManager.unassign(hri);
149
150 while (!closeEventProcessed.get()) {
151 Threads.sleep(100);
152 }
153
154 while (!reopenEventProcessed.get()) {
155 Threads.sleep(100);
156 }
157
158 LOG.info("Done with testReOpenRegion");
159 }
160
161 private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
162 HRegionInfo hri = null;
163 for (HRegionInfo i: regions) {
164 LOG.info(i.getRegionNameAsString());
165 if (!i.isMetaRegion()) {
166 hri = i;
167 break;
168 }
169 }
170 return hri;
171 }
172
173 public static class ReopenEventListener implements EventHandlerListener {
174 private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
175 String regionName;
176 AtomicBoolean eventProcessed;
177 EventType eventType;
178
179 public ReopenEventListener(String regionName,
180 AtomicBoolean eventProcessed, EventType eventType) {
181 this.regionName = regionName;
182 this.eventProcessed = eventProcessed;
183 this.eventType = eventType;
184 }
185
186 @Override
187 public void beforeProcess(EventHandler event) {
188 if(event.getEventType() == eventType) {
189 LOG.info("Received " + eventType + " and beginning to process it");
190 }
191 }
192
193 @Override
194 public void afterProcess(EventHandler event) {
195 LOG.info("afterProcess(" + event + ")");
196 if(event.getEventType() == eventType) {
197 LOG.info("Finished processing " + eventType);
198 String regionName = "";
199 if(eventType == EventType.RS_ZK_REGION_OPENED) {
200 TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
201 regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
202 } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
203 TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
204 regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
205 }
206 if(this.regionName.equals(regionName)) {
207 eventProcessed.set(true);
208 }
209 synchronized(eventProcessed) {
210 eventProcessed.notifyAll();
211 }
212 }
213 }
214 }
215
216 public static class CloseRegionEventListener implements EventHandlerListener {
217 private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
218 String regionToClose;
219 AtomicBoolean closeEventProcessed;
220
221 public CloseRegionEventListener(String regionToClose,
222 AtomicBoolean closeEventProcessed) {
223 this.regionToClose = regionToClose;
224 this.closeEventProcessed = closeEventProcessed;
225 }
226
227 @Override
228 public void afterProcess(EventHandler event) {
229 LOG.info("afterProcess(" + event + ")");
230 if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
231 LOG.info("Finished processing CLOSE REGION");
232 TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
233 if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
234 LOG.info("Setting closeEventProcessed flag");
235 closeEventProcessed.set(true);
236 } else {
237 LOG.info("Region to close didn't match");
238 }
239 }
240 }
241
242 @Override
243 public void beforeProcess(EventHandler event) {
244 if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
245 LOG.info("Received CLOSE RPC and beginning to process it");
246 }
247 }
248 }
249
250
251
252
253
254
255 @Test
256 public void testRSAlreadyProcessingRegion() throws Exception {
257 LOG.info("starting testRSAlreadyProcessingRegion");
258 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
259
260 HRegionServer hr0 =
261 cluster.getLiveRegionServerThreads().get(0).getRegionServer();
262 HRegionServer hr1 =
263 cluster.getLiveRegionServerThreads().get(1).getRegionServer();
264 HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
265
266
267
268
269 ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN");
270
271 AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
272 EventHandlerListener openListener =
273 new ReopenEventListener(hri.getRegionNameAsString(),
274 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
275 cluster.getMaster().executorService.
276 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
277
278
279 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
280 Bytes.toBytes(hr1.getServerName().toString()));
281
282
283 assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
284
285
286 hr1.removeFromRegionsInTransition(hri);
287 reopenEventProcessed.set(false);
288
289
290 hri = getNonMetaRegion(hr1.getOnlineRegions());
291
292 openListener =
293 new ReopenEventListener(hri.getRegionNameAsString(),
294 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
295
296 cluster.getMaster().executorService.
297 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
298
299 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
300 Bytes.toBytes(hr0.getServerName().toString()));
301
302 while (!reopenEventProcessed.get()) {
303 Threads.sleep(100);
304 }
305
306
307 assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
308
309 }
310
311 @Test (timeout=300000) public void testCloseRegion()
312 throws Exception {
313 LOG.info("Running testCloseRegion");
314 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
315 LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
316
317 int rsIdx = 0;
318 HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
319 HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
320 LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
321
322 AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
323 EventHandlerListener listener =
324 new CloseRegionEventListener(hri.getRegionNameAsString(),
325 closeEventProcessed);
326 cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
327
328 cluster.getMaster().assignmentManager.unassign(hri);
329
330 while (!closeEventProcessed.get()) {
331 Threads.sleep(100);
332 }
333 LOG.info("Done with testCloseRegion");
334 }
335
336
337
338
339
340
341 @Test
342 public void testRegionOpenFailsDueToIOException() throws Exception {
343 HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"),
344 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
345 HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
346 TableDescriptors htd = Mockito.mock(TableDescriptors.class);
347 Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors");
348 Whitebox.setInternalState(regionServer, "tableDescriptors", htd);
349 Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any());
350 try {
351 regionServer.openRegion(REGIONINFO);
352 fail("It should throw IOException ");
353 } catch (IOException e) {
354 }
355 Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
356 assertFalse("Region should not be in RIT",
357 regionServer.containsKeyInRegionsInTransition(REGIONINFO));
358 }
359
360 private static void waitUntilAllRegionsAssigned()
361 throws IOException {
362 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
363 HConstants.META_TABLE_NAME);
364 while (true) {
365 int rows = 0;
366 Scan scan = new Scan();
367 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
368 ResultScanner s = meta.getScanner(scan);
369 for (Result r = null; (r = s.next()) != null;) {
370 byte [] b =
371 r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
372 if (b == null || b.length <= 0) {
373 break;
374 }
375 rows++;
376 }
377 s.close();
378
379 if (rows >= countOfRegions) {
380 break;
381 }
382 LOG.info("Found=" + rows);
383 Threads.sleep(1000);
384 }
385 meta.close();
386 }
387
388
389
390
391
392
393
394
395 private static int addToEachStartKey(final int expected) throws IOException {
396 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
397 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
398 HConstants.META_TABLE_NAME);
399 int rows = 0;
400 Scan scan = new Scan();
401 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
402 ResultScanner s = meta.getScanner(scan);
403 for (Result r = null; (r = s.next()) != null;) {
404 byte [] b =
405 r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
406 if (b == null || b.length <= 0) {
407 break;
408 }
409 HRegionInfo hri = Writables.getHRegionInfo(b);
410
411 byte [] row = getStartKey(hri);
412 Put p = new Put(row);
413 p.setWriteToWAL(false);
414 p.add(getTestFamily(), getTestQualifier(), row);
415 t.put(p);
416 rows++;
417 }
418 s.close();
419 Assert.assertEquals(expected, rows);
420 t.close();
421 meta.close();
422 return rows;
423 }
424
425 private static byte [] getStartKey(final HRegionInfo hri) {
426 return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
427 Bytes.toBytes("aaa"): hri.getStartKey();
428 }
429
430 private static byte [] getTestFamily() {
431 return FAMILIES[0];
432 }
433
434 private static byte [] getTestQualifier() {
435 return getTestFamily();
436 }
437
438 public static void main(String args[]) throws Exception {
439 TestZKBasedOpenCloseRegion.beforeAllTests();
440
441 TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
442 test.setup();
443 test.testCloseRegion();
444
445 TestZKBasedOpenCloseRegion.afterAllTests();
446 }
447
448 @org.junit.Rule
449 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
450 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
451 }
452