1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.concurrent.atomic.AtomicBoolean;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.*;
31  import org.apache.hadoop.hbase.client.HTable;
32  import org.apache.hadoop.hbase.client.Put;
33  import org.apache.hadoop.hbase.client.Result;
34  import org.apache.hadoop.hbase.client.ResultScanner;
35  import org.apache.hadoop.hbase.client.Scan;
36  import org.apache.hadoop.hbase.executor.EventHandler;
37  import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
38  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
39  import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
40  import org.apache.hadoop.hbase.regionserver.HRegionServer;
41  import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Threads;
44  import org.apache.hadoop.hbase.util.Writables;
45  import org.junit.AfterClass;
46  import org.junit.Assert;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  import org.mockito.Mockito;
52  import org.mockito.internal.util.reflection.Whitebox;
53  
54  import static org.junit.Assert.assertEquals;
55  import static org.junit.Assert.assertTrue;
56  import static org.junit.Assert.fail;
57  import static org.junit.Assert.assertFalse;
58  
59  /**
60   * Test open and close of regions using zk.
61   */
62  @Category(MediumTests.class)
63  public class TestZKBasedOpenCloseRegion {
64    private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
65    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
67    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
68      Bytes.toBytes("b"), Bytes.toBytes("c")};
69    private static int countOfRegions;
70  
71    @BeforeClass public static void beforeAllTests() throws Exception {
72      Configuration c = TEST_UTIL.getConfiguration();
73      c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class,
74                HRegionServer.class);
75      c.setBoolean("dfs.support.append", true);
76      c.setInt("hbase.regionserver.info.port", 0);
77      TEST_UTIL.startMiniCluster(2);
78      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
79      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
80      countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
81      waitUntilAllRegionsAssigned();
82      addToEachStartKey(countOfRegions);
83      t.close();
84    }
85  
86    @AfterClass public static void afterAllTests() throws Exception {
87      TEST_UTIL.shutdownMiniCluster();
88    }
89  
90    @Before public void setup() throws IOException {
91      if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
92        // Need at least two servers.
93        LOG.info("Started new server=" +
94          TEST_UTIL.getHBaseCluster().startRegionServer());
95  
96      }
97      waitUntilAllRegionsAssigned();
98    }
99  
100   /**
101    * Special HRegionServer used in these tests that allows access to
102    * {@link #addRegionsInTransition(HRegionInfo, String)}.
103    */
104   public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer {
105     public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
106         throws IOException, InterruptedException {
107       super(conf);
108     }
109     @Override
110     public void addRegionsInTransition(HRegionInfo region,
111         String currentAction) throws RegionAlreadyInTransitionException {
112       super.addRegionsInTransition(region, currentAction);
113     }
114   }
115 
116   /**
117    * Test we reopen a region once closed.
118    * @throws Exception
119    */
120   @Test (timeout=300000) public void testReOpenRegion()
121   throws Exception {
122     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
123     LOG.info("Number of region servers = " +
124       cluster.getLiveRegionServerThreads().size());
125 
126     int rsIdx = 0;
127     HRegionServer regionServer =
128       TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
129     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
130     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
131 
132     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
133     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
134 
135     EventHandlerListener closeListener =
136       new ReopenEventListener(hri.getRegionNameAsString(),
137           closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
138     cluster.getMaster().executorService.
139       registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
140 
141     EventHandlerListener openListener =
142       new ReopenEventListener(hri.getRegionNameAsString(),
143           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
144     cluster.getMaster().executorService.
145       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
146 
147     LOG.info("Unassign " + hri.getRegionNameAsString());
148     cluster.getMaster().assignmentManager.unassign(hri);
149 
150     while (!closeEventProcessed.get()) {
151       Threads.sleep(100);
152     }
153 
154     while (!reopenEventProcessed.get()) {
155       Threads.sleep(100);
156     }
157 
158     LOG.info("Done with testReOpenRegion");
159   }
160 
161   private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
162     HRegionInfo hri = null;
163     for (HRegionInfo i: regions) {
164       LOG.info(i.getRegionNameAsString());
165       if (!i.isMetaRegion()) {
166         hri = i;
167         break;
168       }
169     }
170     return hri;
171   }
172 
173   public static class ReopenEventListener implements EventHandlerListener {
174     private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
175     String regionName;
176     AtomicBoolean eventProcessed;
177     EventType eventType;
178 
179     public ReopenEventListener(String regionName,
180         AtomicBoolean eventProcessed, EventType eventType) {
181       this.regionName = regionName;
182       this.eventProcessed = eventProcessed;
183       this.eventType = eventType;
184     }
185 
186     @Override
187     public void beforeProcess(EventHandler event) {
188       if(event.getEventType() == eventType) {
189         LOG.info("Received " + eventType + " and beginning to process it");
190       }
191     }
192 
193     @Override
194     public void afterProcess(EventHandler event) {
195       LOG.info("afterProcess(" + event + ")");
196       if(event.getEventType() == eventType) {
197         LOG.info("Finished processing " + eventType);
198         String regionName = "";
199         if(eventType == EventType.RS_ZK_REGION_OPENED) {
200           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
201           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
202         } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
203           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
204           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
205         }
206         if(this.regionName.equals(regionName)) {
207           eventProcessed.set(true);
208         }
209         synchronized(eventProcessed) {
210           eventProcessed.notifyAll();
211         }
212       }
213     }
214   }
215 
216   public static class CloseRegionEventListener implements EventHandlerListener {
217     private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
218     String regionToClose;
219     AtomicBoolean closeEventProcessed;
220 
221     public CloseRegionEventListener(String regionToClose,
222         AtomicBoolean closeEventProcessed) {
223       this.regionToClose = regionToClose;
224       this.closeEventProcessed = closeEventProcessed;
225     }
226 
227     @Override
228     public void afterProcess(EventHandler event) {
229       LOG.info("afterProcess(" + event + ")");
230       if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
231         LOG.info("Finished processing CLOSE REGION");
232         TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
233         if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
234           LOG.info("Setting closeEventProcessed flag");
235           closeEventProcessed.set(true);
236         } else {
237           LOG.info("Region to close didn't match");
238         }
239       }
240     }
241 
242     @Override
243     public void beforeProcess(EventHandler event) {
244       if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
245         LOG.info("Received CLOSE RPC and beginning to process it");
246       }
247     }
248   }
249 
250   /**
251    * This test shows how a region won't be able to be assigned to a RS
252    * if it's already "processing" it.
253    * @throws Exception
254    */
255   @Test
256   public void testRSAlreadyProcessingRegion() throws Exception {
257     LOG.info("starting testRSAlreadyProcessingRegion");
258     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
259 
260     HRegionServer hr0 =
261         cluster.getLiveRegionServerThreads().get(0).getRegionServer();
262     HRegionServer hr1 =
263         cluster.getLiveRegionServerThreads().get(1).getRegionServer();
264     HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
265 
266     // Fake that hr1 is processing the region. At top of this test we made a
267     // regionserver that gave access addRegionsInTransition. Need to cast as
268     // TestZKBasedOpenCloseRegionRegionServer.
269     ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN");
270 
271     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
272     EventHandlerListener openListener =
273       new ReopenEventListener(hri.getRegionNameAsString(),
274           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
275     cluster.getMaster().executorService.
276       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
277 
278     // now ask the master to move the region to hr1, will fail
279     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
280         Bytes.toBytes(hr1.getServerName().toString()));
281 
282     // make sure the region came back
283     assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
284 
285     // remove the block and reset the boolean
286     hr1.removeFromRegionsInTransition(hri);
287     reopenEventProcessed.set(false);
288     
289     // now try moving a region when there is no region in transition.
290     hri = getNonMetaRegion(hr1.getOnlineRegions());
291 
292     openListener =
293       new ReopenEventListener(hri.getRegionNameAsString(),
294           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
295 
296     cluster.getMaster().executorService.
297       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
298     
299     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
300         Bytes.toBytes(hr0.getServerName().toString()));
301 
302     while (!reopenEventProcessed.get()) {
303       Threads.sleep(100);
304     }
305 
306     // make sure the region has moved from the original RS
307     assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
308 
309   }
310 
311   @Test (timeout=300000) public void testCloseRegion()
312   throws Exception {
313     LOG.info("Running testCloseRegion");
314     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
315     LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
316 
317     int rsIdx = 0;
318     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
319     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
320     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
321 
322     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
323     EventHandlerListener listener =
324       new CloseRegionEventListener(hri.getRegionNameAsString(),
325           closeEventProcessed);
326     cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
327 
328     cluster.getMaster().assignmentManager.unassign(hri);
329 
330     while (!closeEventProcessed.get()) {
331       Threads.sleep(100);
332     }
333     LOG.info("Done with testCloseRegion");
334   }
335 
336   /**
337    * If region open fails with IOException in openRegion() while doing tableDescriptors.get()
338    * the region should not add into regionsInTransitionInRS map
339    * @throws Exception
340    */
341   @Test
342   public void testRegionOpenFailsDueToIOException() throws Exception {
343     HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"),
344         HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
345     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
346     TableDescriptors htd = Mockito.mock(TableDescriptors.class);
347     Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors");
348     Whitebox.setInternalState(regionServer, "tableDescriptors", htd);
349     Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any());
350     try {
351       regionServer.openRegion(REGIONINFO);
352       fail("It should throw IOException ");
353     } catch (IOException e) {
354     }
355     Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
356     assertFalse("Region should not be in RIT",
357         regionServer.containsKeyInRegionsInTransition(REGIONINFO));
358   }
359   
360   private static void waitUntilAllRegionsAssigned()
361   throws IOException {
362     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
363       HConstants.META_TABLE_NAME);
364     while (true) {
365       int rows = 0;
366       Scan scan = new Scan();
367       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
368       ResultScanner s = meta.getScanner(scan);
369       for (Result r = null; (r = s.next()) != null;) {
370         byte [] b =
371           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
372         if (b == null || b.length <= 0) {
373           break;
374         }
375         rows++;
376       }
377       s.close();
378       // If I get to here and all rows have a Server, then all have been assigned.
379       if (rows >= countOfRegions) {
380         break;
381       }
382       LOG.info("Found=" + rows);
383       Threads.sleep(1000);
384     }
385     meta.close();
386   }
387 
388   /*
389    * Add to each of the regions in .META. a value.  Key is the startrow of the
390    * region (except its 'aaa' for first region).  Actual value is the row name.
391    * @param expected
392    * @return
393    * @throws IOException
394    */
395   private static int addToEachStartKey(final int expected) throws IOException {
396     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
397     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
398         HConstants.META_TABLE_NAME);
399     int rows = 0;
400     Scan scan = new Scan();
401     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
402     ResultScanner s = meta.getScanner(scan);
403     for (Result r = null; (r = s.next()) != null;) {
404       byte [] b =
405         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
406       if (b == null || b.length <= 0) {
407         break;
408       }
409       HRegionInfo hri = Writables.getHRegionInfo(b);
410       // If start key, add 'aaa'.
411       byte [] row = getStartKey(hri);
412       Put p = new Put(row);
413       p.setWriteToWAL(false);
414       p.add(getTestFamily(), getTestQualifier(), row);
415       t.put(p);
416       rows++;
417     }
418     s.close();
419     Assert.assertEquals(expected, rows);
420     t.close();
421     meta.close();
422     return rows;
423   }
424 
425   private static byte [] getStartKey(final HRegionInfo hri) {
426     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
427         Bytes.toBytes("aaa"): hri.getStartKey();
428   }
429 
430   private static byte [] getTestFamily() {
431     return FAMILIES[0];
432   }
433 
434   private static byte [] getTestQualifier() {
435     return getTestFamily();
436   }
437 
438   public static void main(String args[]) throws Exception {
439     TestZKBasedOpenCloseRegion.beforeAllTests();
440 
441     TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
442     test.setup();
443     test.testCloseRegion();
444 
445     TestZKBasedOpenCloseRegion.afterAllTests();
446   }
447 
448   @org.junit.Rule
449   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
450     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
451 }
452