1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.concurrent.atomic.AtomicBoolean;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.MiniHBaseCluster;
34  import org.apache.hadoop.hbase.client.HTable;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.ResultScanner;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.executor.EventHandler;
40  import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
41  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
42  import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.Threads;
46  import org.apache.hadoop.hbase.util.Writables;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  
53  /**
54   * Test open and close of regions using zk.
55   */
56  public class TestZKBasedOpenCloseRegion {
57    private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
58    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59    private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
60    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
61      Bytes.toBytes("b"), Bytes.toBytes("c")};
62  
63    @BeforeClass public static void beforeAllTests() throws Exception {
64      Configuration c = TEST_UTIL.getConfiguration();
65      c.setBoolean("dfs.support.append", true);
66      c.setInt("hbase.regionserver.info.port", 0);
67      TEST_UTIL.startMiniCluster(2);
68      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
69      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
70      int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
71      waitUntilAllRegionsAssigned(countOfRegions);
72      addToEachStartKey(countOfRegions);
73    }
74  
75    @AfterClass public static void afterAllTests() throws IOException {
76      TEST_UTIL.shutdownMiniCluster();
77    }
78  
79    @Before public void setup() throws IOException {
80      if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
81        // Need at least two servers.
82        LOG.info("Started new server=" +
83          TEST_UTIL.getHBaseCluster().startRegionServer());
84  
85      }
86    }
87  
88    /**
89     * Test we reopen a region once closed.
90     * @throws Exception
91     */
92    @Test (timeout=300000) public void testReOpenRegion()
93    throws Exception {
94      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
95      LOG.info("Number of region servers = " +
96        cluster.getLiveRegionServerThreads().size());
97  
98      int rsIdx = 0;
99      HRegionServer regionServer =
100       TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
101     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
102     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
103 
104     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
105     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
106 
107     EventHandlerListener closeListener =
108       new ReopenEventListener(hri.getRegionNameAsString(),
109           closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
110     cluster.getMaster().executorService.
111       registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
112 
113     EventHandlerListener openListener =
114       new ReopenEventListener(hri.getRegionNameAsString(),
115           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
116     cluster.getMaster().executorService.
117       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
118 
119     LOG.info("Unassign " + hri.getRegionNameAsString());
120     cluster.getMaster().assignmentManager.unassign(hri);
121 
122     while (!closeEventProcessed.get()) {
123       Threads.sleep(100);
124     }
125 
126     while (!reopenEventProcessed.get()) {
127       Threads.sleep(100);
128     }
129 
130     LOG.info("Done with testReOpenRegion");
131   }
132 
133   private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
134     HRegionInfo hri = null;
135     for (HRegionInfo i: regions) {
136       LOG.info(i.getRegionNameAsString());
137       if (!i.isMetaRegion()) {
138         hri = i;
139         break;
140       }
141     }
142     return hri;
143   }
144 
145   public static class ReopenEventListener implements EventHandlerListener {
146     private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
147     String regionName;
148     AtomicBoolean eventProcessed;
149     EventType eventType;
150 
151     public ReopenEventListener(String regionName,
152         AtomicBoolean eventProcessed, EventType eventType) {
153       this.regionName = regionName;
154       this.eventProcessed = eventProcessed;
155       this.eventType = eventType;
156     }
157 
158     @Override
159     public void beforeProcess(EventHandler event) {
160       if(event.getEventType() == eventType) {
161         LOG.info("Received " + eventType + " and beginning to process it");
162       }
163     }
164 
165     @Override
166     public void afterProcess(EventHandler event) {
167       LOG.info("afterProcess(" + event + ")");
168       if(event.getEventType() == eventType) {
169         LOG.info("Finished processing " + eventType);
170         String regionName = "";
171         if(eventType == EventType.RS_ZK_REGION_OPENED) {
172           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
173           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
174         } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
175           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
176           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
177         }
178         if(this.regionName.equals(regionName)) {
179           eventProcessed.set(true);
180         }
181         synchronized(eventProcessed) {
182           eventProcessed.notifyAll();
183         }
184       }
185     }
186   }
187 
188   @Test (timeout=300000) public void testCloseRegion()
189   throws Exception {
190     LOG.info("Running testCloseRegion");
191     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
192     LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
193 
194     int rsIdx = 0;
195     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
196     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
197     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
198 
199     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
200     EventHandlerListener listener =
201       new CloseRegionEventListener(hri.getRegionNameAsString(),
202           closeEventProcessed);
203     cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
204 
205     cluster.getMaster().assignmentManager.unassign(hri);
206 
207     while (!closeEventProcessed.get()) {
208       Threads.sleep(100);
209     }
210     LOG.info("Done with testCloseRegion");
211   }
212 
213   public static class CloseRegionEventListener implements EventHandlerListener {
214     private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
215     String regionToClose;
216     AtomicBoolean closeEventProcessed;
217 
218     public CloseRegionEventListener(String regionToClose,
219         AtomicBoolean closeEventProcessed) {
220       this.regionToClose = regionToClose;
221       this.closeEventProcessed = closeEventProcessed;
222     }
223 
224     @Override
225     public void afterProcess(EventHandler event) {
226       LOG.info("afterProcess(" + event + ")");
227       if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
228         LOG.info("Finished processing CLOSE REGION");
229         TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
230         if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
231           LOG.info("Setting closeEventProcessed flag");
232           closeEventProcessed.set(true);
233         } else {
234           LOG.info("Region to close didn't match");
235         }
236       }
237     }
238 
239     @Override
240     public void beforeProcess(EventHandler event) {
241       if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
242         LOG.info("Received CLOSE RPC and beginning to process it");
243       }
244     }
245   }
246 
247   private static void waitUntilAllRegionsAssigned(final int countOfRegions)
248   throws IOException {
249     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
250       HConstants.META_TABLE_NAME);
251     while (true) {
252       int rows = 0;
253       Scan scan = new Scan();
254       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
255       ResultScanner s = meta.getScanner(scan);
256       for (Result r = null; (r = s.next()) != null;) {
257         byte [] b =
258           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
259         if (b == null || b.length <= 0) {
260           break;
261         }
262         rows++;
263       }
264       s.close();
265       // If I get to here and all rows have a Server, then all have been assigned.
266       if (rows == countOfRegions) {
267         break;
268       }
269       LOG.info("Found=" + rows);
270       Threads.sleep(1000);
271     }
272   }
273 
274   /*
275    * Add to each of the regions in .META. a value.  Key is the startrow of the
276    * region (except its 'aaa' for first region).  Actual value is the row name.
277    * @param expected
278    * @return
279    * @throws IOException
280    */
281   private static int addToEachStartKey(final int expected) throws IOException {
282     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
283     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
284         HConstants.META_TABLE_NAME);
285     int rows = 0;
286     Scan scan = new Scan();
287     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
288     ResultScanner s = meta.getScanner(scan);
289     for (Result r = null; (r = s.next()) != null;) {
290       byte [] b =
291         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
292       if (b == null || b.length <= 0) {
293         break;
294       }
295       HRegionInfo hri = Writables.getHRegionInfo(b);
296       // If start key, add 'aaa'.
297       byte [] row = getStartKey(hri);
298       Put p = new Put(row);
299       p.add(getTestFamily(), getTestQualifier(), row);
300       t.put(p);
301       rows++;
302     }
303     s.close();
304     Assert.assertEquals(expected, rows);
305     return rows;
306   }
307 
308   private static byte [] getStartKey(final HRegionInfo hri) {
309     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
310         Bytes.toBytes("aaa"): hri.getStartKey();
311   }
312 
313   private static byte [] getTestFamily() {
314     return FAMILIES[0];
315   }
316 
317   private static byte [] getTestQualifier() {
318     return getTestFamily();
319   }
320 
321   public static void main(String args[]) throws Exception {
322     TestZKBasedOpenCloseRegion.beforeAllTests();
323 
324     TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
325     test.setup();
326     test.testCloseRegion();
327 
328     TestZKBasedOpenCloseRegion.afterAllTests();
329   }
330 }