1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.handler;
21
22 import java.io.IOException;
23 import java.util.concurrent.atomic.AtomicBoolean;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.HRegionInfo;
28 import org.apache.hadoop.hbase.HTableDescriptor;
29 import org.apache.hadoop.hbase.Server;
30 import org.apache.hadoop.hbase.executor.EventHandler;
31 import org.apache.hadoop.hbase.regionserver.HRegion;
32 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
33 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
34 import org.apache.hadoop.hbase.util.CancelableProgressable;
35 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
36 import org.apache.zookeeper.KeeperException;
37
38
39
40
41
42
43 public class OpenRegionHandler extends EventHandler {
44 private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
45
46 private final RegionServerServices rsServices;
47
48 private final HRegionInfo regionInfo;
49 private final HTableDescriptor htd;
50
51
52
53
54 private volatile int version = -1;
55
56
57 public OpenRegionHandler(final Server server,
58 final RegionServerServices rsServices, HRegionInfo regionInfo,
59 HTableDescriptor htd) {
60 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, -1);
61 }
62 public OpenRegionHandler(final Server server,
63 final RegionServerServices rsServices, HRegionInfo regionInfo,
64 HTableDescriptor htd, int version) {
65 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
66 version);
67 }
68
69 protected OpenRegionHandler(final Server server,
70 final RegionServerServices rsServices, final HRegionInfo regionInfo,
71 final HTableDescriptor htd, EventType eventType,
72 final int version) {
73 super(server, eventType);
74 this.rsServices = rsServices;
75 this.regionInfo = regionInfo;
76 this.htd = htd;
77 this.version = version;
78 }
79
80 public HRegionInfo getRegionInfo() {
81 return regionInfo;
82 }
83
84 @Override
85 public void process() throws IOException {
86 boolean transitionToFailedOpen = false;
87 boolean openSuccessful = false;
88 try {
89 final String name = regionInfo.getRegionNameAsString();
90 if (this.server.isStopped() || this.rsServices.isStopping()) {
91 return;
92 }
93 final String encodedName = regionInfo.getEncodedName();
94
95
96 HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
97
98
99
100 region = openRegion();
101 if (region == null) {
102 tryTransitionToFailedOpen(regionInfo);
103 transitionToFailedOpen = true;
104 return;
105 }
106 boolean failed = true;
107 if (tickleOpening("post_region_open")) {
108 if (updateMeta(region)) {
109 failed = false;
110 }
111 }
112 if (failed || this.server.isStopped() ||
113 this.rsServices.isStopping()) {
114 cleanupFailedOpen(region);
115 tryTransitionToFailedOpen(regionInfo);
116 transitionToFailedOpen = true;
117 return;
118 }
119
120 if (!transitionToOpened(region)) {
121
122
123
124
125
126
127 cleanupFailedOpen(region);
128 transitionToFailedOpen = true;
129 return;
130 }
131
132 this.rsServices.addToOnlineRegions(region);
133 openSuccessful = true;
134
135 LOG.debug("Opened " + name + " on server:" +
136 this.server.getServerName());
137 } finally {
138 this.rsServices.removeFromRegionsInTransition(this.regionInfo);
139 if (!openSuccessful && !transitionToFailedOpen) {
140 tryTransitionToFailedOpen(regionInfo);
141 }
142 }
143 }
144
145
146
147
148
149
150
151
152 boolean updateMeta(final HRegion r) {
153 if (this.server.isStopped() || this.rsServices.isStopping()) {
154 return false;
155 }
156
157
158 final AtomicBoolean signaller = new AtomicBoolean(false);
159 PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
160 this.server, this.rsServices, signaller);
161 t.start();
162 int assignmentTimeout = this.server.getConfiguration().
163 getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
164
165
166 long timeout = assignmentTimeout * 10;
167 long now = System.currentTimeMillis();
168 long endTime = now + timeout;
169
170
171 long period = Math.max(1, assignmentTimeout/ 3);
172 long lastUpdate = now;
173 boolean tickleOpening = true;
174 while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
175 !this.rsServices.isStopping() && (endTime > now)) {
176 long elapsed = now - lastUpdate;
177 if (elapsed > period) {
178
179 lastUpdate = now;
180 tickleOpening = tickleOpening("post_open_deploy");
181 }
182 synchronized (signaller) {
183 try {
184 signaller.wait(period);
185 } catch (InterruptedException e) {
186
187 }
188 }
189 now = System.currentTimeMillis();
190 }
191
192
193 if (t.isAlive()) {
194 if (!signaller.get()) {
195
196 LOG.debug("Interrupting thread " + t);
197 t.interrupt();
198 }
199 try {
200 t.join();
201 } catch (InterruptedException ie) {
202 LOG.warn("Interrupted joining " +
203 r.getRegionInfo().getRegionNameAsString(), ie);
204 Thread.currentThread().interrupt();
205 }
206 }
207
208
209
210
211 return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
212 }
213
214
215
216
217
218
219
220 static class PostOpenDeployTasksThread extends Thread {
221 private Exception exception = null;
222 private final Server server;
223 private final RegionServerServices services;
224 private final HRegion region;
225 private final AtomicBoolean signaller;
226
227 PostOpenDeployTasksThread(final HRegion region, final Server server,
228 final RegionServerServices services, final AtomicBoolean signaller) {
229 super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
230 this.setDaemon(true);
231 this.server = server;
232 this.services = services;
233 this.region = region;
234 this.signaller = signaller;
235 }
236
237 public void run() {
238 try {
239 this.services.postOpenDeployTasks(this.region,
240 this.server.getCatalogTracker(), false);
241 } catch (Exception e) {
242 LOG.warn("Exception running postOpenDeployTasks; region=" +
243 this.region.getRegionInfo().getEncodedName(), e);
244 this.exception = e;
245 }
246
247 this.signaller.set(true);
248 synchronized (this.signaller) {
249 this.signaller.notify();
250 }
251 }
252
253
254
255
256 Exception getException() {
257 return this.exception;
258 }
259 }
260
261
262
263
264
265
266
267 private boolean transitionToOpened(final HRegion r) throws IOException {
268 boolean result = false;
269 HRegionInfo hri = r.getRegionInfo();
270 final String name = hri.getRegionNameAsString();
271
272 try {
273 if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
274 this.server.getServerName(), this.version) == -1) {
275 LOG.warn("Completed the OPEN of region " + name +
276 " but when transitioning from " +
277 " OPENING to OPENED got a version mismatch, someone else clashed " +
278 "so now unassigning -- closing region on server: " +
279 this.server.getServerName());
280 } else {
281 LOG.debug("region transitioned to opened in zookeeper: " +
282 r.getRegionInfo() + ", server: " + this.server.getServerName());
283 result = true;
284 }
285 } catch (KeeperException e) {
286 LOG.error("Failed transitioning node " + name +
287 " from OPENING to OPENED -- closing region", e);
288 }
289 return result;
290 }
291
292
293
294
295
296
297 private boolean tryTransitionToFailedOpen(final HRegionInfo hri) {
298 boolean result = false;
299 final String name = hri.getRegionNameAsString();
300 try {
301 LOG.info("Opening of region " + hri + " failed, marking as FAILED_OPEN in ZK");
302 if (ZKAssign.transitionNode(
303 this.server.getZooKeeper(), hri,
304 this.server.getServerName(),
305 EventType.RS_ZK_REGION_OPENING,
306 EventType.RS_ZK_REGION_FAILED_OPEN,
307 this.version) == -1) {
308 LOG.warn("Unable to mark region " + hri + " as FAILED_OPEN. " +
309 "It's likely that the master already timed out this open " +
310 "attempt, and thus another RS already has the region.");
311 } else {
312 result = true;
313 }
314 } catch (KeeperException e) {
315 LOG.error("Failed transitioning node " + name +
316 " from OPENING to FAILED_OPEN", e);
317 }
318 return result;
319 }
320
321
322
323
324 HRegion openRegion() {
325 HRegion region = null;
326 try {
327
328
329 region = HRegion.openHRegion(this.regionInfo, this.htd,
330 this.rsServices.getWAL(), this.server.getConfiguration(),
331 this.rsServices,
332 new CancelableProgressable() {
333 public boolean progress() {
334
335
336
337 return tickleOpening("open_region_progress");
338 }
339 });
340 } catch (Throwable t) {
341
342
343
344 LOG.error(
345 "Failed open of region=" + this.regionInfo.getRegionNameAsString()
346 + ", starting to roll back the global memstore size.", t);
347
348 if (this.rsServices != null) {
349 RegionServerAccounting rsAccounting =
350 this.rsServices.getRegionServerAccounting();
351 if (rsAccounting != null) {
352 rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
353 }
354 }
355 }
356 return region;
357 }
358
359 void cleanupFailedOpen(final HRegion region) throws IOException {
360 if (region != null) region.close();
361 }
362
363
364
365
366
367
368
369
370 boolean tickleOpening(final String context) {
371
372 if (!isGoodVersion()) return false;
373 String encodedName = this.regionInfo.getEncodedName();
374 try {
375 this.version =
376 ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
377 this.regionInfo, this.server.getServerName(), this.version);
378 } catch (KeeperException e) {
379 server.abort("Exception refreshing OPENING; region=" + encodedName +
380 ", context=" + context, e);
381 this.version = -1;
382 }
383 boolean b = isGoodVersion();
384 if (!b) {
385 LOG.warn("Failed refreshing OPENING; region=" + encodedName +
386 ", context=" + context);
387 }
388 return b;
389 }
390
391 private boolean isGoodVersion() {
392 return this.version != -1;
393 }
394 }