1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.EOFException;
25 import java.io.IOException;
26 import java.net.ConnectException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.NavigableMap;
33 import java.util.Set;
34 import java.util.SortedMap;
35 import java.util.TreeMap;
36 import java.util.TreeSet;
37 import java.util.concurrent.ConcurrentSkipListMap;
38 import java.util.concurrent.atomic.AtomicInteger;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.hbase.Chore;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HServerAddress;
47 import org.apache.hadoop.hbase.HServerInfo;
48 import org.apache.hadoop.hbase.HTableDescriptor;
49 import org.apache.hadoop.hbase.NotServingRegionException;
50 import org.apache.hadoop.hbase.Server;
51 import org.apache.hadoop.hbase.Stoppable;
52 import org.apache.hadoop.hbase.catalog.CatalogTracker;
53 import org.apache.hadoop.hbase.catalog.MetaReader;
54 import org.apache.hadoop.hbase.catalog.RootLocationEditor;
55 import org.apache.hadoop.hbase.client.Result;
56 import org.apache.hadoop.hbase.executor.ExecutorService;
57 import org.apache.hadoop.hbase.executor.RegionTransitionData;
58 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
59 import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
60 import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
61 import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
62 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
63 import org.apache.hadoop.hbase.util.Bytes;
64 import org.apache.hadoop.hbase.util.Pair;
65 import org.apache.hadoop.hbase.util.Threads;
66 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
67 import org.apache.hadoop.hbase.zookeeper.ZKTable;
68 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
69 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
70 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
71 import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData;
72 import org.apache.hadoop.io.Writable;
73 import org.apache.hadoop.ipc.RemoteException;
74 import org.apache.zookeeper.AsyncCallback;
75 import org.apache.zookeeper.KeeperException;
76 import org.apache.zookeeper.KeeperException.NoNodeException;
77 import org.apache.zookeeper.data.Stat;
78
79
80
81
82
83
84
85
86 public class AssignmentManager extends ZooKeeperListener {
87 private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
88
89 protected Server master;
90
91 private ServerManager serverManager;
92
93 private CatalogTracker catalogTracker;
94
95 private TimeoutMonitor timeoutMonitor;
96
97
98
99
100 private final int maximumAssignmentAttempts;
101
102
103
104
105
106 final ConcurrentSkipListMap<String, RegionState> regionsInTransition =
107 new ConcurrentSkipListMap<String, RegionState>();
108
109
110
111
112
113 final NavigableMap<String, RegionPlan> regionPlans =
114 new TreeMap<String, RegionPlan>();
115
116 private final ZKTable zkTable;
117
118
119
120
121
122
123
124
125 private final NavigableMap<HServerInfo, List<HRegionInfo>> servers =
126 new TreeMap<HServerInfo, List<HRegionInfo>>();
127
128
129
130
131
132
133
134
135 private final SortedMap<HRegionInfo,HServerInfo> regions =
136 new TreeMap<HRegionInfo,HServerInfo>();
137
138 private final ExecutorService executorService;
139
140
141
142
143
144
145
146
147
148
149 public AssignmentManager(Server master, ServerManager serverManager,
150 CatalogTracker catalogTracker, final ExecutorService service)
151 throws KeeperException {
152 super(master.getZooKeeper());
153 this.master = master;
154 this.serverManager = serverManager;
155 this.catalogTracker = catalogTracker;
156 this.executorService = service;
157 Configuration conf = master.getConfiguration();
158 this.timeoutMonitor = new TimeoutMonitor(
159 conf.getInt("hbase.master.assignment.timeoutmonitor.period", 10000),
160 master,
161 conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 30000));
162 Threads.setDaemonThreadRunning(timeoutMonitor,
163 master.getServerName() + ".timeoutMonitor");
164 this.zkTable = new ZKTable(this.master.getZooKeeper());
165 this.maximumAssignmentAttempts =
166 this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
167 }
168
169
170
171
172 public ZKTable getZKTable() {
173
174
175 return this.zkTable;
176 }
177
178
179
180
181
182
183
184 void cleanoutUnassigned() throws IOException, KeeperException {
185
186 ZKAssign.deleteAllNodes(watcher);
187 ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
188 this.watcher.assignmentZNode);
189 }
190
191
192
193
194
195
196
197 void processFailover() throws KeeperException, IOException {
198
199
200
201
202
203
204
205
206
207
208
209 Map<HServerInfo,List<Pair<HRegionInfo,Result>>> deadServers =
210 rebuildUserRegions();
211
212 processDeadServers(deadServers);
213
214 List<String> nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher,
215 watcher.assignmentZNode);
216 if (nodes.isEmpty()) {
217 LOG.info("No regions in transition in ZK to process on failover");
218 return;
219 }
220 LOG.info("Failed-over master needs to process " + nodes.size() +
221 " regions in transition");
222 for (String encodedRegionName: nodes) {
223 processRegionInTransition(encodedRegionName, null);
224 }
225 }
226
227
228
229
230
231
232
233
234
235
236
237
238 boolean processRegionInTransitionAndBlockUntilAssigned(final HRegionInfo hri)
239 throws InterruptedException, KeeperException, IOException {
240 boolean intransistion = processRegionInTransition(hri.getEncodedName(), hri);
241 if (!intransistion) return intransistion;
242 synchronized(this.regionsInTransition) {
243 while (!this.master.isStopped() &&
244 this.regionsInTransition.containsKey(hri.getEncodedName())) {
245 this.regionsInTransition.wait();
246 }
247 }
248 return intransistion;
249 }
250
251
252
253
254
255
256
257
258
259 boolean processRegionInTransition(final String encodedRegionName,
260 final HRegionInfo regionInfo)
261 throws KeeperException, IOException {
262 RegionTransitionData data = ZKAssign.getData(watcher, encodedRegionName);
263 if (data == null) return false;
264 HRegionInfo hri = regionInfo;
265 if (hri == null) {
266 Pair<HRegionInfo, HServerAddress> p =
267 MetaReader.getRegion(catalogTracker, data.getRegionName());
268 if (p == null) return false;
269 hri = p.getFirst();
270 }
271 processRegionsInTransition(data, hri);
272 return true;
273 }
274
275 void processRegionsInTransition(final RegionTransitionData data,
276 final HRegionInfo regionInfo)
277 throws KeeperException {
278 String encodedRegionName = regionInfo.getEncodedName();
279 LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
280 " in state " + data.getEventType());
281 synchronized (regionsInTransition) {
282 switch (data.getEventType()) {
283 case RS_ZK_REGION_CLOSING:
284
285
286 regionsInTransition.put(encodedRegionName, new RegionState(
287 regionInfo, RegionState.State.CLOSING, data.getStamp()));
288 break;
289
290 case RS_ZK_REGION_CLOSED:
291
292 regionsInTransition.put(encodedRegionName, new RegionState(
293 regionInfo, RegionState.State.CLOSED, data.getStamp()));
294 new ClosedRegionHandler(master, this, regionInfo).process();
295 break;
296
297 case M_ZK_REGION_OFFLINE:
298
299 regionsInTransition.put(encodedRegionName, new RegionState(
300 regionInfo, RegionState.State.OFFLINE, data.getStamp()));
301 new ClosedRegionHandler(master, this, regionInfo).process();
302 break;
303
304 case RS_ZK_REGION_OPENING:
305
306
307 regionsInTransition.put(encodedRegionName, new RegionState(
308 regionInfo, RegionState.State.OPENING, data.getStamp()));
309 break;
310
311 case RS_ZK_REGION_OPENED:
312
313 regionsInTransition.put(encodedRegionName, new RegionState(
314 regionInfo, RegionState.State.OPENING, data.getStamp()));
315 HServerInfo hsi = serverManager.getServerInfo(data.getServerName());
316
317
318
319 if (hsi == null) {
320 LOG.warn("Region in transition " + regionInfo.getEncodedName() +
321 " references a server no longer up " + data.getServerName() +
322 "; letting RIT timeout so will be assigned elsewhere");
323 break;
324 }
325 new OpenedRegionHandler(master, this, regionInfo, hsi).process();
326 break;
327 }
328 }
329 }
330
331
332
333
334
335
336
337
338
339
340 private void handleRegion(final RegionTransitionData data) {
341 synchronized(regionsInTransition) {
342 if (data == null || data.getServerName() == null) {
343 LOG.warn("Unexpected NULL input " + data);
344 return;
345 }
346
347 if (data.getServerName().equals(HConstants.HBCK_CODE_NAME)) {
348 handleHBCK(data);
349 return;
350 }
351
352 if (!serverManager.isServerOnline(data.getServerName()) &&
353 !this.master.getServerName().equals(data.getServerName())) {
354 LOG.warn("Attempted to handle region transition for server but " +
355 "server is not online: " + data.getRegionName());
356 return;
357 }
358 String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
359 String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
360 LOG.debug("Handling transition=" + data.getEventType() +
361 ", server=" + data.getServerName() + ", region=" + prettyPrintedRegionName);
362 RegionState regionState = regionsInTransition.get(encodedName);
363 switch (data.getEventType()) {
364 case M_ZK_REGION_OFFLINE:
365
366 break;
367
368 case RS_ZK_REGION_CLOSING:
369
370
371 if (regionState == null ||
372 (!regionState.isPendingClose() && !regionState.isClosing())) {
373 LOG.warn("Received CLOSING for region " + prettyPrintedRegionName +
374 " from server " + data.getServerName() + " but region was in " +
375 " the state " + regionState + " and not " +
376 "in expected PENDING_CLOSE or CLOSING states");
377 return;
378 }
379
380 regionState.update(RegionState.State.CLOSING, data.getStamp());
381 break;
382
383 case RS_ZK_REGION_CLOSED:
384
385 if (regionState == null ||
386 (!regionState.isPendingClose() && !regionState.isClosing())) {
387 LOG.warn("Received CLOSED for region " + prettyPrintedRegionName +
388 " from server " + data.getServerName() + " but region was in " +
389 " the state " + regionState + " and not " +
390 "in expected PENDING_CLOSE or CLOSING states");
391 return;
392 }
393
394
395
396 regionState.update(RegionState.State.CLOSED, data.getStamp());
397 this.executorService.submit(new ClosedRegionHandler(master,
398 this, regionState.getRegion()));
399 break;
400
401 case RS_ZK_REGION_OPENING:
402
403
404 if(regionState == null ||
405 (!regionState.isPendingOpen() && !regionState.isOpening())) {
406 LOG.warn("Received OPENING for region " +
407 prettyPrintedRegionName +
408 " from server " + data.getServerName() + " but region was in " +
409 " the state " + regionState + " and not " +
410 "in expected PENDING_OPEN or OPENING states");
411 return;
412 }
413
414 regionState.update(RegionState.State.OPENING, data.getStamp());
415 break;
416
417 case RS_ZK_REGION_OPENED:
418
419 if(regionState == null ||
420 (!regionState.isPendingOpen() && !regionState.isOpening())) {
421 LOG.warn("Received OPENED for region " +
422 prettyPrintedRegionName +
423 " from server " + data.getServerName() + " but region was in " +
424 " the state " + regionState + " and not " +
425 "in expected PENDING_OPEN or OPENING states");
426 return;
427 }
428
429 regionState.update(RegionState.State.OPEN, data.getStamp());
430 this.executorService.submit(
431 new OpenedRegionHandler(master, this, regionState.getRegion(),
432 this.serverManager.getServerInfo(data.getServerName())));
433 break;
434 }
435 }
436 }
437
438
439
440
441
442
443
444 private void handleHBCK(RegionTransitionData data) {
445 String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
446 LOG.info("Handling HBCK triggered transition=" + data.getEventType() +
447 ", server=" + data.getServerName() + ", region=" +
448 HRegionInfo.prettyPrint(encodedName));
449 RegionState regionState = regionsInTransition.get(encodedName);
450 switch (data.getEventType()) {
451 case M_ZK_REGION_OFFLINE:
452 HRegionInfo regionInfo = null;
453 if (regionState != null) {
454 regionInfo = regionState.getRegion();
455 } else {
456 try {
457 regionInfo = MetaReader.getRegion(catalogTracker,
458 data.getRegionName()).getFirst();
459 } catch (IOException e) {
460 LOG.info("Exception reading META doing HBCK repair operation", e);
461 return;
462 }
463 }
464 LOG.info("HBCK repair is triggering assignment of region=" +
465 regionInfo.getRegionNameAsString());
466
467 assign(regionInfo, false);
468 break;
469
470 default:
471 LOG.warn("Received unexpected region state from HBCK (" +
472 data.getEventType() + ")");
473 break;
474 }
475 }
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 @Override
492 public void nodeCreated(String path) {
493 if(path.startsWith(watcher.assignmentZNode)) {
494 synchronized(regionsInTransition) {
495 try {
496 RegionTransitionData data = ZKAssign.getData(watcher, path);
497 if(data == null) {
498 return;
499 }
500 handleRegion(data);
501 } catch (KeeperException e) {
502 master.abort("Unexpected ZK exception reading unassigned node data", e);
503 }
504 }
505 }
506 }
507
508
509
510
511
512
513
514
515
516
517
518
519
520 @Override
521 public void nodeDataChanged(String path) {
522 if(path.startsWith(watcher.assignmentZNode)) {
523 synchronized(regionsInTransition) {
524 try {
525 RegionTransitionData data = ZKAssign.getData(watcher, path);
526 if(data == null) {
527 return;
528 }
529 handleRegion(data);
530 } catch (KeeperException e) {
531 master.abort("Unexpected ZK exception reading unassigned node data", e);
532 }
533 }
534 }
535 }
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550 @Override
551 public void nodeChildrenChanged(String path) {
552 if(path.equals(watcher.assignmentZNode)) {
553 synchronized(regionsInTransition) {
554 try {
555 List<NodeAndData> newNodes = ZKUtil.watchAndGetNewChildren(watcher,
556 watcher.assignmentZNode);
557 for(NodeAndData newNode : newNodes) {
558 LOG.debug("Handling new unassigned node: " + newNode);
559 handleRegion(RegionTransitionData.fromBytes(newNode.getData()));
560 }
561 } catch(KeeperException e) {
562 master.abort("Unexpected ZK exception reading unassigned children", e);
563 }
564 }
565 }
566 }
567
568
569
570
571
572
573
574
575
576 public void regionOnline(HRegionInfo regionInfo, HServerInfo serverInfo) {
577 synchronized (this.regionsInTransition) {
578 RegionState rs =
579 this.regionsInTransition.remove(regionInfo.getEncodedName());
580 if (rs != null) {
581 this.regionsInTransition.notifyAll();
582 }
583 }
584 synchronized (this.regions) {
585
586 HServerInfo hsi = this.regions.get(regionInfo);
587 if (hsi != null) LOG.warn("Overwriting " + regionInfo.getEncodedName() +
588 " on " + hsi);
589 this.regions.put(regionInfo, serverInfo);
590 addToServers(serverInfo, regionInfo);
591 this.regions.notifyAll();
592 }
593
594 clearRegionPlan(regionInfo);
595
596 updateTimers(serverInfo);
597 }
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612 private void updateTimers(final HServerInfo hsi) {
613
614
615
616
617 Map<String, RegionPlan> copy = new HashMap<String, RegionPlan>();
618 synchronized(this.regionPlans) {
619 copy.putAll(this.regionPlans);
620 }
621 for (Map.Entry<String, RegionPlan> e: copy.entrySet()) {
622 if (!e.getValue().getDestination().equals(hsi)) continue;
623 RegionState rs = null;
624 synchronized (this.regionsInTransition) {
625 rs = this.regionsInTransition.get(e.getKey());
626 }
627 if (rs == null) continue;
628 synchronized (rs) {
629 rs.update(rs.getState());
630 }
631 }
632 }
633
634
635
636
637
638
639
640
641 public void regionOffline(final HRegionInfo regionInfo) {
642 synchronized(this.regionsInTransition) {
643 if (this.regionsInTransition.remove(regionInfo.getEncodedName()) != null) {
644 this.regionsInTransition.notifyAll();
645 }
646 }
647
648 clearRegionPlan(regionInfo);
649 setOffline(regionInfo);
650 }
651
652
653
654
655
656
657
658
659 public void setOffline(HRegionInfo regionInfo) {
660 synchronized (this.regions) {
661 HServerInfo serverInfo = this.regions.remove(regionInfo);
662 if (serverInfo == null) return;
663 List<HRegionInfo> serverRegions = this.servers.get(serverInfo);
664 if (!serverRegions.remove(regionInfo)) {
665 LOG.warn("No " + regionInfo + " on " + serverInfo);
666 }
667 }
668 }
669
670 public void offlineDisabledRegion(HRegionInfo regionInfo) {
671
672 LOG.debug("Table being disabled so deleting ZK node and removing from " +
673 "regions in transition, skipping assignment of region " +
674 regionInfo.getRegionNameAsString());
675 try {
676 if (!ZKAssign.deleteClosedNode(watcher, regionInfo.getEncodedName())) {
677
678 ZKAssign.deleteOfflineNode(watcher, regionInfo.getEncodedName());
679 }
680 } catch (KeeperException.NoNodeException nne) {
681 LOG.debug("Tried to delete closed node for " + regionInfo + " but it " +
682 "does not exist so just offlining");
683 } catch (KeeperException e) {
684 this.master.abort("Error deleting CLOSED node in ZK", e);
685 }
686 regionOffline(regionInfo);
687 }
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709 public void assign(HRegionInfo region, boolean setOfflineInZK) {
710 assign(region, setOfflineInZK, false);
711 }
712
713 public void assign(HRegionInfo region, boolean setOfflineInZK,
714 boolean forceNewPlan) {
715 String tableName = region.getTableDesc().getNameAsString();
716 boolean disabled = this.zkTable.isDisabledTable(tableName);
717 if (disabled || this.zkTable.isDisablingTable(tableName)) {
718 LOG.info("Table " + tableName + (disabled? " disabled;": " disabling;") +
719 " skipping assign of " + region.getRegionNameAsString());
720 offlineDisabledRegion(region);
721 return;
722 }
723 if (this.serverManager.isClusterShutdown()) {
724 LOG.info("Cluster shutdown is set; skipping assign of " +
725 region.getRegionNameAsString());
726 return;
727 }
728 RegionState state = addToRegionsInTransition(region);
729 synchronized (state) {
730 assign(state, setOfflineInZK, forceNewPlan);
731 }
732 }
733
734
735
736
737
738
739
740 void assign(final HServerInfo destination,
741 final List<HRegionInfo> regions) {
742 LOG.debug("Bulk assigning " + regions.size() + " region(s) to " +
743 destination.getServerName());
744
745 List<RegionState> states = new ArrayList<RegionState>(regions.size());
746 synchronized (this.regionsInTransition) {
747 for (HRegionInfo region: regions) {
748 states.add(forceRegionStateToOffline(region));
749 }
750 }
751
752
753 AtomicInteger counter = new AtomicInteger(0);
754 CreateUnassignedAsyncCallback cb =
755 new CreateUnassignedAsyncCallback(this.watcher, destination, counter);
756 for (RegionState state: states) {
757 if (!asyncSetOfflineInZooKeeper(state, cb, state)) {
758 return;
759 }
760 }
761
762 int total = regions.size();
763 for (int oldCounter = 0; true;) {
764 int count = counter.get();
765 if (oldCounter != count) {
766 LOG.info(destination.getServerName() + " unassigned znodes=" + count +
767 " of total=" + total);
768 oldCounter = count;
769 }
770 if (count == total) break;
771 Threads.sleep(1);
772 }
773
774 try {
775
776 this.serverManager.sendRegionOpen(destination, regions);
777 } catch (Throwable t) {
778 this.master.abort("Failed assignment of regions to " + destination, t);
779 return;
780 }
781 LOG.debug("Bulk assigning done for " + destination.getServerName());
782 }
783
784
785
786
787 static class CreateUnassignedAsyncCallback implements AsyncCallback.StringCallback {
788 private final Log LOG = LogFactory.getLog(CreateUnassignedAsyncCallback.class);
789 private final ZooKeeperWatcher zkw;
790 private final HServerInfo destination;
791 private final AtomicInteger counter;
792
793 CreateUnassignedAsyncCallback(final ZooKeeperWatcher zkw,
794 final HServerInfo destination, final AtomicInteger counter) {
795 this.zkw = zkw;
796 this.destination = destination;
797 this.counter = counter;
798 }
799
800 @Override
801 public void processResult(int rc, String path, Object ctx, String name) {
802 if (rc != 0) {
803
804 LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
805 "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
806 this.zkw.abort("Connectionloss writing unassigned at " + path +
807 ", rc=" + rc, null);
808 return;
809 }
810 LOG.debug("rs=" + (RegionState)ctx + ", server=" + this.destination.getServerName());
811
812
813 this.zkw.getZooKeeper().exists(path, this.zkw,
814 new ExistsUnassignedAsyncCallback(this.counter), ctx);
815 }
816 }
817
818
819
820
821
822 static class ExistsUnassignedAsyncCallback implements AsyncCallback.StatCallback {
823 private final Log LOG = LogFactory.getLog(ExistsUnassignedAsyncCallback.class);
824 private final AtomicInteger counter;
825
826 ExistsUnassignedAsyncCallback(final AtomicInteger counter) {
827 this.counter = counter;
828 }
829
830 @Override
831 public void processResult(int rc, String path, Object ctx, Stat stat) {
832 if (rc != 0) {
833
834 LOG.warn("rc != 0 for " + path + " -- retryable connectionloss -- " +
835 "FIX see http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A2");
836 return;
837 }
838 RegionState state = (RegionState)ctx;
839 LOG.debug("rs=" + state);
840
841
842
843
844
845 state.update(RegionState.State.PENDING_OPEN);
846 this.counter.addAndGet(1);
847 }
848 }
849
850
851
852
853
854 private RegionState addToRegionsInTransition(final HRegionInfo region) {
855 synchronized (regionsInTransition) {
856 return forceRegionStateToOffline(region);
857 }
858 }
859
860
861
862
863
864
865
866 private RegionState forceRegionStateToOffline(final HRegionInfo region) {
867 String encodedName = region.getEncodedName();
868 RegionState state = this.regionsInTransition.get(encodedName);
869 if (state == null) {
870 state = new RegionState(region, RegionState.State.OFFLINE);
871 this.regionsInTransition.put(encodedName, state);
872 } else {
873 LOG.debug("Forcing OFFLINE; was=" + state);
874 state.update(RegionState.State.OFFLINE);
875 }
876 return state;
877 }
878
879
880
881
882
883
884
885 private void assign(final RegionState state, final boolean setOfflineInZK,
886 final boolean forceNewPlan) {
887 for (int i = 0; i < this.maximumAssignmentAttempts; i++) {
888 if (setOfflineInZK && !setOfflineInZooKeeper(state)) return;
889 if (this.master.isStopped()) {
890 LOG.debug("Server stopped; skipping assign of " + state);
891 return;
892 }
893 RegionPlan plan = getRegionPlan(state, forceNewPlan);
894 if (plan == null) return;
895 try {
896 LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() +
897 " to " + plan.getDestination().getServerName());
898
899 state.update(RegionState.State.PENDING_OPEN);
900
901 serverManager.sendRegionOpen(plan.getDestination(), state.getRegion());
902 break;
903 } catch (Throwable t) {
904 LOG.warn("Failed assignment of " +
905 state.getRegion().getRegionNameAsString() + " to " +
906 plan.getDestination() + ", trying to assign elsewhere instead; " +
907 "retry=" + i, t);
908
909
910
911 state.update(RegionState.State.OFFLINE);
912
913 if (getRegionPlan(state, plan.getDestination(), true) == null) {
914 LOG.warn("Unable to find a viable location to assign region " +
915 state.getRegion().getRegionNameAsString());
916 return;
917 }
918 }
919 }
920 }
921
922
923
924
925
926
927
928 boolean setOfflineInZooKeeper(final RegionState state) {
929 if (!state.isClosed() && !state.isOffline()) {
930 new RuntimeException("Unexpected state trying to OFFLINE; " + state);
931 this.master.abort("Unexpected state trying to OFFLINE; " + state,
932 new IllegalStateException());
933 return false;
934 }
935 state.update(RegionState.State.OFFLINE);
936 try {
937 if(!ZKAssign.createOrForceNodeOffline(master.getZooKeeper(),
938 state.getRegion(), master.getServerName())) {
939 LOG.warn("Attempted to create/force node into OFFLINE state before " +
940 "completing assignment but failed to do so for " + state);
941 return false;
942 }
943 } catch (KeeperException e) {
944 master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
945 return false;
946 }
947 return true;
948 }
949
950
951
952
953
954
955
956 boolean asyncSetOfflineInZooKeeper(final RegionState state,
957 final AsyncCallback.StringCallback cb, final Object ctx) {
958 if (!state.isClosed() && !state.isOffline()) {
959 new RuntimeException("Unexpected state trying to OFFLINE; " + state);
960 this.master.abort("Unexpected state trying to OFFLINE; " + state,
961 new IllegalStateException());
962 return false;
963 }
964 state.update(RegionState.State.OFFLINE);
965 try {
966 ZKAssign.asyncCreateNodeOffline(master.getZooKeeper(), state.getRegion(),
967 master.getServerName(), cb, ctx);
968 } catch (KeeperException e) {
969 master.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
970 return false;
971 }
972 return true;
973 }
974
975
976
977
978
979
980 RegionPlan getRegionPlan(final RegionState state,
981 final boolean forceNewPlan) {
982 return getRegionPlan(state, null, forceNewPlan);
983 }
984
985
986
987
988
989
990
991
992
993
994 RegionPlan getRegionPlan(final RegionState state,
995 final HServerInfo serverToExclude, final boolean forceNewPlan) {
996
997 String encodedName = state.getRegion().getEncodedName();
998 List<HServerInfo> servers = this.serverManager.getOnlineServersList();
999
1000
1001 if (serverToExclude != null) servers.remove(serverToExclude);
1002 if (servers.isEmpty()) return null;
1003 RegionPlan randomPlan = new RegionPlan(state.getRegion(), null,
1004 LoadBalancer.randomAssignment(servers));
1005 boolean newPlan = false;
1006 RegionPlan existingPlan = null;
1007 synchronized (this.regionPlans) {
1008 existingPlan = this.regionPlans.get(encodedName);
1009 if (forceNewPlan || existingPlan == null
1010 || existingPlan.getDestination() == null
1011 || existingPlan.getDestination().equals(serverToExclude)) {
1012 newPlan = true;
1013 this.regionPlans.put(encodedName, randomPlan);
1014 }
1015 }
1016 if (newPlan) {
1017 LOG.debug("No previous transition plan was found (or we are ignoring " +
1018 "an existing plan) for " + state.getRegion().getRegionNameAsString() +
1019 " so generated a random one; " + randomPlan + "; " +
1020 serverManager.countOfRegionServers() +
1021 " (online=" + serverManager.getOnlineServers().size() +
1022 ", exclude=" + serverToExclude + ") available servers");
1023 return randomPlan;
1024 }
1025 LOG.debug("Using pre-existing plan for region " +
1026 state.getRegion().getRegionNameAsString() + "; plan=" + existingPlan);
1027 return existingPlan;
1028 }
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039 public void unassign(HRegionInfo region) {
1040 unassign(region, false);
1041 }
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 public void unassign(HRegionInfo region, boolean force) {
1054 LOG.debug("Starting unassignment of region " +
1055 region.getRegionNameAsString() + " (offlining)");
1056 synchronized (this.regions) {
1057
1058 if (!regions.containsKey(region)) {
1059 LOG.debug("Attempted to unassign region " +
1060 region.getRegionNameAsString() + " but it is not " +
1061 "currently assigned anywhere");
1062 return;
1063 }
1064 }
1065 String encodedName = region.getEncodedName();
1066
1067 RegionState state;
1068 synchronized (regionsInTransition) {
1069 state = regionsInTransition.get(encodedName);
1070 if (state == null) {
1071 state = new RegionState(region, RegionState.State.PENDING_CLOSE);
1072 regionsInTransition.put(encodedName, state);
1073 } else if (force && state.isPendingClose()) {
1074 LOG.debug("Attempting to unassign region " +
1075 region.getRegionNameAsString() + " which is already pending close "
1076 + "but forcing an additional close");
1077 state.update(RegionState.State.PENDING_CLOSE);
1078 } else {
1079 LOG.debug("Attempting to unassign region " +
1080 region.getRegionNameAsString() + " but it is " +
1081 "already in transition (" + state.getState() + ")");
1082 return;
1083 }
1084 }
1085
1086 HServerInfo server = null;
1087 synchronized (this.regions) {
1088 server = regions.get(region);
1089 }
1090 try {
1091
1092
1093 if (serverManager.sendRegionClose(server, state.getRegion())) {
1094 LOG.debug("Sent CLOSE to " + server + " for region " +
1095 region.getRegionNameAsString());
1096 return;
1097 }
1098
1099 LOG.debug("Server " + server + " region CLOSE RPC returned false for " +
1100 region.getEncodedName());
1101 } catch (NotServingRegionException nsre) {
1102 LOG.info("Server " + server + " returned " + nsre + " for " +
1103 region.getEncodedName());
1104
1105
1106
1107 return;
1108 } catch (ConnectException e) {
1109 LOG.info("Failed connect to " + server + ", message=" + e.getMessage() +
1110 ", region=" + region.getEncodedName());
1111
1112
1113 } catch (java.net.SocketTimeoutException e) {
1114 LOG.info("Server " + server + " returned " + e.getMessage() + " for " +
1115 region.getEncodedName());
1116
1117 } catch (EOFException e) {
1118 LOG.info("Server " + server + " returned " + e.getMessage() + " for " +
1119 region.getEncodedName());
1120
1121 } catch (RemoteException re) {
1122 IOException ioe = re.unwrapRemoteException();
1123 if (ioe instanceof NotServingRegionException) {
1124
1125 LOG.debug("Server " + server + " returned " + ioe + " for " +
1126 region.getEncodedName());
1127 } else if (ioe instanceof EOFException) {
1128
1129 LOG.debug("Server " + server + " returned " + ioe + " for " +
1130 region.getEncodedName());
1131 } else {
1132 this.master.abort("Remote unexpected exception", ioe);
1133 }
1134 } catch (Throwable t) {
1135
1136
1137 this.master.abort("Remote unexpected exception", t);
1138 }
1139 }
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149 public void waitForAssignment(HRegionInfo regionInfo)
1150 throws InterruptedException {
1151 synchronized(regions) {
1152 while(!regions.containsKey(regionInfo)) {
1153 regions.wait();
1154 }
1155 }
1156 }
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168 public void assignRoot() throws KeeperException {
1169 RootLocationEditor.deleteRootLocation(this.master.getZooKeeper());
1170 assign(HRegionInfo.ROOT_REGIONINFO, true);
1171 }
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181 public void assignMeta() {
1182
1183 assign(HRegionInfo.FIRST_META_REGIONINFO, true);
1184 }
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194 public void assignUserRegions(List<HRegionInfo> regions, List<HServerInfo> servers) throws IOException, InterruptedException {
1195 if (regions == null)
1196 return;
1197 Map<HServerInfo, List<HRegionInfo>> bulkPlan = null;
1198
1199 bulkPlan = LoadBalancer.roundRobinAssignment(regions, servers);
1200 LOG.info("Bulk assigning " + regions.size() + " region(s) round-robin across " +
1201 servers.size() + " server(s)");
1202
1203 BulkAssigner ba = new BulkStartupAssigner(this.master, bulkPlan, this);
1204 ba.bulkAssign();
1205 LOG.info("Bulk assigning done");
1206 }
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217 public void assignAllUserRegions() throws IOException, InterruptedException {
1218
1219 List<HServerInfo> servers = serverManager.getOnlineServersList();
1220
1221
1222 Map<HRegionInfo,HServerAddress> allRegions =
1223 MetaReader.fullScan(catalogTracker, this.zkTable.getDisabledTables(), true);
1224 if (allRegions == null || allRegions.isEmpty()) return;
1225
1226
1227 boolean retainAssignment = master.getConfiguration().
1228 getBoolean("hbase.master.startup.retainassign", true);
1229
1230 Map<HServerInfo, List<HRegionInfo>> bulkPlan = null;
1231 if (retainAssignment) {
1232
1233 bulkPlan = LoadBalancer.retainAssignment(allRegions, servers);
1234 } else {
1235
1236 assignUserRegions(new ArrayList<HRegionInfo>(allRegions.keySet()), servers);
1237 return;
1238 }
1239 LOG.info("Bulk assigning " + allRegions.size() + " region(s) across " +
1240 servers.size() + " server(s), retainAssignment=" + retainAssignment);
1241
1242
1243 BulkAssigner ba = new BulkStartupAssigner(this.master, bulkPlan, this);
1244 ba.bulkAssign();
1245 LOG.info("Bulk assigning done");
1246 }
1247
1248
1249
1250
1251 static class BulkStartupAssigner extends BulkAssigner {
1252 private final Map<HServerInfo, List<HRegionInfo>> bulkPlan;
1253 private final AssignmentManager assignmentManager;
1254
1255 BulkStartupAssigner(final Server server,
1256 final Map<HServerInfo, List<HRegionInfo>> bulkPlan,
1257 final AssignmentManager am) {
1258 super(server);
1259 this.bulkPlan = bulkPlan;
1260 this.assignmentManager = am;
1261 }
1262
1263 @Override
1264 public boolean bulkAssign() throws InterruptedException {
1265
1266 this.assignmentManager.timeoutMonitor.bulkAssign(true);
1267 try {
1268 return super.bulkAssign();
1269 } finally {
1270
1271 this.assignmentManager.timeoutMonitor.bulkAssign(false);
1272 }
1273 }
1274
1275 @Override
1276 protected String getThreadNamePrefix() {
1277 return super.getThreadNamePrefix() + "-startup";
1278 }
1279
1280 @Override
1281 protected void populatePool(java.util.concurrent.ExecutorService pool) {
1282 for (Map.Entry<HServerInfo, List<HRegionInfo>> e: this.bulkPlan.entrySet()) {
1283 pool.execute(new SingleServerBulkAssigner(e.getKey(), e.getValue(),
1284 this.assignmentManager));
1285 }
1286 }
1287
1288 protected boolean waitUntilDone(final long timeout)
1289 throws InterruptedException {
1290 return this.assignmentManager.waitUntilNoRegionsInTransition(timeout);
1291 }
1292 }
1293
1294
1295
1296
1297 static class SingleServerBulkAssigner implements Runnable {
1298 private final HServerInfo regionserver;
1299 private final List<HRegionInfo> regions;
1300 private final AssignmentManager assignmentManager;
1301
1302 SingleServerBulkAssigner(final HServerInfo regionserver,
1303 final List<HRegionInfo> regions, final AssignmentManager am) {
1304 this.regionserver = regionserver;
1305 this.regions = regions;
1306 this.assignmentManager = am;
1307 }
1308 @Override
1309 public void run() {
1310 this.assignmentManager.assign(this.regionserver, this.regions);
1311 }
1312 }
1313
1314
1315
1316
1317
1318
1319
1320 boolean waitUntilNoRegionsInTransition(final long timeout)
1321 throws InterruptedException {
1322
1323
1324
1325
1326
1327
1328 long startTime = System.currentTimeMillis();
1329 long remaining = timeout;
1330 synchronized (regionsInTransition) {
1331 while (regionsInTransition.size() > 0 && !this.master.isStopped()
1332 && remaining > 0) {
1333 regionsInTransition.wait(remaining);
1334 remaining = timeout - (System.currentTimeMillis() - startTime);
1335 }
1336 }
1337 return regionsInTransition.isEmpty();
1338 }
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349 private Map<HServerInfo,List<Pair<HRegionInfo,Result>>> rebuildUserRegions()
1350 throws IOException {
1351
1352 List<Result> results = MetaReader.fullScanOfResults(catalogTracker);
1353
1354 Map<HServerInfo,List<Pair<HRegionInfo,Result>>> offlineServers =
1355 new TreeMap<HServerInfo,List<Pair<HRegionInfo,Result>>>();
1356
1357 for (Result result : results) {
1358 Pair<HRegionInfo,HServerInfo> region =
1359 MetaReader.metaRowToRegionPairWithInfo(result);
1360 if (region == null) continue;
1361 HServerInfo regionLocation = region.getSecond();
1362 HRegionInfo regionInfo = region.getFirst();
1363 if (regionLocation == null) {
1364
1365
1366 this.regions.put(regionInfo, null);
1367 } else if (!serverManager.isServerOnline(
1368 regionLocation.getServerName())) {
1369
1370 List<Pair<HRegionInfo,Result>> offlineRegions =
1371 offlineServers.get(regionLocation);
1372 if (offlineRegions == null) {
1373 offlineRegions = new ArrayList<Pair<HRegionInfo,Result>>(1);
1374 offlineServers.put(regionLocation, offlineRegions);
1375 }
1376 offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
1377 } else {
1378
1379 regions.put(regionInfo, regionLocation);
1380 addToServers(regionLocation, regionInfo);
1381 }
1382 }
1383 return offlineServers;
1384 }
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399 private void processDeadServers(
1400 Map<HServerInfo, List<Pair<HRegionInfo, Result>>> deadServers)
1401 throws IOException, KeeperException {
1402 for (Map.Entry<HServerInfo, List<Pair<HRegionInfo,Result>>> deadServer :
1403 deadServers.entrySet()) {
1404 List<Pair<HRegionInfo,Result>> regions = deadServer.getValue();
1405 for (Pair<HRegionInfo,Result> region : regions) {
1406 HRegionInfo regionInfo = region.getFirst();
1407 Result result = region.getSecond();
1408
1409 try {
1410 ZKAssign.createOrForceNodeOffline(watcher, regionInfo,
1411 master.getServerName());
1412 } catch (KeeperException.NoNodeException nne) {
1413
1414 }
1415
1416 ServerShutdownHandler.processDeadRegion(regionInfo, result, this,
1417 this.catalogTracker);
1418 }
1419 }
1420 }
1421
1422
1423
1424
1425
1426
1427 private void addToServers(final HServerInfo hsi, final HRegionInfo hri) {
1428 List<HRegionInfo> hris = servers.get(hsi);
1429 if (hris == null) {
1430 hris = new ArrayList<HRegionInfo>();
1431 servers.put(hsi, hris);
1432 }
1433 hris.add(hri);
1434 }
1435
1436
1437
1438
1439 public NavigableMap<String, RegionState> getRegionsInTransition() {
1440 synchronized (this.regionsInTransition) {
1441 return new TreeMap<String, RegionState>(this.regionsInTransition);
1442 }
1443 }
1444
1445
1446
1447
1448 public boolean isRegionsInTransition() {
1449 synchronized (this.regionsInTransition) {
1450 return !this.regionsInTransition.isEmpty();
1451 }
1452 }
1453
1454
1455
1456
1457
1458
1459 public RegionState isRegionInTransition(final HRegionInfo hri) {
1460 synchronized (this.regionsInTransition) {
1461 return this.regionsInTransition.get(hri.getEncodedName());
1462 }
1463 }
1464
1465
1466
1467
1468
1469
1470
1471 public void clearRegionFromTransition(HRegionInfo hri) {
1472 synchronized (this.regionsInTransition) {
1473 this.regionsInTransition.remove(hri.getEncodedName());
1474 }
1475 synchronized (this.regions) {
1476 this.regions.remove(hri);
1477 for (List<HRegionInfo> regions : this.servers.values()) {
1478 for (int i=0;i<regions.size();i++) {
1479 if (regions.get(i).equals(hri)) {
1480 regions.remove(i);
1481 break;
1482 }
1483 }
1484 }
1485 }
1486 clearRegionPlan(hri);
1487 }
1488
1489
1490
1491
1492 void clearRegionPlan(final HRegionInfo region) {
1493 synchronized (this.regionPlans) {
1494 this.regionPlans.remove(region.getEncodedName());
1495 }
1496 }
1497
1498
1499
1500
1501
1502
1503 public void waitOnRegionToClearRegionsInTransition(final HRegionInfo hri)
1504 throws IOException {
1505 if (isRegionInTransition(hri) == null) return;
1506 RegionState rs = null;
1507
1508
1509 while(!this.master.isStopped() && (rs = isRegionInTransition(hri)) != null) {
1510 Threads.sleep(1000);
1511 LOG.info("Waiting on " + rs + " to clear regions-in-transition");
1512 }
1513 if (this.master.isStopped()) {
1514 LOG.info("Giving up wait on regions in " +
1515 "transition because stoppable.isStopped is set");
1516 }
1517 }
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530 public List<HRegionInfo> getRegionsOfTable(byte[] tableName) {
1531 List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
1532 HRegionInfo boundary =
1533 new HRegionInfo(new HTableDescriptor(tableName), null, null);
1534 synchronized (this.regions) {
1535 for (HRegionInfo regionInfo: this.regions.tailMap(boundary).keySet()) {
1536 if(Bytes.equals(regionInfo.getTableDesc().getName(), tableName)) {
1537 tableRegions.add(regionInfo);
1538 } else {
1539 break;
1540 }
1541 }
1542 }
1543 return tableRegions;
1544 }
1545
1546
1547
1548
1549 public class TimeoutMonitor extends Chore {
1550 private final int timeout;
1551 private boolean bulkAssign = false;
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562 public TimeoutMonitor(final int period, final Stoppable stopper,
1563 final int timeout) {
1564 super("AssignmentTimeoutMonitor", period, stopper);
1565 this.timeout = timeout;
1566 }
1567
1568
1569
1570
1571
1572
1573 public boolean bulkAssign(final boolean bulkAssign) {
1574 boolean result = this.bulkAssign;
1575 this.bulkAssign = bulkAssign;
1576 return result;
1577 }
1578
1579 @Override
1580 protected void chore() {
1581
1582 if (this.bulkAssign) return;
1583 synchronized (regionsInTransition) {
1584
1585 long now = System.currentTimeMillis();
1586 for (RegionState regionState : regionsInTransition.values()) {
1587 if(regionState.getStamp() + timeout <= now) {
1588 HRegionInfo regionInfo = regionState.getRegion();
1589 LOG.info("Regions in transition timed out: " + regionState);
1590
1591 switch (regionState.getState()) {
1592 case CLOSED:
1593 LOG.info("Region " + regionInfo.getEncodedName() +
1594 " has been CLOSED for too long, waiting on queued " +
1595 "ClosedRegionHandler to run or server shutdown");
1596
1597 synchronized(regionState) {
1598 regionState.update(regionState.getState());
1599 }
1600 break;
1601 case OFFLINE:
1602 LOG.info("Region has been OFFLINE for too long, " +
1603 "reassigning " + regionInfo.getRegionNameAsString() +
1604 " to a random server");
1605 assign(regionState.getRegion(), false);
1606 break;
1607 case PENDING_OPEN:
1608 LOG.info("Region has been PENDING_OPEN for too " +
1609 "long, reassigning region=" +
1610 regionInfo.getRegionNameAsString());
1611 assign(regionState.getRegion(), false, true);
1612 break;
1613 case OPENING:
1614 LOG.info("Region has been OPENING for too " +
1615 "long, reassigning region=" +
1616 regionInfo.getRegionNameAsString());
1617
1618 try {
1619 String node = ZKAssign.getNodeName(watcher,
1620 regionInfo.getEncodedName());
1621 Stat stat = new Stat();
1622 RegionTransitionData data = ZKAssign.getDataNoWatch(watcher,
1623 node, stat);
1624 if (data.getEventType() == EventType.RS_ZK_REGION_OPENED) {
1625 LOG.debug("Region has transitioned to OPENED, allowing " +
1626 "watched event handlers to process");
1627 break;
1628 } else if (data.getEventType() !=
1629 EventType.RS_ZK_REGION_OPENING) {
1630 LOG.warn("While timing out a region in state OPENING, " +
1631 "found ZK node in unexpected state: " +
1632 data.getEventType());
1633 break;
1634 }
1635
1636 try {
1637 data = new RegionTransitionData(
1638 EventType.M_ZK_REGION_OFFLINE, regionInfo.getRegionName(),
1639 master.getServerName());
1640 if (ZKUtil.setData(watcher, node, data.getBytes(),
1641 stat.getVersion())) {
1642
1643 ZKUtil.getDataAndWatch(watcher, node);
1644 LOG.info("Successfully transitioned region=" +
1645 regionInfo.getRegionNameAsString() + " into OFFLINE" +
1646 " and forcing a new assignment");
1647 assign(regionState, false, true);
1648 }
1649 } catch (KeeperException.NoNodeException nne) {
1650
1651 }
1652 } catch (KeeperException ke) {
1653 LOG.error("Unexpected ZK exception timing out CLOSING region",
1654 ke);
1655 break;
1656 }
1657 break;
1658 case OPEN:
1659 LOG.error("Region has been OPEN for too long, " +
1660 "we don't know where region was opened so can't do anything");
1661 break;
1662 case PENDING_CLOSE:
1663 LOG.info("Region has been PENDING_CLOSE for too " +
1664 "long, running forced unassign again on region=" +
1665 regionInfo.getRegionNameAsString());
1666 try {
1667
1668
1669 if (!ZKUtil.watchAndCheckExists(watcher,
1670 ZKAssign.getNodeName(watcher,
1671 regionInfo.getEncodedName()))) {
1672 unassign(regionInfo, true);
1673 }
1674 } catch (NoNodeException e) {
1675 LOG.debug("Node no longer existed so not forcing another " +
1676 "unassignment");
1677 } catch (KeeperException e) {
1678 LOG.warn("Unexpected ZK exception timing out a region " +
1679 "close", e);
1680 }
1681 break;
1682 case CLOSING:
1683 LOG.info("Region has been CLOSING for too " +
1684 "long, this should eventually complete or the server will " +
1685 "expire, doing nothing");
1686 break;
1687 }
1688 }
1689 }
1690 }
1691 }
1692 }
1693
1694
1695
1696
1697
1698
1699 public List<RegionState> processServerShutdown(final HServerInfo hsi) {
1700
1701 synchronized (this.regionPlans) {
1702 for (Iterator <Map.Entry<String, RegionPlan>> i =
1703 this.regionPlans.entrySet().iterator(); i.hasNext();) {
1704 Map.Entry<String, RegionPlan> e = i.next();
1705 if (e.getValue().getDestination().equals(hsi)) {
1706
1707 i.remove();
1708 }
1709 }
1710 }
1711
1712
1713
1714 Set<HRegionInfo> deadRegions = null;
1715 List<RegionState> rits = new ArrayList<RegionState>();
1716 synchronized (this.regions) {
1717 List<HRegionInfo> assignedRegions = this.servers.remove(hsi);
1718 if (assignedRegions == null || assignedRegions.isEmpty()) {
1719
1720 return rits;
1721 }
1722 deadRegions = new TreeSet<HRegionInfo>(assignedRegions);
1723 for (HRegionInfo region : deadRegions) {
1724 this.regions.remove(region);
1725 }
1726 }
1727
1728
1729
1730 synchronized (regionsInTransition) {
1731 for (RegionState region : this.regionsInTransition.values()) {
1732 if (deadRegions.remove(region.getRegion())) {
1733 rits.add(region);
1734 }
1735 }
1736 }
1737 return rits;
1738 }
1739
1740
1741
1742
1743
1744
1745
1746
1747 public void handleSplitReport(final HServerInfo hsi, final HRegionInfo parent,
1748 final HRegionInfo a, final HRegionInfo b) {
1749 regionOffline(parent);
1750
1751
1752
1753 try {
1754 RegionTransitionData node = ZKAssign.getDataNoWatch(this.watcher,
1755 parent.getEncodedName(), null);
1756 if (node != null) {
1757 if (node.getEventType().equals(EventType.RS_ZK_REGION_CLOSING)) {
1758 ZKAssign.deleteClosingNode(this.watcher, parent);
1759 } else {
1760 LOG.warn("Split report has RIT node (shouldnt have one): " +
1761 parent + " node: " + node);
1762 }
1763 }
1764 } catch (KeeperException e) {
1765 LOG.warn("Exception while validating RIT during split report", e);
1766 }
1767
1768 regionOnline(a, hsi);
1769 regionOnline(b, hsi);
1770
1771
1772
1773
1774
1775 if (this.zkTable.isDisablingOrDisabledTable(
1776 parent.getTableDesc().getNameAsString())) {
1777 unassign(a);
1778 unassign(b);
1779 }
1780 }
1781
1782
1783
1784
1785
1786
1787 Map<HServerInfo, List<HRegionInfo>> getAssignments() {
1788
1789
1790
1791
1792 Map<HServerInfo, List<HRegionInfo>> result = null;
1793 synchronized (this.regions) {
1794 result = new HashMap<HServerInfo, List<HRegionInfo>>(this.servers.size());
1795 for (Map.Entry<HServerInfo, List<HRegionInfo>> e: this.servers.entrySet()) {
1796 List<HRegionInfo> shallowCopy = new ArrayList<HRegionInfo>(e.getValue());
1797 HServerInfo clone = new HServerInfo(e.getKey());
1798
1799
1800 clone.getLoad().setNumberOfRegions(e.getValue().size());
1801 result.put(clone, shallowCopy);
1802 }
1803 }
1804 return result;
1805 }
1806
1807
1808
1809
1810
1811
1812 Pair<HRegionInfo, HServerInfo> getAssignment(final byte [] encodedRegionName) {
1813 String name = Bytes.toString(encodedRegionName);
1814 synchronized(this.regions) {
1815 for (Map.Entry<HRegionInfo, HServerInfo> e: this.regions.entrySet()) {
1816 if (e.getKey().getEncodedName().equals(name)) {
1817 return new Pair<HRegionInfo, HServerInfo>(e.getKey(), e.getValue());
1818 }
1819 }
1820 }
1821 return null;
1822 }
1823
1824
1825
1826
1827 void balance(final RegionPlan plan) {
1828 synchronized (this.regionPlans) {
1829 this.regionPlans.put(plan.getRegionName(), plan);
1830 }
1831 unassign(plan.getRegionInfo());
1832 }
1833
1834
1835
1836
1837 public static class RegionState implements Writable {
1838 private HRegionInfo region;
1839
1840 public enum State {
1841 OFFLINE,
1842 PENDING_OPEN,
1843 OPENING,
1844 OPEN,
1845 PENDING_CLOSE,
1846 CLOSING,
1847 CLOSED
1848 }
1849
1850 private State state;
1851 private long stamp;
1852
1853 public RegionState() {}
1854
1855 RegionState(HRegionInfo region, State state) {
1856 this(region, state, System.currentTimeMillis());
1857 }
1858
1859 RegionState(HRegionInfo region, State state, long stamp) {
1860 this.region = region;
1861 this.state = state;
1862 this.stamp = stamp;
1863 }
1864
1865 public void update(State state, long stamp) {
1866 this.state = state;
1867 this.stamp = stamp;
1868 }
1869
1870 public void update(State state) {
1871 this.state = state;
1872 this.stamp = System.currentTimeMillis();
1873 }
1874
1875 public State getState() {
1876 return state;
1877 }
1878
1879 public long getStamp() {
1880 return stamp;
1881 }
1882
1883 public HRegionInfo getRegion() {
1884 return region;
1885 }
1886
1887 public boolean isClosing() {
1888 return state == State.CLOSING;
1889 }
1890
1891 public boolean isClosed() {
1892 return state == State.CLOSED;
1893 }
1894
1895 public boolean isPendingClose() {
1896 return state == State.PENDING_CLOSE;
1897 }
1898
1899 public boolean isOpening() {
1900 return state == State.OPENING;
1901 }
1902
1903 public boolean isOpened() {
1904 return state == State.OPEN;
1905 }
1906
1907 public boolean isPendingOpen() {
1908 return state == State.PENDING_OPEN;
1909 }
1910
1911 public boolean isOffline() {
1912 return state == State.OFFLINE;
1913 }
1914
1915 @Override
1916 public String toString() {
1917 return region.getRegionNameAsString() + " state=" + state +
1918 ", ts=" + stamp;
1919 }
1920
1921 @Override
1922 public void readFields(DataInput in) throws IOException {
1923 region = new HRegionInfo();
1924 region.readFields(in);
1925 state = State.valueOf(in.readUTF());
1926 stamp = in.readLong();
1927 }
1928
1929 @Override
1930 public void write(DataOutput out) throws IOException {
1931 region.write(out);
1932 out.writeUTF(state.name());
1933 out.writeLong(stamp);
1934 }
1935 }
1936
1937 public void stop() {
1938 this.timeoutMonitor.interrupt();
1939 }
1940 }