1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.NavigableMap;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HServerInfo;
32 import org.apache.hadoop.hbase.Server;
33 import org.apache.hadoop.hbase.catalog.CatalogTracker;
34 import org.apache.hadoop.hbase.catalog.MetaEditor;
35 import org.apache.hadoop.hbase.catalog.MetaReader;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.executor.EventHandler;
38 import org.apache.hadoop.hbase.master.AssignmentManager;
39 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
40 import org.apache.hadoop.hbase.master.DeadServer;
41 import org.apache.hadoop.hbase.master.MasterServices;
42 import org.apache.hadoop.hbase.master.ServerManager;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.Writables;
45 import org.apache.zookeeper.KeeperException;
46
47
48
49
50
51
52 public class ServerShutdownHandler extends EventHandler {
53 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
54 private final HServerInfo hsi;
55 private final Server server;
56 private final MasterServices services;
57 private final DeadServer deadServers;
58
59 public ServerShutdownHandler(final Server server, final MasterServices services,
60 final DeadServer deadServers, final HServerInfo hsi) {
61 this(server, services, deadServers, hsi, EventType.M_SERVER_SHUTDOWN);
62 }
63
64 ServerShutdownHandler(final Server server, final MasterServices services,
65 final DeadServer deadServers, final HServerInfo hsi, EventType type) {
66 super(server, type);
67 this.hsi = hsi;
68 this.server = server;
69 this.services = services;
70 this.deadServers = deadServers;
71 if (!this.deadServers.contains(hsi.getServerName())) {
72 LOG.warn(hsi.getServerName() + " is NOT in deadservers; it should be!");
73 }
74 }
75
76
77
78
79 boolean isCarryingRoot() {
80 return false;
81 }
82
83
84
85
86 boolean isCarryingMeta() {
87 return false;
88 }
89
90 @Override
91 public void process() throws IOException {
92 final String serverName = this.hsi.getServerName();
93
94 LOG.info("Splitting logs for " + serverName);
95 this.services.getMasterFileSystem().splitLog(serverName);
96
97
98
99
100
101 List<RegionState> regionsInTransition =
102 this.services.getAssignmentManager().processServerShutdown(this.hsi);
103
104
105 if (isCarryingRoot()) {
106 try {
107 this.services.getAssignmentManager().assignRoot();
108 } catch (KeeperException e) {
109 this.server.abort("In server shutdown processing, assigning root", e);
110 throw new IOException("Aborting", e);
111 }
112 }
113
114
115 if (isCarryingMeta()) this.services.getAssignmentManager().assignMeta();
116
117
118
119
120 NavigableMap<HRegionInfo, Result> hris = null;
121 while (!this.server.isStopped()) {
122 try {
123 this.server.getCatalogTracker().waitForMeta();
124 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
125 this.hsi);
126 break;
127 } catch (InterruptedException e) {
128 Thread.currentThread().interrupt();
129 throw new IOException("Interrupted", e);
130 } catch (IOException ioe) {
131 LOG.info("Received exception accessing META during server shutdown of " +
132 serverName + ", retrying META read");
133 }
134 }
135
136
137 for (RegionState rit : regionsInTransition) {
138 if (!rit.isClosing() && !rit.isPendingClose()) {
139 LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
140 " from list of regions to assign because in RIT");
141 hris.remove(rit.getRegion());
142 }
143 }
144
145 LOG.info("Reassigning " + hris.size() + " region(s) that " + serverName +
146 " was carrying (skipping " + regionsInTransition.size() +
147 " regions(s) that are already in transition)");
148
149
150 for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
151 if (processDeadRegion(e.getKey(), e.getValue(),
152 this.services.getAssignmentManager(),
153 this.server.getCatalogTracker())) {
154 this.services.getAssignmentManager().assign(e.getKey(), true);
155 }
156 }
157 this.deadServers.finish(serverName);
158 LOG.info("Finished processing of shutdown of " + serverName);
159 }
160
161
162
163
164
165
166
167
168
169
170
171 public static boolean processDeadRegion(HRegionInfo hri, Result result,
172 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
173 throws IOException {
174
175 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
176 hri.getTableDesc().getNameAsString());
177 if (disabled) return false;
178 if (hri.isOffline() && hri.isSplit()) {
179 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
180 "; checking daughter presence");
181 fixupDaughters(result, assignmentManager, catalogTracker);
182 return false;
183 }
184 return true;
185 }
186
187
188
189
190
191
192
193 static void fixupDaughters(final Result result,
194 final AssignmentManager assignmentManager,
195 final CatalogTracker catalogTracker) throws IOException {
196 fixupDaughter(result, HConstants.SPLITA_QUALIFIER, assignmentManager,
197 catalogTracker);
198 fixupDaughter(result, HConstants.SPLITB_QUALIFIER, assignmentManager,
199 catalogTracker);
200 }
201
202
203
204
205
206
207
208 static void fixupDaughter(final Result result, final byte [] qualifier,
209 final AssignmentManager assignmentManager,
210 final CatalogTracker catalogTracker)
211 throws IOException {
212 HRegionInfo daughter = getHRegionInfo(result, qualifier);
213 if (daughter == null) return;
214 if (isDaughterMissing(catalogTracker, daughter)) {
215 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
216 MetaEditor.addDaughter(catalogTracker, daughter, null);
217
218 assignmentManager.assign(daughter, true);
219 } else {
220 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
221 }
222 }
223
224
225
226
227
228
229
230
231
232 private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier)
233 throws IOException {
234 byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
235 if (bytes == null || bytes.length <= 0) return null;
236 return Writables.getHRegionInfoOrNull(bytes);
237 }
238
239
240
241
242
243
244
245
246 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
247 final HRegionInfo daughter) throws IOException {
248 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
249
250
251
252
253
254
255 byte [] startrow = daughter.getRegionName();
256 MetaReader.fullScan(catalogTracker, visitor, startrow);
257 return !visitor.foundDaughter();
258 }
259
260
261
262
263
264 static class FindDaughterVisitor implements MetaReader.Visitor {
265 private final HRegionInfo daughter;
266 private boolean found = false;
267
268 FindDaughterVisitor(final HRegionInfo daughter) {
269 this.daughter = daughter;
270 }
271
272
273
274
275 boolean foundDaughter() {
276 return this.found;
277 }
278
279 @Override
280 public boolean visit(Result r) throws IOException {
281 HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
282 if (hri == null) {
283 LOG.warn("No serialized HRegionInfo in " + r);
284 return true;
285 }
286
287 if (!Bytes.equals(daughter.getTableDesc().getName(),
288 hri.getTableDesc().getName())) {
289
290 return false;
291 }
292
293 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
294 return false;
295 }
296
297
298
299 this.found = true;
300 return false;
301 }
302 }
303 }