1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Queue;
31 import java.util.Random;
32 import java.util.Set;
33 import java.util.SortedSet;
34 import java.util.TreeMap;
35 import java.util.TreeSet;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HServerAddress;
41 import org.apache.hadoop.hbase.HServerInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.junit.BeforeClass;
46 import org.junit.Test;
47
48 public class TestLoadBalancer {
49 private static final Log LOG = LogFactory.getLog(TestLoadBalancer.class);
50
51 private static LoadBalancer loadBalancer;
52
53 private static Random rand;
54
55 @BeforeClass
56 public static void beforeAllTests() throws Exception {
57 loadBalancer = new LoadBalancer();
58 rand = new Random();
59 }
60
61
62 int [][] clusterStateMocks = new int [][] {
63
64 new int [] { 0 },
65 new int [] { 1 },
66 new int [] { 10 },
67
68 new int [] { 0, 0 },
69 new int [] { 2, 0 },
70 new int [] { 2, 1 },
71 new int [] { 2, 2 },
72 new int [] { 2, 3 },
73 new int [] { 2, 4 },
74 new int [] { 1, 1 },
75 new int [] { 0, 1 },
76 new int [] { 10, 1 },
77 new int [] { 14, 1432 },
78 new int [] { 47, 53 },
79
80 new int [] { 0, 1, 2 },
81 new int [] { 1, 2, 3 },
82 new int [] { 0, 2, 2 },
83 new int [] { 0, 3, 0 },
84 new int [] { 0, 4, 0 },
85 new int [] { 20, 20, 0 },
86
87 new int [] { 0, 1, 2, 3 },
88 new int [] { 4, 0, 0, 0 },
89 new int [] { 5, 0, 0, 0 },
90 new int [] { 6, 6, 0, 0 },
91 new int [] { 6, 2, 0, 0 },
92 new int [] { 6, 1, 0, 0 },
93 new int [] { 6, 0, 0, 0 },
94 new int [] { 4, 4, 4, 7 },
95 new int [] { 4, 4, 4, 8 },
96 new int [] { 0, 0, 0, 7 },
97
98 new int [] { 1, 1, 1, 1, 4 },
99
100 new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
101 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
102 new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
103 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
104 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
105 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
106 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
107 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
108 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
109 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
110 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
111 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
112 new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
113 new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 },
114 new int [] { 1538, 1392, 1561, 1557, 1535, 1553, 1385, 1542, 1619 }
115 };
116
117 int [][] regionsAndServersMocks = new int [][] {
118
119 new int [] { 0, 0 },
120 new int [] { 0, 1 },
121 new int [] { 1, 1 },
122 new int [] { 2, 1 },
123 new int [] { 10, 1 },
124 new int [] { 1, 2 },
125 new int [] { 2, 2 },
126 new int [] { 3, 2 },
127 new int [] { 1, 3 },
128 new int [] { 2, 3 },
129 new int [] { 3, 3 },
130 new int [] { 25, 3 },
131 new int [] { 2, 10 },
132 new int [] { 2, 100 },
133 new int [] { 12, 10 },
134 new int [] { 12, 100 },
135 };
136
137
138
139
140
141
142
143
144
145 @Test
146 public void testBalanceCluster() throws Exception {
147
148 for(int [] mockCluster : clusterStateMocks) {
149 Map<HServerInfo,List<HRegionInfo>> servers = mockClusterServers(mockCluster);
150 LOG.info("Mock Cluster : " + printMock(servers) + " " + printStats(servers));
151 List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
152 List<HServerInfo> balancedCluster = reconcile(servers, plans);
153 LOG.info("Mock Balance : " + printMock(balancedCluster));
154 assertClusterAsBalanced(balancedCluster);
155 for(Map.Entry<HServerInfo, List<HRegionInfo>> entry : servers.entrySet()) {
156 returnRegions(entry.getValue());
157 returnServer(entry.getKey());
158 }
159 }
160
161 }
162
163
164
165
166
167 public void assertClusterAsBalanced(List<HServerInfo> servers) {
168 int numServers = servers.size();
169 int numRegions = 0;
170 int maxRegions = 0;
171 int minRegions = Integer.MAX_VALUE;
172 for(HServerInfo server : servers) {
173 int nr = server.getLoad().getNumberOfRegions();
174 if(nr > maxRegions) {
175 maxRegions = nr;
176 }
177 if(nr < minRegions) {
178 minRegions = nr;
179 }
180 numRegions += nr;
181 }
182 if(maxRegions - minRegions < 2) {
183
184 return;
185 }
186 int min = numRegions / numServers;
187 int max = numRegions % numServers == 0 ? min : min + 1;
188
189 for(HServerInfo server : servers) {
190 assertTrue(server.getLoad().getNumberOfRegions() <= max);
191 assertTrue(server.getLoad().getNumberOfRegions() >= min);
192 }
193 }
194
195
196
197
198
199
200
201
202 @Test
203 public void testImmediateAssignment() throws Exception {
204 for(int [] mock : regionsAndServersMocks) {
205 LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
206 List<HRegionInfo> regions = randomRegions(mock[0]);
207 List<HServerInfo> servers = randomServers(mock[1], 0);
208 Map<HRegionInfo,HServerInfo> assignments =
209 LoadBalancer.immediateAssignment(regions, servers);
210 assertImmediateAssignment(regions, servers, assignments);
211 returnRegions(regions);
212 returnServers(servers);
213 }
214 }
215
216
217
218
219
220
221
222 private void assertImmediateAssignment(List<HRegionInfo> regions,
223 List<HServerInfo> servers, Map<HRegionInfo,HServerInfo> assignments) {
224 for(HRegionInfo region : regions) {
225 assertTrue(assignments.containsKey(region));
226 }
227 }
228
229
230
231
232
233
234
235
236
237 @Test
238 public void testBulkAssignment() throws Exception {
239 for(int [] mock : regionsAndServersMocks) {
240 LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
241 List<HRegionInfo> regions = randomRegions(mock[0]);
242 List<HServerInfo> servers = randomServers(mock[1], 0);
243 Map<HServerInfo,List<HRegionInfo>> assignments =
244 LoadBalancer.roundRobinAssignment(regions, servers);
245 float average = (float)regions.size()/servers.size();
246 int min = (int)Math.floor(average);
247 int max = (int)Math.ceil(average);
248 if(assignments != null && !assignments.isEmpty()) {
249 for(List<HRegionInfo> regionList : assignments.values()) {
250 assertTrue(regionList.size() == min || regionList.size() == max);
251 }
252 }
253 returnRegions(regions);
254 returnServers(servers);
255 }
256 }
257
258
259
260
261
262
263 @Test
264 public void testRetainAssignment() throws Exception {
265
266 List<HServerInfo> servers = randomServers(10, 10);
267 List<HRegionInfo> regions = randomRegions(100);
268 Map<HRegionInfo, HServerAddress> existing =
269 new TreeMap<HRegionInfo, HServerAddress>();
270 for (int i=0;i<regions.size();i++) {
271 existing.put(regions.get(i),
272 servers.get(i % servers.size()).getServerAddress());
273 }
274 Map<HServerInfo, List<HRegionInfo>> assignment =
275 LoadBalancer.retainAssignment(existing, servers);
276 assertRetainedAssignment(existing, servers, assignment);
277
278
279 List<HServerInfo> servers2 = new ArrayList<HServerInfo>(servers);
280 servers2.add(randomServer(10));
281 servers2.add(randomServer(10));
282 assignment = LoadBalancer.retainAssignment(existing, servers2);
283 assertRetainedAssignment(existing, servers2, assignment);
284
285
286 List<HServerInfo> servers3 = new ArrayList<HServerInfo>(servers);
287 servers3.remove(servers3.size()-1);
288 servers3.remove(servers3.size()-2);
289 assignment = LoadBalancer.retainAssignment(existing, servers3);
290 assertRetainedAssignment(existing, servers3, assignment);
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 private void assertRetainedAssignment(
307 Map<HRegionInfo, HServerAddress> existing, List<HServerInfo> servers,
308 Map<HServerInfo, List<HRegionInfo>> assignment) {
309
310 Set<HServerInfo> onlineServerSet = new TreeSet<HServerInfo>(servers);
311 Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
312 for (Map.Entry<HServerInfo, List<HRegionInfo>> a : assignment.entrySet()) {
313 assertTrue("Region assigned to server that was not listed as online",
314 onlineServerSet.contains(a.getKey()));
315 for (HRegionInfo r : a.getValue()) assignedRegions.add(r);
316 }
317 assertEquals(existing.size(), assignedRegions.size());
318
319
320 Set<HServerAddress> onlineAddresses = new TreeSet<HServerAddress>();
321 for (HServerInfo s : servers) onlineAddresses.add(s.getServerAddress());
322 for (Map.Entry<HServerInfo, List<HRegionInfo>> a : assignment.entrySet()) {
323 for (HRegionInfo r : a.getValue()) {
324 HServerAddress address = existing.get(r);
325 if (address != null && onlineAddresses.contains(address)) {
326 assertTrue(a.getKey().getServerAddress().equals(address));
327 }
328 }
329 }
330 }
331
332 private String printStats(Map<HServerInfo, List<HRegionInfo>> servers) {
333 int numServers = servers.size();
334 int totalRegions = 0;
335 for(HServerInfo server : servers.keySet()) {
336 totalRegions += server.getLoad().getNumberOfRegions();
337 }
338 float average = (float)totalRegions / numServers;
339 int max = (int)Math.ceil(average);
340 int min = (int)Math.floor(average);
341 return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
342 }
343
344 private String printMock(Map<HServerInfo, List<HRegionInfo>> servers) {
345 return printMock(Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()])));
346 }
347
348 private String printMock(List<HServerInfo> balancedCluster) {
349 SortedSet<HServerInfo> sorted = new TreeSet<HServerInfo>(balancedCluster);
350 HServerInfo [] arr = sorted.toArray(new HServerInfo[sorted.size()]);
351 StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
352 sb.append("{ ");
353 for(int i=0;i<arr.length;i++) {
354 if(i != 0) {
355 sb.append(" , ");
356 }
357 sb.append(arr[i].getLoad().getNumberOfRegions());
358 }
359 sb.append(" }");
360 return sb.toString();
361 }
362
363
364
365
366
367
368
369
370 private List<HServerInfo> reconcile(
371 Map<HServerInfo, List<HRegionInfo>> servers, List<RegionPlan> plans) {
372 if(plans != null) {
373 for(RegionPlan plan : plans) {
374 plan.getSource().getLoad().setNumberOfRegions(
375 plan.getSource().getLoad().getNumberOfRegions() - 1);
376 plan.getDestination().getLoad().setNumberOfRegions(
377 plan.getDestination().getLoad().getNumberOfRegions() + 1);
378 }
379 }
380 return Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()]));
381 }
382
383 private Map<HServerInfo, List<HRegionInfo>> mockClusterServers(
384 int [] mockCluster) {
385 int numServers = mockCluster.length;
386 Map<HServerInfo,List<HRegionInfo>> servers =
387 new TreeMap<HServerInfo,List<HRegionInfo>>();
388 for(int i=0;i<numServers;i++) {
389 int numRegions = mockCluster[i];
390 HServerInfo server = randomServer(numRegions);
391 List<HRegionInfo> regions = randomRegions(numRegions);
392 servers.put(server, regions);
393 }
394 return servers;
395 }
396
397 private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
398
399 private List<HRegionInfo> randomRegions(int numRegions) {
400 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
401 byte [] start = new byte[16];
402 byte [] end = new byte[16];
403 rand.nextBytes(start);
404 rand.nextBytes(end);
405 for(int i=0;i<numRegions;i++) {
406 if(!regionQueue.isEmpty()) {
407 regions.add(regionQueue.poll());
408 continue;
409 }
410 Bytes.putInt(start, 0, numRegions << 1);
411 Bytes.putInt(end, 0, (numRegions << 1) + 1);
412 HRegionInfo hri = new HRegionInfo(
413 new HTableDescriptor(Bytes.toBytes("table")), start, end);
414 regions.add(hri);
415 }
416 return regions;
417 }
418
419 private void returnRegions(List<HRegionInfo> regions) {
420 regionQueue.addAll(regions);
421 }
422
423 private Queue<HServerInfo> serverQueue = new LinkedList<HServerInfo>();
424
425 private HServerInfo randomServer(int numRegions) {
426 if(!serverQueue.isEmpty()) {
427 HServerInfo server = this.serverQueue.poll();
428 server.getLoad().setNumberOfRegions(numRegions);
429 return server;
430 }
431 String host = "127.0.0.1";
432 int port = rand.nextInt(60000);
433 long startCode = rand.nextLong();
434 HServerInfo hsi =
435 new HServerInfo(new HServerAddress(host, port), startCode, port, host);
436 hsi.getLoad().setNumberOfRegions(numRegions);
437 return hsi;
438 }
439
440 private List<HServerInfo> randomServers(int numServers, int numRegionsPerServer) {
441 List<HServerInfo> servers = new ArrayList<HServerInfo>(numServers);
442 for(int i=0;i<numServers;i++) {
443 servers.add(randomServer(numRegionsPerServer));
444 }
445 return servers;
446 }
447
448 private void returnServer(HServerInfo server) {
449 serverQueue.add(server);
450 }
451
452 private void returnServers(List<HServerInfo> servers) {
453 serverQueue.addAll(servers);
454 }
455 }