1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.LinkedList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Queue;
31  import java.util.Random;
32  import java.util.Set;
33  import java.util.SortedSet;
34  import java.util.TreeMap;
35  import java.util.TreeSet;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HServerAddress;
41  import org.apache.hadoop.hbase.HServerInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  
48  public class TestLoadBalancer {
49    private static final Log LOG = LogFactory.getLog(TestLoadBalancer.class);
50  
51    private static LoadBalancer loadBalancer;
52  
53    private static Random rand;
54  
55    @BeforeClass
56    public static void beforeAllTests() throws Exception {
57      loadBalancer = new LoadBalancer();
58      rand = new Random();
59    }
60  
61    // int[testnum][servernumber] -> numregions
62    int [][] clusterStateMocks = new int [][] {
63        // 1 node
64        new int [] { 0 },
65        new int [] { 1 },
66        new int [] { 10 },
67        // 2 node
68        new int [] { 0, 0 },
69        new int [] { 2, 0 },
70        new int [] { 2, 1 },
71        new int [] { 2, 2 },
72        new int [] { 2, 3 },
73        new int [] { 2, 4 },
74        new int [] { 1, 1 },
75        new int [] { 0, 1 },
76        new int [] { 10, 1 },
77        new int [] { 14, 1432 },
78        new int [] { 47, 53 },
79        // 3 node
80        new int [] { 0, 1, 2 },
81        new int [] { 1, 2, 3 },
82        new int [] { 0, 2, 2 },
83        new int [] { 0, 3, 0 },
84        new int [] { 0, 4, 0 },
85        new int [] { 20, 20, 0 },
86        // 4 node
87        new int [] { 0, 1, 2, 3 },
88        new int [] { 4, 0, 0, 0 },
89        new int [] { 5, 0, 0, 0 },
90        new int [] { 6, 6, 0, 0 },
91        new int [] { 6, 2, 0, 0 },
92        new int [] { 6, 1, 0, 0 },
93        new int [] { 6, 0, 0, 0 },
94        new int [] { 4, 4, 4, 7 },
95        new int [] { 4, 4, 4, 8 },
96        new int [] { 0, 0, 0, 7 },
97        // 5 node
98        new int [] { 1, 1, 1, 1, 4 },
99        // more nodes
100       new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
101       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
102       new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
103       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
104       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
105       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
106       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
107       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
108       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
109       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
110       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
111       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
112       new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
113       new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 },
114       new int [] { 1538, 1392, 1561, 1557, 1535, 1553, 1385, 1542, 1619 }
115   };
116 
117   int [][] regionsAndServersMocks = new int [][] {
118       // { num regions, num servers }
119       new int [] { 0, 0 },
120       new int [] { 0, 1 },
121       new int [] { 1, 1 },
122       new int [] { 2, 1 },
123       new int [] { 10, 1 },
124       new int [] { 1, 2 },
125       new int [] { 2, 2 },
126       new int [] { 3, 2 },
127       new int [] { 1, 3 },
128       new int [] { 2, 3 },
129       new int [] { 3, 3 },
130       new int [] { 25, 3 },
131       new int [] { 2, 10 },
132       new int [] { 2, 100 },
133       new int [] { 12, 10 },
134       new int [] { 12, 100 },
135   };
136 
137   /**
138    * Test the load balancing algorithm.
139    *
140    * Invariant is that all servers should be hosting either
141    * floor(average) or ceiling(average)
142    *
143    * @throws Exception
144    */
145   @Test
146   public void testBalanceCluster() throws Exception {
147 
148     for(int [] mockCluster : clusterStateMocks) {
149       Map<HServerInfo,List<HRegionInfo>> servers = mockClusterServers(mockCluster);
150       LOG.info("Mock Cluster : " + printMock(servers) + " " + printStats(servers));
151       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
152       List<HServerInfo> balancedCluster = reconcile(servers, plans);
153       LOG.info("Mock Balance : " + printMock(balancedCluster));
154       assertClusterAsBalanced(balancedCluster);
155       for(Map.Entry<HServerInfo, List<HRegionInfo>> entry : servers.entrySet()) {
156         returnRegions(entry.getValue());
157         returnServer(entry.getKey());
158       }
159     }
160 
161   }
162 
163   /**
164    * Invariant is that all servers have between floor(avg) and ceiling(avg)
165    * number of regions.
166    */
167   public void assertClusterAsBalanced(List<HServerInfo> servers) {
168     int numServers = servers.size();
169     int numRegions = 0;
170     int maxRegions = 0;
171     int minRegions = Integer.MAX_VALUE;
172     for(HServerInfo server : servers) {
173       int nr = server.getLoad().getNumberOfRegions();
174       if(nr > maxRegions) {
175         maxRegions = nr;
176       }
177       if(nr < minRegions) {
178         minRegions = nr;
179       }
180       numRegions += nr;
181     }
182     if(maxRegions - minRegions < 2) {
183       // less than 2 between max and min, can't balance
184       return;
185     }
186     int min = numRegions / numServers;
187     int max = numRegions % numServers == 0 ? min : min + 1;
188 
189     for(HServerInfo server : servers) {
190       assertTrue(server.getLoad().getNumberOfRegions() <= max);
191       assertTrue(server.getLoad().getNumberOfRegions() >= min);
192     }
193   }
194 
195   /**
196    * Tests immediate assignment.
197    *
198    * Invariant is that all regions have an assignment.
199    *
200    * @throws Exception
201    */
202   @Test
203   public void testImmediateAssignment() throws Exception {
204     for(int [] mock : regionsAndServersMocks) {
205       LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
206       List<HRegionInfo> regions = randomRegions(mock[0]);
207       List<HServerInfo> servers = randomServers(mock[1], 0);
208       Map<HRegionInfo,HServerInfo> assignments =
209         LoadBalancer.immediateAssignment(regions, servers);
210       assertImmediateAssignment(regions, servers, assignments);
211       returnRegions(regions);
212       returnServers(servers);
213     }
214   }
215 
216   /**
217    * All regions have an assignment.
218    * @param regions
219    * @param servers
220    * @param assignments
221    */
222   private void assertImmediateAssignment(List<HRegionInfo> regions,
223       List<HServerInfo> servers, Map<HRegionInfo,HServerInfo> assignments) {
224     for(HRegionInfo region : regions) {
225       assertTrue(assignments.containsKey(region));
226     }
227   }
228 
229   /**
230    * Tests the bulk assignment used during cluster startup.
231    *
232    * Round-robin.  Should yield a balanced cluster so same invariant as the load
233    * balancer holds, all servers holding either floor(avg) or ceiling(avg).
234    *
235    * @throws Exception
236    */
237   @Test
238   public void testBulkAssignment() throws Exception {
239     for(int [] mock : regionsAndServersMocks) {
240       LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
241       List<HRegionInfo> regions = randomRegions(mock[0]);
242       List<HServerInfo> servers = randomServers(mock[1], 0);
243       Map<HServerInfo,List<HRegionInfo>> assignments =
244         LoadBalancer.roundRobinAssignment(regions, servers);
245       float average = (float)regions.size()/servers.size();
246       int min = (int)Math.floor(average);
247       int max = (int)Math.ceil(average);
248       if(assignments != null && !assignments.isEmpty()) {
249         for(List<HRegionInfo> regionList : assignments.values()) {
250           assertTrue(regionList.size() == min || regionList.size() == max);
251         }
252       }
253       returnRegions(regions);
254       returnServers(servers);
255     }
256   }
257 
258   /**
259    * Test the cluster startup bulk assignment which attempts to retain
260    * assignment info.
261    * @throws Exception
262    */
263   @Test
264   public void testRetainAssignment() throws Exception {
265     // Test simple case where all same servers are there
266     List<HServerInfo> servers = randomServers(10, 10);
267     List<HRegionInfo> regions = randomRegions(100);
268     Map<HRegionInfo, HServerAddress> existing =
269       new TreeMap<HRegionInfo, HServerAddress>();
270     for (int i=0;i<regions.size();i++) {
271       existing.put(regions.get(i),
272           servers.get(i % servers.size()).getServerAddress());
273     }
274     Map<HServerInfo, List<HRegionInfo>> assignment =
275       LoadBalancer.retainAssignment(existing, servers);
276     assertRetainedAssignment(existing, servers, assignment);
277 
278     // Include two new servers that were not there before
279     List<HServerInfo> servers2 = new ArrayList<HServerInfo>(servers);
280     servers2.add(randomServer(10));
281     servers2.add(randomServer(10));
282     assignment = LoadBalancer.retainAssignment(existing, servers2);
283     assertRetainedAssignment(existing, servers2, assignment);
284 
285     // Remove two of the servers that were previously there
286     List<HServerInfo> servers3 = new ArrayList<HServerInfo>(servers);
287     servers3.remove(servers3.size()-1);
288     servers3.remove(servers3.size()-2);
289     assignment = LoadBalancer.retainAssignment(existing, servers3);
290     assertRetainedAssignment(existing, servers3, assignment);
291   }
292 
293   /**
294    * Asserts a valid retained assignment plan.
295    * <p>
296    * Must meet the following conditions:
297    * <ul>
298    *   <li>Every input region has an assignment, and to an online server
299    *   <li>If a region had an existing assignment to a server with the same
300    *       address a a currently online server, it will be assigned to it
301    * </ul>
302    * @param existing
303    * @param servers
304    * @param assignment
305    */
306   private void assertRetainedAssignment(
307       Map<HRegionInfo, HServerAddress> existing, List<HServerInfo> servers,
308       Map<HServerInfo, List<HRegionInfo>> assignment) {
309     // Verify condition 1, every region assigned, and to online server
310     Set<HServerInfo> onlineServerSet = new TreeSet<HServerInfo>(servers);
311     Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
312     for (Map.Entry<HServerInfo, List<HRegionInfo>> a : assignment.entrySet()) {
313       assertTrue("Region assigned to server that was not listed as online",
314           onlineServerSet.contains(a.getKey()));
315       for (HRegionInfo r : a.getValue()) assignedRegions.add(r);
316     }
317     assertEquals(existing.size(), assignedRegions.size());
318 
319     // Verify condition 2, if server had existing assignment, must have same
320     Set<HServerAddress> onlineAddresses = new TreeSet<HServerAddress>();
321     for (HServerInfo s : servers) onlineAddresses.add(s.getServerAddress());
322     for (Map.Entry<HServerInfo, List<HRegionInfo>> a : assignment.entrySet()) {
323       for (HRegionInfo r : a.getValue()) {
324         HServerAddress address = existing.get(r);
325         if (address != null && onlineAddresses.contains(address)) {
326           assertTrue(a.getKey().getServerAddress().equals(address));
327         }
328       }
329     }
330   }
331 
332   private String printStats(Map<HServerInfo, List<HRegionInfo>> servers) {
333     int numServers = servers.size();
334     int totalRegions = 0;
335     for(HServerInfo server : servers.keySet()) {
336       totalRegions += server.getLoad().getNumberOfRegions();
337     }
338     float average = (float)totalRegions / numServers;
339     int max = (int)Math.ceil(average);
340     int min = (int)Math.floor(average);
341     return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
342   }
343 
344   private String printMock(Map<HServerInfo, List<HRegionInfo>> servers) {
345     return printMock(Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()])));
346   }
347 
348   private String printMock(List<HServerInfo> balancedCluster) {
349     SortedSet<HServerInfo> sorted = new TreeSet<HServerInfo>(balancedCluster);
350     HServerInfo [] arr = sorted.toArray(new HServerInfo[sorted.size()]);
351     StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
352     sb.append("{ ");
353     for(int i=0;i<arr.length;i++) {
354       if(i != 0) {
355         sb.append(" , ");
356       }
357       sb.append(arr[i].getLoad().getNumberOfRegions());
358     }
359     sb.append(" }");
360     return sb.toString();
361   }
362 
363   /**
364    * This assumes the RegionPlan HSI instances are the same ones in the map, so
365    * actually no need to even pass in the map, but I think it's clearer.
366    * @param servers
367    * @param plans
368    * @return
369    */
370   private List<HServerInfo> reconcile(
371       Map<HServerInfo, List<HRegionInfo>> servers, List<RegionPlan> plans) {
372     if(plans != null) {
373       for(RegionPlan plan : plans) {
374         plan.getSource().getLoad().setNumberOfRegions(
375             plan.getSource().getLoad().getNumberOfRegions() - 1);
376         plan.getDestination().getLoad().setNumberOfRegions(
377             plan.getDestination().getLoad().getNumberOfRegions() + 1);
378       }
379     }
380     return Arrays.asList(servers.keySet().toArray(new HServerInfo[servers.size()]));
381   }
382 
383   private Map<HServerInfo, List<HRegionInfo>> mockClusterServers(
384       int [] mockCluster) {
385     int numServers = mockCluster.length;
386     Map<HServerInfo,List<HRegionInfo>> servers =
387       new TreeMap<HServerInfo,List<HRegionInfo>>();
388     for(int i=0;i<numServers;i++) {
389       int numRegions = mockCluster[i];
390       HServerInfo server = randomServer(numRegions);
391       List<HRegionInfo> regions = randomRegions(numRegions);
392       servers.put(server, regions);
393     }
394     return servers;
395   }
396 
397   private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
398 
399   private List<HRegionInfo> randomRegions(int numRegions) {
400     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
401     byte [] start = new byte[16];
402     byte [] end = new byte[16];
403     rand.nextBytes(start);
404     rand.nextBytes(end);
405     for(int i=0;i<numRegions;i++) {
406       if(!regionQueue.isEmpty()) {
407         regions.add(regionQueue.poll());
408         continue;
409       }
410       Bytes.putInt(start, 0, numRegions << 1);
411       Bytes.putInt(end, 0, (numRegions << 1) + 1);
412       HRegionInfo hri = new HRegionInfo(
413           new HTableDescriptor(Bytes.toBytes("table")), start, end);
414       regions.add(hri);
415     }
416     return regions;
417   }
418 
419   private void returnRegions(List<HRegionInfo> regions) {
420     regionQueue.addAll(regions);
421   }
422 
423   private Queue<HServerInfo> serverQueue = new LinkedList<HServerInfo>();
424 
425   private HServerInfo randomServer(int numRegions) {
426     if(!serverQueue.isEmpty()) {
427       HServerInfo server = this.serverQueue.poll();
428       server.getLoad().setNumberOfRegions(numRegions);
429       return server;
430     }
431     String host = "127.0.0.1";
432     int port = rand.nextInt(60000);
433     long startCode = rand.nextLong();
434     HServerInfo hsi =
435       new HServerInfo(new HServerAddress(host, port), startCode, port, host);
436     hsi.getLoad().setNumberOfRegions(numRegions);
437     return hsi;
438   }
439 
440   private List<HServerInfo> randomServers(int numServers, int numRegionsPerServer) {
441     List<HServerInfo> servers = new ArrayList<HServerInfo>(numServers);
442     for(int i=0;i<numServers;i++) {
443       servers.add(randomServer(numRegionsPerServer));
444     }
445     return servers;
446   }
447 
448   private void returnServer(HServerInfo server) {
449     serverQueue.add(server);
450   }
451 
452   private void returnServers(List<HServerInfo> servers) {
453     serverQueue.addAll(servers);
454   }
455 }