View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Comparator;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableSet;
28  import java.util.Random;
29  import java.util.TreeMap;
30  import java.util.TreeSet;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.BlockLocation;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HServerAddress;
40  import org.apache.hadoop.hbase.HServerInfo;
41  
42  /**
43   * Makes decisions about the placement and movement of Regions across
44   * RegionServers.
45   *
46   * <p>Cluster-wide load balancing will occur only when there are no regions in
47   * transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
48   *
49   * <p>Inline region placement with {@link #immediateAssignment} can be used when
50   * the Master needs to handle closed regions that it currently does not have
51   * a destination set for.  This can happen during master failover.
52   *
53   * <p>On cluster startup, bulk assignment can be used to determine
54   * locations for all Regions in a cluster.
55   *
56   * <p>This classes produces plans for the {@link AssignmentManager} to execute.
57   */
58  public class LoadBalancer {
59    private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
60    private static final Random rand = new Random();
61  
62    /**
63     * Generate a global load balancing plan according to the specified map of
64     * server information to the most loaded regions of each server.
65     *
66     * The load balancing invariant is that all servers are within 1 region of the
67     * average number of regions per server.  If the average is an integer number,
68     * all servers will be balanced to the average.  Otherwise, all servers will
69     * have either floor(average) or ceiling(average) regions.
70     *
71     * The algorithm is currently implemented as such:
72     *
73     * <ol>
74     * <li>Determine the two valid numbers of regions each server should have,
75     *     <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average).
76     *
77     * <li>Iterate down the most loaded servers, shedding regions from each so
78     *     each server hosts exactly <b>MAX</b> regions.  Stop once you reach a
79     *     server that already has &lt;= <b>MAX</b> regions.
80     *
81     * <li>Iterate down the least loaded servers, assigning regions so each server
82     *     has exactly </b>MIN</b> regions.  Stop once you reach a server that
83     *     already has &gt;= <b>MIN</b> regions.
84     *
85     *     Regions being assigned to underloaded servers are those that were shed
86     *     in the previous step.  It is possible that there were not enough
87     *     regions shed to fill each underloaded server to <b>MIN</b>.  If so we
88     *     end up with a number of regions required to do so, <b>neededRegions</b>.
89     *
90     *     It is also possible that we were able fill each underloaded but ended
91     *     up with regions that were unassigned from overloaded servers but that
92     *     still do not have assignment.
93     *
94     *     If neither of these conditions hold (no regions needed to fill the
95     *     underloaded servers, no regions leftover from overloaded servers),
96     *     we are done and return.  Otherwise we handle these cases below.
97     *
98     * <li>If <b>neededRegions</b> is non-zero (still have underloaded servers),
99     *     we iterate the most loaded servers again, shedding a single server from
100    *     each (this brings them from having <b>MAX</b> regions to having
101    *     <b>MIN</b> regions).
102    *
103    * <li>We now definitely have more regions that need assignment, either from
104    *     the previous step or from the original shedding from overloaded servers.
105    *
106    *     Iterate the least loaded servers filling each to <b>MIN</b>.
107    *
108    * <li>If we still have more regions that need assignment, again iterate the
109    *     least loaded servers, this time giving each one (filling them to
110    *     </b>MAX</b>) until we run out.
111    *
112    * <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions.
113    *
114    *     In addition, any server hosting &gt;= <b>MAX</b> regions is guaranteed
115    *     to end up with <b>MAX</b> regions at the end of the balancing.  This
116    *     ensures the minimal number of regions possible are moved.
117    * </ol>
118    *
119    * TODO: We can at-most reassign the number of regions away from a particular
120    *       server to be how many they report as most loaded.
121    *       Should we just keep all assignment in memory?  Any objections?
122    *       Does this mean we need HeapSize on HMaster?  Or just careful monitor?
123    *       (current thinking is we will hold all assignments in memory)
124    *
125    * @param clusterState Map of regionservers and their load/region information to
126    *                   a list of their most loaded regions
127    * @return a list of regions to be moved, including source and destination,
128    *         or null if cluster is already balanced
129    */
130   public List<RegionPlan> balanceCluster(
131       Map<HServerInfo,List<HRegionInfo>> clusterState) {
132     long startTime = System.currentTimeMillis();
133 
134     // Make a map sorted by load and count regions
135     TreeMap<HServerInfo,List<HRegionInfo>> serversByLoad =
136       new TreeMap<HServerInfo,List<HRegionInfo>>(
137           new HServerInfo.LoadComparator());
138     int numServers = clusterState.size();
139     if (numServers == 0) {
140       LOG.debug("numServers=0 so skipping load balancing");
141       return null;
142     }
143     int numRegions = 0;
144     // Iterate so we can count regions as we build the map
145     for(Map.Entry<HServerInfo, List<HRegionInfo>> server:
146         clusterState.entrySet()) {
147       server.getKey().getLoad().setNumberOfRegions(server.getValue().size());
148       numRegions += server.getKey().getLoad().getNumberOfRegions();
149       serversByLoad.put(server.getKey(), server.getValue());
150     }
151 
152     // Check if we even need to do any load balancing
153     float average = (float)numRegions / numServers; // for logging
154     int min = numRegions / numServers;
155     int max = numRegions % numServers == 0 ? min : min + 1;
156     if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= max &&
157        serversByLoad.firstKey().getLoad().getNumberOfRegions() >= min) {
158       // Skipped because no server outside (min,max) range
159       LOG.info("Skipping load balancing.  servers=" + numServers + " " +
160           "regions=" + numRegions + " average=" + average + " " +
161           "mostloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions() +
162           " leastloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions());
163       return null;
164     }
165 
166     // Balance the cluster
167     // TODO: Look at data block locality or a more complex load to do this
168     List<RegionPlan> regionsToMove = new ArrayList<RegionPlan>();
169     int regionidx = 0; // track the index in above list for setting destination
170 
171     // Walk down most loaded, pruning each to the max
172     int serversOverloaded = 0;
173     Map<HServerInfo,BalanceInfo> serverBalanceInfo =
174       new TreeMap<HServerInfo,BalanceInfo>();
175     for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
176       serversByLoad.descendingMap().entrySet()) {
177       HServerInfo serverInfo = server.getKey();
178       int regionCount = serverInfo.getLoad().getNumberOfRegions();
179       if(regionCount <= max) {
180         serverBalanceInfo.put(serverInfo, new BalanceInfo(0, 0));
181         break;
182       }
183       serversOverloaded++;
184       List<HRegionInfo> regions = server.getValue();
185       int numToOffload = Math.min(regionCount - max, regions.size());
186       int numTaken = 0;
187       for (HRegionInfo hri: regions) {
188         // Don't rebalance meta regions.
189         if (hri.isMetaRegion()) continue;
190         regionsToMove.add(new RegionPlan(hri, serverInfo, null));
191         numTaken++;
192         if (numTaken >= numToOffload) break;
193       }
194       serverBalanceInfo.put(serverInfo,
195           new BalanceInfo(numToOffload, (-1)*numTaken));
196     }
197 
198     // Walk down least loaded, filling each to the min
199     int serversUnderloaded = 0; // number of servers that get new regions
200     int neededRegions = 0; // number of regions needed to bring all up to min
201     for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
202       serversByLoad.entrySet()) {
203       int regionCount = server.getKey().getLoad().getNumberOfRegions();
204       if(regionCount >= min) {
205         break;
206       }
207       serversUnderloaded++;
208       int numToTake = min - regionCount;
209       int numTaken = 0;
210       while(numTaken < numToTake && regionidx < regionsToMove.size()) {
211         regionsToMove.get(regionidx).setDestination(server.getKey());
212         numTaken++;
213         regionidx++;
214       }
215       serverBalanceInfo.put(server.getKey(), new BalanceInfo(0, numTaken));
216       // If we still want to take some, increment needed
217       if(numTaken < numToTake) {
218         neededRegions += (numToTake - numTaken);
219       }
220     }
221 
222     // If none needed to fill all to min and none left to drain all to max,
223     // we are done
224     if(neededRegions == 0 && regionidx == regionsToMove.size()) {
225       long endTime = System.currentTimeMillis();
226       LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
227           "Moving " + regionsToMove.size() + " regions off of " +
228           serversOverloaded + " overloaded servers onto " +
229           serversUnderloaded + " less loaded servers");
230       return regionsToMove;
231     }
232 
233     // Need to do a second pass.
234     // Either more regions to assign out or servers that are still underloaded
235 
236     // If we need more to fill min, grab one from each most loaded until enough
237     if (neededRegions != 0) {
238       // Walk down most loaded, grabbing one from each until we get enough
239       for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
240         serversByLoad.descendingMap().entrySet()) {
241         BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
242         int idx =
243           balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
244         if (idx >= server.getValue().size()) break;
245         HRegionInfo region = server.getValue().get(idx);
246         if (region.isMetaRegion()) continue; // Don't move meta regions.
247         regionsToMove.add(new RegionPlan(region, server.getKey(), null));
248         if(--neededRegions == 0) {
249           // No more regions needed, done shedding
250           break;
251         }
252       }
253     }
254 
255     // Now we have a set of regions that must be all assigned out
256     // Assign each underloaded up to the min, then if leftovers, assign to max
257 
258     // Walk down least loaded, assigning to each to fill up to min
259     for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
260       serversByLoad.entrySet()) {
261       int regionCount = server.getKey().getLoad().getNumberOfRegions();
262       if (regionCount >= min) break;
263       BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
264       if(balanceInfo != null) {
265         regionCount += balanceInfo.getNumRegionsAdded();
266       }
267       if(regionCount >= min) {
268         continue;
269       }
270       int numToTake = min - regionCount;
271       int numTaken = 0;
272       while(numTaken < numToTake && regionidx < regionsToMove.size()) {
273         regionsToMove.get(regionidx).setDestination(server.getKey());
274         numTaken++;
275         regionidx++;
276       }
277     }
278 
279     // If we still have regions to dish out, assign underloaded to max
280     if(regionidx != regionsToMove.size()) {
281       for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
282         serversByLoad.entrySet()) {
283         int regionCount = server.getKey().getLoad().getNumberOfRegions();
284         if(regionCount >= max) {
285           break;
286         }
287         regionsToMove.get(regionidx).setDestination(server.getKey());
288         regionidx++;
289         if(regionidx == regionsToMove.size()) {
290           break;
291         }
292       }
293     }
294 
295     long endTime = System.currentTimeMillis();
296 
297     if (regionidx != regionsToMove.size() || neededRegions != 0) {
298       // Emit data so can diagnose how balancer went astray.
299       LOG.warn("regionidx=" + regionidx + ", regionsToMove=" + regionsToMove.size() +
300       ", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded +
301       ", serversUnderloaded=" + serversUnderloaded);
302       StringBuilder sb = new StringBuilder();
303       for (Map.Entry<HServerInfo, List<HRegionInfo>> e: clusterState.entrySet()) {
304         if (sb.length() > 0) sb.append(", ");
305         sb.append(e.getKey().getServerName());
306         sb.append(" ");
307         sb.append(e.getValue().size());
308       }
309       LOG.warn("Input " + sb.toString());
310     }
311 
312     // All done!
313     LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
314         "Moving " + regionsToMove.size() + " regions off of " +
315         serversOverloaded + " overloaded servers onto " +
316         serversUnderloaded + " less loaded servers");
317 
318     return regionsToMove;
319   }
320 
321   /**
322    * Stores additional per-server information about the regions added/removed
323    * during the run of the balancing algorithm.
324    *
325    * For servers that receive additional regions, we are not updating the number
326    * of regions in HServerInfo once we decide to reassign regions to a server,
327    * but we need this information later in the algorithm.  This is stored in
328    * <b>numRegionsAdded</b>.
329    *
330    * For servers that shed regions, we need to track which regions we have
331    * already shed.  <b>nextRegionForUnload</b> contains the index in the list
332    * of regions on the server that is the next to be shed.
333    */
334   private static class BalanceInfo {
335 
336     private final int nextRegionForUnload;
337     private final int numRegionsAdded;
338 
339     public BalanceInfo(int nextRegionForUnload, int numRegionsAdded) {
340       this.nextRegionForUnload = nextRegionForUnload;
341       this.numRegionsAdded = numRegionsAdded;
342     }
343 
344     public int getNextRegionForUnload() {
345       return nextRegionForUnload;
346     }
347 
348     public int getNumRegionsAdded() {
349       return numRegionsAdded;
350     }
351   }
352 
353   /**
354    * Generates a bulk assignment plan to be used on cluster startup using a
355    * simple round-robin assignment.
356    * <p>
357    * Takes a list of all the regions and all the servers in the cluster and
358    * returns a map of each server to the regions that it should be assigned.
359    * <p>
360    * Currently implemented as a round-robin assignment.  Same invariant as
361    * load balancing, all servers holding floor(avg) or ceiling(avg).
362    *
363    * TODO: Use block locations from HDFS to place regions with their blocks
364    *
365    * @param regions all regions
366    * @param servers all servers
367    * @return map of server to the regions it should take, or null if no
368    *         assignment is possible (ie. no regions or no servers)
369    */
370   public static Map<HServerInfo,List<HRegionInfo>> roundRobinAssignment(
371       List<HRegionInfo> regions, List<HServerInfo> servers) {
372     if(regions.size() == 0 || servers.size() == 0) {
373       return null;
374     }
375     Map<HServerInfo,List<HRegionInfo>> assignments =
376       new TreeMap<HServerInfo,List<HRegionInfo>>();
377     int numRegions = regions.size();
378     int numServers = servers.size();
379     int max = (int)Math.ceil((float)numRegions/numServers);
380     int serverIdx = 0;
381     if (numServers > 1) {
382       serverIdx = rand.nextInt(numServers);
383     }
384     int regionIdx = 0;
385     for (int j = 0; j < numServers; j++) {
386       HServerInfo server = servers.get((j+serverIdx) % numServers);
387       List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
388       for (int i=regionIdx; i<numRegions; i += numServers) {
389         serverRegions.add(regions.get(i % numRegions));
390       }
391       assignments.put(server, serverRegions);
392       regionIdx++;
393     }
394     return assignments;
395   }
396 
397   /**
398    * Generates a bulk assignment startup plan, attempting to reuse the existing
399    * assignment information from META, but adjusting for the specified list of
400    * available/online servers available for assignment.
401    * <p>
402    * Takes a map of all regions to their existing assignment from META.  Also
403    * takes a list of online servers for regions to be assigned to.  Attempts to
404    * retain all assignment, so in some instances initial assignment will not be
405    * completely balanced.
406    * <p>
407    * Any leftover regions without an existing server to be assigned to will be
408    * assigned randomly to available servers.
409    * @param regions regions and existing assignment from meta
410    * @param servers available servers
411    * @return map of servers and regions to be assigned to them
412    */
413   public static Map<HServerInfo, List<HRegionInfo>> retainAssignment(
414       Map<HRegionInfo, HServerAddress> regions, List<HServerInfo> servers) {
415     Map<HServerInfo, List<HRegionInfo>> assignments =
416       new TreeMap<HServerInfo, List<HRegionInfo>>();
417     // Build a map of server addresses to server info so we can match things up
418     Map<HServerAddress, HServerInfo> serverMap =
419       new TreeMap<HServerAddress, HServerInfo>();
420     for (HServerInfo server : servers) {
421       serverMap.put(server.getServerAddress(), server);
422       assignments.put(server, new ArrayList<HRegionInfo>());
423     }
424     for (Map.Entry<HRegionInfo, HServerAddress> region : regions.entrySet()) {
425       HServerAddress hsa = region.getValue();
426       HServerInfo server = hsa == null? null: serverMap.get(hsa);
427       if (server != null) {
428         assignments.get(server).add(region.getKey());
429       } else {
430         assignments.get(servers.get(rand.nextInt(assignments.size()))).add(
431             region.getKey());
432       }
433     }
434     return assignments;
435   }
436 
437   /**
438    * Find the block locations for all of the files for the specified region.
439    *
440    * Returns an ordered list of hosts that are hosting the blocks for this
441    * region.  The weight of each host is the sum of the block lengths of all
442    * files on that host, so the first host in the list is the server which
443    * holds the most bytes of the given region's HFiles.
444    *
445    * TODO: Make this work.  Need to figure out how to match hadoop's hostnames
446    *       given for block locations with our HServerAddress.
447    * TODO: Use the right directory for the region
448    * TODO: Use getFileBlockLocations on the files not the directory
449    *
450    * @param fs the filesystem
451    * @param region region
452    * @return ordered list of hosts holding blocks of the specified region
453    * @throws IOException if any filesystem errors
454    */
455   @SuppressWarnings("unused")
456   private List<String> getTopBlockLocations(FileSystem fs, HRegionInfo region)
457   throws IOException {
458     String encodedName = region.getEncodedName();
459     Path path = new Path("/hbase/table/" + encodedName);
460     FileStatus status = fs.getFileStatus(path);
461     BlockLocation [] blockLocations =
462       fs.getFileBlockLocations(status, 0, status.getLen());
463     Map<HostAndWeight,HostAndWeight> hostWeights =
464       new TreeMap<HostAndWeight,HostAndWeight>(new HostAndWeight.HostComparator());
465     for(BlockLocation bl : blockLocations) {
466       String [] hosts = bl.getHosts();
467       long len = bl.getLength();
468       for(String host : hosts) {
469         HostAndWeight haw = hostWeights.get(host);
470         if(haw == null) {
471           haw = new HostAndWeight(host, len);
472           hostWeights.put(haw, haw);
473         } else {
474           haw.addWeight(len);
475         }
476       }
477     }
478     NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
479         new HostAndWeight.WeightComparator());
480     orderedHosts.addAll(hostWeights.values());
481     List<String> topHosts = new ArrayList<String>(orderedHosts.size());
482     for(HostAndWeight haw : orderedHosts.descendingSet()) {
483       topHosts.add(haw.getHost());
484     }
485     return topHosts;
486   }
487 
488   /**
489    * Stores the hostname and weight for that hostname.
490    *
491    * This is used when determining the physical locations of the blocks making
492    * up a region.
493    *
494    * To make a prioritized list of the hosts holding the most data of a region,
495    * this class is used to count the total weight for each host.  The weight is
496    * currently just the size of the file.
497    */
498   private static class HostAndWeight {
499 
500     private final String host;
501     private long weight;
502 
503     public HostAndWeight(String host, long weight) {
504       this.host = host;
505       this.weight = weight;
506     }
507 
508     public void addWeight(long weight) {
509       this.weight += weight;
510     }
511 
512     public String getHost() {
513       return host;
514     }
515 
516     public long getWeight() {
517       return weight;
518     }
519 
520     private static class HostComparator implements Comparator<HostAndWeight> {
521       @Override
522       public int compare(HostAndWeight l, HostAndWeight r) {
523         return l.getHost().compareTo(r.getHost());
524       }
525     }
526 
527     private static class WeightComparator implements Comparator<HostAndWeight> {
528       @Override
529       public int compare(HostAndWeight l, HostAndWeight r) {
530         if(l.getWeight() == r.getWeight()) {
531           return l.getHost().compareTo(r.getHost());
532         }
533         return l.getWeight() < r.getWeight() ? -1 : 1;
534       }
535     }
536   }
537 
538   /**
539    * Generates an immediate assignment plan to be used by a new master for
540    * regions in transition that do not have an already known destination.
541    *
542    * Takes a list of regions that need immediate assignment and a list of
543    * all available servers.  Returns a map of regions to the server they
544    * should be assigned to.
545    *
546    * This method will return quickly and does not do any intelligent
547    * balancing.  The goal is to make a fast decision not the best decision
548    * possible.
549    *
550    * Currently this is random.
551    *
552    * @param regions
553    * @param servers
554    * @return map of regions to the server it should be assigned to
555    */
556   public static Map<HRegionInfo,HServerInfo> immediateAssignment(
557       List<HRegionInfo> regions, List<HServerInfo> servers) {
558     Map<HRegionInfo,HServerInfo> assignments =
559       new TreeMap<HRegionInfo,HServerInfo>();
560     for(HRegionInfo region : regions) {
561       assignments.put(region, servers.get(rand.nextInt(servers.size())));
562     }
563     return assignments;
564   }
565 
566   public static HServerInfo randomAssignment(List<HServerInfo> servers) {
567     if (servers == null || servers.isEmpty()) {
568       LOG.warn("Wanted to do random assignment but no servers to assign to");
569       return null;
570     }
571     return servers.get(rand.nextInt(servers.size()));
572   }
573 
574   /**
575    * Stores the plan for the move of an individual region.
576    *
577    * Contains info for the region being moved, info for the server the region
578    * should be moved from, and info for the server the region should be moved
579    * to.
580    *
581    * The comparable implementation of this class compares only the region
582    * information and not the source/dest server info.
583    */
584   public static class RegionPlan implements Comparable<RegionPlan> {
585     private final HRegionInfo hri;
586     private final HServerInfo source;
587     private HServerInfo dest;
588 
589     /**
590      * Instantiate a plan for a region move, moving the specified region from
591      * the specified source server to the specified destination server.
592      *
593      * Destination server can be instantiated as null and later set
594      * with {@link #setDestination(HServerInfo)}.
595      *
596      * @param hri region to be moved
597      * @param source regionserver region should be moved from
598      * @param dest regionserver region should be moved to
599      */
600     public RegionPlan(final HRegionInfo hri, HServerInfo source, HServerInfo dest) {
601       this.hri = hri;
602       this.source = source;
603       this.dest = dest;
604     }
605 
606     /**
607      * Set the destination server for the plan for this region.
608      */
609     public void setDestination(HServerInfo dest) {
610       this.dest = dest;
611     }
612 
613     /**
614      * Get the source server for the plan for this region.
615      * @return server info for source
616      */
617     public HServerInfo getSource() {
618       return source;
619     }
620 
621     /**
622      * Get the destination server for the plan for this region.
623      * @return server info for destination
624      */
625     public HServerInfo getDestination() {
626       return dest;
627     }
628 
629     /**
630      * Get the encoded region name for the region this plan is for.
631      * @return Encoded region name
632      */
633     public String getRegionName() {
634       return this.hri.getEncodedName();
635     }
636 
637     public HRegionInfo getRegionInfo() {
638       return this.hri;
639     }
640 
641     /**
642      * Compare the region info.
643      * @param o region plan you are comparing against
644      */
645     @Override
646     public int compareTo(RegionPlan o) {
647       return getRegionName().compareTo(o.getRegionName());
648     }
649 
650     @Override
651     public String toString() {
652       return "hri=" + this.hri.getRegionNameAsString() + ", src=" +
653         (this.source == null? "": this.source.getServerName()) +
654         ", dest=" + (this.dest == null? "": this.dest.getServerName());
655     }
656   }
657 }