View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import java.io.IOException;
21  import java.io.InterruptedIOException;
22  import java.net.ConnectException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
31  import org.apache.hadoop.hbase.ServerName;
32  import org.apache.hadoop.hbase.client.Delete;
33  import org.apache.hadoop.hbase.client.HTable;
34  import org.apache.hadoop.hbase.client.Mutation;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.PairOfSameType;
39  import org.apache.hadoop.hbase.util.Threads;
40  import org.apache.hadoop.hbase.util.Writables;
41  
42  /**
43   * Writes region and assignment information to <code>.META.</code>.
44   * TODO: Put MetaReader and MetaEditor together; doesn't make sense having
45   * them distinct.
46   */
47  public class MetaEditor {
48    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
49    // its only used to get its Configuration so we can get associated
50    // Connection.
51    private static final Log LOG = LogFactory.getLog(MetaEditor.class);
52  
53    private static Put makePutFromRegionInfo(HRegionInfo regionInfo)
54    throws IOException {
55      Put put = new Put(regionInfo.getRegionName());
56      put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
57          Writables.getBytes(regionInfo));
58      return put;
59    }
60  
61    /**
62     * Put the passed <code>p</code> to the <code>.META.</code> table.
63     * @param ct CatalogTracker on whose back we will ride the edit.
64     * @param p Put to add to .META.
65     * @throws IOException
66     */
67    static void putToMetaTable(final CatalogTracker ct, final Put p)
68    throws IOException {
69      put(MetaReader.getMetaHTable(ct), p);
70    }
71  
72    /**
73     * Put the passed <code>p</code> to the <code>.META.</code> table.
74     * @param ct CatalogTracker on whose back we will ride the edit.
75     * @param p Put to add to .META.
76     * @throws IOException
77     */
78    static void putToRootTable(final CatalogTracker ct, final Put p)
79    throws IOException {
80      put(MetaReader.getRootHTable(ct), p);
81    }
82  
83    /**
84     * Put the passed <code>p</code> to a catalog table.
85     * @param ct CatalogTracker on whose back we will ride the edit.
86     * @param p Put to add
87     * @throws IOException
88     */
89    static void putToCatalogTable(final CatalogTracker ct, final Put p)
90    throws IOException {
91      HTable t = MetaReader.getCatalogHTable(ct, p.getRow());
92      put(t, p);
93    }
94  
95    /**
96     * @param t Table to use (will be closed when done).
97     * @param p
98     * @throws IOException
99     */
100   private static void put(final HTable t, final Put p) throws IOException {
101     try {
102       t.put(p);
103     } finally {
104       t.close();
105     }
106   }
107 
108   /**
109    * Put the passed <code>ps</code> to the <code>.META.</code> table.
110    * @param ct CatalogTracker on whose back we will ride the edit.
111    * @param ps Put to add to .META.
112    * @throws IOException
113    */
114   static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
115   throws IOException {
116     HTable t = MetaReader.getMetaHTable(ct);
117     try {
118       t.put(ps);
119     } finally {
120       t.close();
121     }
122   }
123 
124   /**
125    * Delete the passed <code>d</code> from the <code>.META.</code> table.
126    * @param ct CatalogTracker on whose back we will ride the edit.
127    * @param d Delete to add to .META.
128    * @throws IOException
129    */
130   static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
131       throws IOException {
132     List<Delete> dels = new ArrayList<Delete>(1);
133     dels.add(d);
134     deleteFromMetaTable(ct, dels);
135   }
136 
137   /**
138    * Delete the passed <code>deletes</code> from the <code>.META.</code> table.
139    * @param ct CatalogTracker on whose back we will ride the edit.
140    * @param deletes Deletes to add to .META.  This list should support #remove.
141    * @throws IOException
142    */
143   public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
144       throws IOException {
145     HTable t = MetaReader.getMetaHTable(ct);
146     try {
147       t.delete(deletes);
148     } finally {
149       t.close();
150     }
151   }
152 
153   /**
154    * Execute the passed <code>mutations</code> against <code>.META.</code> table.
155    * @param ct CatalogTracker on whose back we will ride the edit.
156    * @param mutations Puts and Deletes to execute on .META.
157    * @throws IOException
158    */
159   static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
160       throws IOException {
161     HTable t = MetaReader.getMetaHTable(ct);
162     try {
163       t.batch(mutations);
164     } catch (InterruptedException e) {
165       InterruptedIOException ie = new InterruptedIOException(e.getMessage());
166       ie.initCause(e);
167       throw ie;
168     } finally {
169       t.close();
170     }
171   }
172 
173   /**
174    * Adds a META row for the specified new region.
175    * @param regionInfo region information
176    * @throws IOException if problem connecting or updating meta
177    */
178   public static void addRegionToMeta(CatalogTracker catalogTracker,
179       HRegionInfo regionInfo)
180   throws IOException {
181     putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
182     LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
183   }
184 
185   /**
186    * Adds a META row for each of the specified new regions.
187    * @param catalogTracker CatalogTracker
188    * @param regionInfos region information list
189    * @throws IOException if problem connecting or updating meta
190    */
191   public static void addRegionsToMeta(CatalogTracker catalogTracker,
192       List<HRegionInfo> regionInfos)
193   throws IOException {
194     List<Put> puts = new ArrayList<Put>();
195     for (HRegionInfo regionInfo : regionInfos) {
196       puts.add(makePutFromRegionInfo(regionInfo));
197     }
198     putsToMetaTable(catalogTracker, puts);
199     LOG.info("Added " + puts.size() + " regions in META");
200   }
201 
202   /**
203    * Offline parent in meta.
204    * Used when splitting.
205    * @param catalogTracker
206    * @param parent
207    * @param a Split daughter region A
208    * @param b Split daughter region B
209    * @throws NotAllMetaRegionsOnlineException
210    * @throws IOException
211    */
212   public static void offlineParentInMeta(CatalogTracker catalogTracker,
213       HRegionInfo parent, final HRegionInfo a, final HRegionInfo b)
214   throws NotAllMetaRegionsOnlineException, IOException {
215     HRegionInfo copyOfParent = new HRegionInfo(parent);
216     copyOfParent.setOffline(true);
217     copyOfParent.setSplit(true);
218     Put put = new Put(copyOfParent.getRegionName());
219     addRegionInfo(put, copyOfParent);
220     put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
221       Writables.getBytes(a));
222     put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
223       Writables.getBytes(b));
224     putToMetaTable(catalogTracker, put);
225     LOG.info("Offlined parent region " + parent.getRegionNameAsString() +
226       " in META");
227   }
228 
229   public static void addDaughter(final CatalogTracker catalogTracker,
230       final HRegionInfo regionInfo, final ServerName sn)
231   throws NotAllMetaRegionsOnlineException, IOException {
232     Put put = new Put(regionInfo.getRegionName());
233     addRegionInfo(put, regionInfo);
234     if (sn != null) addLocation(put, sn);
235     putToMetaTable(catalogTracker, put);
236     LOG.info("Added daughter " + regionInfo.getRegionNameAsString() +
237       (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
238   }
239 
240   /**
241    * Updates the location of the specified META region in ROOT to be the
242    * specified server hostname and startcode.
243    * <p>
244    * Uses passed catalog tracker to get a connection to the server hosting
245    * ROOT and makes edits to that region.
246    *
247    * @param catalogTracker catalog tracker
248    * @param regionInfo region to update location of
249    * @param sn Server name
250    * @throws IOException
251    * @throws ConnectException Usually because the regionserver carrying .META.
252    * is down.
253    * @throws NullPointerException Because no -ROOT- server connection
254    */
255   public static void updateMetaLocation(CatalogTracker catalogTracker,
256       HRegionInfo regionInfo, ServerName sn)
257   throws IOException, ConnectException {
258     updateLocation(catalogTracker, regionInfo, sn);
259   }
260 
261   /**
262    * Updates the location of the specified region in META to be the specified
263    * server hostname and startcode.
264    * <p>
265    * Uses passed catalog tracker to get a connection to the server hosting
266    * META and makes edits to that region.
267    *
268    * @param catalogTracker catalog tracker
269    * @param regionInfo region to update location of
270    * @param sn Server name
271    * @throws IOException
272    */
273   public static void updateRegionLocation(CatalogTracker catalogTracker,
274       HRegionInfo regionInfo, ServerName sn)
275   throws IOException {
276     updateLocation(catalogTracker, regionInfo, sn);
277   }
278 
279   /**
280    * Updates the location of the specified region to be the specified server.
281    * <p>
282    * Connects to the specified server which should be hosting the specified
283    * catalog region name to perform the edit.
284    *
285    * @param catalogTracker
286    * @param regionInfo region to update location of
287    * @param sn Server name
288    * @throws IOException In particular could throw {@link java.net.ConnectException}
289    * if the server is down on other end.
290    */
291   private static void updateLocation(final CatalogTracker catalogTracker,
292       HRegionInfo regionInfo, ServerName sn)
293   throws IOException {
294     Put put = new Put(regionInfo.getRegionName());
295     addLocation(put, sn);
296     putToCatalogTable(catalogTracker, put);
297     LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
298       " with server=" + sn);
299   }
300 
301   /**
302    * Deletes the specified region from META.
303    * @param catalogTracker
304    * @param regionInfo region to be deleted from META
305    * @throws IOException
306    */
307   public static void deleteRegion(CatalogTracker catalogTracker,
308       HRegionInfo regionInfo)
309   throws IOException {
310     Delete delete = new Delete(regionInfo.getRegionName());
311     deleteFromMetaTable(catalogTracker, delete);
312     LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
313   }
314 
315   /**
316    * Deletes the specified regions from META.
317    * @param catalogTracker
318    * @param regionsInfo list of regions to be deleted from META
319    * @throws IOException
320    */
321   public static void deleteRegions(CatalogTracker catalogTracker,
322       List<HRegionInfo> regionsInfo) throws IOException {
323     List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
324     for (HRegionInfo hri: regionsInfo) {
325       deletes.add(new Delete(hri.getRegionName()));
326     }
327     deleteFromMetaTable(catalogTracker, deletes);
328     LOG.info("Deleted from META, regions: " + regionsInfo);
329   }
330 
331   /**
332    * Adds and Removes the specified regions from .META.
333    * @param catalogTracker
334    * @param regionsToRemove list of regions to be deleted from META
335    * @param regionsToAdd list of regions to be added to META
336    * @throws IOException
337    */
338   public static void mutateRegions(CatalogTracker catalogTracker,
339       final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
340       throws IOException {
341     List<Mutation> mutation = new ArrayList<Mutation>();
342     if (regionsToRemove != null) {
343       for (HRegionInfo hri: regionsToRemove) {
344         mutation.add(new Delete(hri.getRegionName()));
345       }
346     }
347     if (regionsToAdd != null) {
348       for (HRegionInfo hri: regionsToAdd) {
349         mutation.add(makePutFromRegionInfo(hri));
350       }
351     }
352     mutateMetaTable(catalogTracker, mutation);
353     if (regionsToRemove != null && regionsToRemove.size() > 0) {
354       LOG.debug("Deleted from META, regions: " + regionsToRemove);
355     }
356     if (regionsToAdd != null && regionsToAdd.size() > 0) {
357       LOG.debug("Add to META, regions: " + regionsToAdd);
358     }
359   }
360 
361   /**
362    * Overwrites the specified regions from hbase:meta
363    * @param catalogTracker
364    * @param regionInfos list of regions to be added to META
365    * @throws IOException
366    */
367   public static void overwriteRegions(CatalogTracker catalogTracker,
368       List<HRegionInfo> regionInfos) throws IOException {
369     deleteRegions(catalogTracker, regionInfos);
370     // Why sleep? This is the easiest way to ensure that the previous deletes does not
371     // eclipse the following puts, that might happen in the same ts from the server.
372     // See HBASE-9906, and HBASE-9879. Once either HBASE-9879, HBASE-8770 is fixed,
373     // or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep.
374     Threads.sleep(20);
375     addRegionsToMeta(catalogTracker, regionInfos);
376     LOG.info("Overwritten " + regionInfos);
377   }
378 
379   public static HRegionInfo getHRegionInfo(
380       Result data) throws IOException {
381     byte [] bytes =
382       data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
383     if (bytes == null) return null;
384     HRegionInfo info = Writables.getHRegionInfo(bytes);
385     LOG.info("Current INFO from scan results = " + info);
386     return info;
387   }
388 
389   /**
390    * Returns the daughter regions by reading from the corresponding columns of the .META. table
391    * Result. If the region is not a split parent region, it returns PairOfSameType(null, null).
392    */
393   public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
394     HRegionInfo splitA = Writables.getHRegionInfoOrNull(
395         data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER));
396     HRegionInfo splitB = Writables.getHRegionInfoOrNull(
397         data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER));
398     return new PairOfSameType<HRegionInfo>(splitA, splitB);
399   }
400 
401   private static Put addRegionInfo(final Put p, final HRegionInfo hri)
402   throws IOException {
403     p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
404         Writables.getBytes(hri));
405     return p;
406   }
407 
408   private static Put addLocation(final Put p, final ServerName sn) {
409     p.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
410       Bytes.toBytes(sn.getHostAndPort()));
411     p.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
412       Bytes.toBytes(sn.getStartcode()));
413     return p;
414   }
415 }