View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import java.io.IOException;
21  import java.util.LinkedList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.Chore;
31  import org.apache.hadoop.hbase.RemoteExceptionHandler;
32  import org.apache.hadoop.hbase.Stoppable;
33  import org.apache.hadoop.hbase.util.FSUtils;
34  
35  /**
36   * Abstract Cleaner that uses a chain of delegates to clean a directory of files
37   * @param <T> Cleaner delegate class that is dynamically loaded from configuration
38   */
39  public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
40  
41    private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
42  
43    private final FileSystem fs;
44    private final Path oldFileDir;
45    private final Configuration conf;
46    List<T> cleanersChain;
47  
48    /**
49     * @param name name of the chore being run
50     * @param sleepPeriod the period of time to sleep between each run
51     * @param s the stopper
52     * @param conf configuration to use
53     * @param fs handle to the FS
54     * @param oldFileDir the path to the archived files
55     * @param confKey configuration key for the classes to instantiate
56     */
57    public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
58        FileSystem fs, Path oldFileDir, String confKey) {
59      super(name, sleepPeriod, s);
60      this.fs = fs;
61      this.oldFileDir = oldFileDir;
62      this.conf = conf;
63  
64      initCleanerChain(confKey);
65    }
66  
67    /**
68     * Validate the file to see if it even belongs in the directory. If it is valid, then the file
69     * will go through the cleaner delegates, but otherwise the file is just deleted.
70     * @param file full {@link Path} of the file to be checked
71     * @return <tt>true</tt> if the file is valid, <tt>false</tt> otherwise
72     */
73    protected abstract boolean validate(Path file);
74  
75    /**
76     * Instanitate and initialize all the file cleaners set in the configuration
77     * @param confKey key to get the file cleaner classes from the configuration
78     */
79    private void initCleanerChain(String confKey) {
80      this.cleanersChain = new LinkedList<T>();
81      String[] logCleaners = conf.getStrings(confKey);
82      if (logCleaners != null) {
83        for (String className : logCleaners) {
84          T logCleaner = newFileCleaner(className, conf);
85          if (logCleaner != null) {
86            LOG.debug("initialize cleaner=" + className);
87            this.cleanersChain.add(logCleaner);
88          }
89        }
90      }
91    }
92  
93    /**
94     * A utility method to create new instances of LogCleanerDelegate based on the class name of the
95     * LogCleanerDelegate.
96     * @param className fully qualified class name of the LogCleanerDelegate
97     * @param conf
98     * @return the new instance
99     */
100   public T newFileCleaner(String className, Configuration conf) {
101     try {
102       Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
103         FileCleanerDelegate.class);
104       @SuppressWarnings("unchecked")
105       T cleaner = (T) c.newInstance();
106       cleaner.setConf(conf);
107       return cleaner;
108     } catch (Exception e) {
109       LOG.warn("Can NOT create CleanerDelegate: " + className, e);
110       // skipping if can't instantiate
111       return null;
112     }
113   }
114 
115   @Override
116   protected void chore() {
117     try {
118       FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
119       // if the path (file or directory) doesn't exist, then we can just return
120       if (files == null) return;
121       // loop over the found files and see if they should be deleted
122       for (FileStatus file : files) {
123         try {
124           if (file.isDir()) checkAndDeleteDirectory(file.getPath());
125           else checkAndDelete(file.getPath());
126         } catch (IOException e) {
127           e = RemoteExceptionHandler.checkIOException(e);
128           LOG.warn("Error while cleaning the logs", e);
129         }
130       }
131     } catch (IOException e) {
132       LOG.warn("Failed to get status of:" + oldFileDir);
133     }
134 
135   }
136 
137   /**
138    * Attempt to delete a directory and all files under that directory. Each child file is passed
139    * through the delegates to see if it can be deleted. If the directory has not children when the
140    * cleaners have finished it is deleted.
141    * <p>
142    * If new children files are added between checks of the directory, the directory will <b>not</b>
143    * be deleted.
144    * @param toCheck directory to check
145    * @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
146    * @throws IOException if there is an unexpected filesystem error
147    */
148   public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
149     if (LOG.isTraceEnabled()) {
150       LOG.trace("Checking directory: " + toCheck);
151     }
152     FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
153     // if the directory doesn't exist, then we are done
154     if (children == null) {
155       try {
156         return fs.delete(toCheck, false);
157       } catch (IOException e) {
158         if (LOG.isTraceEnabled()) {
159           LOG.trace("Couldn't delete directory: " + toCheck, e);
160         }
161       }
162       // couldn't delete w/o exception, so we can't return success.
163       return false;
164     }     
165 
166     boolean canDeleteThis = true;
167     for (FileStatus child : children) {
168       Path path = child.getPath();
169       // attempt to delete all the files under the directory
170       if (child.isDir()) {
171         if (!checkAndDeleteDirectory(path)) {
172           canDeleteThis = false;
173         }
174       }
175       // otherwise we can just check the file
176       else if (!checkAndDelete(path)) {
177         canDeleteThis = false;
178       }
179     }
180 
181     // if the directory has children, we can't delete it, so we are done
182     if (!canDeleteThis) return false;
183 
184     // otherwise, all the children (that we know about) have been deleted, so we should try to
185     // delete this directory. However, don't do so recursively so we don't delete files that have
186     // been added since we last checked.
187     try {
188       return fs.delete(toCheck, false);
189     } catch (IOException e) {
190       if (LOG.isTraceEnabled()) {
191         LOG.trace("Couldn't delete directory: " + toCheck, e);
192       }
193     }
194 
195     // couldn't delete w/o exception, so we can't return success.
196     return false;
197   }
198 
199   /**
200    * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
201    * necessary.
202    * @param filePath path of the file to check (and possibly delete)
203    * @throws IOException if cann't delete a file because of a filesystem issue
204    * @throws IllegalArgumentException if the file is a directory and has children
205    */
206   private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
207     // first check to see if the path is valid
208     if (!validate(filePath)) {
209       LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
210       boolean success = this.fs.delete(filePath, true);
211       if (!success) LOG.warn("Attempted to delete:" + filePath
212           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
213 
214       return success;
215     }
216     // check each of the cleaners for the file
217     for (T cleaner : cleanersChain) {
218       if (cleaner.isStopped() || this.stopper.isStopped()) {
219         LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
220             + this.oldFileDir);
221         return false;
222       }
223 
224       if (!cleaner.isFileDeletable(filePath)) {
225         // this file is not deletable, then we are done
226         if (LOG.isTraceEnabled()) {
227           LOG.trace(filePath + " is not deletable according to:" + cleaner);
228         }
229         return false;
230       }
231     }
232     // delete this file if it passes all the cleaners
233     if (LOG.isTraceEnabled()) {
234       LOG.trace("Removing:" + filePath + " from archive");
235     }
236     boolean success = this.fs.delete(filePath, false);
237     if (!success) {
238       LOG.warn("Attempted to delete:" + filePath
239           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
240     }
241     return success;
242   }
243 
244   @Override
245   public void cleanup() {
246     for (T lc : this.cleanersChain) {
247       try {
248         lc.stop("Exiting");
249       } catch (Throwable t) {
250         LOG.warn("Stopping", t);
251       }
252     }
253   }
254 }