1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileStatus;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.Chore;
31 import org.apache.hadoop.hbase.RemoteExceptionHandler;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.util.FSUtils;
34
35
36
37
38
39 public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
40
41 private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
42
43 private final FileSystem fs;
44 private final Path oldFileDir;
45 private final Configuration conf;
46 List<T> cleanersChain;
47
48
49
50
51
52
53
54
55
56
57 public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
58 FileSystem fs, Path oldFileDir, String confKey) {
59 super(name, sleepPeriod, s);
60 this.fs = fs;
61 this.oldFileDir = oldFileDir;
62 this.conf = conf;
63
64 initCleanerChain(confKey);
65 }
66
67
68
69
70
71
72
73 protected abstract boolean validate(Path file);
74
75
76
77
78
79 private void initCleanerChain(String confKey) {
80 this.cleanersChain = new LinkedList<T>();
81 String[] logCleaners = conf.getStrings(confKey);
82 if (logCleaners != null) {
83 for (String className : logCleaners) {
84 T logCleaner = newFileCleaner(className, conf);
85 if (logCleaner != null) {
86 LOG.debug("initialize cleaner=" + className);
87 this.cleanersChain.add(logCleaner);
88 }
89 }
90 }
91 }
92
93
94
95
96
97
98
99
100 public T newFileCleaner(String className, Configuration conf) {
101 try {
102 Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
103 FileCleanerDelegate.class);
104 @SuppressWarnings("unchecked")
105 T cleaner = (T) c.newInstance();
106 cleaner.setConf(conf);
107 return cleaner;
108 } catch (Exception e) {
109 LOG.warn("Can NOT create CleanerDelegate: " + className, e);
110
111 return null;
112 }
113 }
114
115 @Override
116 protected void chore() {
117 try {
118 FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
119
120 if (files == null) return;
121
122 for (FileStatus file : files) {
123 try {
124 if (file.isDir()) checkAndDeleteDirectory(file.getPath());
125 else checkAndDelete(file.getPath());
126 } catch (IOException e) {
127 e = RemoteExceptionHandler.checkIOException(e);
128 LOG.warn("Error while cleaning the logs", e);
129 }
130 }
131 } catch (IOException e) {
132 LOG.warn("Failed to get status of:" + oldFileDir);
133 }
134
135 }
136
137
138
139
140
141
142
143
144
145
146
147
148 public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
149 if (LOG.isTraceEnabled()) {
150 LOG.trace("Checking directory: " + toCheck);
151 }
152 FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
153
154 if (children == null) {
155 try {
156 return fs.delete(toCheck, false);
157 } catch (IOException e) {
158 if (LOG.isTraceEnabled()) {
159 LOG.trace("Couldn't delete directory: " + toCheck, e);
160 }
161 }
162
163 return false;
164 }
165
166 boolean canDeleteThis = true;
167 for (FileStatus child : children) {
168 Path path = child.getPath();
169
170 if (child.isDir()) {
171 if (!checkAndDeleteDirectory(path)) {
172 canDeleteThis = false;
173 }
174 }
175
176 else if (!checkAndDelete(path)) {
177 canDeleteThis = false;
178 }
179 }
180
181
182 if (!canDeleteThis) return false;
183
184
185
186
187 try {
188 return fs.delete(toCheck, false);
189 } catch (IOException e) {
190 if (LOG.isTraceEnabled()) {
191 LOG.trace("Couldn't delete directory: " + toCheck, e);
192 }
193 }
194
195
196 return false;
197 }
198
199
200
201
202
203
204
205
206 private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
207
208 if (!validate(filePath)) {
209 LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
210 boolean success = this.fs.delete(filePath, true);
211 if (!success) LOG.warn("Attempted to delete:" + filePath
212 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
213
214 return success;
215 }
216
217 for (T cleaner : cleanersChain) {
218 if (cleaner.isStopped() || this.stopper.isStopped()) {
219 LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
220 + this.oldFileDir);
221 return false;
222 }
223
224 if (!cleaner.isFileDeletable(filePath)) {
225
226 if (LOG.isTraceEnabled()) {
227 LOG.trace(filePath + " is not deletable according to:" + cleaner);
228 }
229 return false;
230 }
231 }
232
233 if (LOG.isTraceEnabled()) {
234 LOG.trace("Removing:" + filePath + " from archive");
235 }
236 boolean success = this.fs.delete(filePath, false);
237 if (!success) {
238 LOG.warn("Attempted to delete:" + filePath
239 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
240 }
241 return success;
242 }
243
244 @Override
245 public void cleanup() {
246 for (T lc : this.cleanersChain) {
247 try {
248 lc.stop("Exiting");
249 } catch (Throwable t) {
250 LOG.warn("Stopping", t);
251 }
252 }
253 }
254 }