001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.hadoop;
016
017 import java.io.IOException;
018 import java.net.URISyntaxException;
019 import java.util.List;
020
021 import org.apache.hadoop.conf.Configuration;
022 import org.apache.hadoop.fs.FileStatus;
023 import org.apache.hadoop.fs.FileSystem;
024 import org.apache.hadoop.fs.Path;
025 import org.apache.hadoop.fs.permission.FsPermission;
026 import org.apache.oozie.action.ActionExecutor;
027 import org.apache.oozie.action.ActionExecutorException;
028 import org.apache.oozie.client.WorkflowAction;
029 import org.apache.oozie.service.HadoopAccessorException;
030 import org.apache.oozie.service.HadoopAccessorService;
031 import org.apache.oozie.service.Services;
032 import org.apache.oozie.util.XmlUtils;
033 import org.jdom.Element;
034
035 /**
036 * File system action executor. <p/> This executes the file system mkdir, move and delete commands
037 */
038 public class FsActionExecutor extends ActionExecutor {
039
040 public FsActionExecutor() {
041 super("fs");
042 }
043
044 Path getPath(Element element, String attribute) {
045 String str = element.getAttributeValue(attribute).trim();
046 return new Path(str);
047 }
048
049 void validatePath(Path path, boolean withScheme) throws ActionExecutorException {
050 String scheme = path.toUri().getScheme();
051 if (withScheme) {
052 if (scheme == null) {
053 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001",
054 "Missing scheme in path [{0}]", path);
055 }
056 else {
057 if (!scheme.equals("hdfs")) {
058 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002",
059 "Scheme [{0}] not support in path [{1}]", scheme, path);
060 }
061 }
062 }
063 else {
064 if (scheme != null) {
065 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS003",
066 "Scheme [{0}] not allowed in path [{1}]", scheme, path);
067 }
068 }
069 }
070
071 @SuppressWarnings("unchecked")
072 void doOperations(Context context, Element element) throws ActionExecutorException {
073 try {
074 FileSystem fs = context.getAppFileSystem();
075 boolean recovery = fs.exists(getRecoveryPath(context));
076 if (!recovery) {
077 fs.mkdirs(getRecoveryPath(context));
078 }
079 for (Element commandElement : (List<Element>) element.getChildren()) {
080 String command = commandElement.getName();
081 if (command.equals("mkdir")) {
082 Path path = getPath(commandElement, "path");
083 mkdir(context, path);
084 }
085 else {
086 if (command.equals("delete")) {
087 Path path = getPath(commandElement, "path");
088 delete(context, path);
089 }
090 else {
091 if (command.equals("move")) {
092 Path source = getPath(commandElement, "source");
093 Path target = getPath(commandElement, "target");
094 move(context, source, target, recovery);
095 }
096 else {
097 if (command.equals("chmod")) {
098 Path path = getPath(commandElement, "path");
099 String str = commandElement.getAttributeValue("dir-files");
100 boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
101 String permissionsMask = commandElement.getAttributeValue("permissions").trim();
102 chmod(context, path, permissionsMask, dirFiles);
103 }
104 }
105 }
106 }
107 }
108 }
109 catch (Exception ex) {
110 throw convertException(ex);
111 }
112 }
113
114 /**
115 * @param path
116 * @param context
117 * @return FileSystem
118 * @throws HadoopAccessorException
119 */
120 private FileSystem getFileSystemFor(Path path, Context context) throws HadoopAccessorException {
121 String user = context.getWorkflow().getUser();
122 String group = context.getWorkflow().getGroup();
123 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
124 new Configuration());
125 }
126
127 /**
128 * @param path
129 * @param user
130 * @param group
131 * @return FileSystem
132 * @throws HadoopAccessorException
133 */
134 private FileSystem getFileSystemFor(Path path, String user, String group) throws HadoopAccessorException {
135 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
136 new Configuration());
137 }
138
139 void mkdir(Context context, Path path) throws ActionExecutorException {
140 try {
141 validatePath(path, true);
142 FileSystem fs = getFileSystemFor(path, context);
143
144 if (!fs.exists(path)) {
145 if (!fs.mkdirs(path)) {
146 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS004",
147 "mkdir, path [{0}] could not create directory", path);
148 }
149 }
150 }
151 catch (Exception ex) {
152 throw convertException(ex);
153 }
154 }
155
156 /**
157 * Delete path
158 *
159 * @param context
160 * @param path
161 * @throws ActionExecutorException
162 */
163 public void delete(Context context, Path path) throws ActionExecutorException {
164 try {
165 validatePath(path, true);
166 FileSystem fs = getFileSystemFor(path, context);
167
168 if (fs.exists(path)) {
169 if (!fs.delete(path, true)) {
170 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
171 "delete, path [{0}] could not delete path", path);
172 }
173 }
174 }
175 catch (Exception ex) {
176 throw convertException(ex);
177 }
178 }
179
180 /**
181 * Delete path
182 *
183 * @param user
184 * @param group
185 * @param path
186 * @throws ActionExecutorException
187 */
188 public void delete(String user, String group, Path path) throws ActionExecutorException {
189 try {
190 validatePath(path, true);
191 FileSystem fs = getFileSystemFor(path, user, group);
192
193 if (fs.exists(path)) {
194 if (!fs.delete(path, true)) {
195 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
196 "delete, path [{0}] could not delete path", path);
197 }
198 }
199 }
200 catch (Exception ex) {
201 throw convertException(ex);
202 }
203 }
204
205 /**
206 * Move source to target
207 *
208 * @param context
209 * @param source
210 * @param target
211 * @param recovery
212 * @throws ActionExecutorException
213 */
214 public void move(Context context, Path source, Path target, boolean recovery) throws ActionExecutorException {
215 try {
216 validatePath(source, true);
217 validatePath(target, false);
218 FileSystem fs = getFileSystemFor(source, context);
219
220 if (!fs.exists(source) && !recovery) {
221 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS006",
222 "move, source path [{0}] does not exist", source);
223 }
224
225 Path path = new Path(source, target);
226 if (fs.exists(path) && !recovery) {
227 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007",
228 "move, target path [{0}] already exists", target);
229 }
230
231 if (!fs.rename(source, target) && !recovery) {
232 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008",
233 "move, could not move [{0}] to [{1}]", source, target);
234 }
235 }
236 catch (Exception ex) {
237 throw convertException(ex);
238 }
239 }
240
241 void chmod(Context context, Path path, String permissions, boolean dirFiles) throws ActionExecutorException {
242 try {
243 validatePath(path, true);
244 FileSystem fs = getFileSystemFor(path, context);
245
246 if (!fs.exists(path)) {
247 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009",
248 "chmod, path [{0}] does not exist", path);
249 }
250
251 FileStatus pathStatus = fs.getFileStatus(path);
252
253 Path[] paths;
254 if (dirFiles && pathStatus.isDir()) {
255 FileStatus[] filesStatus = fs.listStatus(path);
256 paths = new Path[filesStatus.length];
257 for (int i = 0; i < filesStatus.length; i++) {
258 paths[i] = filesStatus[i].getPath();
259 }
260 }
261 else {
262 paths = new Path[]{path};
263 }
264
265 FsPermission newFsPermission = createShortPermission(permissions, path);
266 fs.setPermission(path, newFsPermission);
267 for (Path p : paths) {
268 fs.setPermission(p, newFsPermission);
269 }
270 }
271 catch (Exception ex) {
272 throw convertException(ex);
273 }
274 }
275
276 FsPermission createShortPermission(String permissions, Path path) throws ActionExecutorException {
277 if (permissions.length() == 3) {
278 char user = permissions.charAt(0);
279 char group = permissions.charAt(1);
280 char other = permissions.charAt(2);
281 int useri = user - '0';
282 int groupi = group - '0';
283 int otheri = other - '0';
284 int mask = useri * 100 + groupi * 10 + otheri;
285 short omask = Short.parseShort(Integer.toString(mask), 8);
286 return new FsPermission(omask);
287 }
288 else {
289 if (permissions.length() == 10) {
290 return FsPermission.valueOf(permissions);
291 }
292 else {
293 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS010",
294 "chmod, path [{0}] invalid permissions mask [{1}]", path, permissions);
295 }
296 }
297 }
298
299 @Override
300 public void check(Context context, WorkflowAction action) throws ActionExecutorException {
301 }
302
303 @Override
304 public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
305 }
306
307 @Override
308 public void start(Context context, WorkflowAction action) throws ActionExecutorException {
309 try {
310 context.setStartData("-", "-", "-");
311 Element actionXml = XmlUtils.parseXml(action.getConf());
312 doOperations(context, actionXml);
313 context.setExecutionData("OK", null);
314 }
315 catch (Exception ex) {
316 throw convertException(ex);
317 }
318 }
319
320 @Override
321 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
322 String externalStatus = action.getExternalStatus();
323 WorkflowAction.Status status = externalStatus.equals("OK") ? WorkflowAction.Status.OK :
324 WorkflowAction.Status.ERROR;
325 context.setEndData(status, getActionSignal(status));
326 if (!context.getProtoActionConf().getBoolean("oozie.action.keep.action.dir", false)) {
327 try {
328 FileSystem fs = context.getAppFileSystem();
329 fs.delete(context.getActionDir(), true);
330 }
331 catch (Exception ex) {
332 throw convertException(ex);
333 }
334 }
335 }
336
337 @Override
338 public boolean isCompleted(String externalStatus) {
339 return true;
340 }
341
342 /**
343 * @param context
344 * @return
345 * @throws HadoopAccessorException
346 * @throws IOException
347 * @throws URISyntaxException
348 */
349 public Path getRecoveryPath(Context context) throws HadoopAccessorException, IOException, URISyntaxException {
350 return new Path(context.getActionDir(), "fs-" + context.getRecoveryId());
351 }
352
353 }