001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action;
016
017 import org.apache.hadoop.fs.FileSystem;
018 import org.apache.hadoop.fs.Path;
019 import org.apache.hadoop.conf.Configuration;
020 import org.apache.oozie.client.WorkflowAction;
021 import org.apache.oozie.client.WorkflowJob;
022 import org.apache.oozie.util.ELEvaluator;
023 import org.apache.oozie.util.ParamChecker;
024 import org.apache.oozie.util.XLog;
025 import org.apache.oozie.service.HadoopAccessorException;
026 import org.apache.oozie.service.Services;
027
028 import java.io.IOException;
029 import java.net.URISyntaxException;
030 import java.util.HashMap;
031 import java.util.Map;
032 import java.util.Properties;
033 import java.util.LinkedHashMap;
034
035 /**
036 * Base action executor class. <p/> All the action executors should extend this class.
037 */
038 public abstract class ActionExecutor {
039
040 /**
041 * Configuration prefix for action executor (sub-classes) properties.
042 */
043 public static final String CONF_PREFIX = "oozie.action.";
044
045 /**
046 * Error code used by {@link #convertException} when there is not register error information for an exception.
047 */
048 public static final String ERROR_OTHER = "OTHER";
049
050 private static class ErrorInfo {
051 ActionExecutorException.ErrorType errorType;
052 String errorCode;
053
054 private ErrorInfo(ActionExecutorException.ErrorType errorType, String errorCode) {
055 this.errorType = errorType;
056 this.errorCode = errorCode;
057 }
058 }
059
060 private static boolean initMode = false;
061 private static Map<String, Map<Class, ErrorInfo>> ERROR_INFOS = new HashMap<String, Map<Class, ErrorInfo>>();
062
063 /**
064 * Context information passed to the ActionExecutor methods.
065 */
066 public interface Context {
067
068 /**
069 * Create the callback URL for the action.
070 *
071 * @param externalStatusVar variable for the caller to inject the external status.
072 * @return the callback URL.
073 */
074 public String getCallbackUrl(String externalStatusVar);
075
076 /**
077 * Return a proto configuration for actions with auth properties already set.
078 *
079 * @return a proto configuration for actions with auth properties already set.
080 */
081 public Configuration getProtoActionConf();
082
083 /**
084 * Return the workflow job.
085 *
086 * @return the workflow job.
087 */
088 public WorkflowJob getWorkflow();
089
090 /**
091 * Return an ELEvaluator with the context injected.
092 *
093 * @return configured ELEvaluator.
094 */
095 public ELEvaluator getELEvaluator();
096
097 /**
098 * Set a workflow action variable. <p/> Convenience method that prefixes the variable name with the action name
099 * plus a '.'.
100 *
101 * @param name variable name.
102 * @param value variable value, <code>null</code> removes the variable.
103 */
104 public void setVar(String name, String value);
105
106 /**
107 * Get a workflow action variable. <p/> Convenience method that prefixes the variable name with the action name
108 * plus a '.'.
109 *
110 * @param name variable name.
111 * @return the variable value, <code>null</code> if not set.
112 */
113 public String getVar(String name);
114
115 /**
116 * Set the action tracking information for an successfully started action.
117 *
118 * @param externalId the action external ID.
119 * @param trackerUri the action tracker URI.
120 * @param consoleUrl the action console URL.
121 */
122 void setStartData(String externalId, String trackerUri, String consoleUrl);
123
124 /**
125 * Set the action execution completion information for an action. The action status is set to {@link
126 * org.apache.oozie.client.WorkflowAction.Status#DONE}
127 *
128 * @param externalStatus the action external end status.
129 * @param actionData the action data on completion, <code>null</code> if none.
130 */
131 void setExecutionData(String externalStatus, Properties actionData);
132
133 /**
134 * Set the action end completion information for a completed action.
135 *
136 * @param status the action end status, it can be {@link org.apache.oozie.client.WorkflowAction.Status#OK} or
137 * {@link org.apache.oozie.client.WorkflowAction.Status#ERROR}.
138 * @param signalValue the action external end status.
139 */
140 void setEndData(WorkflowAction.Status status, String signalValue);
141
142 /**
143 * Return if the executor invocation is a retry or not.
144 *
145 * @return if the executor invocation is a retry or not.
146 */
147 boolean isRetry();
148
149 /**
150 * Sets the external status for the action in context.
151 *
152 * @param externalStatus the external status.
153 */
154 void setExternalStatus(String externalStatus);
155
156 /**
157 * Get the Action Recovery ID.
158 *
159 * @return recovery ID.
160 */
161 String getRecoveryId();
162
163 /*
164 * @return the path that will be used to store action specific data
165 * @throws IOException @throws URISyntaxException @throws HadoopAccessorException
166 */
167 public Path getActionDir() throws HadoopAccessorException, IOException, URISyntaxException;
168
169 /**
170 * @return filesystem handle for the application deployment fs.
171 * @throws IOException
172 * @throws URISyntaxException
173 * @throws HadoopAccessorException
174 */
175 public FileSystem getAppFileSystem() throws HadoopAccessorException, IOException, URISyntaxException;
176
177 public void setErrorInfo(String str, String exMsg);
178 }
179
180 /**
181 * Define the default maximum number of retry attempts for transient errors (total attempts = 1 + MAX_RETRIES).
182 */
183 public static final int MAX_RETRIES = 3;
184
185 /**
186 * Define the default inteval in seconds between retries.
187 */
188 public static final long RETRY_INTERVAL = 60;
189
190 private String type;
191 private int maxRetries;
192 private long retryInterval;
193
194 /**
195 * Create an action executor with default retry parameters.
196 *
197 * @param type action executor type.
198 */
199 protected ActionExecutor(String type) {
200 this(type, MAX_RETRIES, RETRY_INTERVAL);
201 }
202
203 /**
204 * Create an action executor.
205 *
206 * @param type action executor type.
207 * @param retryAttempts retry attempts.
208 * @param retryInterval retry interval, in seconds.
209 */
210 protected ActionExecutor(String type, int retryAttempts, long retryInterval) {
211 this.type = ParamChecker.notEmpty(type, "type");
212 this.maxRetries = retryAttempts;
213 this.retryInterval = retryInterval;
214 }
215
216 /**
217 * Clear all init settings for all action types.
218 */
219 public static void resetInitInfo() {
220 if (!initMode) {
221 throw new IllegalStateException("Error, action type info locked");
222 }
223 ERROR_INFOS.clear();
224 }
225
226 /**
227 * Enable action type initialization.
228 */
229 public static void enableInit() {
230 initMode = true;
231 }
232
233 /**
234 * Disable action type initialization.
235 */
236 public static void disableInit() {
237 initMode = false;
238 }
239
240 /**
241 * Invoked once at system initialization time. <p/> It can be used to register error information for the expected
242 * exceptions. Exceptions should be register from subclasses to superclasses to ensure proper detection, same thing
243 * that it is done in a normal catch. <p/> This method should invoke the {@link #registerError} method to register
244 * all its possible errors. <p/> Subclasses overriding must invoke super.
245 */
246 public void initActionType() {
247 ERROR_INFOS.put(getType(), new LinkedHashMap<Class, ErrorInfo>());
248 }
249
250 /**
251 * Return the system ID, this ID is defined in Oozie configuration.
252 *
253 * @return the system ID.
254 */
255 public String getOozieSystemId() {
256 return Services.get().getSystemId();
257 }
258
259 /**
260 * Return the runtime directory of the Oozie instance. <p/> The directory is created under TMP and it is always a
261 * new directory per system initialization.
262 *
263 * @return the runtime directory of the Oozie instance.
264 */
265 public String getOozieRuntimeDir() {
266 return Services.get().getRuntimeDir();
267 }
268
269 /**
270 * Return Oozie configuration. <p/> This is useful for actions that need access to configuration properties.
271 *
272 * @return Oozie configuration.
273 */
274 public Configuration getOozieConf() {
275 return Services.get().getConf();
276 }
277
278 /**
279 * Register error handling information for an exception.
280 *
281 * @param exClass excpetion class name (to work in case of a particular exception not being in the classpath, needed
282 * to be able to handle multiple version of Hadoop or other JARs used by executors with the same codebase).
283 * @param errorType error type for the exception.
284 * @param errorCode error code for the exception.
285 */
286 protected void registerError(String exClass, ActionExecutorException.ErrorType errorType, String errorCode) {
287 if (!initMode) {
288 throw new IllegalStateException("Error, action type info locked");
289 }
290 try {
291 Class klass = Thread.currentThread().getContextClassLoader().loadClass(exClass);
292 Map<Class, ErrorInfo> executorErrorInfo = ERROR_INFOS.get(getType());
293 executorErrorInfo.put(klass, new ErrorInfo(errorType, errorCode));
294 }
295 catch (ClassNotFoundException ex) {
296 XLog.getLog(getClass()).warn(
297 "Exception [{0}] no in classpath, ActionExecutor [{1}] will handled it as ERROR", exClass,
298 getType());
299 }
300 }
301
302 /**
303 * Return the action executor type.
304 *
305 * @return the action executor type.
306 */
307 public String getType() {
308 return type;
309 }
310
311 /**
312 * Return the maximum number of retries for the action executor.
313 *
314 * @return the maximum number of retries for the action executor.
315 */
316 public int getMaxRetries() {
317 return maxRetries;
318 }
319
320 /**
321 * Set the maximum number of retries for the action executor.
322 *
323 * @param maxRetries the maximum number of retries.
324 */
325 public void setMaxRetries(int maxRetries) {
326 this.maxRetries = maxRetries;
327 }
328
329 /**
330 * Return the retry interval for the action executor in seconds.
331 *
332 * @return the retry interval for the action executor in seconds.
333 */
334 public long getRetryInterval() {
335 return retryInterval;
336 }
337
338 /**
339 * Sets the retry interval for the action executor.
340 *
341 * @param retryInterval retry interval in seconds.
342 */
343 public void setRetryInterval(long retryInterval) {
344 this.retryInterval = retryInterval;
345 }
346
347 /**
348 * Utility method to handle exceptions in the {@link #start}, {@link #end}, {@link #kill} and {@link #check} methods
349 * <p/> It uses the error registry to convert exceptions to {@link ActionExecutorException}s.
350 *
351 * @param ex exception to convert.
352 * @return ActionExecutorException converted exception.
353 */
354 @SuppressWarnings({"ThrowableInstanceNeverThrown"})
355 protected ActionExecutorException convertException(Exception ex) {
356 if (ex instanceof ActionExecutorException) {
357 return (ActionExecutorException) ex;
358 }
359 for (Map.Entry<Class, ErrorInfo> errorInfo : ERROR_INFOS.get(getType()).entrySet()) {
360 if (errorInfo.getKey().isInstance(ex)) {
361 return new ActionExecutorException(errorInfo.getValue().errorType, errorInfo.getValue().errorCode,
362 "{0}", ex.getMessage(), ex);
363 }
364 }
365 String errorCode = ex.getClass().getName();
366 errorCode = errorCode.substring(errorCode.lastIndexOf(".") + 1);
367 return new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, errorCode, "{0}", ex.getMessage(),
368 ex);
369 }
370
371 /**
372 * Convenience method that return the signal for an Action based on the action status.
373 *
374 * @param status action status.
375 * @return the action signal.
376 */
377 protected String getActionSignal(WorkflowAction.Status status) {
378 switch (status) {
379 case OK:
380 return "OK";
381 case ERROR:
382 case KILLED:
383 return "ERROR";
384 default:
385 throw new IllegalArgumentException("Action status for signal can only be OK or ERROR");
386 }
387 }
388
389 /**
390 * Return the path that will be used to store action specific data
391 *
392 * @param jobId Worfklow ID
393 * @param action Action
394 * @param key An Identifier
395 * @param temp temp directory flag
396 * @return A string that has the path
397 */
398 protected String getActionDirPath(String jobId, WorkflowAction action, String key, boolean temp) {
399 String name = jobId + "/" + action.getName() + "--" + key;
400 if (temp) {
401 name += ".temp";
402 }
403 return getOozieSystemId() + "/" + name;
404 }
405
406 /**
407 * Return the path that will be used to store action specific data.
408 *
409 * @param jobId Workflow ID
410 * @param action Action
411 * @param key An identifier
412 * @param temp Temp directory flag
413 * @return Path to the directory
414 */
415 public Path getActionDir(String jobId, WorkflowAction action, String key, boolean temp) {
416 return new Path(getActionDirPath(jobId, action, key, temp));
417 }
418
419 /**
420 * Start an action. <p/> The {@link Context#setStartData} method must be called within this method. <p/> If the
421 * action has completed, the {@link Context#setExecutionData} method must be called within this method.
422 *
423 * @param context executor context.
424 * @param action the action to start.
425 * @throws ActionExecutorException thrown if the action could not start.
426 */
427 public abstract void start(Context context, WorkflowAction action) throws ActionExecutorException;
428
429 /**
430 * End an action after it has executed. <p/> The {@link Context#setEndData} method must be called within this
431 * method.
432 *
433 * @param context executor context.
434 * @param action the action to end.
435 * @throws ActionExecutorException thrown if the action could not end.
436 */
437 public abstract void end(Context context, WorkflowAction action) throws ActionExecutorException;
438
439 /**
440 * Check if an action has completed. This method must be implemented by Async Action Executors. <p/> If the action
441 * has completed, the {@link Context#setExecutionData} method must be called within this method. <p/> If the action
442 * has not completed, the {@link Context#setExternalStatus} method must be called within this method.
443 *
444 * @param context executor context.
445 * @param action the action to end.
446 * @throws ActionExecutorException thrown if the action could not be checked.
447 */
448 public abstract void check(Context context, WorkflowAction action) throws ActionExecutorException;
449
450 /**
451 * Kill an action. <p/> The {@link Context#setEndData} method must be called within this method.
452 *
453 * @param context executor context.
454 * @param action the action to kill.
455 * @throws ActionExecutorException thrown if the action could not be killed.
456 */
457 public abstract void kill(Context context, WorkflowAction action) throws ActionExecutorException;
458
459 /**
460 * Return if the external status indicates that the action has completed.
461 *
462 * @param externalStatus external status to check.
463 * @return if the external status indicates that the action has completed.
464 */
465 public abstract boolean isCompleted(String externalStatus);
466
467 }