001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie;
016
017 import org.apache.oozie.util.XLogStreamer;
018 import org.apache.oozie.service.XLogService;
019 import org.apache.oozie.service.DagXLogInfoService;
020 import org.apache.hadoop.conf.Configuration;
021 import org.apache.oozie.client.CoordinatorJob;
022 import org.apache.oozie.client.WorkflowJob;
023 import org.apache.oozie.client.OozieClient;
024 import org.apache.oozie.command.wf.CompletedActionCommand;
025 import org.apache.oozie.command.CommandException;
026 import org.apache.oozie.command.Command;
027 import org.apache.oozie.command.wf.JobCommand;
028 import org.apache.oozie.command.wf.JobsCommand;
029 import org.apache.oozie.command.wf.KillCommand;
030 import org.apache.oozie.command.wf.ReRunCommand;
031 import org.apache.oozie.command.wf.ResumeCommand;
032 import org.apache.oozie.command.wf.SubmitCommand;
033 import org.apache.oozie.command.wf.SubmitHttpCommand;
034 import org.apache.oozie.command.wf.SubmitPigCommand;
035 import org.apache.oozie.command.wf.SubmitMRCommand;
036 import org.apache.oozie.command.wf.StartCommand;
037 import org.apache.oozie.command.wf.SuspendCommand;
038 import org.apache.oozie.command.wf.DefinitionCommand;
039 import org.apache.oozie.command.wf.ExternalIdCommand;
040 import org.apache.oozie.command.wf.WorkflowActionInfoCommand;
041 import org.apache.oozie.service.Services;
042 import org.apache.oozie.service.CallableQueueService;
043 import org.apache.oozie.util.ParamChecker;
044 import org.apache.oozie.util.XConfiguration;
045 import org.apache.oozie.util.XLog;
046
047 import java.io.Writer;
048 import java.util.List;
049 import java.util.Properties;
050 import java.util.Set;
051 import java.util.HashSet;
052 import java.util.StringTokenizer;
053 import java.util.Map;
054 import java.util.HashMap;
055 import java.util.ArrayList;
056 import java.io.IOException;
057
058 /**
059 * The DagEngine bean provides all the DAG engine functionality for WS calls.
060 */
061 public class DagEngine extends BaseEngine {
062
063 private static final int HIGH_PRIORITY = 2;
064
065 /**
066 * Create a system Dag engine, with no user and no group.
067 */
068 public DagEngine() {
069 }
070
071 /**
072 * Create a Dag engine to perform operations on behave of a user.
073 *
074 * @param user user name.
075 * @param authToken the authentication token.
076 */
077 public DagEngine(String user, String authToken) {
078 this.user = ParamChecker.notEmpty(user, "user");
079 this.authToken = ParamChecker.notEmpty(authToken, "authToken");
080 }
081
082 /**
083 * Submit a workflow job. <p/> It validates configuration properties.
084 *
085 * @param conf job configuration.
086 * @param startJob indicates if the job should be started or not.
087 * @return the job Id.
088 * @throws DagEngineException thrown if the job could not be created.
089 */
090 @Override
091 public String submitJob(Configuration conf, boolean startJob) throws DagEngineException {
092 validateSubmitConfiguration(conf);
093 SubmitCommand submit = new SubmitCommand(conf, getAuthToken());
094 try {
095 String jobId = submit.call();
096 if (startJob) {
097 start(jobId);
098 }
099 return jobId;
100 }
101 catch (CommandException ex) {
102 throw new DagEngineException(ex);
103 }
104 }
105
106 /**
107 * Submit a pig/mapreduce job through HTTP.
108 * <p/>
109 * It validates configuration properties.
110 *
111 * @param conf job configuration.
112 * @param jobType job type - can be "pig" or "mapreduce".
113 * @return the job Id.
114 * @throws DagEngineException thrown if the job could not be created.
115 */
116 public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException {
117 validateSubmitConfiguration(conf);
118
119 SubmitHttpCommand submit = null;
120 if (jobType.equals("pig")) {
121 submit = new SubmitPigCommand(conf, getAuthToken());
122 }
123 else if (jobType.equals("mapreduce")) {
124 submit = new SubmitMRCommand(conf, getAuthToken());
125 }
126
127 try {
128 String jobId = submit.call();
129 start(jobId);
130 return jobId;
131 }
132 catch (CommandException ex) {
133 throw new DagEngineException(ex);
134 }
135 }
136
137 public static void main(String[] args) throws Exception {
138 // Configuration conf = new XConfiguration(IOUtils.getResourceAsReader(
139 // "org/apache/oozie/coord/conf.xml", -1));
140
141 Configuration conf = new XConfiguration();
142
143 // String appXml =
144 // IOUtils.getResourceAsString("org/apache/oozie/coord/test1.xml", -1);
145 conf.set(OozieClient.APP_PATH, "file:///Users/danielwo/oozie/workflows/examples/seed/workflows/map-reduce");
146 conf.set(OozieClient.USER_NAME, "danielwo");
147 conf.set(OozieClient.GROUP_NAME, "other");
148
149 conf.set("inputDir", " blah ");
150
151 // System.out.println("appXml :"+ appXml + "\n conf :"+ conf);
152 new Services().init();
153 try {
154 DagEngine de = new DagEngine("me", "TESTING_WF");
155 String jobId = de.submitJob(conf, true);
156 System.out.println("WF Job Id " + jobId);
157
158 Thread.sleep(20000);
159 }
160 finally {
161 Services.get().destroy();
162 }
163 }
164
165 private void validateSubmitConfiguration(Configuration conf) throws DagEngineException {
166 if (conf.get(OozieClient.APP_PATH) == null) {
167 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
168 }
169 }
170
171 /**
172 * Start a job.
173 *
174 * @param jobId job Id.
175 * @throws DagEngineException thrown if the job could not be started.
176 */
177 @Override
178 public void start(String jobId) throws DagEngineException {
179 // Changing to synchronous call from asynchronous queuing to prevent the
180 // loss of command if the queue is full or the queue is lost in case of
181 // failure.
182 try {
183 new StartCommand(jobId).call();
184 }
185 catch (CommandException e) {
186 throw new DagEngineException(e);
187 }
188 }
189
190 /**
191 * Resume a job.
192 *
193 * @param jobId job Id.
194 * @throws DagEngineException thrown if the job could not be resumed.
195 */
196 @Override
197 public void resume(String jobId) throws DagEngineException {
198 // Changing to synchronous call from asynchronous queuing to prevent the
199 // loss of command if the queue is full or the queue is lost in case of
200 // failure.
201 try {
202 new ResumeCommand(jobId).call();
203 }
204 catch (CommandException e) {
205 throw new DagEngineException(e);
206 }
207 }
208
209 /**
210 * Suspend a job.
211 *
212 * @param jobId job Id.
213 * @throws DagEngineException thrown if the job could not be suspended.
214 */
215 @Override
216 public void suspend(String jobId) throws DagEngineException {
217 // Changing to synchronous call from asynchronous queuing to prevent the
218 // loss of command if the queue is full or the queue is lost in case of
219 // failure.
220 try {
221 new SuspendCommand(jobId).call();
222 }
223 catch (CommandException e) {
224 throw new DagEngineException(e);
225 }
226 }
227
228 /**
229 * Kill a job.
230 *
231 * @param jobId job Id.
232 * @throws DagEngineException thrown if the job could not be killed.
233 */
234 @Override
235 public void kill(String jobId) throws DagEngineException {
236 // Changing to synchronous call from asynchronous queuing to prevent the
237 // loss of command if the queue is full or the queue is lost in case of
238 // failure.
239 try {
240 new KillCommand(jobId).call();
241 XLog.getLog(getClass()).info("User " + user + " killed the WF job " + jobId);
242 }
243 catch (CommandException e) {
244 throw new DagEngineException(e);
245 }
246 }
247
248 /* (non-Javadoc)
249 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
250 */
251 @Override
252 public void change(String jobId, String changeValue) throws DagEngineException {
253 // This code should not be reached.
254 throw new DagEngineException(ErrorCode.E1017);
255 }
256
257 /**
258 * Rerun a job.
259 *
260 * @param jobId job Id to rerun.
261 * @param conf configuration information for the rerun.
262 * @throws DagEngineException thrown if the job could not be rerun.
263 */
264 @Override
265 public void reRun(String jobId, Configuration conf) throws DagEngineException {
266 try {
267 validateReRunConfiguration(conf);
268 new ReRunCommand(jobId, conf, getAuthToken()).call();
269 start(jobId);
270 }
271 catch (CommandException ex) {
272 throw new DagEngineException(ex);
273 }
274 }
275
276 private void validateReRunConfiguration(Configuration conf) throws DagEngineException {
277 if (conf.get(OozieClient.APP_PATH) == null) {
278 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
279 }
280 if (conf.get(OozieClient.RERUN_SKIP_NODES) == null) {
281 throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES);
282 }
283 }
284
285 /**
286 * Process an action callback.
287 *
288 * @param actionId the action Id.
289 * @param externalStatus the action external status.
290 * @param actionData the action output data, <code>null</code> if none.
291 * @throws DagEngineException thrown if the callback could not be processed.
292 */
293 public void processCallback(String actionId, String externalStatus, Properties actionData)
294 throws DagEngineException {
295 XLog.Info.get().clearParameter(XLogService.GROUP);
296 XLog.Info.get().clearParameter(XLogService.USER);
297 Command<Void, ?> command = new CompletedActionCommand(actionId, externalStatus, actionData, HIGH_PRIORITY);
298 if (!Services.get().get(CallableQueueService.class).queue(command)) {
299 XLog.getLog(this.getClass()).warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback");
300 }
301 }
302
303 /**
304 * Return the info about a job.
305 *
306 * @param jobId job Id.
307 * @return the workflow job info.
308 * @throws DagEngineException thrown if the job info could not be obtained.
309 */
310 @Override
311 public WorkflowJob getJob(String jobId) throws DagEngineException {
312 try {
313 return new JobCommand(jobId).call();
314 }
315 catch (CommandException ex) {
316 throw new DagEngineException(ex);
317 }
318 }
319
320 /**
321 * Return the info about a job with actions subset.
322 *
323 * @param jobId job Id
324 * @param start starting from this index in the list of actions belonging to the job
325 * @param length number of actions to be returned
326 * @return the workflow job info.
327 * @throws DagEngineException thrown if the job info could not be obtained.
328 */
329 @Override
330 public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException {
331 try {
332 return new JobCommand(jobId, start, length).call();
333 }
334 catch (CommandException ex) {
335 throw new DagEngineException(ex);
336 }
337 }
338
339 /**
340 * Return the a job definition.
341 *
342 * @param jobId job Id.
343 * @return the job definition.
344 * @throws DagEngineException thrown if the job definition could no be obtained.
345 */
346 @Override
347 public String getDefinition(String jobId) throws DagEngineException {
348 try {
349 return new DefinitionCommand(jobId).call();
350 }
351 catch (CommandException ex) {
352 throw new DagEngineException(ex);
353 }
354 }
355
356 /**
357 * Stream the log of a job.
358 *
359 * @param jobId job Id.
360 * @param writer writer to stream the log to.
361 * @throws IOException thrown if the log cannot be streamed.
362 * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
363 */
364 @Override
365 public void streamLog(String jobId, Writer writer) throws IOException, DagEngineException {
366 XLogStreamer.Filter filter = new XLogStreamer.Filter();
367 filter.setParameter(DagXLogInfoService.JOB, jobId);
368 WorkflowJob job = getJob(jobId);
369 Services.get().get(XLogService.class).streamLog(filter, job.getStartTime(), job.getEndTime(), writer);
370 }
371
372 private static final Set<String> FILTER_NAMES = new HashSet<String>();
373
374 static {
375 FILTER_NAMES.add(OozieClient.FILTER_USER);
376 FILTER_NAMES.add(OozieClient.FILTER_NAME);
377 FILTER_NAMES.add(OozieClient.FILTER_GROUP);
378 FILTER_NAMES.add(OozieClient.FILTER_STATUS);
379 }
380
381 /**
382 * Validate a jobs filter.
383 *
384 * @param filter filter to validate.
385 * @return the parsed filter.
386 * @throws DagEngineException thrown if the filter is invalid.
387 */
388 protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException {
389 Map<String, List<String>> map = new HashMap<String, List<String>>();
390 if (filter != null) {
391 StringTokenizer st = new StringTokenizer(filter, ";");
392 while (st.hasMoreTokens()) {
393 String token = st.nextToken();
394 if (token.contains("=")) {
395 String[] pair = token.split("=");
396 if (pair.length != 2) {
397 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
398 }
399 if (!FILTER_NAMES.contains(pair[0])) {
400 throw new DagEngineException(ErrorCode.E0420, filter, XLog
401 .format("invalid name [{0}]", pair[0]));
402 }
403 if (pair[0].equals("status")) {
404 try {
405 WorkflowJob.Status.valueOf(pair[1]);
406 }
407 catch (IllegalArgumentException ex) {
408 throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]",
409 pair[1]));
410 }
411 }
412 List<String> list = map.get(pair[0]);
413 if (list == null) {
414 list = new ArrayList<String>();
415 map.put(pair[0], list);
416 }
417 list.add(pair[1]);
418 }
419 else {
420 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
421 }
422 }
423 }
424 return map;
425 }
426
427 /**
428 * Return the info about a set of jobs.
429 *
430 * @param filterStr job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax.
431 * @param start offset, base 1.
432 * @param len number of jobs to return.
433 * @return job info for all matching jobs, the jobs don't contain node action information.
434 * @throws DagEngineException thrown if the jobs info could not be obtained.
435 */
436 @SuppressWarnings("unchecked")
437 public WorkflowsInfo getJobs(String filterStr, int start, int len) throws DagEngineException {
438 Map<String, List<String>> filter = parseFilter(filterStr);
439 try {
440 return new JobsCommand(filter, start, len).call();
441 }
442 catch (CommandException dce) {
443 throw new DagEngineException(dce);
444 }
445 }
446
447 /**
448 * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes.
449 *
450 * @param externalId external ID provided at job submission time.
451 * @return the associated workflow job ID if any, <code>null</code> if none.
452 * @throws DagEngineException thrown if the lookup could not be done.
453 */
454 @Override
455 public String getJobIdForExternalId(String externalId) throws DagEngineException {
456 try {
457 return new ExternalIdCommand(externalId).call();
458 }
459 catch (CommandException dce) {
460 throw new DagEngineException(dce);
461 }
462 }
463
464 @Override
465 public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException {
466 throw new BaseEngineException(new XException(ErrorCode.E0301));
467 }
468
469 @Override
470 public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BaseEngineException {
471 throw new BaseEngineException(new XException(ErrorCode.E0301));
472 }
473
474 public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException {
475 try {
476 return new WorkflowActionInfoCommand(actionId).call();
477 }
478 catch (CommandException ex) {
479 throw new BaseEngineException(ex);
480 }
481 }
482
483 @Override
484 public String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException {
485 return null;
486 }
487 }