001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.coord;
016
017 import java.io.StringReader;
018 import java.util.Date;
019 import java.util.List;
020
021 import org.apache.hadoop.conf.Configuration;
022 import org.apache.oozie.CoordinatorActionBean;
023 import org.apache.oozie.ErrorCode;
024 import org.apache.oozie.client.CoordinatorAction;
025 import org.apache.oozie.command.CommandException;
026 import org.apache.oozie.coord.CoordELEvaluator;
027 import org.apache.oozie.coord.CoordELFunctions;
028 import org.apache.oozie.coord.CoordUtils;
029 import org.apache.oozie.coord.CoordinatorJobException;
030 import org.apache.oozie.coord.SyncCoordAction;
031 import org.apache.oozie.coord.TimeUnit;
032 import org.apache.oozie.service.Services;
033 import org.apache.oozie.service.UUIDService;
034 import org.apache.oozie.util.DateUtils;
035 import org.apache.oozie.util.ELEvaluator;
036 import org.apache.oozie.util.XConfiguration;
037 import org.apache.oozie.util.XmlUtils;
038 import org.jdom.Element;
039
040 public class CoordCommandUtils {
041 public static int CURRENT = 0;
042 public static int LATEST = 1;
043 public static int FUTURE = 2;
044 public static int UNEXPECTED = -1;
045 public static final String RESOLVED_UNRESOLVED_SEPARATOR = ";";
046
047 /**
048 * parse a function like coord:latest(n)/future() and return the 'n'.
049 * <p/>
050 * @param function
051 * @param event
052 * @param appInst
053 * @param conf
054 * @param restArg
055 * @return int instanceNumber
056 * @throws Exception
057 */
058 public static int getInstanceNumber(String function, Element event, SyncCoordAction appInst, Configuration conf,
059 StringBuilder restArg) throws Exception {
060 ELEvaluator eval = CoordELEvaluator
061 .createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf);
062 String newFunc = CoordELFunctions.evalAndWrap(eval, function);
063 int funcType = getFuncType(newFunc);
064 if (funcType == CURRENT || funcType == LATEST) {
065 return parseOneArg(newFunc);
066 }
067 else {
068 return parseMoreArgs(newFunc, restArg);
069 }
070 }
071
072 private static int parseOneArg(String funcName) throws Exception {
073 int firstPos = funcName.indexOf("(");
074 int lastPos = funcName.lastIndexOf(")");
075 if (firstPos >= 0 && lastPos > firstPos) {
076 String tmp = funcName.substring(firstPos + 1, lastPos).trim();
077 if (tmp.length() > 0) {
078 return Integer.parseInt(tmp);
079 }
080 }
081 throw new RuntimeException("Unformatted function :" + funcName);
082 }
083
084 private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception {
085 int firstPos = funcName.indexOf("(");
086 int secondPos = funcName.lastIndexOf(",");
087 int lastPos = funcName.lastIndexOf(")");
088 if (firstPos >= 0 && secondPos > firstPos) {
089 String tmp = funcName.substring(firstPos + 1, secondPos).trim();
090 if (tmp.length() > 0) {
091 restArg.append(funcName.substring(secondPos + 1, lastPos).trim());
092 return Integer.parseInt(tmp);
093 }
094 }
095 throw new RuntimeException("Unformatted function :" + funcName);
096 }
097
098 /**
099 * @param EL function name
100 * @return type of EL function
101 */
102 public static int getFuncType(String function) {
103 if (function.indexOf("current") >= 0) {
104 return CURRENT;
105 }
106 else if (function.indexOf("latest") >= 0) {
107 return LATEST;
108 }
109 else if (function.indexOf("future") >= 0) {
110 return FUTURE;
111 }
112 return UNEXPECTED;
113 // throw new RuntimeException("Unexpected instance name "+ function);
114 }
115
116 /**
117 * @param startInst: EL function name
118 * @param endInst: EL function name
119 * @throws CommandException if both are not the same function
120 */
121 public static void checkIfBothSameType(String startInst, String endInst) throws CommandException {
122 if (getFuncType(startInst) != getFuncType(endInst)) {
123 throw new CommandException(ErrorCode.E1010,
124 " start-instance and end-instance both should be either latest or current or future\n"
125 + " start " + startInst + " and end " + endInst);
126 }
127 }
128
129 /**
130 * Resolve list of <instance> </instance> tags.
131 *
132 * @param event
133 * @param instances
134 * @param actionInst
135 * @param conf
136 * @param eval: ELEvalautor
137 * @throws Exception
138 */
139 public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst,
140 Configuration conf, ELEvaluator eval) throws Exception {
141 for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) {
142 if (instances.length() > 0) {
143 instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
144 }
145 instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval));
146 }
147 event.removeChildren("instance", event.getNamespace());
148 }
149
150 /**
151 * Resolve <start-instance> <end-insatnce> tag. Don't resolve any
152 * latest()/future()
153 *
154 * @param event
155 * @param instances
156 * @param appInst
157 * @param conf
158 * @param eval: ELEvalautor
159 * @throws Exception
160 */
161 public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst,
162 Configuration conf, ELEvaluator eval) throws Exception {
163 Element eStartInst = event.getChild("start-instance", event.getNamespace());
164 Element eEndInst = event.getChild("end-instance", event.getNamespace());
165 if (eStartInst != null && eEndInst != null) {
166 String strStart = eStartInst.getTextTrim();
167 String strEnd = eEndInst.getTextTrim();
168 checkIfBothSameType(strStart, strEnd);
169 StringBuilder restArg = new StringBuilder(); // To store rest
170 // arguments for
171 // future
172 // function
173 int startIndex = getInstanceNumber(strStart, event, appInst, conf, restArg);
174 restArg.delete(0, restArg.length());
175 int endIndex = getInstanceNumber(strEnd, event, appInst, conf, restArg);
176 if (startIndex > endIndex) {
177 throw new CommandException(ErrorCode.E1010,
178 " start-instance should be equal or earlier than the end-instance \n"
179 + XmlUtils.prettyPrint(event));
180 }
181 int funcType = getFuncType(strStart);
182 if (funcType == CURRENT) {
183 // Everything could be resolved NOW. no latest() ELs
184 for (int i = endIndex; i >= startIndex; i--) {
185 String matInstance = materializeInstance(event, "${coord:current(" + i + ")}", appInst, conf, eval);
186 if (matInstance == null || matInstance.length() == 0) {
187 // Earlier than dataset's initial instance
188 break;
189 }
190 if (instances.length() > 0) {
191 instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
192 }
193 instances.append(matInstance);
194 }
195 }
196 else { // latest(n)/future() EL is present
197 for (; startIndex <= endIndex; startIndex++) {
198 if (instances.length() > 0) {
199 instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
200 }
201 if (funcType == LATEST) {
202 instances.append("${coord:latest(" + startIndex + ")}");
203 }
204 else { // For future
205 instances.append("${coord:future(" + startIndex + ",'" + restArg + "')}");
206 }
207 }
208 }
209 // Remove start-instance and end-instances
210 event.removeChild("start-instance", event.getNamespace());
211 event.removeChild("end-instance", event.getNamespace());
212 }
213 }
214
215 /**
216 * Materialize one instance like current(-2)
217 *
218 * @param event : <data-in>
219 * @param expr : instance like current(-1)
220 * @param appInst : application specific info
221 * @param conf
222 * @param evalInst :ELEvaluator
223 * @return materialized date string
224 * @throws Exception
225 */
226 public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf,
227 ELEvaluator evalInst) throws Exception {
228 if (event == null) {
229 return null;
230 }
231 // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event,
232 // appInst, conf);
233 return CoordELFunctions.evalAndWrap(evalInst, expr);
234 }
235
236 /**
237 * Create two new tags with <uris> and <unresolved-instances>.
238 *
239 * @param event
240 * @param instances
241 * @param dependencyList
242 * @throws Exception
243 */
244 public static void separateResolvedAndUnresolved(Element event, StringBuilder instances, StringBuffer dependencyList)
245 throws Exception {
246 StringBuilder unresolvedInstances = new StringBuilder();
247 StringBuilder urisWithDoneFlag = new StringBuilder();
248 String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag);
249 if (uris.length() > 0) {
250 Element uriInstance = new Element("uris", event.getNamespace());
251 uriInstance.addContent(uris);
252 event.getContent().add(1, uriInstance);
253 if (dependencyList.length() > 0) {
254 dependencyList.append(CoordELFunctions.INSTANCE_SEPARATOR);
255 }
256 dependencyList.append(urisWithDoneFlag);
257 }
258 if (unresolvedInstances.length() > 0) {
259 Element elemInstance = new Element("unresolved-instances", event.getNamespace());
260 elemInstance.addContent(unresolvedInstances.toString());
261 event.getContent().add(1, elemInstance);
262 }
263 }
264
265 /**
266 * The function create a list of URIs separated by "," using the instances
267 * time stamp and URI-template
268 *
269 * @param event : <data-in> event
270 * @param instances : List of time stamp separated by ","
271 * @param unresolvedInstances : list of instance with latest function
272 * @param urisWithDoneFlag : list of URIs with the done flag appended
273 * @return : list of URIs separated by ";" as a string.
274 * @throws Exception
275 */
276 public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances,
277 StringBuilder urisWithDoneFlag) throws Exception {
278 if (instances == null || instances.length() == 0) {
279 return "";
280 }
281 String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR);
282 StringBuilder uris = new StringBuilder();
283
284 Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag",
285 event.getNamespace());
286 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
287
288 for (int i = 0; i < instanceList.length; i++) {
289 if(instanceList[i].trim().length() == 0) {
290 continue;
291 }
292 int funcType = getFuncType(instanceList[i]);
293 if (funcType == LATEST || funcType == FUTURE) {
294 if (unresolvedInstances.length() > 0) {
295 unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR);
296 }
297 unresolvedInstances.append(instanceList[i]);
298 continue;
299 }
300 ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]);
301 if (uris.length() > 0) {
302 uris.append(CoordELFunctions.INSTANCE_SEPARATOR);
303 urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR);
304 }
305
306 String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace())
307 .getChild("uri-template", event.getNamespace()).getTextTrim());
308 uris.append(uriPath);
309 if (doneFlag.length() > 0) {
310 uriPath += "/" + doneFlag;
311 }
312 urisWithDoneFlag.append(uriPath);
313 }
314 return uris.toString();
315 }
316
317 /**
318 * @param eSla
319 * @param nominalTime
320 * @param conf
321 * @return boolean to determine whether the SLA element is present or not
322 * @throws CoordinatorJobException
323 */
324 public static boolean materializeSLA(Element eSla, Date nominalTime, Configuration conf)
325 throws CoordinatorJobException {
326 if (eSla == null) {
327 // eAppXml.getNamespace("sla"));
328 return false;
329 }
330 try {
331 ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(nominalTime, conf);
332 List<Element> elemList = eSla.getChildren();
333 for (Element elem : elemList) {
334 String updated;
335 try {
336 updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim());
337 }
338 catch (Exception e) {
339 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
340 }
341 elem.removeContent();
342 elem.addContent(updated);
343 }
344 }
345 catch (Exception e) {
346 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
347 }
348 return true;
349 }
350
351 /**
352 * Materialize one instance for specific nominal time. It includes: 1.
353 * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize
354 * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and
355 * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag
356 *
357 * @param jobId coordinator job id
358 * @param dryrun true if it is dryrun
359 * @param eAction frequency unexploded-job
360 * @param nominalTime materialization time
361 * @param instanceCount instance numbers
362 * @param conf job configuration
363 * @param actionBean CoordinatorActionBean to materialize
364 * @return one materialized action for specific nominal time
365 * @throws Exception
366 */
367 @SuppressWarnings("unchecked")
368 public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime,
369 int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception {
370 String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + "");
371 SyncCoordAction appInst = new SyncCoordAction();
372 appInst.setActionId(actionId);
373 appInst.setName(eAction.getAttributeValue("name"));
374 appInst.setNominalTime(nominalTime);
375 int frequency = Integer.parseInt(eAction.getAttributeValue("frequency"));
376 appInst.setFrequency(frequency);
377 appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit")));
378 appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone")));
379 appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration")));
380
381 StringBuffer dependencyList = new StringBuffer();
382
383 Element inputList = eAction.getChild("input-events", eAction.getNamespace());
384 List<Element> dataInList = null;
385 if (inputList != null) {
386 dataInList = inputList.getChildren("data-in", eAction.getNamespace());
387 materializeDataEvents(dataInList, appInst, conf, dependencyList);
388 }
389
390 Element outputList = eAction.getChild("output-events", eAction.getNamespace());
391 List<Element> dataOutList = null;
392 if (outputList != null) {
393 dataOutList = outputList.getChildren("data-out", eAction.getNamespace());
394 StringBuffer tmp = new StringBuffer();
395 // no dependency checks
396 materializeDataEvents(dataOutList, appInst, conf, tmp);
397 }
398
399 eAction.removeAttribute("start");
400 eAction.removeAttribute("end");
401 eAction.setAttribute("instance-number", Integer.toString(instanceCount));
402 eAction.setAttribute("action-nominal-time", DateUtils.formatDateUTC(nominalTime));
403
404 boolean isSla = CoordCommandUtils.materializeSLA(eAction.getChild("action", eAction.getNamespace()).getChild(
405 "info", eAction.getNamespace("sla")), nominalTime, conf);
406
407 // Setting up action bean
408 actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString());
409 actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString());
410 actionBean.setCreatedTime(new Date());
411 actionBean.setJobId(jobId);
412 actionBean.setId(actionId);
413 actionBean.setLastModifiedTime(new Date());
414 actionBean.setStatus(CoordinatorAction.Status.WAITING);
415 actionBean.setActionNumber(instanceCount);
416 actionBean.setMissingDependencies(dependencyList.toString());
417 actionBean.setNominalTime(nominalTime);
418 if (isSla == true) {
419 actionBean.setSlaXml(XmlUtils.prettyPrint(
420 eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla")))
421 .toString());
422 }
423
424 // actionBean.setTrackerUri(trackerUri);//TOOD:
425 // actionBean.setConsoleUrl(consoleUrl); //TODO:
426 // actionBean.setType(type);//TODO:
427 // actionBean.setErrorInfo(errorCode, errorMessage); //TODO:
428 // actionBean.setExternalStatus(externalStatus);//TODO
429 if (!dryrun) {
430 return XmlUtils.prettyPrint(eAction).toString();
431 }
432 else {
433 String action = XmlUtils.prettyPrint(eAction).toString();
434 CoordActionInputCheckCommand coordActionInput = new CoordActionInputCheckCommand(actionBean.getId());
435 StringBuilder actionXml = new StringBuilder(action);
436 StringBuilder existList = new StringBuilder();
437 StringBuilder nonExistList = new StringBuilder();
438 StringBuilder nonResolvedList = new StringBuilder();
439 getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList);
440 Date actualTime = new Date();
441 Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
442 coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf, actualTime);
443 return actionXml.toString();
444 }
445 }
446
447 /**
448 * Materialize all <input-events>/<data-in> or <output-events>/<data-out>
449 * tags Create uris for resolved instances. Create unresolved instance for
450 * latest()/future().
451 *
452 * @param events
453 * @param appInst
454 * @param conf
455 * @throws Exception
456 */
457 public static void materializeDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf,
458 StringBuffer dependencyList) throws Exception {
459
460 if (events == null) {
461 return;
462 }
463 StringBuffer unresolvedList = new StringBuffer();
464 for (Element event : events) {
465 StringBuilder instances = new StringBuilder();
466 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
467 // Handle list of instance tag
468 resolveInstances(event, instances, appInst, conf, eval);
469 // Handle start-instance and end-instance
470 resolveInstanceRange(event, instances, appInst, conf, eval);
471 // Separate out the unresolved instances
472 separateResolvedAndUnresolved(event, instances, dependencyList);
473 String tmpUnresolved = event.getChildTextTrim("unresolved-instances", event.getNamespace());
474 if (tmpUnresolved != null) {
475 if (unresolvedList.length() > 0) {
476 unresolvedList.append(CoordELFunctions.INSTANCE_SEPARATOR);
477 }
478 unresolvedList.append(tmpUnresolved);
479 }
480 }
481 if (unresolvedList.length() > 0) {
482 dependencyList.append(RESOLVED_UNRESOLVED_SEPARATOR);
483 dependencyList.append(unresolvedList);
484 }
485 return;
486 }
487
488 /**
489 * Get resolved string from missDepList
490 *
491 * @param missDepList
492 * @param resolved
493 * @param unresolved
494 * @return resolved string
495 */
496 public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) {
497 if (missDepList != null) {
498 int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR);
499 if (index < 0) {
500 resolved.append(missDepList);
501 }
502 else {
503 resolved.append(missDepList.substring(0, index));
504 unresolved.append(missDepList.substring(index + 1));
505 }
506 }
507 return resolved.toString();
508 }
509
510 }