001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.coord;
016
017 import java.io.IOException;
018 import java.util.Calendar;
019 import java.util.Date;
020 import java.util.TimeZone;
021
022 import org.apache.hadoop.conf.Configuration;
023 import org.apache.hadoop.fs.Path;
024
025 import org.apache.oozie.client.OozieClient;
026 import org.apache.oozie.util.DateUtils;
027 import org.apache.oozie.util.ELEvaluator;
028 import org.apache.oozie.util.ParamChecker;
029 import org.apache.oozie.util.XLog;
030 import org.apache.oozie.service.HadoopAccessorException;
031 import org.apache.oozie.service.Services;
032 import org.apache.oozie.service.HadoopAccessorService;
033
034 /**
035 * This class implements the EL function related to coordinator
036 */
037
038 public class CoordELFunctions {
039 final private static String DATASET = "oozie.coord.el.dataset.bean";
040 final private static String COORD_ACTION = "oozie.coord.el.app.bean";
041 final public static String CONFIGURATION = "oozie.coord.el.conf";
042 // INSTANCE_SEPARATOR is used to separate multiple directories into one tag.
043 final public static String INSTANCE_SEPARATOR = "#";
044 final public static String DIR_SEPARATOR = ",";
045 // TODO: in next release, support flexibility
046 private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS";
047
048 /**
049 * Used in defining the frequency in 'day' unit. <p/> domain: <code> val > 0</code> and should be integer.
050 *
051 * @param val frequency in number of days.
052 * @return number of days and also set the frequency timeunit to "day"
053 */
054 public static int ph1_coord_days(int val) {
055 val = ParamChecker.checkGTZero(val, "n");
056 ELEvaluator eval = ELEvaluator.getCurrent();
057 eval.setVariable("timeunit", TimeUnit.DAY);
058 eval.setVariable("endOfDuration", TimeUnit.NONE);
059 return val;
060 }
061
062 /**
063 * Used in defining the frequency in 'month' unit. <p/> domain: <code> val > 0</code> and should be integer.
064 *
065 * @param val frequency in number of months.
066 * @return number of months and also set the frequency timeunit to "month"
067 */
068 public static int ph1_coord_months(int val) {
069 val = ParamChecker.checkGTZero(val, "n");
070 ELEvaluator eval = ELEvaluator.getCurrent();
071 eval.setVariable("timeunit", TimeUnit.MONTH);
072 eval.setVariable("endOfDuration", TimeUnit.NONE);
073 return val;
074 }
075
076 /**
077 * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val > 0</code> and should
078 * be integer.
079 *
080 * @param val frequency in number of hours.
081 * @return number of minutes and also set the frequency timeunit to "minute"
082 */
083 public static int ph1_coord_hours(int val) {
084 val = ParamChecker.checkGTZero(val, "n");
085 ELEvaluator eval = ELEvaluator.getCurrent();
086 eval.setVariable("timeunit", TimeUnit.MINUTE);
087 eval.setVariable("endOfDuration", TimeUnit.NONE);
088 return val * 60;
089 }
090
091 /**
092 * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val > 0</code> and should be integer.
093 *
094 * @param val frequency in number of minutes.
095 * @return number of minutes and also set the frequency timeunit to "minute"
096 */
097 public static int ph1_coord_minutes(int val) {
098 val = ParamChecker.checkGTZero(val, "n");
099 ELEvaluator eval = ELEvaluator.getCurrent();
100 eval.setVariable("timeunit", TimeUnit.MINUTE);
101 eval.setVariable("endOfDuration", TimeUnit.NONE);
102 return val;
103 }
104
105 /**
106 * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will
107 * start at 00:00 hour of each day. <p/> domain: <code> val > 0</code> and should be integer.
108 *
109 * @param val frequency in number of days.
110 * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day"
111 */
112 public static int ph1_coord_endOfDays(int val) {
113 val = ParamChecker.checkGTZero(val, "n");
114 ELEvaluator eval = ELEvaluator.getCurrent();
115 eval.setVariable("timeunit", TimeUnit.DAY);
116 eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY);
117 return val;
118 }
119
120 /**
121 * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will
122 * start at first day of each month at 00:00 hour. <p/> domain: <code> val > 0</code> and should be integer.
123 *
124 * @param val: frequency in number of months.
125 * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month"
126 */
127 public static int ph1_coord_endOfMonths(int val) {
128 val = ParamChecker.checkGTZero(val, "n");
129 ELEvaluator eval = ELEvaluator.getCurrent();
130 eval.setVariable("timeunit", TimeUnit.MONTH);
131 eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH);
132 return val;
133 }
134
135 /**
136 * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/>
137 * 1. Timezone of both dataset and job <p/> 2. Action creation Time
138 *
139 * @return difference in minutes (DataSet TZ Offset - Application TZ offset)
140 */
141 public static int ph2_coord_tzOffset() {
142 Date actionCreationTime = getActionCreationtime();
143 TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ");
144 TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ");
145 // Apply the TZ into Calendar object
146 Calendar dsTime = Calendar.getInstance(dsTZ);
147 dsTime.setTime(actionCreationTime);
148 Calendar jobTime = Calendar.getInstance(jobTZ);
149 jobTime.setTime(actionCreationTime);
150 return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60);
151 }
152
153 public static int ph3_coord_tzOffset() {
154 return ph2_coord_tzOffset();
155 }
156
157 /**
158 * Returns the a date string while given a base date in 'strBaseDate',
159 * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH).
160 *
161 * @param strBaseDate -- base date
162 * @param offset -- any number
163 * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
164 * @return date string
165 * @throws Exception
166 */
167 public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
168 Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
169 StringBuilder buffer = new StringBuilder();
170 baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset);
171 buffer.append(DateUtils.formatDateUTC(baseCalDate));
172 return buffer.toString();
173 }
174
175 public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
176 return ph2_coord_dateOffset(strBaseDate, offset, unit);
177 }
178
179 /**
180 * Determine the date-time in UTC of n-th future available dataset instance
181 * from nominal Time but not beyond the instance specified as 'instance.
182 * <p/>
183 * It depends on:
184 * <p/>
185 * 1. Data set frequency
186 * <p/>
187 * 2. Data set Time unit (day, month, minute)
188 * <p/>
189 * 3. Data set Time zone/DST
190 * <p/>
191 * 4. End Day/Month flag
192 * <p/>
193 * 5. Data set initial instance
194 * <p/>
195 * 6. Action Creation Time
196 * <p/>
197 * 7. Existence of dataset's directory
198 *
199 * @param n :instance count
200 * <p/>
201 * domain: n >= 0, n is integer
202 * @param instance: How many future instance it should check? value should
203 * be >=0
204 * @return date-time in UTC of the n-th instance
205 * <p/>
206 * @throws Exception
207 */
208 public static String ph3_coord_future(int n, int instance) throws Exception {
209 ParamChecker.checkGEZero(n, "future:n");
210 ParamChecker.checkGTZero(instance, "future:instance");
211 if (isSyncDataSet()) {// For Sync Dataset
212 return coord_future_sync(n, instance);
213 }
214 else {
215 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
216 }
217 }
218
219 private static String coord_future_sync(int n, int instance) throws Exception {
220 ELEvaluator eval = ELEvaluator.getCurrent();
221 String retVal = "";
222 int datasetFrequency = (int) getDSFrequency();// in minutes
223 TimeUnit dsTimeUnit = getDSTimeUnit();
224 int[] instCount = new int[1];
225 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
226 if (nominalInstanceCal != null) {
227 Calendar initInstance = getInitialInstanceCal();
228 nominalInstanceCal = (Calendar) initInstance.clone();
229 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
230
231 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
232 if (ds == null) {
233 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
234 }
235 String uriTemplate = ds.getUriTemplate();
236 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
237 if (conf == null) {
238 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
239 }
240 int available = 0, checkedInstance = 0;
241 boolean resolved = false;
242 String user = ParamChecker
243 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
244 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
245 OozieClient.GROUP_NAME);
246 String doneFlag = ds.getDoneFlag();
247 while (instance >= checkedInstance) {
248 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
249 String uriPath = uriEval.evaluate(uriTemplate, String.class);
250 String pathWithDoneFlag = uriPath;
251 if (doneFlag.length() > 0) {
252 pathWithDoneFlag += "/" + doneFlag;
253 }
254 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
255 XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag);
256 if (available == n) {
257 XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag);
258 resolved = true;
259 retVal = DateUtils.formatDateUTC(nominalInstanceCal);
260 eval.setVariable("resolved_path", uriPath);
261 break;
262 }
263 available++;
264 }
265 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
266 // -datasetFrequency);
267 nominalInstanceCal = (Calendar) initInstance.clone();
268 instCount[0]++;
269 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
270 checkedInstance++;
271 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
272 }
273 if (!resolved) {
274 // return unchanged future function with variable 'is_resolved'
275 // to 'false'
276 eval.setVariable("is_resolved", Boolean.FALSE);
277 retVal = "${coord:future(" + n + ", " + instance + ")}";
278 }
279 else {
280 eval.setVariable("is_resolved", Boolean.TRUE);
281 }
282 }
283 else {// No feasible nominal time
284 eval.setVariable("is_resolved", Boolean.TRUE);
285 retVal = "";
286 }
287 return retVal;
288 }
289
290 /**
291 * Return nominal time or Action Creation Time.
292 * <p/>
293 *
294 * @return coordinator action creation or materialization date time
295 * @throws Exception if unable to format the Date object to String
296 */
297 public static String ph2_coord_nominalTime() throws Exception {
298 ELEvaluator eval = ELEvaluator.getCurrent();
299 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
300 "Coordinator Action");
301 return DateUtils.formatDateUTC(action.getNominalTime());
302 }
303
304 public static String ph3_coord_nominalTime() throws Exception {
305 return ph2_coord_nominalTime();
306 }
307
308 /**
309 * Return Action Id. <p/>
310 *
311 * @return coordinator action Id
312 */
313 public static String ph2_coord_actionId() throws Exception {
314 ELEvaluator eval = ELEvaluator.getCurrent();
315 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
316 "Coordinator Action");
317 return action.getActionId();
318 }
319
320 public static String ph3_coord_actionId() throws Exception {
321 return ph2_coord_actionId();
322 }
323
324 /**
325 * Return Job Name. <p/>
326 *
327 * @return coordinator name
328 */
329 public static String ph2_coord_name() throws Exception {
330 ELEvaluator eval = ELEvaluator.getCurrent();
331 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
332 "Coordinator Action");
333 return action.getName();
334 }
335
336 public static String ph3_coord_name() throws Exception {
337 return ph2_coord_name();
338 }
339
340 /**
341 * Return Action Start time. <p/>
342 *
343 * @return coordinator action start time
344 * @throws Exception if unable to format the Date object to String
345 */
346 public static String ph2_coord_actualTime() throws Exception {
347 ELEvaluator eval = ELEvaluator.getCurrent();
348 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
349 if (coordAction == null) {
350 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
351 }
352 return DateUtils.formatDateUTC(coordAction.getActualTime());
353 }
354
355 public static String ph3_coord_actualTime() throws Exception {
356 return ph2_coord_actualTime();
357 }
358
359 /**
360 * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level
361 * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
362 * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
363 * unresolved, this function will echo back the original function <p/> otherwise it sends the uris.
364 *
365 * @param dataInName : Datain name
366 * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest)
367 * , echo back <p/> the function without resolving the function.
368 */
369 public static String ph3_coord_dataIn(String dataInName) {
370 String uris = "";
371 ELEvaluator eval = ELEvaluator.getCurrent();
372 uris = (String) eval.getVariable(".datain." + dataInName);
373 Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
374 if (unresolved != null && unresolved.booleanValue() == true) {
375 return "${coord:dataIn('" + dataInName + "')}";
376 }
377 return uris;
378 }
379
380 /**
381 * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level
382 * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris.
383 *
384 * @param dataOutName : Dataout name
385 * @return the list of URI's separated by INSTANCE_SEPARATOR
386 */
387 public static String ph3_coord_dataOut(String dataOutName) {
388 String uris = "";
389 ELEvaluator eval = ELEvaluator.getCurrent();
390 uris = (String) eval.getVariable(".dataout." + dataOutName);
391 return uris;
392 }
393
394 /**
395 * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2.
396 * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data
397 * set initial instance <p/> 6. Action Creation Time
398 *
399 * @param n instance count domain: n is integer
400 * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance
401 * of DS
402 * @throws Exception
403 */
404 public static String ph2_coord_current(int n) throws Exception {
405 if (isSyncDataSet()) { // For Sync Dataset
406 return coord_current_sync(n);
407 }
408 else {
409 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
410 }
411 }
412
413 /**
414 * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency
415 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
416 * Data set initial instance <p/> 6. Action Creation Time
417 *
418 * @param n instance count <p/> domain: n is integer
419 * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
420 * @throws Exception
421 */
422 public static int ph2_coord_hoursInDay(int n) throws Exception {
423 int datasetFrequency = (int) getDSFrequency();
424 // /Calendar nominalInstanceCal =
425 // getCurrentInstance(getActionCreationtime());
426 Calendar nominalInstanceCal = getEffectiveNominalTime();
427 if (nominalInstanceCal == null) {
428 return -1;
429 }
430 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
431 /*
432 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
433 * { return -1; }
434 */
435 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
436 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
437 return DateUtils.hoursInDay(nominalInstanceCal);
438 }
439
440 public static int ph3_coord_hoursInDay(int n) throws Exception {
441 return ph2_coord_hoursInDay(n);
442 }
443
444 /**
445 * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency .
446 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
447 * Data set initial instance <p/> 6. Action Creation Time
448 *
449 * @param n instance count. domain: n is integer
450 * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
451 * @throws Exception
452 */
453 public static int ph2_coord_daysInMonth(int n) throws Exception {
454 int datasetFrequency = (int) getDSFrequency();// in minutes
455 // Calendar nominalInstanceCal =
456 // getCurrentInstance(getActionCreationtime());
457 Calendar nominalInstanceCal = getEffectiveNominalTime();
458 if (nominalInstanceCal == null) {
459 return -1;
460 }
461 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
462 /*
463 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
464 * { return -1; }
465 */
466 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
467 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
468 return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH);
469 }
470
471 public static int ph3_coord_daysInMonth(int n) throws Exception {
472 return ph2_coord_daysInMonth(n);
473 }
474
475 /**
476 * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set
477 * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month
478 * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory
479 *
480 * @param n :instance count <p/> domain: n > 0, n is integer
481 * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than
482 * Initial-Instance of DS
483 * @throws Exception
484 */
485 public static String ph3_coord_latest(int n) throws Exception {
486 if (n > 0) {
487 throw new IllegalArgumentException("paramter should be <= 0 but it is " + n);
488 }
489 if (isSyncDataSet()) {// For Sync Dataset
490 return coord_latest_sync(n);
491 }
492 else {
493 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
494 }
495 }
496
497 /**
498 * Configure an evaluator with data set and application specific information. <p/> Helper method of associating
499 * dataset and application object
500 *
501 * @param evaluator : to set variables
502 * @param ds : Data Set object
503 * @param coordAction : Application instance
504 */
505 public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) {
506 evaluator.setVariable(COORD_ACTION, coordAction);
507 evaluator.setVariable(DATASET, ds);
508 }
509
510 /**
511 * Helper method to wrap around with "${..}". <p/>
512 *
513 *
514 * @param eval :EL evaluator
515 * @param expr : expression to evaluate
516 * @return Resolved expression or echo back the same expression
517 * @throws Exception
518 */
519 public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception {
520 try {
521 eval.setVariable(".wrap", null);
522 String result = eval.evaluate(expr, String.class);
523 if (eval.getVariable(".wrap") != null) {
524 return "${" + result + "}";
525 }
526 else {
527 return result;
528 }
529 }
530 catch (Exception e) {
531 throw new Exception("Unable to evaluate :" + expr + ":\n", e);
532 }
533 }
534
535 // Set of echo functions
536
537 public static String ph1_coord_current_echo(String n) {
538 return echoUnResolved("current", n);
539 }
540
541 public static String ph2_coord_current_echo(String n) {
542 return echoUnResolved("current", n);
543 }
544
545 public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) {
546 return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit);
547 }
548
549 public static String ph1_coord_latest_echo(String n) {
550 return echoUnResolved("latest", n);
551 }
552
553 public static String ph2_coord_latest_echo(String n) {
554 return ph1_coord_latest_echo(n);
555 }
556
557 public static String ph1_coord_future_echo(String n, String instance) {
558 return echoUnResolved("future", n + ", " + instance + "");
559 }
560
561 public static String ph2_coord_future_echo(String n, String instance) {
562 return ph1_coord_future_echo(n, instance);
563 }
564
565 public static String ph1_coord_dataIn_echo(String n) {
566 ELEvaluator eval = ELEvaluator.getCurrent();
567 String val = (String) eval.getVariable("oozie.dataname." + n);
568 if (val == null || val.equals("data-in") == false) {
569 XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid");
570 throw new RuntimeException("data_in_name " + n + " is not valid");
571 }
572 return echoUnResolved("dataIn", "'" + n + "'");
573 }
574
575 public static String ph1_coord_dataOut_echo(String n) {
576 ELEvaluator eval = ELEvaluator.getCurrent();
577 String val = (String) eval.getVariable("oozie.dataname." + n);
578 if (val == null || val.equals("data-out") == false) {
579 XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
580 throw new RuntimeException("data_out_name " + n + " is not valid");
581 }
582 return echoUnResolved("dataOut", "'" + n + "'");
583 }
584
585 public static String ph1_coord_nominalTime_echo() {
586 return echoUnResolved("nominalTime", "");
587 }
588
589 public static String ph1_coord_nominalTime_echo_wrap() {
590 // return "${coord:nominalTime()}"; // no resolution
591 return echoUnResolved("nominalTime", "");
592 }
593
594 public static String ph1_coord_nominalTime_echo_fixed() {
595 return "2009-03-06T010:00"; // Dummy resolution
596 }
597
598 public static String ph1_coord_actionId_echo() {
599 return echoUnResolved("actionId", "");
600 }
601
602 public static String ph1_coord_name_echo() {
603 return echoUnResolved("name", "");
604 }
605
606 // The following echo functions are not used in any phases yet
607 // They are here for future purpose.
608 public static String coord_minutes_echo(String n) {
609 return echoUnResolved("minutes", n);
610 }
611
612 public static String coord_hours_echo(String n) {
613 return echoUnResolved("hours", n);
614 }
615
616 public static String coord_days_echo(String n) {
617 return echoUnResolved("days", n);
618 }
619
620 public static String coord_endOfDay_echo(String n) {
621 return echoUnResolved("endOfDay", n);
622 }
623
624 public static String coord_months_echo(String n) {
625 return echoUnResolved("months", n);
626 }
627
628 public static String coord_endOfMonth_echo(String n) {
629 return echoUnResolved("endOfMonth", n);
630 }
631
632 public static String coord_actualTime_echo() {
633 return echoUnResolved("actualTime", "");
634 }
635
636 // This echo function will always return "24" for validation only.
637 // This evaluation ****should not**** replace the original XML
638 // Create a temporary string and validate the function
639 // This is **required** for evaluating an expression like
640 // coord:HoursInDay(0) + 3
641 // actual evaluation will happen in phase 2 or phase 3.
642 public static String ph1_coord_hoursInDay_echo(String n) {
643 return "24";
644 // return echoUnResolved("hoursInDay", n);
645 }
646
647 // This echo function will always return "30" for validation only.
648 // This evaluation ****should not**** replace the original XML
649 // Create a temporary string and validate the function
650 // This is **required** for evaluating an expression like
651 // coord:daysInMonth(0) + 3
652 // actual evaluation will happen in phase 2 or phase 3.
653 public static String ph1_coord_daysInMonth_echo(String n) {
654 // return echoUnResolved("daysInMonth", n);
655 return "30";
656 }
657
658 // This echo function will always return "3" for validation only.
659 // This evaluation ****should not**** replace the original XML
660 // Create a temporary string and validate the function
661 // This is **required** for evaluating an expression like coord:tzOffset + 2
662 // actual evaluation will happen in phase 2 or phase 3.
663 public static String ph1_coord_tzOffset_echo() {
664 // return echoUnResolved("tzOffset", "");
665 return "3";
666 }
667
668 // Local methods
669 /**
670 * @param n
671 * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the
672 * Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset.
673 * @throws Exception
674 */
675 private static String coord_current_sync(int n) throws Exception {
676 int datasetFrequency = getDSFrequency();// in minutes
677 TimeUnit dsTimeUnit = getDSTimeUnit();
678 int[] instCount = new int[1];// used as pass by ref
679 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
680 if (nominalInstanceCal == null) {
681 return "";
682 }
683 nominalInstanceCal = getInitialInstanceCal();
684 int absInstanceCount = instCount[0] + n;
685 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount);
686
687 if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
688 return "";
689 }
690 String str = DateUtils.formatDateUTC(nominalInstanceCal);
691 return str;
692 }
693
694 /**
695 * @param offset
696 * @return n-th available latest instance Date-Time for SYNC data-set
697 * @throws Exception
698 */
699 private static String coord_latest_sync(int offset) throws Exception {
700 if (offset > 0) {
701 throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0"
702 + offset);
703 }
704 ELEvaluator eval = ELEvaluator.getCurrent();
705 String retVal = "";
706 int datasetFrequency = (int) getDSFrequency();// in minutes
707 TimeUnit dsTimeUnit = getDSTimeUnit();
708 int[] instCount = new int[1];
709 Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
710 if (nominalInstanceCal != null) {
711 Calendar initInstance = getInitialInstanceCal();
712 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
713 if (ds == null) {
714 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
715 }
716 String uriTemplate = ds.getUriTemplate();
717 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
718 if (conf == null) {
719 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
720 }
721 int available = 0;
722 boolean resolved = false;
723 String user = ParamChecker
724 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
725 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
726 OozieClient.GROUP_NAME);
727 String doneFlag = ds.getDoneFlag();
728 while (nominalInstanceCal.compareTo(initInstance) >= 0) {
729 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
730 String uriPath = uriEval.evaluate(uriTemplate, String.class);
731 String pathWithDoneFlag = uriPath;
732 if (doneFlag.length() > 0) {
733 pathWithDoneFlag += "/" + doneFlag;
734 }
735 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
736 XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag);
737 if (available == offset) {
738 XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
739 resolved = true;
740 retVal = DateUtils.formatDateUTC(nominalInstanceCal);
741 eval.setVariable("resolved_path", uriPath);
742 break;
743 }
744
745 available--;
746 }
747 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
748 // -datasetFrequency);
749 nominalInstanceCal = (Calendar) initInstance.clone();
750 instCount[0]--;
751 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
752 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
753 }
754 if (!resolved) {
755 // return unchanged latest function with variable 'is_resolved'
756 // to 'false'
757 eval.setVariable("is_resolved", Boolean.FALSE);
758 retVal = "${coord:latest(" + offset + ")}";
759 }
760 else {
761 eval.setVariable("is_resolved", Boolean.TRUE);
762 }
763 }
764 else {// No feasible nominal time
765 eval.setVariable("is_resolved", Boolean.FALSE);
766 }
767 return retVal;
768 }
769
770 // TODO : Not an efficient way. In a loop environment, we could do something
771 // outside the loop
772 /**
773 * Check whether a URI path exists
774 *
775 * @param sPath
776 * @param conf
777 * @return
778 * @throws IOException
779 */
780
781 private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf)
782 throws IOException, HadoopAccessorException {
783 // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE;
784 Path path = new Path(sPath);
785 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
786 new Configuration()).exists(path);
787 }
788
789 /**
790 * @param tm
791 * @return a new Evaluator to be used for URI-template evaluation
792 */
793 private static ELEvaluator getUriEvaluator(Calendar tm) {
794 ELEvaluator retEval = new ELEvaluator();
795 retEval.setVariable("YEAR", tm.get(Calendar.YEAR));
796 retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm
797 .get(Calendar.MONTH) + 1));
798 retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm
799 .get(Calendar.DAY_OF_MONTH));
800 retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm
801 .get(Calendar.HOUR_OF_DAY));
802 retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm
803 .get(Calendar.MINUTE));
804 return retEval;
805 }
806
807 /**
808 * @return whether a data set is SYNCH or ASYNC
809 */
810 private static boolean isSyncDataSet() {
811 ELEvaluator eval = ELEvaluator.getCurrent();
812 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
813 if (ds == null) {
814 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
815 }
816 return ds.getType().equalsIgnoreCase("SYNC");
817 }
818
819 /**
820 * Check whether a function should be resolved.
821 *
822 * @param functionName
823 * @param n
824 * @return null if the functionName needs to be resolved otherwise return the calling function unresolved.
825 */
826 private static String checkIfResolved(String functionName, String n) {
827 ELEvaluator eval = ELEvaluator.getCurrent();
828 String replace = (String) eval.getVariable("resolve_" + functionName);
829 if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't
830 // resolve
831 // return "${coord:" + functionName + "(" + n +")}"; //Unresolved
832 eval.setVariable(".wrap", "true");
833 return "coord:" + functionName + "(" + n + ")"; // Unresolved
834 }
835 return null; // Resolved it
836 }
837
838 private static String echoUnResolved(String functionName, String n) {
839 return echoUnResolvedPre(functionName, n, "coord:");
840 }
841
842 private static String echoUnResolvedPre(String functionName, String n, String prefix) {
843 ELEvaluator eval = ELEvaluator.getCurrent();
844 eval.setVariable(".wrap", "true");
845 return prefix + functionName + "(" + n + ")"; // Unresolved
846 }
847
848 /**
849 * @return the initial instance of a DataSet in DATE
850 */
851 private static Date getInitialInstance() {
852 return getInitialInstanceCal().getTime();
853 // return ds.getInitInstance();
854 }
855
856 /**
857 * @return the initial instance of a DataSet in Calendar
858 */
859 private static Calendar getInitialInstanceCal() {
860 ELEvaluator eval = ELEvaluator.getCurrent();
861 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
862 if (ds == null) {
863 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
864 }
865 Calendar effInitTS = Calendar.getInstance();
866 effInitTS.setTime(ds.getInitInstance());
867 effInitTS.setTimeZone(ds.getTimeZone());
868 // To adjust EOD/EOM
869 DateUtils.moveToEnd(effInitTS, getDSEndOfFlag());
870 return effInitTS;
871 // return ds.getInitInstance();
872 }
873
874 /**
875 * @return Nominal or action creation Time when all the dependencies of an application instance are met.
876 */
877 private static Date getActionCreationtime() {
878 ELEvaluator eval = ELEvaluator.getCurrent();
879 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
880 if (coordAction == null) {
881 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
882 }
883 return coordAction.getNominalTime();
884 }
885
886 /**
887 * @return Actual Time when all the dependencies of an application instance are met.
888 */
889 private static Date getActualTime() {
890 ELEvaluator eval = ELEvaluator.getCurrent();
891 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
892 if (coordAction == null) {
893 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
894 }
895 return coordAction.getActualTime();
896 }
897
898 /**
899 * @return TimeZone for the application or job.
900 */
901 private static TimeZone getJobTZ() {
902 ELEvaluator eval = ELEvaluator.getCurrent();
903 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
904 if (coordAction == null) {
905 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
906 }
907 return coordAction.getTimeZone();
908 }
909
910 /**
911 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
912 *
913 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
914 * the dataset.
915 */
916 private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
917 Date datasetInitialInstance = getInitialInstance();
918 TimeUnit dsTimeUnit = getDSTimeUnit();
919 TimeZone dsTZ = getDatasetTZ();
920 // Convert Date to Calendar for corresponding TZ
921 Calendar current = Calendar.getInstance();
922 current.setTime(datasetInitialInstance);
923 current.setTimeZone(dsTZ);
924
925 Calendar calEffectiveTime = Calendar.getInstance();
926 calEffectiveTime.setTime(effectiveTime);
927 calEffectiveTime.setTimeZone(dsTZ);
928 instanceCount[0] = 0;
929 if (current.compareTo(calEffectiveTime) > 0) {
930 // Nominal Time < initial Instance
931 // TODO: getClass() call doesn't work from static method.
932 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
933 // current.getTime());
934 return null;
935 }
936 Calendar origCurrent = (Calendar) current.clone();
937 while (current.compareTo(calEffectiveTime) <= 0) {
938 current = (Calendar) origCurrent.clone();
939 instanceCount[0]++;
940 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
941 }
942 instanceCount[0]--;
943
944 current = (Calendar) origCurrent.clone();
945 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
946 return current;
947 }
948
949 private static Calendar getEffectiveNominalTime() {
950 Date datasetInitialInstance = getInitialInstance();
951 TimeZone dsTZ = getDatasetTZ();
952 // Convert Date to Calendar for corresponding TZ
953 Calendar current = Calendar.getInstance();
954 current.setTime(datasetInitialInstance);
955 current.setTimeZone(dsTZ);
956
957 Calendar calEffectiveTime = Calendar.getInstance();
958 calEffectiveTime.setTime(getActionCreationtime());
959 calEffectiveTime.setTimeZone(dsTZ);
960 if (current.compareTo(calEffectiveTime) > 0) {
961 // Nominal Time < initial Instance
962 // TODO: getClass() call doesn't work from static method.
963 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
964 // current.getTime());
965 return null;
966 }
967 return calEffectiveTime;
968 }
969
970 /**
971 * @return dataset frequency in minutes
972 */
973 private static int getDSFrequency() {
974 ELEvaluator eval = ELEvaluator.getCurrent();
975 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
976 if (ds == null) {
977 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
978 }
979 return ds.getFrequency();
980 }
981
982 /**
983 * @return dataset TimeUnit
984 */
985 private static TimeUnit getDSTimeUnit() {
986 ELEvaluator eval = ELEvaluator.getCurrent();
987 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
988 if (ds == null) {
989 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
990 }
991 return ds.getTimeUnit();
992 }
993
994 /**
995 * @return dataset TimeZone
996 */
997 private static TimeZone getDatasetTZ() {
998 ELEvaluator eval = ELEvaluator.getCurrent();
999 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1000 if (ds == null) {
1001 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1002 }
1003 return ds.getTimeZone();
1004 }
1005
1006 /**
1007 * @return dataset TimeUnit
1008 */
1009 private static TimeUnit getDSEndOfFlag() {
1010 ELEvaluator eval = ELEvaluator.getCurrent();
1011 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1012 if (ds == null) {
1013 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1014 }
1015 return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration();
1016 }
1017
1018 }