001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.coord;
016
017 import java.util.Calendar;
018 import java.util.Date;
019 import java.util.HashMap;
020 import java.util.Iterator;
021 import java.util.List;
022 import java.util.Map;
023 import java.util.TimeZone;
024
025 import org.apache.hadoop.conf.Configuration;
026 import org.apache.oozie.service.ELService;
027 import org.apache.oozie.service.Services;
028 import org.apache.oozie.util.DateUtils;
029 import org.apache.oozie.util.ELEvaluator;
030 import org.apache.oozie.util.XLog;
031 import org.apache.oozie.util.XmlUtils;
032 import org.jdom.Element;
033
034 /**
035 * This class provide different evaluators required at different stages
036 */
037 public class CoordELEvaluator {
038 public static final Integer MINUTE = 1;
039 public static final Integer HOUR = 60 * MINUTE;
040
041 /**
042 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
043 * 1)
044 *
045 * @param conf : Configuration containing property variables
046 * @return configured ELEvaluator
047 */
048 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
049 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
050 setConfigToEval(eval, conf);
051 return eval;
052 }
053
054 /**
055 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
056 *
057 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
058 * @param appInst : Application Instance related information such as Action creation Time
059 * @param conf :Configuration to substitute any variables
060 * @return configured ELEvaluator
061 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
062 */
063 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
064 throws Exception {
065 return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
066 }
067
068 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
069 Configuration conf) throws Exception {
070 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
071 setConfigToEval(eval, conf);
072 SyncCoordDataset ds = getDSObject(event);
073 CoordELFunctions.configureEvaluator(eval, ds, appInst);
074 return eval;
075 }
076
077 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
078 HashMap<String, String> dataNameList) throws Exception {
079 ELEvaluator eval = createELEvaluatorForGroup(conf, group);
080 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
081 String key = it.next();
082 String value = dataNameList.get(key);
083 eval.setVariable("oozie.dataname." + key, value);
084 }
085 return eval;
086 }
087
088 /**
089 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
090 * should be resolved when all other data dependencies are met.
091 *
092 * @param actualTime : Action start time
093 * @param nominalTime : Action creation time
094 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
095 * @param conf :Configuration to substitute any variables
096 * @return configured ELEvaluator
097 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
098 */
099 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
100 throws Exception {
101 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
102 setConfigToEval(eval, conf);
103 SyncCoordDataset ds = getDSObject(dEvent);
104 SyncCoordAction appInst = new SyncCoordAction();// TODO:
105 appInst.setNominalTime(nominalTime);
106 appInst.setActualTime(actualTime);// TODO:
107 CoordELFunctions.configureEvaluator(eval, ds, appInst);
108 // Configuration tmpConf = new Configuration();
109 Configuration tmpConf = CoordUtils.getHadoopConf(conf);
110 // TODO:Set hadoop properties
111 eval.setVariable(CoordELFunctions.CONFIGURATION, tmpConf);
112 return eval;
113 }
114
115 /**
116 * Create a SLA evaluator to be used during Materialization
117 *
118 * @param nominalTime
119 * @param conf
120 * @return
121 * @throws Exception
122 */
123 public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
124 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
125 setConfigToEval(eval, conf);
126 SyncCoordAction appInst = new SyncCoordAction();// TODO:
127 appInst.setNominalTime(nominalTime);
128 CoordELFunctions.configureEvaluator(eval, null, appInst);
129 return eval;
130 }
131
132 /**
133 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
134 *
135 * @param eJob : XML element for the application instance
136 * @param conf :Configuration to substitute any variables
137 * @return configured ELEvaluator
138 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
139 */
140 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
141 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
142 setConfigToEval(e, conf);
143 SyncCoordAction appInst = new SyncCoordAction();
144 String strNominalTime = eJob.getAttributeValue("action-nominal-time");
145 if (strNominalTime != null) {
146 appInst.setNominalTime(DateUtils.parseDateUTC(strNominalTime));
147 appInst.setActionId(actionId);
148 appInst.setName(eJob.getAttributeValue("name"));
149 }
150 CoordELFunctions.configureEvaluator(e, null, appInst);
151 Element events = eJob.getChild("input-events", eJob.getNamespace());
152 if (events != null) {
153 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
154 if (data.getChild("uris", data.getNamespace()) != null) {
155 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
156 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
157 e.setVariable(".datain." + data.getAttributeValue("name"), uris);
158 }
159 else {
160 }
161 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
162 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
163 // check
164 // null
165 }
166 }
167 }
168 events = eJob.getChild("output-events", eJob.getNamespace());
169 if (events != null) {
170 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
171 if (data.getChild("uris", data.getNamespace()) != null) {
172 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
173 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
174 e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
175 }
176 else {
177 }// TODO
178 if (data.getChild("unresolved-instances", data.getNamespace()) != null) {
179 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
180 // check
181 // null
182 }
183 }
184 }
185 return e;
186 }
187
188 /**
189 * Create a new Evaluator to resolve URI temple with time specific constant
190 *
191 * @param strDate : Date-time
192 * @return configured ELEvaluator
193 * @throws Exception If there is any date-time string in wrong format, the exception is thrown
194 */
195 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
196 ELEvaluator eval = new ELEvaluator();
197 Calendar date = Calendar.getInstance(TimeZone.getTimeZone("UTC")); // TODO:UTC
198 // always???
199 date.setTime(DateUtils.parseDateUTC(strDate));
200 eval.setVariable("YEAR", date.get(Calendar.YEAR));
201 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
202 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
203 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
204 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
205 return eval;
206 }
207
208 /**
209 * Create Dataset object using the Dataset XML information
210 *
211 * @param eData
212 * @return
213 * @throws Exception
214 */
215 private static SyncCoordDataset getDSObject(Element eData) throws Exception {
216 SyncCoordDataset ds = new SyncCoordDataset();
217 Element eDataset = eData.getChild("dataset", eData.getNamespace());
218 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
219 Date initInstance = DateUtils.parseDateUTC(eDataset.getAttributeValue("initial-instance"));
220 ds.setInitInstance(initInstance);
221 if (eDataset.getAttributeValue("frequency") != null) {
222 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
223 ds.setFrequency(frequency);
224 ds.setType("SYNC");
225 if (eDataset.getAttributeValue("freq_timeunit") == null) {
226 throw new RuntimeException("No freq_timeunit defined in data set definition\n"
227 + XmlUtils.prettyPrint(eDataset));
228 }
229 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
230 if (eDataset.getAttributeValue("timezone") == null) {
231 throw new RuntimeException("No timezone defined in data set definition\n"
232 + XmlUtils.prettyPrint(eDataset));
233 }
234 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
235 if (eDataset.getAttributeValue("end_of_duration") == null) {
236 throw new RuntimeException("No end_of_duration defined in data set definition\n"
237 + XmlUtils.prettyPrint(eDataset));
238 }
239 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
240
241 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
242 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
243 ds.setDoneFlag(doneFlag);
244 }
245 else {
246 ds.setType("ASYNC");
247 }
248 String name = eDataset.getAttributeValue("name");
249 ds.setName(name);
250 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
251 // eData.getNamespace()));
252 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
253 ds.setUriTemplate(uriTemplate);
254 // ds.setTimeUnit(TimeUnit.MINUTES);
255 return ds;
256 }
257
258 /**
259 * Set all job configurations properties into evaluator.
260 *
261 * @param eval : Evaluator to set variables
262 * @param conf : configurations to set Evaluator
263 */
264 private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
265 for (Map.Entry<String, String> entry : conf) {
266 eval.setVariable(entry.getKey(), entry.getValue());
267 }
268 }
269
270 /**
271 * make any one digit number to two digit string pre-appending a"0"
272 *
273 * @param num : number to make sting
274 * @return :String of length at least two digit.
275 */
276 private static String make2Digits(int num) {
277 String ret = "" + num;
278 if (num <= 9) {
279 ret = "0" + ret;
280 }
281 return ret;
282 }
283 }