001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.coord;
016
017 import java.io.IOException;
018 import java.io.StringReader;
019 import java.util.Date;
020 import java.util.List;
021
022 import org.apache.hadoop.conf.Configuration;
023 import org.apache.hadoop.fs.Path;
024 import org.apache.oozie.CoordinatorActionBean;
025 import org.apache.oozie.ErrorCode;
026 import org.apache.oozie.client.CoordinatorAction;
027 import org.apache.oozie.client.OozieClient;
028 import org.apache.oozie.command.CommandException;
029 import org.apache.oozie.coord.CoordELEvaluator;
030 import org.apache.oozie.coord.CoordELFunctions;
031 import org.apache.oozie.service.HadoopAccessorException;
032 import org.apache.oozie.service.HadoopAccessorService;
033 import org.apache.oozie.service.Services;
034 import org.apache.oozie.store.CoordinatorStore;
035 import org.apache.oozie.store.StoreException;
036 import org.apache.oozie.util.DateUtils;
037 import org.apache.oozie.util.ELEvaluator;
038 import org.apache.oozie.util.Instrumentation;
039 import org.apache.oozie.util.ParamChecker;
040 import org.apache.oozie.util.XConfiguration;
041 import org.apache.oozie.util.XLog;
042 import org.apache.oozie.util.XmlUtils;
043 import org.jdom.Element;
044
045 public class CoordActionInputCheckCommand extends CoordinatorCommand<Void> {
046
047 private String actionId;
048 private final XLog log = XLog.getLog(getClass());
049 private int COMMAND_REQUEUE_INTERVAL = 60000; // 1 minute
050 private CoordinatorActionBean coordAction = null;
051
052 public CoordActionInputCheckCommand(String actionId) {
053 super("coord_action_input", "coord_action_input", 1, XLog.STD);
054 this.actionId = actionId;
055 }
056
057 @Override
058 protected Void call(CoordinatorStore store) throws StoreException, CommandException {
059 log.debug("After store.get() for action ID " + actionId + " : " + coordAction.getStatus());
060 // this action should only get processed if current time >
061 // materialization time
062 // otherwise, requeue this action after 30 seconds
063 Date nominalTime = coordAction.getNominalTime();
064 Date currentTime = new Date();
065 if (nominalTime.compareTo(currentTime) > 0) {
066 log.info("[" + actionId
067 + "]::ActionInputCheck:: nominal Time is newer than current time, so requeue and wait. Current="
068 + currentTime + ", nominal=" + nominalTime);
069 queueCallable(new CoordActionInputCheckCommand(coordAction.getId()), Math.max(
070 (nominalTime.getTime() - currentTime.getTime()), COMMAND_REQUEUE_INTERVAL));
071 // update lastModifiedTime
072 store.updateCoordinatorAction(coordAction);
073 return null;
074 }
075 if (coordAction.getStatus() == CoordinatorActionBean.Status.WAITING) {
076 log.info("[" + actionId + "]::ActionInputCheck:: Action is in WAITING state.");
077 StringBuilder actionXml = new StringBuilder(coordAction.getActionXml());// job.getXml();
078 Instrumentation.Cron cron = new Instrumentation.Cron();
079 try {
080 Configuration actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
081 cron.start();
082 StringBuilder existList = new StringBuilder();
083 StringBuilder nonExistList = new StringBuilder();
084 StringBuilder nonResolvedList = new StringBuilder();
085 CoordCommandUtils.getResolvedList(coordAction.getMissingDependencies(), nonExistList,
086 nonResolvedList);
087
088 log.info("[" + actionId + "]::ActionInputCheck:: Missing deps:" + nonExistList.toString() + " "
089 + nonResolvedList.toString());
090 Date actualTime = new Date();
091 boolean status = checkInput(actionXml, existList, nonExistList, actionConf, actualTime);
092 coordAction.setLastModifiedTime(actualTime);
093 coordAction.setActionXml(actionXml.toString());
094 if (nonResolvedList.length() > 0 && status == false) {
095 nonExistList.append(CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR).append(
096 nonResolvedList);
097 }
098 coordAction.setMissingDependencies(nonExistList.toString());
099 if (status == true) {
100 coordAction.setStatus(CoordinatorAction.Status.READY);
101 // pass jobID to the ReadyCommand
102 queueCallable(new CoordActionReadyCommand(coordAction.getJobId()), 100);
103 }
104 else {
105 long waitingTime = (actualTime.getTime() - coordAction.getNominalTime().getTime()) / (60 * 1000);
106 int timeOut = coordAction.getTimeOut();
107 if ((timeOut >= 0) && (waitingTime > timeOut)) {
108 queueCallable(new CoordActionTimeOut(coordAction), 100);
109 coordAction.setStatus(CoordinatorAction.Status.TIMEDOUT);
110 }
111 else {
112 queueCallable(new CoordActionInputCheckCommand(coordAction.getId()), COMMAND_REQUEUE_INTERVAL);
113 }
114 }
115 store.updateCoordActionMin(coordAction);
116 }
117 catch (Exception e) {
118 log.warn(actionId + ": Exception occurs: " + e + " STORE is active " + store.isActive(), e);
119 throw new CommandException(ErrorCode.E1005, e.getMessage(), e);
120 }
121 cron.stop();
122 }
123 else {
124 log.info("[" + actionId + "]::ActionInputCheck:: Ignoring action. Should be in WAITING state, but state="
125 + coordAction.getStatus());
126 }
127 return null;
128 }
129
130 protected boolean checkInput(StringBuilder actionXml, StringBuilder existList, StringBuilder nonExistList,
131 Configuration conf, Date actualTime) throws Exception {
132 Element eAction = XmlUtils.parseXml(actionXml.toString());
133 boolean allExist = checkResolvedUris(eAction, existList, nonExistList, conf);
134 if (allExist) {
135 log.debug("[" + actionId + "]::ActionInputCheck:: Checking Latest/future");
136 allExist = checkUnresolvedInstances(eAction, conf, actualTime);
137 }
138 if (allExist == true) {
139 materializeDataProperties(eAction, conf);
140 actionXml.replace(0, actionXml.length(), XmlUtils.prettyPrint(eAction).toString());
141 }
142 return allExist;
143 }
144
145 /**
146 * Materialize data properties defined in <action> tag. it includes dataIn(<DS>) and dataOut(<DS>) it creates a list
147 * of files that will be needed.
148 *
149 * @param eAction
150 * @param conf
151 * @throws Exception
152 * @update modify 'Action' element with appropriate list of files.
153 */
154 private void materializeDataProperties(Element eAction, Configuration conf) throws Exception {
155 ELEvaluator eval = CoordELEvaluator.createDataEvaluator(eAction, conf, actionId);
156 Element configElem = eAction.getChild("action", eAction.getNamespace()).getChild("workflow",
157 eAction.getNamespace()).getChild("configuration", eAction.getNamespace());
158 if (configElem != null) {
159 for (Element propElem : (List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
160 resolveTagContents("value", propElem, eval);
161 }
162 }
163 }
164
165 private void resolveTagContents(String tagName, Element elem, ELEvaluator eval) throws Exception {
166 if (elem == null) {
167 return;
168 }
169 Element tagElem = elem.getChild(tagName, elem.getNamespace());
170 if (tagElem != null) {
171 String updated = CoordELFunctions.evalAndWrap(eval, tagElem.getText());
172 tagElem.removeContent();
173 tagElem.addContent(updated);
174 }
175 else {
176 log.warn(" Value NOT FOUND " + tagName);
177 }
178 }
179
180 private boolean checkUnresolvedInstances(Element eAction, Configuration actionConf, Date actualTime)
181 throws Exception {
182 String strAction = XmlUtils.prettyPrint(eAction).toString();
183 Date nominalTime = DateUtils.parseDateUTC(eAction.getAttributeValue("action-nominal-time"));
184 StringBuffer resultedXml = new StringBuffer();
185
186 boolean ret;
187 Element inputList = eAction.getChild("input-events", eAction.getNamespace());
188 if (inputList != null) {
189 ret = materializeUnresolvedEvent(inputList.getChildren("data-in", eAction.getNamespace()),
190 nominalTime, actualTime, actionConf);
191 if (ret == false) {
192 resultedXml.append(strAction);
193 return false;
194 }
195 }
196
197 // Using latest() or future() in output-event is not intuitive.
198 // We need to make
199 // sure, this assumption is correct.
200 Element outputList = eAction.getChild("output-events", eAction.getNamespace());
201 if (outputList != null) {
202 for (Element dEvent : (List<Element>) outputList.getChildren("data-out", eAction.getNamespace())) {
203 if (dEvent.getChild("unresolved-instances", dEvent.getNamespace()) != null) {
204 throw new CommandException(ErrorCode.E1006, "coord:latest()/future()",
205 " not permitted in output-event ");
206 }
207 }
208 /*
209 * ret = materializeUnresolvedEvent( (List<Element>)
210 * outputList.getChildren("data-out", eAction.getNamespace()),
211 * actualTime, nominalTime, actionConf); if (ret == false) {
212 * resultedXml.append(strAction); return false; }
213 */
214 }
215 return true;
216 }
217
218 private boolean materializeUnresolvedEvent(List<Element> eDataEvents, Date nominalTime, Date actualTime,
219 Configuration conf) throws Exception {
220 for (Element dEvent : eDataEvents) {
221 if (dEvent.getChild("unresolved-instances", dEvent.getNamespace()) == null) {
222 continue;
223 }
224 ELEvaluator eval = CoordELEvaluator.createLazyEvaluator(actualTime, nominalTime, dEvent, conf);
225 String uresolvedInstance = dEvent.getChild("unresolved-instances", dEvent.getNamespace()).getTextTrim();
226 String unresolvedList[] = uresolvedInstance.split(CoordELFunctions.INSTANCE_SEPARATOR);
227 StringBuffer resolvedTmp = new StringBuffer();
228 for (int i = 0; i < unresolvedList.length; i++) {
229 String ret = CoordELFunctions.evalAndWrap(eval, unresolvedList[i]);
230 Boolean isResolved = (Boolean) eval.getVariable("is_resolved");
231 if (isResolved == false) {
232 log.info("[" + actionId + "]::Cannot resolve: " + ret);
233 return false;
234 }
235 if (resolvedTmp.length() > 0) {
236 resolvedTmp.append(CoordELFunctions.INSTANCE_SEPARATOR);
237 }
238 resolvedTmp.append((String) eval.getVariable("resolved_path"));
239 }
240 if (resolvedTmp.length() > 0) {
241 if (dEvent.getChild("uris", dEvent.getNamespace()) != null) {
242 resolvedTmp.append(CoordELFunctions.INSTANCE_SEPARATOR).append(
243 dEvent.getChild("uris", dEvent.getNamespace()).getTextTrim());
244 dEvent.removeChild("uris", dEvent.getNamespace());
245 }
246 Element uriInstance = new Element("uris", dEvent.getNamespace());
247 uriInstance.addContent(resolvedTmp.toString());
248 dEvent.getContent().add(1, uriInstance);
249 }
250 dEvent.removeChild("unresolved-instances", dEvent.getNamespace());
251 }
252
253 return true;
254 }
255
256 private boolean checkResolvedUris(Element eAction, StringBuilder existList, StringBuilder nonExistList,
257 Configuration conf) throws IOException {
258
259 log.info("[" + actionId + "]::ActionInputCheck:: In checkResolvedUris...");
260 Element inputList = eAction.getChild("input-events", eAction.getNamespace());
261 if (inputList != null) {
262 // List<Element> eDataEvents = inputList.getChildren("data-in",
263 // eAction.getNamespace());
264 // for (Element event : eDataEvents) {
265 // Element uris = event.getChild("uris", event.getNamespace());
266 if (nonExistList.length() > 0) {
267 checkListOfPaths(existList, nonExistList, conf);
268 }
269 // }
270 return nonExistList.length() == 0;
271 }
272 return true;
273 }
274
275 private boolean checkListOfPaths(StringBuilder existList, StringBuilder nonExistList, Configuration conf)
276 throws IOException {
277
278 log.info("[" + actionId + "]::ActionInputCheck:: In checkListOfPaths for: " + nonExistList.toString());
279
280 String[] uriList = nonExistList.toString().split(CoordELFunctions.INSTANCE_SEPARATOR);
281 nonExistList.delete(0, nonExistList.length());
282 boolean allExists = true;
283 String existSeparator = "", nonExistSeparator = "";
284 for (int i = 0; i < uriList.length; i++) {
285 if (allExists) {
286 allExists = pathExists(uriList[i], conf);
287 log.info("[" + actionId + "]::ActionInputCheck:: File:" + uriList[i] + ", Exists? :" + allExists);
288 }
289 if (allExists) {
290 existList.append(existSeparator).append(uriList[i]);
291 existSeparator = CoordELFunctions.INSTANCE_SEPARATOR;
292 }
293 else {
294 nonExistList.append(nonExistSeparator).append(uriList[i]);
295 nonExistSeparator = CoordELFunctions.INSTANCE_SEPARATOR;
296 }
297 }
298 return allExists;
299 }
300
301 private boolean pathExists(String sPath, Configuration actionConf) throws IOException {
302 log.debug("checking for the file " + sPath);
303 Path path = new Path(sPath);
304 String user = ParamChecker.notEmpty(actionConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
305 String group = ParamChecker.notEmpty(actionConf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
306 try {
307 return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
308 new Configuration()).exists(path);
309 }
310 catch (HadoopAccessorException e) {
311 throw new IOException(e);
312 }
313 }
314
315 /**
316 * The function create a list of URIs separated by "," using the instances time stamp and URI-template
317 *
318 * @param event : <data-in> event
319 * @param instances : List of time stamp seprated by ","
320 * @param unresolvedInstances : list of instance with latest/future function
321 * @return : list of URIs separated by ",".
322 * @throws Exception
323 */
324 private String createURIs(Element event, String instances, StringBuilder unresolvedInstances) throws Exception {
325 if (instances == null || instances.length() == 0) {
326 return "";
327 }
328 String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR);
329 StringBuilder uris = new StringBuilder();
330
331 for (int i = 0; i < instanceList.length; i++) {
332 int funcType = CoordCommandUtils.getFuncType(instanceList[i]);
333 if (funcType == CoordCommandUtils.LATEST || funcType == CoordCommandUtils.FUTURE) {
334 if (unresolvedInstances.length() > 0) {
335 unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR);
336 }
337 unresolvedInstances.append(instanceList[i]);
338 continue;
339 }
340 ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]);
341 // uris.append(eval.evaluate(event.getChild("dataset",
342 // event.getNamespace()).getChild("uri-template",
343 // event.getNamespace()).getTextTrim(), String.class));
344 if (uris.length() > 0) {
345 uris.append(CoordELFunctions.INSTANCE_SEPARATOR);
346 }
347 uris.append(CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace()).getChild(
348 "uri-template", event.getNamespace()).getTextTrim()));
349 }
350 return uris.toString();
351 }
352
353 @Override
354 protected Void execute(CoordinatorStore store) throws StoreException, CommandException {
355 log.info("STARTED CoordActionInputCheckCommand for actionid=" + actionId);
356 try {
357 coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId);
358 setLogInfo(coordAction);
359 if (lock(coordAction.getJobId())) {
360 call(store);
361 }
362 else {
363 queueCallable(new CoordActionInputCheckCommand(actionId), LOCK_FAILURE_REQUEUE_INTERVAL);
364 log.warn("CoordActionInputCheckCommand lock was not acquired - failed jobId=" + coordAction.getJobId()
365 + ", actionId=" + actionId + ". Requeing the same.");
366 }
367 }
368 catch (InterruptedException e) {
369 queueCallable(new CoordActionInputCheckCommand(actionId), LOCK_FAILURE_REQUEUE_INTERVAL);
370 log.warn("CoordActionInputCheckCommand lock acquiring failed with exception " + e.getMessage() + " for jobId="
371 + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same.");
372 }
373 finally {
374 log.info("ENDED CoordActionInputCheckCommand for actionid=" + actionId);
375 }
376 return null;
377 }
378
379 /**
380 * @param args
381 * @throws Exception
382 */
383 public static void main(String[] args) throws Exception {
384 new Services().init();
385 String actionId = "0000000-091221141623042-oozie-dani-C@4";
386 try {
387 new CoordActionInputCheckCommand(actionId).call();
388 Thread.sleep(10000);
389 }
390 finally {
391 new Services().destroy();
392 }
393 }
394
395 }