001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.coord;
016
017 import org.apache.hadoop.conf.Configuration;
018
019 import org.apache.oozie.client.CoordinatorAction;
020 import org.apache.oozie.CoordinatorActionBean;
021 import org.apache.oozie.DagEngine;
022 import org.apache.oozie.DagEngineException;
023 import org.apache.oozie.ErrorCode;
024 import org.apache.oozie.command.CommandException;
025 import org.apache.oozie.service.DagEngineService;
026 import org.apache.oozie.store.StoreException;
027 import org.apache.oozie.store.CoordinatorStore;
028 import org.apache.oozie.service.Services;
029 import org.apache.oozie.util.ParamChecker;
030 import org.apache.oozie.util.XLog;
031 import org.apache.oozie.util.XmlUtils;
032 import org.apache.oozie.util.XConfiguration;
033 import org.apache.oozie.util.db.SLADbOperations;
034 import org.apache.oozie.client.SLAEvent.SlaAppType;
035 import org.apache.oozie.client.SLAEvent.Status;
036
037 import org.jdom.Element;
038 import org.jdom.JDOMException;
039
040 import java.io.IOException;
041 import java.io.StringReader;
042
043 public class CoordActionStartCommand extends CoordinatorCommand<Void> {
044
045 public static final String EL_ERROR = "EL_ERROR";
046 public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
047 public static final String COULD_NOT_START = "COULD_NOT_START";
048 public static final String START_DATA_MISSING = "START_DATA_MISSING";
049 public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
050
051 private final XLog log = XLog.getLog(getClass());
052 private String actionId = null;
053 private String user = null;
054 private String authToken = null;
055 private CoordinatorActionBean coordAction = null;
056
057 public CoordActionStartCommand(String id, String user, String token) {
058 super("coord_action_start", "coord_action_start", 1, XLog.OPS);
059 this.actionId = ParamChecker.notEmpty(id, "id");
060 this.user = ParamChecker.notEmpty(user, "user");
061 this.authToken = ParamChecker.notEmpty(token, "token");
062 }
063
064 /**
065 * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
066 * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
067 * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
068 * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
069 * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
070 * Merge Action createdConf with actionXml to create new runConf with replaced variables
071 *
072 * @param action CoordinatorActionBean
073 * @return Configuration
074 * @throws CommandException
075 */
076 private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
077 String createdConf = action.getCreatedConf();
078 String actionXml = action.getActionXml();
079 Element workflowProperties = null;
080 try {
081 workflowProperties = XmlUtils.parseXml(actionXml);
082 }
083 catch (JDOMException e1) {
084 log.warn("Configuration parse error in:" + actionXml);
085 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
086 }
087 // generate the 'runConf' for this action
088 // Step 1: runConf = createdConf
089 Configuration runConf = null;
090 try {
091 runConf = new XConfiguration(new StringReader(createdConf));
092 }
093 catch (IOException e1) {
094 log.warn("Configuration parse error in:" + createdConf);
095 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
096 }
097 // Step 2: Merge local properties into runConf
098 // extract 'property' tags under 'configuration' block in the
099 // coordinator.xml (saved in actionxml column)
100 // convert Element to XConfiguration
101 Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace())
102 .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
103 workflowProperties.getNamespace());
104 if (configElement != null) {
105 String strConfig = XmlUtils.prettyPrint(configElement).toString();
106 Configuration localConf;
107 try {
108 localConf = new XConfiguration(new StringReader(strConfig));
109 }
110 catch (IOException e1) {
111 log.warn("Configuration parse error in:" + strConfig);
112 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
113 }
114
115 // copy configuration properties in coordinator.xml to the runConf
116 XConfiguration.copy(localConf, runConf);
117 }
118
119 // Step 3: Extract value of 'app-path' in actionxml, and save it as a
120 // new property called 'oozie.wf.application.path'
121 // WF Engine requires the path to the workflow.xml to be saved under
122 // this property name
123 String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow",
124 workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue();
125 runConf.set("oozie.wf.application.path", appPath);
126 return runConf;
127 }
128
129 protected Void call(CoordinatorStore store) throws StoreException, CommandException {
130 boolean makeFail = true;
131 String errCode = "";
132 String errMsg = "";
133 ParamChecker.notEmpty(user, "user");
134 ParamChecker.notEmpty(authToken, "authToken");
135
136 // CoordinatorActionBean coordAction = store.getCoordinatorAction(id, true);
137 log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
138 if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
139 // log.debug("getting.. job id: " + coordAction.getJobId());
140 // create merged runConf to pass to WF Engine
141 Configuration runConf = mergeConfig(coordAction);
142 coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
143 // log.debug("%%% merged runconf=" +
144 // XmlUtils.prettyPrint(runConf).toString());
145 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
146 try {
147 boolean startJob = true;
148 Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
149 SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), store, Status.STARTED,
150 SlaAppType.COORDINATOR_ACTION);
151 String wfId = dagEngine.submitJob(conf, startJob);
152 coordAction.setStatus(CoordinatorAction.Status.RUNNING);
153 coordAction.setExternalId(wfId);
154 store.updateCoordinatorAction(coordAction);
155 makeFail = false;
156 }
157 catch (StoreException se) {
158 makeFail = false;
159 throw se;
160 }
161 catch (DagEngineException dee) {
162 errMsg = dee.getMessage();
163 errCode = "E1005";
164 log.warn("can not create DagEngine for submitting jobs", dee);
165 }
166 catch (CommandException ce) {
167 errMsg = ce.getMessage();
168 errCode = ce.getErrorCode().toString();
169 log.warn("command exception occured ", ce);
170 }
171 catch (java.io.IOException ioe) {
172 errMsg = ioe.getMessage();
173 errCode = "E1005";
174 log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
175 }
176 catch (Exception ex) {
177 errMsg = ex.getMessage();
178 errCode = "E1005";
179 log.warn("can not create DagEngine for submitting jobs", ex);
180 }
181 finally {
182 if (makeFail == true) { // No DB exception occurs
183 log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
184 coordAction.setStatus(CoordinatorAction.Status.FAILED);
185 if (errMsg.length() > 254) { // Because table column size is 255
186 errMsg = errMsg.substring(0, 255);
187 }
188 coordAction.setErrorMessage(errMsg);
189 coordAction.setErrorCode(errCode);
190 store.updateCoordinatorAction(coordAction);
191 queueCallable(new CoordActionReadyCommand(coordAction.getJobId()));
192 }
193 }
194 }
195 return null;
196 }
197
198 @Override
199 protected Void execute(CoordinatorStore store) throws StoreException, CommandException {
200 log.info("STARTED CoordActionStartCommand actionId=" + actionId);
201 try {
202 coordAction = store.getEntityManager().find(CoordinatorActionBean.class, actionId);
203 setLogInfo(coordAction);
204 if (lock(coordAction.getJobId())) {
205 call(store);
206 }
207 else {
208 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
209 log.warn("CoordActionStartCommand lock was not acquired - failed jobId=" + coordAction.getJobId()
210 + ", actionId=" + actionId + ". Requeing the same.");
211 }
212 }
213 catch (InterruptedException e) {
214 queueCallable(new CoordActionStartCommand(actionId, user, authToken), LOCK_FAILURE_REQUEUE_INTERVAL);
215 log.warn("CoordActionStartCommand lock acquiring failed with exception " + e.getMessage() + " for jobId="
216 + coordAction.getJobId() + ", actionId=" + actionId + " Requeing the same.");
217 }
218 finally {
219 log.info("ENDED CoordActionStartCommand actionId=" + actionId);
220 }
221 return null;
222 }
223 }