001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.oozie;
016
017 import org.apache.oozie.client.OozieClientException;
018 import org.apache.oozie.action.ActionExecutor;
019 import org.apache.oozie.action.ActionExecutorException;
020 import org.apache.oozie.DagEngine;
021 import org.apache.oozie.LocalOozieClient;
022 import org.apache.oozie.WorkflowJobBean;
023 import org.apache.oozie.service.DagEngineService;
024 import org.apache.oozie.service.WorkflowAppService;
025 import org.apache.oozie.client.WorkflowAction;
026 import org.apache.oozie.client.OozieClient;
027 import org.apache.oozie.client.WorkflowJob;
028 import org.apache.oozie.command.CommandException;
029 import org.apache.oozie.util.PropertiesUtils;
030 import org.apache.oozie.util.XmlUtils;
031 import org.apache.oozie.util.XConfiguration;
032 import org.apache.oozie.util.XLog;
033 import org.apache.oozie.service.Services;
034 import org.apache.hadoop.conf.Configuration;
035 import org.jdom.Element;
036 import org.jdom.Namespace;
037
038 import java.io.StringReader;
039 import java.io.IOException;
040 import java.util.Set;
041 import java.util.HashSet;
042
043 public class SubWorkflowActionExecutor extends ActionExecutor {
044 public static final String ACTION_TYPE = "sub-workflow";
045 public static final String LOCAL = "local";
046
047 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
048
049 static {
050 String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
051 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
052 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
053 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS};
054
055 String[] badDefaultProps = {PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
056 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME};
057 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
058 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
059 }
060
061 protected SubWorkflowActionExecutor() {
062 super(ACTION_TYPE);
063 }
064
065 public void initActionType() {
066 super.initActionType();
067 }
068
069 protected OozieClient getWorkflowClient(Context context, String oozieUri) {
070 OozieClient oozieClient;
071 if (oozieUri.equals(LOCAL)) {
072 WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
073 String user = workflow.getUser();
074 String group = workflow.getGroup();
075 String authToken = workflow.getAuthToken();
076 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
077 oozieClient = new LocalOozieClient(dagEngine);
078 }
079 else {
080 // TODO we need to add authToken to the WC for the remote case
081 oozieClient = new OozieClient(oozieUri);
082 }
083 return oozieClient;
084 }
085
086 protected void injectInline(Element eConf, Configuration subWorkflowConf) throws IOException,
087 ActionExecutorException {
088 if (eConf != null) {
089 String strConf = XmlUtils.prettyPrint(eConf).toString();
090 Configuration conf = new XConfiguration(new StringReader(strConf));
091 try {
092 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_DEFAULT_PROPERTIES);
093 }
094 catch (CommandException ex) {
095 throw convertException(ex);
096 }
097 XConfiguration.copy(conf, subWorkflowConf);
098 }
099 }
100
101 @SuppressWarnings("unchecked")
102 protected void injectCallback(Context context, Configuration conf) {
103 String callback = context.getCallbackUrl("$status");
104 if (conf.get(OozieClient.WORKFLOW_NOTIFICATION_URL) != null) {
105 XLog.getLog(getClass())
106 .warn("Sub-Workflow configuration has a custom job end notification URI, overriding");
107 }
108 conf.set(OozieClient.WORKFLOW_NOTIFICATION_URL, callback);
109 }
110
111 protected void injectRecovery(String externalId, Configuration conf) {
112 conf.set(OozieClient.EXTERNAL_ID, externalId);
113 }
114
115 protected String checkIfRunning(OozieClient oozieClient, String extId) throws OozieClientException {
116 String jobId = oozieClient.getJobId(extId);
117 if (jobId.equals("")) {
118 return null;
119 }
120 return jobId;
121 }
122
123 public void start(Context context, WorkflowAction action) throws ActionExecutorException {
124 try {
125 Element eConf = XmlUtils.parseXml(action.getConf());
126 Namespace ns = eConf.getNamespace();
127 Element e = eConf.getChild("oozie", ns);
128 String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
129 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
130 String subWorkflowId = null;
131 String extId = context.getRecoveryId();
132 String runningJobId = null;
133 if (extId != null) {
134 runningJobId = checkIfRunning(oozieClient, extId);
135 }
136 if (runningJobId == null) {
137 String appPath = eConf.getChild("app-path", ns).getTextTrim();
138
139 XConfiguration subWorkflowConf = new XConfiguration();
140 if (eConf.getChild(("propagate-configuration"), ns) != null) {
141 Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
142 XConfiguration.copy(parentConf, subWorkflowConf);
143 }
144
145 // the proto has the necessary credentials
146 Configuration protoActionConf = context.getProtoActionConf();
147 XConfiguration.copy(protoActionConf, subWorkflowConf);
148 subWorkflowConf.set(OozieClient.APP_PATH, appPath);
149 injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
150 injectCallback(context, subWorkflowConf);
151 injectRecovery(extId, subWorkflowConf);
152
153 subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
154 }
155 else {
156 subWorkflowId = runningJobId;
157 }
158 WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
159 String consoleUrl = workflow.getConsoleUrl();
160 context.setStartData(subWorkflowId, oozieUri, consoleUrl);
161 if (runningJobId != null) {
162 check(context, action);
163 }
164 }
165 catch (Exception ex) {
166 throw convertException(ex);
167 }
168 }
169
170 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
171 try {
172 String externalStatus = action.getExternalStatus();
173 WorkflowAction.Status status = externalStatus.equals("SUCCEEDED") ? WorkflowAction.Status.OK
174 : WorkflowAction.Status.ERROR;
175 context.setEndData(status, getActionSignal(status));
176 }
177 catch (Exception ex) {
178 throw convertException(ex);
179 }
180 }
181
182 public void check(Context context, WorkflowAction action) throws ActionExecutorException {
183 try {
184 String subWorkflowId = action.getExternalId();
185 String oozieUri = action.getTrackerUri();
186 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
187 WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId);
188 WorkflowJob.Status status = subWorkflow.getStatus();
189 switch (status) {
190 case FAILED:
191 case KILLED:
192 case SUCCEEDED:
193 context.setExecutionData(status.toString(), null);
194 break;
195 default:
196 context.setExternalStatus(status.toString());
197 break;
198 }
199 }
200 catch (Exception ex) {
201 throw convertException(ex);
202 }
203 }
204
205 public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
206 try {
207 String subWorkflowId = action.getExternalId();
208 String oozieUri = action.getTrackerUri();
209 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
210 oozieClient.kill(subWorkflowId);
211 context.setEndData(WorkflowAction.Status.KILLED, getActionSignal(WorkflowAction.Status.KILLED));
212 }
213 catch (Exception ex) {
214 throw convertException(ex);
215 }
216 }
217
218 private static Set<String> FINAL_STATUS = new HashSet<String>();
219
220 static {
221 FINAL_STATUS.add("SUCCEEDED");
222 FINAL_STATUS.add("KILLED");
223 FINAL_STATUS.add("FAILED");
224 }
225
226 public boolean isCompleted(String externalStatus) {
227 return FINAL_STATUS.contains(externalStatus);
228 }
229 }