001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.wf;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.hadoop.fs.Path;
019 import org.apache.hadoop.fs.FileSystem;
020 import org.apache.oozie.WorkflowJobBean;
021 import org.apache.oozie.ErrorCode;
022 import org.apache.oozie.service.HadoopAccessorException;
023 import org.apache.oozie.service.WorkflowStoreService;
024 import org.apache.oozie.service.WorkflowAppService;
025 import org.apache.oozie.service.HadoopAccessorService;
026 import org.apache.oozie.service.Services;
027 import org.apache.oozie.service.DagXLogInfoService;
028 import org.apache.oozie.util.XLog;
029 import org.apache.oozie.util.ParamChecker;
030 import org.apache.oozie.util.XConfiguration;
031 import org.apache.oozie.util.XmlUtils;
032 import org.apache.oozie.command.Command;
033 import org.apache.oozie.command.CommandException;
034 import org.apache.oozie.command.coord.CoordSubmitCommand;
035 import org.apache.oozie.coord.CoordELFunctions;
036 import org.apache.oozie.coord.CoordinatorJobException;
037 import org.apache.oozie.service.ELService;
038 import org.apache.oozie.service.SchemaService;
039 import org.apache.oozie.service.WorkflowAppService;
040 import org.apache.oozie.service.DagXLogInfoService;
041 import org.apache.oozie.service.WorkflowStoreService;
042 import org.apache.oozie.store.StoreException;
043 import org.apache.oozie.store.Store;
044 import org.apache.oozie.store.WorkflowStore;
045 import org.apache.oozie.workflow.WorkflowApp;
046 import org.apache.oozie.workflow.WorkflowException;
047 import org.apache.oozie.workflow.WorkflowInstance;
048 import org.apache.oozie.workflow.WorkflowLib;
049 import org.apache.oozie.util.ELEvaluator;
050 import org.apache.oozie.util.ParamChecker;
051 import org.apache.oozie.util.PropertiesUtils;
052 import org.apache.oozie.util.XLog;
053 import org.apache.oozie.util.XmlUtils;
054 import org.apache.oozie.util.XConfiguration;
055 import org.apache.oozie.util.db.SLADbOperations;
056 import org.apache.oozie.service.Services;
057 import org.apache.oozie.service.SchemaService.SchemaName;
058 import org.apache.oozie.client.OozieClient;
059 import org.apache.oozie.client.WorkflowJob;
060 import org.apache.oozie.client.SLAEvent.SlaAppType;
061 import org.jdom.Element;
062 import org.jdom.JDOMException;
063 import org.jdom.Namespace;
064
065 import java.util.Date;
066 import java.util.List;
067 import java.util.Map;
068 import java.util.Set;
069 import java.util.HashSet;
070 import java.util.Map;
071 import java.io.IOException;
072
073 public class SubmitCommand extends WorkflowCommand<String> {
074 public static final String CONFIG_DEFAULT = "config-default.xml";
075
076 private Configuration conf;
077 private String authToken;
078
079 public SubmitCommand(Configuration conf, String authToken) {
080 super("submit", "submit", 1, XLog.STD);
081 this.conf = ParamChecker.notNull(conf, "conf");
082 this.authToken = ParamChecker.notEmpty(authToken, "authToken");
083 }
084
085 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
086 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
087
088 static {
089 String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
090 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
091 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
092 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS};
093 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
094
095 String[] badDefaultProps = {PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
096 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME};
097 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
098 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
099 }
100
101 @Override
102 protected String call(WorkflowStore store) throws StoreException, CommandException {
103 incrJobCounter(1);
104 WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
105 try {
106 XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
107 WorkflowApp app = wps.parseDef(conf, authToken);
108 XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken);
109 WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
110
111 Path configDefault = new Path(conf.get(OozieClient.APP_PATH), CONFIG_DEFAULT);
112
113 String user = conf.get(OozieClient.USER_NAME);
114 String group = conf.get(OozieClient.GROUP_NAME);
115 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group,
116 configDefault.toUri(), new Configuration());
117
118 if (fs.exists(configDefault)) {
119 try {
120 Configuration defaultConf = new XConfiguration(fs.open(configDefault));
121 PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
122 XConfiguration.injectDefaults(defaultConf, conf);
123 }
124 catch (IOException ex) {
125 throw new IOException("default configuration file, " + ex.getMessage(), ex);
126 }
127 }
128
129 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
130
131 // Resolving all variables in the job properties.
132 // This ensures the Hadoop Configuration semantics is preserved.
133 XConfiguration resolvedVarsConf = new XConfiguration();
134 for (Map.Entry<String, String> entry : conf) {
135 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
136 }
137 conf = resolvedVarsConf;
138
139 WorkflowInstance wfInstance;
140 try {
141 wfInstance = workflowLib.createInstance(app, conf);
142 }
143 catch (WorkflowException e) {
144 throw new StoreException(e);
145 }
146
147 Configuration conf = wfInstance.getConf();
148 // System.out.println("WF INSTANCE CONF:");
149 // System.out.println(XmlUtils.prettyPrint(conf).toString());
150
151 WorkflowJobBean workflow = new WorkflowJobBean();
152 workflow.setId(wfInstance.getId());
153 workflow.setAppName(app.getName());
154 workflow.setAppPath(conf.get(OozieClient.APP_PATH));
155 workflow.setConf(XmlUtils.prettyPrint(conf).toString());
156 workflow.setProtoActionConf(protoActionConf.toXmlString());
157 workflow.setCreatedTime(new Date());
158 workflow.setLastModifiedTime(new Date());
159 workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
160 workflow.setStatus(WorkflowJob.Status.PREP);
161 workflow.setRun(0);
162 workflow.setUser(conf.get(OozieClient.USER_NAME));
163 workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
164 workflow.setAuthToken(authToken);
165 workflow.setWorkflowInstance(wfInstance);
166 workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
167
168 setLogInfo(workflow);
169 Element wfElem = XmlUtils.parseXml(app.getDefinition());
170 ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
171 String jobSlaXml = verifySlaElements(wfElem, evalSla);
172 writeSLARegistration(jobSlaXml, workflow.getId(), workflow.getUser(), workflow.getGroup(), store);
173 workflow.setSlaXml(jobSlaXml);
174 // System.out.println("SlaXml :"+ slaXml);
175
176 store.insertWorkflow(workflow);
177
178 // Configuration conf1 = workflow.getWorkflowInstance().getConf();
179 // System.out.println("WF1 INSTANCE CONF:");
180 // System.out.println(XmlUtils.prettyPrint(conf1).toString());
181 // Add WF_JOB SLA Registration event
182
183 return workflow.getId();
184 }
185 catch (WorkflowException ex) {
186 throw new CommandException(ex);
187 }
188 catch (HadoopAccessorException ex) {
189 throw new CommandException(ex);
190 }
191 catch (Exception ex) {
192 throw new CommandException(ErrorCode.E0803, ex);
193 }
194 }
195
196 private String verifySlaElements(Element eWfJob, ELEvaluator evalSla) throws CommandException {
197 String jobSlaXml = "";
198 // String prefix = XmlUtils.getNamespacePrefix(eWfJob,
199 // SchemaService.SLA_NAME_SPACE_URI);
200 // Validate WF job
201 Element eSla = eWfJob.getChild("info", Namespace.getNamespace(SchemaService.SLA_NAME_SPACE_URI));
202 if (eSla != null) {
203 jobSlaXml = resolveSla(eSla, evalSla);
204 }
205
206 // Validate all actions
207 for (Element action : (List<Element>) eWfJob.getChildren("action", eWfJob.getNamespace())) {
208 eSla = action.getChild("info", Namespace.getNamespace(SchemaService.SLA_NAME_SPACE_URI));
209 if (eSla != null) {
210 resolveSla(eSla, evalSla);
211 }
212 }
213 return jobSlaXml;
214 }
215
216 private void writeSLARegistration(String slaXml, String id, String user, String group, Store store)
217 throws CommandException {
218 try {
219 if (slaXml != null && slaXml.length() > 0) {
220 Element eSla = XmlUtils.parseXml(slaXml);
221 SLADbOperations.writeSlaRegistrationEvent(eSla, store, id, SlaAppType.WORKFLOW_JOB, user, group);
222 }
223 }
224 catch (Exception e) {
225 // TODO Auto-generated catch block
226 e.printStackTrace();
227 throw new CommandException(ErrorCode.E1007, "workflow " + id, e);
228 }
229 }
230
231 public static String resolveSla(Element eSla, ELEvaluator evalSla) throws CommandException {
232 // EL evaluation
233 String slaXml = XmlUtils.prettyPrint(eSla).toString();
234 try {
235 slaXml = XmlUtils.removeComments(slaXml);
236 slaXml = evalSla.evaluate(slaXml, String.class);
237 XmlUtils.validateData(slaXml, SchemaName.SLA_ORIGINAL);
238 return slaXml;
239 }
240 catch (Exception e) {
241 throw new CommandException(ErrorCode.E1004, "Validation erro :" + e.getMessage(), e);
242 }
243 }
244
245 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
246 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
247 for (Map.Entry<String, String> entry : conf) {
248 eval.setVariable(entry.getKey(), entry.getValue());
249 }
250 return eval;
251 }
252
253 }