001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.wf;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.oozie.WorkflowJobBean;
019 import org.apache.oozie.ErrorCode;
020 import org.apache.oozie.service.WorkflowStoreService;
021 import org.apache.oozie.service.WorkflowAppService;
022 import org.apache.oozie.service.Services;
023 import org.apache.oozie.service.DagXLogInfoService;
024 import org.apache.oozie.util.XLog;
025 import org.apache.oozie.util.ParamChecker;
026 import org.apache.oozie.util.XConfiguration;
027 import org.apache.oozie.util.XmlUtils;
028 import org.apache.oozie.command.CommandException;
029 import org.apache.oozie.store.StoreException;
030 import org.apache.oozie.store.WorkflowStore;
031 import org.apache.oozie.workflow.WorkflowApp;
032 import org.apache.oozie.workflow.WorkflowException;
033 import org.apache.oozie.workflow.WorkflowInstance;
034 import org.apache.oozie.workflow.WorkflowLib;
035 import org.apache.oozie.util.PropertiesUtils;
036 import org.apache.oozie.client.OozieClient;
037 import org.apache.oozie.client.WorkflowJob;
038 import org.apache.oozie.client.XOozieClient;
039 import org.jdom.Element;
040 import org.jdom.Namespace;
041
042 import java.util.Date;
043 import java.util.Map;
044 import java.util.Set;
045 import java.util.HashSet;
046
047 public abstract class SubmitHttpCommand extends WorkflowCommand<String> {
048 protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>();
049 protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
050
051 static {
052 MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
053 MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
054 MANDATORY_OOZIE_CONFS.add(XOozieClient.LIBPATH);
055
056 OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES);
057 OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES);
058 }
059
060 private Configuration conf;
061 private String authToken;
062
063 public SubmitHttpCommand(String name, String type, Configuration conf, String authToken) {
064 super(name, type, 1, XLog.STD);
065 this.conf = ParamChecker.notNull(conf, "conf");
066 this.authToken = ParamChecker.notEmpty(authToken, "authToken");
067 }
068
069 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
070 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
071
072 static {
073 String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
074 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
075 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
076 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
077 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
078
079 String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
080 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME };
081 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
082 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
083 }
084
085 /**
086 * Generate workflow xml from conf object
087 *
088 * @param conf the configuration object
089 * @return workflow xml def string representation
090 */
091 abstract protected String getWorkflowXml(Configuration conf);
092
093 @Override
094 protected String call(WorkflowStore store) throws StoreException, CommandException {
095 incrJobCounter(1);
096 WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
097 try {
098 XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
099 String wfXml = getWorkflowXml(conf);
100 XLog.getLog(getClass()).debug("workflow xml created on the server side is :\n");
101 XLog.getLog(getClass()).debug(wfXml);
102 WorkflowApp app = wps.parseDef(wfXml);
103 XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken);
104 WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
105
106 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
107
108 // Resolving all variables in the job properties.
109 // This ensures the Hadoop Configuration semantics is preserved.
110 XConfiguration resolvedVarsConf = new XConfiguration();
111 for (Map.Entry<String, String> entry : conf) {
112 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
113 }
114 conf = resolvedVarsConf;
115
116 WorkflowInstance wfInstance;
117 try {
118 wfInstance = workflowLib.createInstance(app, conf);
119 }
120 catch (WorkflowException e) {
121 throw new StoreException(e);
122 }
123
124 Configuration conf = wfInstance.getConf();
125
126 WorkflowJobBean workflow = new WorkflowJobBean();
127 workflow.setId(wfInstance.getId());
128 workflow.setAppName(app.getName());
129 workflow.setAppPath(conf.get(OozieClient.APP_PATH));
130 workflow.setConf(XmlUtils.prettyPrint(conf).toString());
131 workflow.setProtoActionConf(protoActionConf.toXmlString());
132 workflow.setCreatedTime(new Date());
133 workflow.setLastModifiedTime(new Date());
134 workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
135 workflow.setStatus(WorkflowJob.Status.PREP);
136 workflow.setRun(0);
137 workflow.setUser(conf.get(OozieClient.USER_NAME));
138 workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
139 workflow.setAuthToken(authToken);
140 workflow.setWorkflowInstance(wfInstance);
141 workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
142
143 setLogInfo(workflow);
144 store.insertWorkflow(workflow);
145
146 return workflow.getId();
147 }
148 catch (WorkflowException ex) {
149 throw new CommandException(ex);
150 }
151 catch (Exception ex) {
152 throw new CommandException(ErrorCode.E0803, ex);
153 }
154 }
155
156 static private void addSection(Element X, Namespace ns, String filesStr, String tagName) {
157 if (filesStr != null) {
158 String[] files = filesStr.split(",");
159 for (String f : files) {
160 Element tagElement = new Element(tagName, ns);
161 if (f.contains("#")) {
162 tagElement.addContent(f);
163 }
164 else {
165 String filename = f.substring(f.lastIndexOf("/") + 1, f.length());
166 if (filename == null || filename.isEmpty()) {
167 tagElement.addContent(f);
168 }
169 else {
170 tagElement.addContent(f + "#" + filename);
171 }
172 }
173 X.addContent(tagElement);
174 }
175 }
176 }
177
178 /**
179 * Add file section in X.
180 *
181 * @param parent XML element to be appended
182 * @param conf Configuration object
183 * @param ns XML element namespace
184 */
185 static void addFileSection(Element X, Configuration conf, Namespace ns) {
186 String filesStr = conf.get(XOozieClient.FILES);
187 addSection(X, ns, filesStr, "file");
188 }
189
190 /**
191 * Add archive section in X.
192 *
193 * @param parent XML element to be appended
194 * @param conf Configuration object
195 * @param ns XML element namespace
196 */
197 static void addArchiveSection(Element X, Configuration conf, Namespace ns) {
198 String archivesStr = conf.get(XOozieClient.ARCHIVES);
199 addSection(X, ns, archivesStr, "archive");
200 }
201 }