001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.wf;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.oozie.service.WorkflowAppService;
019 import org.apache.oozie.util.XmlUtils;
020 import org.jdom.Element;
021 import org.jdom.Namespace;
022 import org.apache.oozie.client.XOozieClient;
023
024 import java.util.HashSet;
025 import java.util.Iterator;
026 import java.util.Map;
027 import java.util.Set;
028
029 public class SubmitMRCommand extends SubmitHttpCommand {
030 private static final Set<String> SKIPPED_CONFS = new HashSet<String>();
031
032 public SubmitMRCommand(Configuration conf, String authToken) {
033 super("submitMR", "submitMR", conf, authToken);
034 }
035
036 static {
037 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER);
038 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_UGI);
039 SKIPPED_CONFS.add(XOozieClient.JT);
040 SKIPPED_CONFS.add(XOozieClient.NN);
041 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_JT_KERBEROS_NAME);
042 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_NN_KERBEROS_NAME);
043 }
044
045 private Element generateConfigurationSection(Configuration conf, Namespace ns) {
046 Element configuration = null;
047 Iterator<Map.Entry<String, String>> iter = conf.iterator();
048 while (iter.hasNext()) {
049 Map.Entry<String, String> entry = iter.next();
050 String name = entry.getKey();
051 if (MANDATORY_OOZIE_CONFS.contains(name) || OPTIONAL_OOZIE_CONFS.contains(name)
052 || SKIPPED_CONFS.contains(name)) {
053 continue;
054 }
055
056 if (configuration == null) {
057 configuration = new Element("configuration", ns);
058 }
059
060 String value = entry.getValue();
061 Element property = new Element("property", ns);
062 Element nameElement = new Element("name", ns);
063 nameElement.addContent(name != null ? name : "");
064 property.addContent(nameElement);
065 Element valueElement = new Element("value", ns);
066 valueElement.addContent(value != null ? value : "");
067 property.addContent(valueElement);
068 configuration.addContent(property);
069 }
070
071 return configuration;
072 }
073
074 private Element generateMRSection(Configuration conf, Namespace ns) {
075 Element mapreduce = new Element("map-reduce", ns);
076 Element jt = new Element("job-tracker", ns);
077 jt.addContent(conf.get(XOozieClient.JT));
078 mapreduce.addContent(jt);
079 Element nn = new Element("name-node", ns);
080 nn.addContent(conf.get(XOozieClient.NN));
081 mapreduce.addContent(nn);
082
083 if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN,
084 // LIBPATH
085 // configuration section
086 Element configuration = generateConfigurationSection(conf, ns);
087 if (configuration != null) {
088 mapreduce.addContent(configuration);
089 }
090
091 // file section
092 addFileSection(mapreduce, conf, ns);
093
094 // archive section
095 addArchiveSection(mapreduce, conf, ns);
096 }
097
098 return mapreduce;
099 }
100
101 /*
102 * (non-Javadoc)
103 *
104 * @see
105 * org.apache.oozie.command.wf.SubmitHttpCommand#getWorkflowXml(org.apache
106 * .hadoop.conf.Configuration)
107 */
108 @Override
109 protected String getWorkflowXml(Configuration conf) {
110 for (String key : MANDATORY_OOZIE_CONFS) {
111 String value = conf.get(key);
112 if (value == null) {
113 throw new RuntimeException(key + " is not specified");
114 }
115 }
116
117 Namespace ns = Namespace.getNamespace("uri:oozie:workflow:0.2");
118 Element root = new Element("workflow-app", ns);
119 root.setAttribute("name", "oozie-mapreduce");
120
121 Element start = new Element("start", ns);
122 start.setAttribute("to", "hadoop1");
123 root.addContent(start);
124
125 Element action = new Element("action", ns);
126 action.setAttribute("name", "hadoop1");
127
128 Element mapreduce = generateMRSection(conf, ns);
129 action.addContent(mapreduce);
130
131 Element ok = new Element("ok", ns);
132 ok.setAttribute("to", "end");
133 action.addContent(ok);
134
135 Element error = new Element("error", ns);
136 error.setAttribute("to", "fail");
137 action.addContent(error);
138
139 root.addContent(action);
140
141 Element kill = new Element("kill", ns);
142 kill.setAttribute("name", "fail");
143 Element message = new Element("message", ns);
144 message.addContent("Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]");
145 kill.addContent(message);
146 root.addContent(kill);
147
148 Element end = new Element("end", ns);
149 end.setAttribute("name", "end");
150 root.addContent(end);
151
152 return XmlUtils.prettyPrint(root).toString();
153 }
154 }