001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.hadoop;
016
017 import java.util.List;
018
019 import org.apache.hadoop.conf.Configuration;
020 import org.apache.hadoop.fs.Path;
021 import org.apache.hadoop.mapred.Counters;
022 import org.apache.hadoop.mapred.JobClient;
023 import org.apache.hadoop.mapred.JobConf;
024 import org.apache.hadoop.mapred.JobID;
025 import org.apache.hadoop.mapred.RunningJob;
026 import org.apache.oozie.action.ActionExecutorException;
027 import org.apache.oozie.client.WorkflowAction;
028 import org.apache.oozie.util.XConfiguration;
029 import org.apache.oozie.util.XLog;
030 import org.apache.oozie.util.XmlUtils;
031 import org.jdom.Element;
032 import org.jdom.Namespace;
033 import org.json.simple.JSONObject;
034
035 public class MapReduceActionExecutor extends JavaActionExecutor {
036
037 public static final String HADOOP_COUNTERS = "hadoop.counters";
038 private XLog log = XLog.getLog(getClass());
039
040 public MapReduceActionExecutor() {
041 super("map-reduce");
042 }
043
044 protected List<Class> getLauncherClasses() {
045 List<Class> classes = super.getLauncherClasses();
046 classes.add(LauncherMain.class);
047 classes.add(MapReduceMain.class);
048 classes.add(StreamingMain.class);
049 classes.add(PipesMain.class);
050 return classes;
051 }
052
053 protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
054 String mainClass;
055 Namespace ns = actionXml.getNamespace();
056 if (actionXml.getChild("streaming", ns) != null) {
057 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, StreamingMain.class.getName());
058 }
059 else {
060 if (actionXml.getChild("pipes", ns) != null) {
061 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
062 }
063 else {
064 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
065 }
066 }
067 return mainClass;
068 }
069
070 @Override
071 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
072 super.setupLauncherConf(conf, actionXml, appPath, context);
073 conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
074 return conf;
075 }
076
077 @SuppressWarnings("unchecked")
078 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
079 throws ActionExecutorException {
080 Namespace ns = actionXml.getNamespace();
081 if (actionXml.getChild("streaming", ns) != null) {
082 Element streamingXml = actionXml.getChild("streaming", ns);
083 String mapper = streamingXml.getChildTextTrim("mapper", ns);
084 String reducer = streamingXml.getChildTextTrim("reducer", ns);
085 String recordReader = streamingXml.getChildTextTrim("record-reader", ns);
086 List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns);
087 String[] recordReaderMapping = new String[list.size()];
088 for (int i = 0; i < list.size(); i++) {
089 recordReaderMapping[i] = list.get(i).getTextTrim();
090 }
091 list = (List<Element>) streamingXml.getChildren("env", ns);
092 String[] env = new String[list.size()];
093 for (int i = 0; i < list.size(); i++) {
094 env[i] = list.get(i).getTextTrim();
095 }
096 StreamingMain.setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env);
097 }
098 else {
099 if (actionXml.getChild("pipes", ns) != null) {
100 Element pipesXml = actionXml.getChild("pipes", ns);
101 String map = pipesXml.getChildTextTrim("map", ns);
102 String reduce = pipesXml.getChildTextTrim("reduce", ns);
103 String inputFormat = pipesXml.getChildTextTrim("inputformat", ns);
104 String partitioner = pipesXml.getChildTextTrim("partitioner", ns);
105 String writer = pipesXml.getChildTextTrim("writer", ns);
106 String program = pipesXml.getChildTextTrim("program", ns);
107 PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program);
108 }
109 }
110 actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
111 return actionConf;
112 }
113
114 @Override
115 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
116 super.end(context, action);
117 JobClient jobClient = null;
118 boolean exception = false;
119 try {
120 if (action.getStatus() == WorkflowAction.Status.OK) {
121 Element actionXml = XmlUtils.parseXml(action.getConf());
122 Configuration conf = createBaseHadoopConf(context, actionXml);
123 JobConf jobConf = new JobConf();
124 XConfiguration.copy(conf, jobConf);
125 jobClient = createJobClient(context, jobConf);
126 RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId()));
127 if (runningJob == null) {
128 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002",
129 "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", action
130 .getExternalId(), action.getId());
131 }
132
133 // TODO this has to be done in a better way
134 if (!runningJob.getJobName().startsWith("oozie:action:")) {
135 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR001",
136 "ID swap should have happened in launcher job [{0}]", action.getExternalId());
137 }
138 Counters counters = runningJob.getCounters();
139 if (counters != null) {
140 JSONObject json = counterstoJson(counters);
141 context.setVar(HADOOP_COUNTERS, json.toJSONString());
142 }
143 else {
144
145 context.setVar(HADOOP_COUNTERS, "");
146
147 XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]", action.getExternalId());
148 }
149 }
150 }
151 catch (Exception ex) {
152 exception = true;
153 throw convertException(ex);
154 }
155 finally {
156 if (jobClient != null) {
157 try {
158 jobClient.close();
159 }
160 catch (Exception e) {
161 if (exception) {
162 log.error("JobClient error: ", e);
163 }
164 else {
165 throw convertException(e);
166 }
167 }
168 }
169 }
170 }
171
172 @SuppressWarnings("unchecked")
173 private JSONObject counterstoJson(Counters counters) {
174
175 if (counters == null) {
176 return null;
177 }
178
179 JSONObject groups = new JSONObject();
180 for (String gName : counters.getGroupNames()) {
181 JSONObject group = new JSONObject();
182 for (Counters.Counter counter : counters.getGroup(gName)) {
183 String cName = counter.getName();
184 Long cValue = counter.getCounter();
185 group.put(cName, cValue);
186 }
187 groups.put(gName, group);
188 }
189 return groups;
190 }
191
192 }