001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.service;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.hadoop.fs.FileStatus;
019 import org.apache.hadoop.fs.FileSystem;
020 import org.apache.hadoop.fs.Path;
021 import org.apache.hadoop.fs.PathFilter;
022 import org.apache.oozie.client.OozieClient;
023 import org.apache.oozie.client.XOozieClient;
024 import org.apache.oozie.command.CommandException;
025 import org.apache.oozie.workflow.WorkflowApp;
026 import org.apache.oozie.workflow.WorkflowException;
027 import org.apache.oozie.util.IOUtils;
028 import org.apache.oozie.util.XConfiguration;
029 import org.apache.oozie.util.XLog;
030 import org.apache.oozie.ErrorCode;
031
032 import java.io.IOException;
033 import java.io.InputStreamReader;
034 import java.io.Reader;
035 import java.io.StringWriter;
036 import java.net.URI;
037 import java.net.URISyntaxException;
038 import java.util.ArrayList;
039 import java.util.List;
040 import java.util.Map;
041
042 /**
043 * Service that provides application workflow definition reading from the path and creation of the proto configuration.
044 */
045 public abstract class WorkflowAppService implements Service {
046 public static final String APP_LIB_PATH_LIST = "oozie.wf.application.lib";
047
048 public static final String HADOOP_UGI = "hadoop.job.ugi";
049
050 public static final String HADOOP_USER = "user.name";
051
052 public static final String HADOOP_JT_KERBEROS_NAME = "mapreduce.jobtracker.kerberos.principal";
053
054 public static final String HADOOP_NN_KERBEROS_NAME = "dfs.namenode.kerberos.principal";
055
056 /**
057 * Initialize the workflow application service.
058 *
059 * @param services services instance.
060 */
061 public void init(Services services) {
062 }
063
064 /**
065 * Destroy the workflow application service.
066 */
067 public void destroy() {
068 }
069
070 /**
071 * Return the public interface for workflow application service.
072 *
073 * @return {@link WorkflowAppService}.
074 */
075 public Class<? extends Service> getInterface() {
076 return WorkflowAppService.class;
077 }
078
079 /**
080 * Read workflow definition.
081 *
082 * @param appPath application path.
083 * @param user user name.
084 * @param group group name.
085 * @param autToken authentication token.
086 * @return workflow definition.
087 * @throws WorkflowException thrown if the definition could not be read.
088 */
089 protected String readDefinition(String appPath, String user, String group, String autToken)
090 throws WorkflowException {
091 try {
092 URI uri = new URI(appPath);
093 FileSystem fs = Services.get().get(HadoopAccessorService.class).
094 createFileSystem(user, group, uri, new Configuration());
095 Reader reader = new InputStreamReader(fs.open(new Path(uri.getPath(), "workflow.xml")));
096 StringWriter writer = new StringWriter();
097 IOUtils.copyCharStream(reader, writer);
098 return writer.toString();
099
100 }
101 catch (IOException ex) {
102 throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
103 }
104 catch (URISyntaxException ex) {
105 throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
106 }
107 catch (HadoopAccessorException ex) {
108 throw new WorkflowException(ex);
109 }
110 catch (Exception ex) {
111 throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
112 }
113 }
114
115 /**
116 * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be
117 * added to distributed cache. These paths include .jar,.so and the resource file paths.
118 *
119 * @param jobConf job configuration.
120 * @param authToken authentication token.
121 * @return proto configuration.
122 * @throws WorkflowException thrown if the proto action configuration could not be created.
123 */
124 public XConfiguration createProtoActionConf(Configuration jobConf, String authToken) throws WorkflowException {
125 XConfiguration conf = new XConfiguration();
126 try {
127 String user = jobConf.get(OozieClient.USER_NAME);
128 String group = jobConf.get(OozieClient.GROUP_NAME);
129 String hadoopUgi = user + "," + group;
130
131 conf.set(OozieClient.USER_NAME, user);
132 conf.set(OozieClient.GROUP_NAME, group);
133 conf.set(HADOOP_UGI, hadoopUgi);
134
135 if (Services.get().getConf().getBoolean("oozie.service.HadoopAccessorService.kerberos.enabled", false)) {
136 conf.set(HADOOP_JT_KERBEROS_NAME, jobConf.get(HADOOP_JT_KERBEROS_NAME));
137 conf.set(HADOOP_NN_KERBEROS_NAME, jobConf.get(HADOOP_NN_KERBEROS_NAME));
138 }
139
140 URI uri = new URI(jobConf.get(OozieClient.APP_PATH));
141
142 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, conf);
143
144 Path appPath = new Path(uri.getPath());
145 XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(XOozieClient.LIBPATH));
146 XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath);
147
148 List<String> filePaths = null;
149 if (jobConf.get(XOozieClient.LIBPATH) != null) { // This is a HTTP submission job;
150 filePaths = getLibFiles(fs, appPath);
151 } else {
152 filePaths = getLibFiles(fs, new Path(appPath + "/lib"));
153 }
154
155 conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
156
157 //Add all properties start with 'oozie.'
158 for (Map.Entry<String, String> entry : jobConf) {
159 if (entry.getKey().startsWith("oozie.")) {
160 String name = entry.getKey();
161 String value = entry.getValue();
162 conf.set(name, value);
163 }
164 }
165 return conf;
166 }
167 catch (IOException ex) {
168 throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
169 }
170 catch (URISyntaxException ex) {
171 throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
172 }
173 catch (HadoopAccessorException ex) {
174 throw new WorkflowException(ex);
175 }
176 catch (Exception ex) {
177 throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH),
178 ex.getMessage(), ex);
179 }
180 }
181
182 /**
183 * Parse workflow definition.
184 *
185 * @param jobConf job configuration.
186 * @param authToken authentication token.
187 * @return workflow application.
188 * @throws WorkflowException thrown if the workflow application could not be parsed.
189 */
190 public abstract WorkflowApp parseDef(Configuration jobConf, String authToken) throws WorkflowException;
191
192 /**
193 * Parse workflow definition.
194 * @param wfXml workflow.
195 * @return workflow application.
196 * @throws WorkflowException thrown if the workflow application could not be parsed.
197 */
198 public abstract WorkflowApp parseDef(String wfXml) throws WorkflowException;
199
200 /**
201 * Get all library paths.
202 *
203 * @param fs file system object.
204 * @param libPath hdfs library path.
205 * @return list of paths.
206 * @throws IOException thrown if the lib paths could not be obtained.
207 */
208 private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
209 List<String> libPaths = new ArrayList<String>();
210 FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
211
212 for (FileStatus file : files) {
213 libPaths.add((String) file.getPath().toUri().getPath().trim());
214 }
215 return libPaths;
216 }
217
218 /*
219 * Filter class doing no filtering.
220 * We dont need define this class, but seems fs.listStatus() is not working properly without this.
221 * So providing this dummy no filtering Filter class.
222 */
223 private class NoPathFilter implements PathFilter {
224 @Override
225 public boolean accept(Path path) {
226 return true;
227 }
228 }
229 }