001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.client;
016
017 import java.io.BufferedReader;
018 import java.io.File;
019 import java.io.FileReader;
020 import java.io.IOException;
021 import java.io.InputStreamReader;
022 import java.net.HttpURLConnection;
023 import java.util.Properties;
024
025 import org.apache.oozie.client.rest.JsonTags;
026 import org.apache.oozie.client.rest.RestConstants;
027 import org.json.simple.JSONObject;
028 import org.json.simple.JSONValue;
029
030 public class XOozieClient extends OozieClient {
031
032 public static final String JT = "mapred.job.tracker";
033
034 public static final String NN = "fs.default.name";
035
036 public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
037
038 public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal";
039
040 public static final String LIBPATH = "oozie.libpath";
041
042 public static final String PIG_SCRIPT = "oozie.pig.script";
043
044 public static final String PIG_OPTIONS = "oozie.pig.options";
045
046 public static final String FILES = "oozie.files";
047
048 public static final String ARCHIVES = "oozie.archives";
049
050 protected XOozieClient() {
051 }
052
053 /**
054 * Create an eXtended Workflow client instance.
055 *
056 * @param oozieUrl URL of the Oozie instance it will interact with.
057 */
058 public XOozieClient(String oozieUrl) {
059 super(oozieUrl);
060 }
061
062 private String readPigScript(String script) throws IOException {
063 if (!new File(script).exists()) {
064 throw new IOException("Error: Pig script file [" + script + "] does not exist");
065 }
066
067 BufferedReader br = null;
068 try {
069 br = new BufferedReader(new FileReader(script));
070 StringBuilder sb = new StringBuilder();
071 String line;
072 while ((line = br.readLine()) != null) {
073 sb.append(line + "\n");
074 }
075 return sb.toString();
076 }
077 finally {
078 try {
079 br.close();
080 }
081 catch (IOException ex) {
082 System.err.println("Error: " + ex.getMessage());
083 }
084 }
085 }
086
087 static void setStrings(Properties conf, String key, String[] values) {
088 if (values != null) {
089 conf.setProperty(key + ".size", (new Integer(values.length)).toString());
090 for (int i = 0; i < values.length; i++) {
091 conf.setProperty(key + "." + i, values[i]);
092 }
093 }
094 }
095
096 private void validateHttpSbumitConf(Properties conf) {
097 String JT = conf.getProperty(XOozieClient.JT);
098 if (JT == null) {
099 throw new RuntimeException("jobtracker is not specified in conf");
100 }
101
102 String NN = conf.getProperty(XOozieClient.NN);
103 if (NN == null) {
104 throw new RuntimeException("namenode is not specified in conf");
105 }
106
107 String libPath = conf.getProperty(XOozieClient.LIBPATH);
108 if (libPath == null) {
109 throw new RuntimeException("libpath is not specified in conf");
110 }
111 if (!libPath.startsWith("hdfs://")) {
112 String newLibPath = NN + libPath;
113 conf.setProperty(XOozieClient.LIBPATH, newLibPath);
114 }
115 }
116
117 /**
118 * Submit a Pig job via HTTP.
119 *
120 * @param conf job configuration.
121 * @param pigScriptFile pig script file.
122 * @param pigArgs pig arguments string.
123 * @return the job Id.
124 * @throws OozieClientException thrown if the job could not be submitted.
125 */
126 public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException {
127 if (conf == null) {
128 throw new IllegalArgumentException("conf cannot be null");
129 }
130 if (pigScriptFile == null) {
131 throw new IllegalArgumentException("pigScriptFile cannot be null");
132 }
133
134 validateHttpSbumitConf(conf);
135
136 conf.setProperty(XOozieClient.PIG_SCRIPT, readPigScript(pigScriptFile));
137 setStrings(conf, XOozieClient.PIG_OPTIONS, pigArgs);
138
139 return (new HttpJobSubmit(conf, "pig")).call();
140 }
141
142 /**
143 * Submit a Map/Reduce job via HTTP.
144 *
145 * @param conf job configuration.
146 * @return the job Id.
147 * @throws OozieClientException thrown if the job could not be submitted.
148 */
149 public String submitMapReduce(Properties conf) throws OozieClientException {
150 if (conf == null) {
151 throw new IllegalArgumentException("conf cannot be null");
152 }
153
154 validateHttpSbumitConf(conf);
155
156 return (new HttpJobSubmit(conf, "mapreduce")).call();
157 }
158
159 private class HttpJobSubmit extends ClientCallable<String> {
160 private Properties conf;
161
162 HttpJobSubmit(Properties conf, String jobType) {
163 super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.JOBTYPE_PARAM, jobType));
164 this.conf = notNull(conf, "conf");
165 }
166
167 @Override
168 protected String call(HttpURLConnection conn) throws IOException, OozieClientException {
169 conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE);
170 writeToXml(conf, conn.getOutputStream());
171 if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) {
172 JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
173 return (String) json.get(JsonTags.JOB_ID);
174 }
175 if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
176 handleError(conn);
177 }
178 return null;
179 }
180 }
181
182 /**
183 * set LIBPATH for HTTP submission job.
184 *
185 * @param conf Configuration object.
186 * @param path lib HDFS path.
187 */
188 public void setLib(Properties conf, String pathStr) {
189 conf.setProperty(XOozieClient.LIBPATH, pathStr);
190 }
191
192 /**
193 * The equivalent to <file> tag in oozie's workflow xml.
194 *
195 * @param conf Configuration object.
196 * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
197 * For example, "/user/oozie/parameter_file#myparams". If no "#..." is specified, file name will be used as
198 * symbolic link name.
199 */
200 public void addFile(Properties conf, String file) {
201 if (file == null || file.length() == 0) {
202 throw new IllegalArgumentException("file cannot be null or empty");
203 }
204 String files = conf.getProperty(FILES);
205 conf.setProperty(FILES, files == null ? file : files + "," + file);
206 }
207
208 /**
209 * The equivalent to <archive> tag in oozie's workflow xml.
210 *
211 * @param conf Configuration object.
212 * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
213 * For example, "/user/oozie/udf1.jar#my.jar". If no "#..." is specified, file name will be used as
214 * symbolic link name.
215 */
216 public void addArchive(Properties conf, String file) {
217 if (file == null || file.length() == 0) {
218 throw new IllegalArgumentException("file cannot be null or empty");
219 }
220 String files = conf.getProperty(ARCHIVES);
221 conf.setProperty(ARCHIVES, files == null ? file : files + "," + file);
222 }
223 }