001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import java.io.BufferedReader;
021 import java.io.BufferedWriter;
022 import java.io.File;
023 import java.io.FileOutputStream;
024 import java.io.FileReader;
025 import java.io.FileWriter;
026 import java.io.IOException;
027 import java.io.OutputStream;
028 import java.util.ArrayList;
029 import java.util.HashMap;
030 import java.util.HashSet;
031 import java.util.List;
032 import java.util.Map;
033 import java.util.Map.Entry;
034 import java.util.Set;
035 import java.util.regex.Matcher;
036 import java.util.regex.Pattern;
037
038 import org.apache.hadoop.conf.Configuration;
039 import org.apache.hadoop.fs.Path;
040 import org.apache.hadoop.hive.cli.CliDriver;
041
042 public class HiveMain extends LauncherMain {
043
044 public static final String HIVE_L4J_PROPS = "hive-log4j.properties";
045 public static final String HIVE_EXEC_L4J_PROPS = "hive-exec-log4j.properties";
046 public static final String HIVE_DEFAULT_CONF = "hive-default.xml";
047 public static final String HIVE_SITE_CONF = "hive-site.xml";
048 public static final String HIVE_SCRIPT = "oozie.hive.script";
049 public static final String HIVE_PARAMS = "oozie.hive.params";
050
051 public static void main(String[] args) throws Exception {
052 run(HiveMain.class, args);
053 }
054
055 private Configuration initActionConf() {
056 // Loading action conf prepared by Oozie
057 Configuration hiveConf = new Configuration(false);
058
059 String actionXml = System.getProperty("oozie.action.conf.xml");
060
061 if (actionXml == null) {
062 throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
063 }
064 if (!new File(actionXml).exists()) {
065 throw new RuntimeException("Action Configuration XML file ["
066 + actionXml + "] does not exist");
067 } else {
068 System.out.println("Using action configuration file " + actionXml);
069 }
070
071 hiveConf.addResource(new Path("file:///", actionXml));
072
073 // Propagate delegation related props from launcher job to Hive job
074 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
075 hiveConf.set("mapreduce.job.credentials.binary",
076 System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
077 System.out.println("------------------------");
078 System.out.println("Setting env property for mapreduce.job.credentials.binary to:"
079 + System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
080 System.out.println("------------------------");
081 System.setProperty("mapreduce.job.credentials.binary",
082 System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
083 } else {
084 System.out.println("Non-Kerberos execution");
085 }
086
087 // Have to explicitly unset this property or Hive will not set it.
088 hiveConf.set("mapred.job.name", "");
089
090 // See https://issues.apache.org/jira/browse/HIVE-1411
091 hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
092
093 return hiveConf;
094 }
095
096 protected void run(String[] args) throws Exception {
097 System.out.println();
098 System.out.println("Oozie Hive action configuration");
099 System.out.println("=================================================================");
100
101 Configuration hiveConf = initActionConf();
102
103 // Write the action configuration out to hive-site.xml
104 OutputStream os = new FileOutputStream(HIVE_SITE_CONF);
105 hiveConf.writeXml(os);
106 os.close();
107
108 System.out.println();
109 System.out.println("Hive Configuration Properties:");
110 System.out.println("------------------------");
111 for (Entry<String, String> entry : hiveConf) {
112 System.out.println(entry.getKey() + "=" + entry.getValue());
113 }
114 System.out.flush();
115 System.out.println("------------------------");
116 System.out.println();
117
118 List<String> arguments = new ArrayList<String>();
119 String scriptPath = hiveConf.get(HIVE_SCRIPT);
120
121 if (scriptPath == null) {
122 throw new RuntimeException("Action Configuration does not have [oozie.hive.script] property");
123 }
124
125 if (!new File(scriptPath).exists()) {
126 throw new RuntimeException("Hive script file [" + scriptPath + "] does not exist");
127 }
128
129 //
130 // Prepare the Hive Script
131 //
132 System.out.println("Hive script [" + scriptPath + "] content: ");
133 System.out.println("------------------------");
134 System.out.println(readStringFromFile(scriptPath));
135 System.out.println("------------------------");
136 System.out.println();
137
138 Map<String, String> varMap = new HashMap<String, String>();
139
140 for (String param : MapReduceMain.getStrings(hiveConf, HIVE_PARAMS)) {
141 System.out.println("Param Expression: '" + param + "'");
142
143 int idx = param.indexOf('=');
144 if (idx == -1) {
145 throw new RuntimeException("Parameter expression must contain an assignment: "
146 + param);
147 } else if (idx == 0) {
148 throw new RuntimeException("Parameter value not specified: " + param);
149 }
150 String var = param.substring(0, idx);
151 String val = param.substring(idx + 1, param.length());
152 System.out.println("var = " + var + ", val = " + val);
153 varMap.put(var, val);
154 }
155
156 if (varMap.size() > 0) {
157 System.out.println("Substituting parameter values...");
158 String subScript = substituteVars(varMap, readStringFromFile(scriptPath));
159 scriptPath = scriptPath + ".sub";
160 writeStringToFile(scriptPath, subScript);
161
162 System.out.println("Substituted script [" + scriptPath + "] content: ");
163 System.out.println("------------------------");
164 System.out.println(readStringFromFile(scriptPath));
165 System.out.println("------------------------");
166 System.out.println();
167 }
168
169 arguments.add("-f");
170 arguments.add(scriptPath);
171
172
173 System.out.println("Hive command arguments :");
174 for (String arg : arguments) {
175 System.out.println(" " + arg);
176 }
177
178 System.out.println("=================================================================");
179 System.out.println();
180 System.out.println(">>> Invoking Hive command line now >>>");
181 System.out.println();
182 System.out.flush();
183
184 runHive(arguments.toArray(new String[arguments.size()]));
185
186 System.out.println("\n<<< Invocation of Hive command completed <<<\n");
187 }
188
189 private void runHive(String[] args) throws Exception {
190 CliDriver.main(args);
191 }
192
193 public static void setHiveScript(Configuration conf, String script, String[] params) {
194 conf.set(HIVE_SCRIPT, script);
195 MapReduceMain.setStrings(conf, HIVE_PARAMS, params);
196 }
197
198 private static String readStringFromFile(String filePath) throws IOException {
199 String line;
200 BufferedReader br = new BufferedReader(new FileReader(filePath));
201 StringBuilder sb = new StringBuilder();
202 String sep = System.getProperty("line.separator");
203 while ((line = br.readLine()) != null) {
204 sb.append(line).append(sep);
205 }
206 return sb.toString();
207 }
208
209 private static void writeStringToFile(String filePath, String str) throws IOException {
210 BufferedWriter out = new BufferedWriter(new FileWriter(filePath));
211 out.write(str);
212 out.close();
213 }
214
215
216 // Ripped from oahc.Configuration with slight modifications.
217 // Note:
218 // Rather than break at a small pattern limit, we need Hive action to
219 // allow large substitution limit but break if there is cycle
220 // in substitution.
221 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
222 static int MAX_VARIABLE_SUBST_LIMIT = 10000;
223
224 static String substituteVars(Map<String, String> varMap, String expr) {
225 Matcher match = varPat.matcher("");
226 String eval = expr;
227 Set<String> evalSet = new HashSet<String>();
228 for(int s = 0; s < MAX_VARIABLE_SUBST_LIMIT; s++) {
229 if (evalSet.contains(eval)) {
230 //Cyclic resolution pattern detected. Return current expression.
231 return eval;
232 }
233 evalSet.add(eval);
234 match.reset(eval);
235 if (!match.find()) {
236 return eval;
237 }
238 String var = match.group();
239 var = var.substring(2, var.length()-1); // remove ${ .. }
240 String val = varMap.get(var);
241 if (null == val) {
242 return eval; // return literal ${var}: var is unbound
243 }
244 // substitute
245 eval = eval.substring(0, match.start())+val+eval.substring(match.end());
246 }
247 throw new RuntimeException(
248 "Maximum variable substitution limit exceeded: "
249 + MAX_VARIABLE_SUBST_LIMIT + " " + expr);
250 }
251 }