001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF;
021 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS;
022 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS;
023 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF;
024 import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
025
026 import java.io.IOException;
027 import java.io.StringReader;
028 import java.util.List;
029
030 import org.apache.hadoop.conf.Configuration;
031 import org.apache.hadoop.fs.FileStatus;
032 import org.apache.hadoop.fs.FileSystem;
033 import org.apache.hadoop.fs.Path;
034 import org.apache.oozie.action.ActionExecutorException;
035 import org.apache.oozie.client.WorkflowAction;
036 import org.apache.oozie.util.XConfiguration;
037 import org.apache.oozie.util.XmlUtils;
038 import org.jdom.Element;
039 import org.jdom.JDOMException;
040 import org.jdom.Namespace;
041
042 public class HiveActionExecutor extends JavaActionExecutor {
043
044 public HiveActionExecutor() {
045 super("hive");
046 }
047
048 @Override
049 protected List<Class> getLauncherClasses() {
050 List<Class> classes = super.getLauncherClasses();
051 classes.add(LauncherMain.class);
052 classes.add(MapReduceMain.class);
053 classes.add(HiveMain.class);
054 return classes;
055 }
056
057 @Override
058 protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
059 return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, HiveMain.class.getName());
060 }
061
062 @Override
063 void injectActionCallback(Context context, Configuration launcherConf) {
064 }
065
066 protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath)
067 throws ActionExecutorException {
068 try {
069 if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) {
070 return;
071 }
072 Path parentPath = resourcePath.getParent();
073 String resourceName = resourcePath.getName();
074 addToCache(conf, parentPath, resourceName + "#" + resourceName, false);
075 } catch (Exception ex) {
076 throw convertException(ex);
077 }
078 }
079
080 @Override
081 protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
082 throws ActionExecutorException {
083 try {
084 super.setupLauncherConf(conf, actionXml, appPath, context);
085 Namespace ns = actionXml.getNamespace();
086
087 // Look for hive-log4j.properties and hive-exec-log4j.properties
088 // in the configuration directory and add them to the distributed
089 // cache.
090 Element e = actionXml.getChild("conf-dir", ns);
091 if (e != null) {
092 Path hiveConfPath = new Path(appPath, e.getTextTrim());
093 FileSystem fs = getActionFileSystem(context, actionXml);
094 FileStatus status = fs.getFileStatus(hiveConfPath);
095 if (status.isDir()) {
096 addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS));
097 addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS));
098 }
099 }
100
101 // Make the query file available to Hive.
102 String script = actionXml.getChild("script", ns).getTextTrim();
103 String scriptName = new Path(script).getName();
104 addToCache(conf, appPath, script + "#" + scriptName, false);
105
106 return conf;
107 } catch (IOException ex) {
108 throw convertException(ex);
109 }
110 }
111
112 @Override
113 @SuppressWarnings("unchecked")
114 Configuration setupActionConf(Configuration actionConf, Context context,
115 Element actionXml, Path appPath) throws ActionExecutorException {
116
117 try {
118 // Read hive-default.xml, hive-site.xml, and any configuration
119 // properties defined in the workflow and overlay these properties
120 // onto the action configuration in this order.
121 Namespace ns = actionXml.getNamespace();
122 Element e = actionXml.getChild("conf-dir", ns);
123 if (e != null) {
124 Path hiveConfPath = new Path(appPath, e.getTextTrim());
125 FileSystem fs = getActionFileSystem(context, actionXml);
126 FileStatus status = fs.getFileStatus(hiveConfPath);
127 if (status.isDir()) {
128 loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_DEFAULT_CONF));
129 loadConfiguration(actionConf, fs, new Path(hiveConfPath, HIVE_SITE_CONF));
130 }
131 }
132
133 e = actionXml.getChild("configuration", ns);
134 if (e != null) {
135 String strConf = XmlUtils.prettyPrint(e).toString();
136 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
137 checkForDisallowedProps(inlineConf, "inline configuration");
138 XConfiguration.copy(inlineConf, actionConf);
139 }
140
141 String script = actionXml.getChild("script", ns).getTextTrim();
142 String scriptName = new Path(script).getName();
143
144 List<Element> params = actionXml.getChildren("param", ns);
145 String[] strParams = new String[params.size()];
146 for (int i = 0; i < params.size(); i++) {
147 strParams[i] = params.get(i).getTextTrim();
148 }
149
150 HiveMain.setHiveScript(actionConf, scriptName, strParams);
151 return actionConf;
152 } catch (IOException e) {
153 throw convertException(e);
154 }
155 }
156
157 private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath)
158 throws ActionExecutorException {
159 try {
160 if (!fs.exists(confPath) || !fs.isFile(confPath)) {
161 return;
162 }
163 Configuration newConf = new XConfiguration(fs.open(confPath));
164 checkForDisallowedProps(newConf, confPath.getName());
165 XConfiguration.copy(newConf, baseConf);
166 } catch (IOException ex) {
167 throw convertException(ex);
168 }
169 }
170
171 @Override
172 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
173 return true;
174 }
175
176 }