001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import java.io.IOException;
021 import java.io.StringReader;
022 import java.util.List;
023
024 import org.apache.hadoop.conf.Configuration;
025 import org.apache.hadoop.fs.FileStatus;
026 import org.apache.hadoop.fs.FileSystem;
027 import org.apache.hadoop.fs.Path;
028 import org.apache.oozie.action.ActionExecutorException;
029 import org.apache.oozie.client.WorkflowAction;
030 import org.apache.oozie.util.XConfiguration;
031 import org.apache.oozie.util.XmlUtils;
032 import org.jdom.Element;
033 import org.jdom.JDOMException;
034 import org.jdom.Namespace;
035
036 import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_DEFAULT_CONF;
037 import static org.apache.oozie.action.hadoop.SqoopMain.SQOOP_SITE_CONF;
038 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_DEFAULT_CONF;
039 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_EXEC_L4J_PROPS;
040 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_L4J_PROPS;
041 import static org.apache.oozie.action.hadoop.HiveMain.HIVE_SITE_CONF;
042 import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
043
044 public class SqoopActionExecutor extends JavaActionExecutor {
045
046
047 public SqoopActionExecutor() {
048 super("sqoop");
049 }
050
051 @Override
052 protected List<Class> getLauncherClasses() {
053 List<Class> classes = super.getLauncherClasses();
054 classes.add(LauncherMain.class);
055 classes.add(MapReduceMain.class);
056 classes.add(SqoopMain.class);
057 return classes;
058 }
059
060 @Override
061 protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
062 return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, SqoopMain.class.getName());
063 }
064
065 @Override
066 void injectActionCallback(Context context, Configuration launcherConf) {
067 }
068
069 protected void addResourceToCache(Configuration conf, FileSystem fs, Path resourcePath)
070 throws ActionExecutorException {
071 try {
072 if (!fs.exists(resourcePath) || !fs.isFile(resourcePath)) {
073 return;
074 }
075 Path parentPath = resourcePath.getParent();
076 String resourceName = resourcePath.getName();
077 addToCache(conf, parentPath, resourceName + "#" + resourceName, false);
078 } catch (Exception ex) {
079 throw convertException(ex);
080 }
081 }
082
083 @Override
084 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
085 throws ActionExecutorException {
086 try {
087 super.setupLauncherConf(conf, actionXml, appPath, context);
088 Namespace ns = actionXml.getNamespace();
089
090 // Look for hive-log4j.properties and hive-exec-log4j.properties
091 // in the configuration directory and add them to the distributed
092 // cache.
093 Element e = actionXml.getChild("conf-dir", ns);
094 if (e != null) {
095 Path hiveConfPath = new Path(appPath, e.getTextTrim());
096 FileSystem fs = getActionFileSystem(context, actionXml);
097 FileStatus status = fs.getFileStatus(hiveConfPath);
098 if (status.isDir()) {
099 addResourceToCache(conf, fs, new Path(hiveConfPath,HIVE_L4J_PROPS));
100 addResourceToCache(conf, fs, new Path(hiveConfPath, HIVE_EXEC_L4J_PROPS));
101 }
102 }
103 return conf;
104 } catch (IOException ex) {
105 throw convertException(ex);
106 }
107 }
108
109 @Override
110 @SuppressWarnings("unchecked")
111 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
112 throws ActionExecutorException {
113 super.setupActionConf(actionConf, context, actionXml, appPath);
114 Namespace ns = actionXml.getNamespace();
115
116 // Read sqoop-default.xml, sqoop-site.xml, hive-default.xml, hive-site.xml,
117 // and any configuration properties defined in the workflow and overlay
118 // these properties onto the action configuration in this order.
119 Element e = actionXml.getChild("conf-dir", ns);
120 try {
121 if (e != null) {
122 Path sqoopConfPath = new Path(appPath, e.getTextTrim());
123 FileSystem fs = getActionFileSystem(context, actionXml);
124 FileStatus status = fs.getFileStatus(sqoopConfPath);
125 if (status.isDir()) {
126 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_DEFAULT_CONF));
127 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, SQOOP_SITE_CONF));
128 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_DEFAULT_CONF));
129 loadConfiguration(actionConf, fs, new Path(sqoopConfPath, HIVE_SITE_CONF));
130 }
131 }
132
133 e = actionXml.getChild("configuration", ns);
134 if (e != null) {
135 String strConf = XmlUtils.prettyPrint(e).toString();
136 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
137 checkForDisallowedProps(inlineConf, "inline configuration");
138 XConfiguration.copy(inlineConf, actionConf);
139 }
140 } catch (IOException ex) {
141 throw convertException(ex);
142 }
143
144 String command = actionXml.getChild("command", ns).getTextTrim();
145
146 SqoopMain.setSqoopCommand(actionConf, command);
147 return actionConf;
148 }
149
150 private void loadConfiguration(Configuration baseConf, FileSystem fs, Path confPath)
151 throws ActionExecutorException {
152 try {
153 if (!fs.exists(confPath) || !fs.isFile(confPath)) {
154 return;
155 }
156 Configuration newConf = new XConfiguration(fs.open(confPath));
157 checkForDisallowedProps(newConf, confPath.getName());
158 XConfiguration.copy(newConf, baseConf);
159 } catch (IOException ex) {
160 throw convertException(ex);
161 }
162 }
163
164 @Override
165 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
166 return true;
167 }
168 }