001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.service;
016
017 import org.apache.hadoop.mapred.JobClient;
018 import org.apache.hadoop.mapred.JobConf;
019 import org.apache.hadoop.fs.FileSystem;
020 import org.apache.hadoop.fs.Path;
021 import org.apache.hadoop.conf.Configuration;
022 import org.apache.hadoop.security.UserGroupInformation;
023 import org.apache.hadoop.filecache.DistributedCache;
024 import org.apache.oozie.ErrorCode;
025 import org.apache.oozie.util.ParamChecker;
026 import org.apache.oozie.util.XConfiguration;
027 import org.apache.oozie.util.XLog;
028
029 import java.io.IOException;
030 import java.net.URI;
031 import java.net.URISyntaxException;
032 import java.security.PrivilegedExceptionAction;
033 import java.util.Set;
034 import java.util.HashSet;
035
036 /**
037 * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The
038 * default accessor used is the base accessor which just injects the UGI into the configuration instance used to
039 * create/obtain JobClient and ileSystem instances. <p/> The HadoopAccess class to use can be configured in the
040 * <code>oozie-site.xml</code> using the <code>oozie.service.HadoopAccessorService.accessor.class</code> property.
041 */
042 public class HadoopAccessorService implements Service {
043
044 public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService.";
045 public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist";
046 public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist";
047
048 private Set<String> jobTrackerWhitelist = new HashSet<String>();
049 private Set<String> nameNodeWhitelist = new HashSet<String>();
050
051 public void init(Services services) throws ServiceException {
052 for (String name : services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)) {
053 String tmp = name.toLowerCase().trim();
054 if (tmp.length() == 0) {
055 continue;
056 }
057 jobTrackerWhitelist.add(tmp);
058 }
059 XLog.getLog(getClass()).info(
060 "JOB_TRACKER_WHITELIST :" + services.getConf().getStringCollection(JOB_TRACKER_WHITELIST)
061 + ", Total entries :" + jobTrackerWhitelist.size());
062 for (String name : services.getConf().getStringCollection(NAME_NODE_WHITELIST)) {
063 String tmp = name.toLowerCase().trim();
064 if (tmp.length() == 0) {
065 continue;
066 }
067 nameNodeWhitelist.add(tmp);
068 }
069 XLog.getLog(getClass()).info(
070 "NAME_NODE_WHITELIST :" + services.getConf().getStringCollection(NAME_NODE_WHITELIST)
071 + ", Total entries :" + nameNodeWhitelist.size());
072 init(services.getConf());
073 }
074
075 public void init(Configuration serviceConf) throws ServiceException {
076 }
077
078 public void destroy() {
079 }
080
081 public Class<? extends Service> getInterface() {
082 return HadoopAccessorService.class;
083 }
084
085 /**
086 * Return a JobClient created with the provided user/group.
087 *
088 * @param conf JobConf with all necessary information to create the
089 * JobClient.
090 * @return JobClient created with the provided user/group.
091 * @throws HadoopAccessorException if the client could not be created.
092 */
093 public JobClient createJobClient(String user, String group, JobConf conf) throws HadoopAccessorException {
094 validateJobTracker(conf.get("mapred.job.tracker"));
095 conf = createConfiguration(user, group, conf);
096 try {
097 return new JobClient(conf);
098 }
099 catch (IOException e) {
100 throw new HadoopAccessorException(ErrorCode.E0902, e);
101 }
102 }
103
104 /**
105 * Return a FileSystem created with the provided user/group.
106 *
107 * @param conf Configuration with all necessary information to create the
108 * FileSystem.
109 * @return FileSystem created with the provided user/group.
110 * @throws HadoopAccessorException if the filesystem could not be created.
111 */
112 public FileSystem createFileSystem(String user, String group, Configuration conf) throws HadoopAccessorException {
113 try {
114 validateNameNode(new URI(conf.get("fs.default.name")).getAuthority());
115 conf = createConfiguration(user, group, conf);
116 return FileSystem.get(conf);
117 }
118 catch (IOException e) {
119 throw new HadoopAccessorException(ErrorCode.E0902, e);
120 }
121 catch (URISyntaxException e) {
122 throw new HadoopAccessorException(ErrorCode.E0902, e);
123 }
124 }
125
126 /**
127 * Return a FileSystem created with the provided user/group for the
128 * specified URI.
129 *
130 * @param uri file system URI.
131 * @param conf Configuration with all necessary information to create the
132 * FileSystem.
133 * @return FileSystem created with the provided user/group.
134 * @throws HadoopAccessorException if the filesystem could not be created.
135 */
136 public FileSystem createFileSystem(String user, String group, URI uri, Configuration conf)
137 throws HadoopAccessorException {
138 validateNameNode(uri.getAuthority());
139 conf = createConfiguration(user, group, conf);
140 try {
141 return FileSystem.get(uri, conf);
142 }
143 catch (IOException e) {
144 throw new HadoopAccessorException(ErrorCode.E0902, e);
145 }
146 }
147
148 /**
149 * Validate Job tracker
150 * @param jobTrackerUri
151 * @throws HadoopAccessorException
152 */
153 protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException {
154 validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900);
155 }
156
157 /**
158 * Validate Namenode list
159 * @param nameNodeUri
160 * @throws HadoopAccessorException
161 */
162 protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException {
163 validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901);
164 }
165
166 private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException {
167 if (uri != null) {
168 uri = uri.toLowerCase().trim();
169 if (whitelist.size() > 0 && !whitelist.contains(uri)) {
170 throw new HadoopAccessorException(error, uri);
171 }
172 }
173 }
174
175 @SuppressWarnings("unchecked")
176 private <C extends Configuration> C createConfiguration(String user, String group, C conf) {
177 ParamChecker.notEmpty(user, "user");
178 ParamChecker.notEmpty(group, "group");
179 C fsConf = (C) ((conf instanceof JobConf) ? new JobConf() : new Configuration());
180 XConfiguration.copy(conf, fsConf);
181 fsConf.set("user.name", user);
182 fsConf.set("hadoop.job.ugi", user + "," + group);
183 return fsConf;
184 }
185
186 /**
187 * Add a file to the ClassPath via the DistributedCache.
188 */
189 public void addFileToClassPath(String user, String group, final Path file, final Configuration conf)
190 throws IOException {
191 Configuration defaultConf = createConfiguration(user, group, conf);
192 DistributedCache.addFileToClassPath(file, defaultConf);
193 DistributedCache.addFileToClassPath(file, conf);
194 }
195
196 }