001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapreduce; 020 021 import java.io.IOException; 022 import java.net.InetSocketAddress; 023 import java.security.PrivilegedExceptionAction; 024 import java.util.ArrayList; 025 import java.util.List; 026 import java.util.ServiceLoader; 027 028 import org.apache.commons.logging.Log; 029 import org.apache.commons.logging.LogFactory; 030 import org.apache.hadoop.classification.InterfaceAudience; 031 import org.apache.hadoop.classification.InterfaceStability; 032 import org.apache.hadoop.conf.Configuration; 033 import org.apache.hadoop.fs.FileSystem; 034 import org.apache.hadoop.fs.Path; 035 import org.apache.hadoop.io.Text; 036 import org.apache.hadoop.ipc.RemoteException; 037 import org.apache.hadoop.mapred.JobConf; 038 import org.apache.hadoop.mapred.Master; 039 import org.apache.hadoop.mapreduce.protocol.ClientProtocol; 040 import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider; 041 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; 042 import org.apache.hadoop.mapreduce.util.ConfigUtil; 043 import org.apache.hadoop.mapreduce.v2.LogParams; 044 import org.apache.hadoop.net.NetUtils; 045 import org.apache.hadoop.security.AccessControlException; 046 import org.apache.hadoop.security.UserGroupInformation; 047 import org.apache.hadoop.security.token.SecretManager.InvalidToken; 048 import org.apache.hadoop.security.token.Token; 049 050 /** 051 * Provides a way to access information about the map/reduce cluster. 052 */ 053 @InterfaceAudience.Public 054 @InterfaceStability.Evolving 055 public class Cluster { 056 057 @InterfaceStability.Evolving 058 public static enum JobTrackerStatus {INITIALIZING, RUNNING}; 059 060 private ClientProtocolProvider clientProtocolProvider; 061 private ClientProtocol client; 062 private UserGroupInformation ugi; 063 private Configuration conf; 064 private FileSystem fs = null; 065 private Path sysDir = null; 066 private Path stagingAreaDir = null; 067 private Path jobHistoryDir = null; 068 private static final Log LOG = LogFactory.getLog(Cluster.class); 069 070 private static ServiceLoader<ClientProtocolProvider> frameworkLoader = 071 ServiceLoader.load(ClientProtocolProvider.class); 072 073 static { 074 ConfigUtil.loadResources(); 075 } 076 077 public Cluster(Configuration conf) throws IOException { 078 this(null, conf); 079 } 080 081 public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) 082 throws IOException { 083 this.conf = conf; 084 this.ugi = UserGroupInformation.getCurrentUser(); 085 initialize(jobTrackAddr, conf); 086 } 087 088 private void initialize(InetSocketAddress jobTrackAddr, Configuration conf) 089 throws IOException { 090 091 synchronized (frameworkLoader) { 092 for (ClientProtocolProvider provider : frameworkLoader) { 093 LOG.debug("Trying ClientProtocolProvider : " 094 + provider.getClass().getName()); 095 ClientProtocol clientProtocol = null; 096 try { 097 if (jobTrackAddr == null) { 098 clientProtocol = provider.create(conf); 099 } else { 100 clientProtocol = provider.create(jobTrackAddr, conf); 101 } 102 103 if (clientProtocol != null) { 104 clientProtocolProvider = provider; 105 client = clientProtocol; 106 LOG.debug("Picked " + provider.getClass().getName() 107 + " as the ClientProtocolProvider"); 108 break; 109 } 110 else { 111 LOG.debug("Cannot pick " + provider.getClass().getName() 112 + " as the ClientProtocolProvider - returned null protocol"); 113 } 114 } 115 catch (Exception e) { 116 LOG.info("Failed to use " + provider.getClass().getName() 117 + " due to error: " + e.getMessage()); 118 } 119 } 120 } 121 122 if (null == clientProtocolProvider || null == client) { 123 throw new IOException( 124 "Cannot initialize Cluster. Please check your configuration for " 125 + MRConfig.FRAMEWORK_NAME 126 + " and the correspond server addresses."); 127 } 128 } 129 130 ClientProtocol getClient() { 131 return client; 132 } 133 134 Configuration getConf() { 135 return conf; 136 } 137 138 /** 139 * Close the <code>Cluster</code>. 140 */ 141 public synchronized void close() throws IOException { 142 clientProtocolProvider.close(client); 143 } 144 145 private Job[] getJobs(JobStatus[] stats) throws IOException { 146 List<Job> jobs = new ArrayList<Job>(); 147 for (JobStatus stat : stats) { 148 jobs.add(Job.getInstance(this, stat, new JobConf(stat.getJobFile()))); 149 } 150 return jobs.toArray(new Job[0]); 151 } 152 153 /** 154 * Get the file system where job-specific files are stored 155 * 156 * @return object of FileSystem 157 * @throws IOException 158 * @throws InterruptedException 159 */ 160 public synchronized FileSystem getFileSystem() 161 throws IOException, InterruptedException { 162 if (this.fs == null) { 163 try { 164 this.fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 165 public FileSystem run() throws IOException, InterruptedException { 166 final Path sysDir = new Path(client.getSystemDir()); 167 return sysDir.getFileSystem(getConf()); 168 } 169 }); 170 } catch (InterruptedException e) { 171 throw new RuntimeException(e); 172 } 173 } 174 return fs; 175 } 176 177 /** 178 * Get job corresponding to jobid. 179 * 180 * @param jobId 181 * @return object of {@link Job} 182 * @throws IOException 183 * @throws InterruptedException 184 */ 185 public Job getJob(JobID jobId) throws IOException, InterruptedException { 186 JobStatus status = client.getJobStatus(jobId); 187 if (status != null) { 188 return Job.getInstance(this, status, new JobConf(status.getJobFile())); 189 } 190 return null; 191 } 192 193 /** 194 * Get all the queues in cluster. 195 * 196 * @return array of {@link QueueInfo} 197 * @throws IOException 198 * @throws InterruptedException 199 */ 200 public QueueInfo[] getQueues() throws IOException, InterruptedException { 201 return client.getQueues(); 202 } 203 204 /** 205 * Get queue information for the specified name. 206 * 207 * @param name queuename 208 * @return object of {@link QueueInfo} 209 * @throws IOException 210 * @throws InterruptedException 211 */ 212 public QueueInfo getQueue(String name) 213 throws IOException, InterruptedException { 214 return client.getQueue(name); 215 } 216 217 /** 218 * Get log parameters for the specified jobID or taskAttemptID 219 * @param jobID the job id. 220 * @param taskAttemptID the task attempt id. Optional. 221 * @return the LogParams 222 * @throws IOException 223 * @throws InterruptedException 224 */ 225 public LogParams getLogParams(JobID jobID, TaskAttemptID taskAttemptID) 226 throws IOException, InterruptedException { 227 return client.getLogFileParams(jobID, taskAttemptID); 228 } 229 230 /** 231 * Get current cluster status. 232 * 233 * @return object of {@link ClusterMetrics} 234 * @throws IOException 235 * @throws InterruptedException 236 */ 237 public ClusterMetrics getClusterStatus() throws IOException, InterruptedException { 238 return client.getClusterMetrics(); 239 } 240 241 /** 242 * Get all active trackers in the cluster. 243 * 244 * @return array of {@link TaskTrackerInfo} 245 * @throws IOException 246 * @throws InterruptedException 247 */ 248 public TaskTrackerInfo[] getActiveTaskTrackers() 249 throws IOException, InterruptedException { 250 return client.getActiveTrackers(); 251 } 252 253 /** 254 * Get blacklisted trackers. 255 * 256 * @return array of {@link TaskTrackerInfo} 257 * @throws IOException 258 * @throws InterruptedException 259 */ 260 public TaskTrackerInfo[] getBlackListedTaskTrackers() 261 throws IOException, InterruptedException { 262 return client.getBlacklistedTrackers(); 263 } 264 265 /** 266 * Get all the jobs in cluster. 267 * 268 * @return array of {@link Job} 269 * @throws IOException 270 * @throws InterruptedException 271 * @deprecated Use {@link #getAllJobStatuses()} instead. 272 */ 273 @Deprecated 274 public Job[] getAllJobs() throws IOException, InterruptedException { 275 return getJobs(client.getAllJobs()); 276 } 277 278 /** 279 * Get job status for all jobs in the cluster. 280 * @return job status for all jobs in cluster 281 * @throws IOException 282 * @throws InterruptedException 283 */ 284 public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException { 285 return client.getAllJobs(); 286 } 287 288 /** 289 * Grab the jobtracker system directory path where 290 * job-specific files will be placed. 291 * 292 * @return the system directory where job-specific files are to be placed. 293 */ 294 public Path getSystemDir() throws IOException, InterruptedException { 295 if (sysDir == null) { 296 sysDir = new Path(client.getSystemDir()); 297 } 298 return sysDir; 299 } 300 301 /** 302 * Grab the jobtracker's view of the staging directory path where 303 * job-specific files will be placed. 304 * 305 * @return the staging directory where job-specific files are to be placed. 306 */ 307 public Path getStagingAreaDir() throws IOException, InterruptedException { 308 if (stagingAreaDir == null) { 309 stagingAreaDir = new Path(client.getStagingAreaDir()); 310 } 311 return stagingAreaDir; 312 } 313 314 /** 315 * Get the job history file path for a given job id. The job history file at 316 * this path may or may not be existing depending on the job completion state. 317 * The file is present only for the completed jobs. 318 * @param jobId the JobID of the job submitted by the current user. 319 * @return the file path of the job history file 320 * @throws IOException 321 * @throws InterruptedException 322 */ 323 public String getJobHistoryUrl(JobID jobId) throws IOException, 324 InterruptedException { 325 if (jobHistoryDir == null) { 326 jobHistoryDir = new Path(client.getJobHistoryDir()); 327 } 328 return new Path(jobHistoryDir, jobId.toString() + "_" 329 + ugi.getShortUserName()).toString(); 330 } 331 332 /** 333 * Gets the Queue ACLs for current user 334 * @return array of QueueAclsInfo object for current user. 335 * @throws IOException 336 */ 337 public QueueAclsInfo[] getQueueAclsForCurrentUser() 338 throws IOException, InterruptedException { 339 return client.getQueueAclsForCurrentUser(); 340 } 341 342 /** 343 * Gets the root level queues. 344 * @return array of JobQueueInfo object. 345 * @throws IOException 346 */ 347 public QueueInfo[] getRootQueues() throws IOException, InterruptedException { 348 return client.getRootQueues(); 349 } 350 351 /** 352 * Returns immediate children of queueName. 353 * @param queueName 354 * @return array of JobQueueInfo which are children of queueName 355 * @throws IOException 356 */ 357 public QueueInfo[] getChildQueues(String queueName) 358 throws IOException, InterruptedException { 359 return client.getChildQueues(queueName); 360 } 361 362 /** 363 * Get the JobTracker's status. 364 * 365 * @return {@link JobTrackerStatus} of the JobTracker 366 * @throws IOException 367 * @throws InterruptedException 368 */ 369 public JobTrackerStatus getJobTrackerStatus() throws IOException, 370 InterruptedException { 371 return client.getJobTrackerStatus(); 372 } 373 374 /** 375 * Get the tasktracker expiry interval for the cluster 376 * @return the expiry interval in msec 377 */ 378 public long getTaskTrackerExpiryInterval() throws IOException, 379 InterruptedException { 380 return client.getTaskTrackerExpiryInterval(); 381 } 382 383 /** 384 * Get a delegation token for the user from the JobTracker. 385 * @param renewer the user who can renew the token 386 * @return the new token 387 * @throws IOException 388 */ 389 public Token<DelegationTokenIdentifier> 390 getDelegationToken(Text renewer) throws IOException, InterruptedException{ 391 Token<DelegationTokenIdentifier> result = 392 client.getDelegationToken(renewer); 393 394 if (result == null) { 395 return result; 396 } 397 398 InetSocketAddress addr = Master.getMasterAddress(conf); 399 StringBuilder service = new StringBuilder(); 400 service.append(NetUtils.normalizeHostName(addr.getAddress(). 401 getHostAddress())); 402 service.append(':'); 403 service.append(addr.getPort()); 404 result.setService(new Text(service.toString())); 405 return result; 406 } 407 408 /** 409 * Renew a delegation token 410 * @param token the token to renew 411 * @return the new expiration time 412 * @throws InvalidToken 413 * @throws IOException 414 * @deprecated Use {@link Token#renew} instead 415 */ 416 public long renewDelegationToken(Token<DelegationTokenIdentifier> token 417 ) throws InvalidToken, IOException, 418 InterruptedException { 419 try { 420 return client.renewDelegationToken(token); 421 } catch (RemoteException re) { 422 throw re.unwrapRemoteException(InvalidToken.class, 423 AccessControlException.class); 424 } 425 } 426 427 /** 428 * Cancel a delegation token from the JobTracker 429 * @param token the token to cancel 430 * @throws IOException 431 * @deprecated Use {@link Token#cancel} instead 432 */ 433 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token 434 ) throws IOException, 435 InterruptedException { 436 try { 437 client.cancelDelegationToken(token); 438 } catch (RemoteException re) { 439 throw re.unwrapRemoteException(InvalidToken.class, 440 AccessControlException.class); 441 } 442 } 443 444 }