001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.mapred; 019 020 import java.io.FileNotFoundException; 021 import java.io.IOException; 022 import java.net.InetSocketAddress; 023 import java.net.URL; 024 import java.util.ArrayList; 025 import java.util.Collection; 026 import java.util.List; 027 028 import org.apache.hadoop.classification.InterfaceAudience; 029 import org.apache.hadoop.classification.InterfaceStability; 030 import org.apache.hadoop.conf.Configuration; 031 import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo; 032 import org.apache.hadoop.mapreduce.Cluster; 033 import org.apache.hadoop.mapreduce.ClusterMetrics; 034 import org.apache.hadoop.mapreduce.Job; 035 import org.apache.hadoop.mapreduce.QueueInfo; 036 import org.apache.hadoop.mapreduce.TaskTrackerInfo; 037 import org.apache.hadoop.mapreduce.TaskType; 038 import org.apache.hadoop.mapreduce.filecache.DistributedCache; 039 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; 040 import org.apache.hadoop.mapreduce.tools.CLI; 041 import org.apache.hadoop.mapreduce.util.ConfigUtil; 042 import org.apache.hadoop.fs.FileSystem; 043 import org.apache.hadoop.fs.Path; 044 import org.apache.hadoop.io.Text; 045 import org.apache.hadoop.security.token.Token; 046 import org.apache.hadoop.security.token.TokenRenewer; 047 import org.apache.hadoop.security.token.SecretManager.InvalidToken; 048 import org.apache.hadoop.util.Tool; 049 import org.apache.hadoop.util.ToolRunner; 050 051 /** 052 * <code>JobClient</code> is the primary interface for the user-job to interact 053 * with the cluster. 054 * 055 * <code>JobClient</code> provides facilities to submit jobs, track their 056 * progress, access component-tasks' reports/logs, get the Map-Reduce cluster 057 * status information etc. 058 * 059 * <p>The job submission process involves: 060 * <ol> 061 * <li> 062 * Checking the input and output specifications of the job. 063 * </li> 064 * <li> 065 * Computing the {@link InputSplit}s for the job. 066 * </li> 067 * <li> 068 * Setup the requisite accounting information for the {@link DistributedCache} 069 * of the job, if necessary. 070 * </li> 071 * <li> 072 * Copying the job's jar and configuration to the map-reduce system directory 073 * on the distributed file-system. 074 * </li> 075 * <li> 076 * Submitting the job to the cluster and optionally monitoring 077 * it's status. 078 * </li> 079 * </ol></p> 080 * 081 * Normally the user creates the application, describes various facets of the 082 * job via {@link JobConf} and then uses the <code>JobClient</code> to submit 083 * the job and monitor its progress. 084 * 085 * <p>Here is an example on how to use <code>JobClient</code>:</p> 086 * <p><blockquote><pre> 087 * // Create a new JobConf 088 * JobConf job = new JobConf(new Configuration(), MyJob.class); 089 * 090 * // Specify various job-specific parameters 091 * job.setJobName("myjob"); 092 * 093 * job.setInputPath(new Path("in")); 094 * job.setOutputPath(new Path("out")); 095 * 096 * job.setMapperClass(MyJob.MyMapper.class); 097 * job.setReducerClass(MyJob.MyReducer.class); 098 * 099 * // Submit the job, then poll for progress until the job is complete 100 * JobClient.runJob(job); 101 * </pre></blockquote></p> 102 * 103 * <h4 id="JobControl">Job Control</h4> 104 * 105 * <p>At times clients would chain map-reduce jobs to accomplish complex tasks 106 * which cannot be done via a single map-reduce job. This is fairly easy since 107 * the output of the job, typically, goes to distributed file-system and that 108 * can be used as the input for the next job.</p> 109 * 110 * <p>However, this also means that the onus on ensuring jobs are complete 111 * (success/failure) lies squarely on the clients. In such situations the 112 * various job-control options are: 113 * <ol> 114 * <li> 115 * {@link #runJob(JobConf)} : submits the job and returns only after 116 * the job has completed. 117 * </li> 118 * <li> 119 * {@link #submitJob(JobConf)} : only submits the job, then poll the 120 * returned handle to the {@link RunningJob} to query status and make 121 * scheduling decisions. 122 * </li> 123 * <li> 124 * {@link JobConf#setJobEndNotificationURI(String)} : setup a notification 125 * on job-completion, thus avoiding polling. 126 * </li> 127 * </ol></p> 128 * 129 * @see JobConf 130 * @see ClusterStatus 131 * @see Tool 132 * @see DistributedCache 133 * @deprecated Use {@link Job} and {@link Cluster} instead 134 */ 135 @Deprecated 136 @InterfaceAudience.Public 137 @InterfaceStability.Stable 138 public class JobClient extends CLI { 139 public static enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL } 140 private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; 141 142 static{ 143 ConfigUtil.loadResources(); 144 } 145 146 /** 147 * A NetworkedJob is an implementation of RunningJob. It holds 148 * a JobProfile object to provide some info, and interacts with the 149 * remote service to provide certain functionality. 150 */ 151 static class NetworkedJob implements RunningJob { 152 Job job; 153 /** 154 * We store a JobProfile and a timestamp for when we last 155 * acquired the job profile. If the job is null, then we cannot 156 * perform any of the tasks. The job might be null if the cluster 157 * has completely forgotten about the job. (eg, 24 hours after the 158 * job completes.) 159 */ 160 public NetworkedJob(JobStatus status, Cluster cluster) throws IOException { 161 job = Job.getInstance(cluster, status, new JobConf(status.getJobFile())); 162 } 163 164 public NetworkedJob(Job job) throws IOException { 165 this.job = job; 166 } 167 168 public Configuration getConfiguration() { 169 return job.getConfiguration(); 170 } 171 172 /** 173 * An identifier for the job 174 */ 175 public JobID getID() { 176 return JobID.downgrade(job.getJobID()); 177 } 178 179 /** @deprecated This method is deprecated and will be removed. Applications should 180 * rather use {@link #getID()}.*/ 181 @Deprecated 182 public String getJobID() { 183 return getID().toString(); 184 } 185 186 /** 187 * The user-specified job name 188 */ 189 public String getJobName() { 190 return job.getJobName(); 191 } 192 193 /** 194 * The name of the job file 195 */ 196 public String getJobFile() { 197 return job.getJobFile(); 198 } 199 200 /** 201 * A URL where the job's status can be seen 202 */ 203 public String getTrackingURL() { 204 return job.getTrackingURL(); 205 } 206 207 /** 208 * A float between 0.0 and 1.0, indicating the % of map work 209 * completed. 210 */ 211 public float mapProgress() throws IOException { 212 try { 213 return job.mapProgress(); 214 } catch (InterruptedException ie) { 215 throw new IOException(ie); 216 } 217 } 218 219 /** 220 * A float between 0.0 and 1.0, indicating the % of reduce work 221 * completed. 222 */ 223 public float reduceProgress() throws IOException { 224 try { 225 return job.reduceProgress(); 226 } catch (InterruptedException ie) { 227 throw new IOException(ie); 228 } 229 } 230 231 /** 232 * A float between 0.0 and 1.0, indicating the % of cleanup work 233 * completed. 234 */ 235 public float cleanupProgress() throws IOException { 236 try { 237 return job.cleanupProgress(); 238 } catch (InterruptedException ie) { 239 throw new IOException(ie); 240 } 241 } 242 243 /** 244 * A float between 0.0 and 1.0, indicating the % of setup work 245 * completed. 246 */ 247 public float setupProgress() throws IOException { 248 try { 249 return job.setupProgress(); 250 } catch (InterruptedException ie) { 251 throw new IOException(ie); 252 } 253 } 254 255 /** 256 * Returns immediately whether the whole job is done yet or not. 257 */ 258 public synchronized boolean isComplete() throws IOException { 259 try { 260 return job.isComplete(); 261 } catch (InterruptedException ie) { 262 throw new IOException(ie); 263 } 264 } 265 266 /** 267 * True iff job completed successfully. 268 */ 269 public synchronized boolean isSuccessful() throws IOException { 270 try { 271 return job.isSuccessful(); 272 } catch (InterruptedException ie) { 273 throw new IOException(ie); 274 } 275 } 276 277 /** 278 * Blocks until the job is finished 279 */ 280 public void waitForCompletion() throws IOException { 281 try { 282 job.waitForCompletion(false); 283 } catch (InterruptedException ie) { 284 throw new IOException(ie); 285 } catch (ClassNotFoundException ce) { 286 throw new IOException(ce); 287 } 288 } 289 290 /** 291 * Tells the service to get the state of the current job. 292 */ 293 public synchronized int getJobState() throws IOException { 294 try { 295 return job.getJobState().getValue(); 296 } catch (InterruptedException ie) { 297 throw new IOException(ie); 298 } 299 } 300 301 /** 302 * Tells the service to terminate the current job. 303 */ 304 public synchronized void killJob() throws IOException { 305 try { 306 job.killJob(); 307 } catch (InterruptedException ie) { 308 throw new IOException(ie); 309 } 310 } 311 312 313 /** Set the priority of the job. 314 * @param priority new priority of the job. 315 */ 316 public synchronized void setJobPriority(String priority) 317 throws IOException { 318 try { 319 job.setPriority( 320 org.apache.hadoop.mapreduce.JobPriority.valueOf(priority)); 321 } catch (InterruptedException ie) { 322 throw new IOException(ie); 323 } 324 } 325 326 /** 327 * Kill indicated task attempt. 328 * @param taskId the id of the task to kill. 329 * @param shouldFail if true the task is failed and added to failed tasks list, otherwise 330 * it is just killed, w/o affecting job failure status. 331 */ 332 public synchronized void killTask(TaskAttemptID taskId, 333 boolean shouldFail) throws IOException { 334 try { 335 if (shouldFail) { 336 job.failTask(taskId); 337 } else { 338 job.killTask(taskId); 339 } 340 } catch (InterruptedException ie) { 341 throw new IOException(ie); 342 } 343 } 344 345 /** @deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}*/ 346 @Deprecated 347 public synchronized void killTask(String taskId, boolean shouldFail) throws IOException { 348 killTask(TaskAttemptID.forName(taskId), shouldFail); 349 } 350 351 /** 352 * Fetch task completion events from cluster for this job. 353 */ 354 public synchronized TaskCompletionEvent[] getTaskCompletionEvents( 355 int startFrom) throws IOException { 356 try { 357 org.apache.hadoop.mapreduce.TaskCompletionEvent[] acls = 358 job.getTaskCompletionEvents(startFrom, 10); 359 TaskCompletionEvent[] ret = new TaskCompletionEvent[acls.length]; 360 for (int i = 0 ; i < acls.length; i++ ) { 361 ret[i] = TaskCompletionEvent.downgrade(acls[i]); 362 } 363 return ret; 364 } catch (InterruptedException ie) { 365 throw new IOException(ie); 366 } 367 } 368 369 /** 370 * Dump stats to screen 371 */ 372 @Override 373 public String toString() { 374 return job.toString(); 375 } 376 377 /** 378 * Returns the counters for this job 379 */ 380 public Counters getCounters() throws IOException { 381 try { 382 Counters result = null; 383 org.apache.hadoop.mapreduce.Counters temp = job.getCounters(); 384 if(temp != null) { 385 result = Counters.downgrade(temp); 386 } 387 return result; 388 } catch (InterruptedException ie) { 389 throw new IOException(ie); 390 } 391 } 392 393 @Override 394 public String[] getTaskDiagnostics(TaskAttemptID id) throws IOException { 395 try { 396 return job.getTaskDiagnostics(id); 397 } catch (InterruptedException ie) { 398 throw new IOException(ie); 399 } 400 } 401 402 public String getHistoryUrl() throws IOException { 403 try { 404 return job.getHistoryUrl(); 405 } catch (InterruptedException ie) { 406 throw new IOException(ie); 407 } 408 } 409 410 public boolean isRetired() throws IOException { 411 try { 412 return job.isRetired(); 413 } catch (InterruptedException ie) { 414 throw new IOException(ie); 415 } 416 } 417 418 boolean monitorAndPrintJob() throws IOException, InterruptedException { 419 return job.monitorAndPrintJob(); 420 } 421 } 422 423 Cluster cluster; 424 425 /** 426 * Create a job client. 427 */ 428 public JobClient() { 429 } 430 431 /** 432 * Build a job client with the given {@link JobConf}, and connect to the 433 * default cluster 434 * 435 * @param conf the job configuration. 436 * @throws IOException 437 */ 438 public JobClient(JobConf conf) throws IOException { 439 init(conf); 440 } 441 442 /** 443 * Build a job client with the given {@link Configuration}, 444 * and connect to the default cluster 445 * 446 * @param conf the configuration. 447 * @throws IOException 448 */ 449 public JobClient(Configuration conf) throws IOException { 450 init(new JobConf(conf)); 451 } 452 453 /** 454 * Connect to the default cluster 455 * @param conf the job configuration. 456 * @throws IOException 457 */ 458 public void init(JobConf conf) throws IOException { 459 setConf(conf); 460 cluster = new Cluster(conf); 461 } 462 463 @InterfaceAudience.Private 464 public static class Renewer extends TokenRenewer { 465 466 @Override 467 public boolean handleKind(Text kind) { 468 return DelegationTokenIdentifier.MAPREDUCE_DELEGATION_KIND.equals(kind); 469 } 470 471 @SuppressWarnings("unchecked") 472 @Override 473 public long renew(Token<?> token, Configuration conf 474 ) throws IOException, InterruptedException { 475 return new Cluster(conf). 476 renewDelegationToken((Token<DelegationTokenIdentifier>) token); 477 } 478 479 @SuppressWarnings("unchecked") 480 @Override 481 public void cancel(Token<?> token, Configuration conf 482 ) throws IOException, InterruptedException { 483 new Cluster(conf). 484 cancelDelegationToken((Token<DelegationTokenIdentifier>) token); 485 } 486 487 @Override 488 public boolean isManaged(Token<?> token) throws IOException { 489 return true; 490 } 491 492 } 493 494 /** 495 * Build a job client, connect to the indicated job tracker. 496 * 497 * @param jobTrackAddr the job tracker to connect to. 498 * @param conf configuration. 499 */ 500 public JobClient(InetSocketAddress jobTrackAddr, 501 Configuration conf) throws IOException { 502 cluster = new Cluster(jobTrackAddr, conf); 503 } 504 505 /** 506 * Close the <code>JobClient</code>. 507 */ 508 public synchronized void close() throws IOException { 509 cluster.close(); 510 } 511 512 /** 513 * Get a filesystem handle. We need this to prepare jobs 514 * for submission to the MapReduce system. 515 * 516 * @return the filesystem handle. 517 */ 518 public synchronized FileSystem getFs() throws IOException { 519 try { 520 return cluster.getFileSystem(); 521 } catch (InterruptedException ie) { 522 throw new IOException(ie); 523 } 524 } 525 526 /** 527 * Get a handle to the Cluster 528 */ 529 public Cluster getClusterHandle() { 530 return cluster; 531 } 532 533 /** 534 * Submit a job to the MR system. 535 * 536 * This returns a handle to the {@link RunningJob} which can be used to track 537 * the running-job. 538 * 539 * @param jobFile the job configuration. 540 * @return a handle to the {@link RunningJob} which can be used to track the 541 * running-job. 542 * @throws FileNotFoundException 543 * @throws InvalidJobConfException 544 * @throws IOException 545 */ 546 public RunningJob submitJob(String jobFile) throws FileNotFoundException, 547 InvalidJobConfException, 548 IOException { 549 // Load in the submitted job details 550 JobConf job = new JobConf(jobFile); 551 return submitJob(job); 552 } 553 554 /** 555 * Submit a job to the MR system. 556 * This returns a handle to the {@link RunningJob} which can be used to track 557 * the running-job. 558 * 559 * @param conf the job configuration. 560 * @return a handle to the {@link RunningJob} which can be used to track the 561 * running-job. 562 * @throws FileNotFoundException 563 * @throws IOException 564 */ 565 public RunningJob submitJob(JobConf conf) throws FileNotFoundException, 566 IOException { 567 try { 568 conf.setBooleanIfUnset("mapred.mapper.new-api", false); 569 conf.setBooleanIfUnset("mapred.reducer.new-api", false); 570 Job job = Job.getInstance(conf); 571 job.submit(); 572 return new NetworkedJob(job); 573 } catch (InterruptedException ie) { 574 throw new IOException("interrupted", ie); 575 } catch (ClassNotFoundException cnfe) { 576 throw new IOException("class not found", cnfe); 577 } 578 } 579 580 /** 581 * Get an {@link RunningJob} object to track an ongoing job. Returns 582 * null if the id does not correspond to any known job. 583 * 584 * @param jobid the jobid of the job. 585 * @return the {@link RunningJob} handle to track the job, null if the 586 * <code>jobid</code> doesn't correspond to any known job. 587 * @throws IOException 588 */ 589 public RunningJob getJob(JobID jobid) throws IOException { 590 try { 591 Job job = cluster.getJob(jobid); 592 if (job != null) { 593 JobStatus status = JobStatus.downgrade(job.getStatus()); 594 if (status != null) { 595 return new NetworkedJob(status, cluster); 596 } 597 } 598 } catch (InterruptedException ie) { 599 throw new IOException(ie); 600 } 601 return null; 602 } 603 604 /**@deprecated Applications should rather use {@link #getJob(JobID)}. 605 */ 606 @Deprecated 607 public RunningJob getJob(String jobid) throws IOException { 608 return getJob(JobID.forName(jobid)); 609 } 610 611 private static final TaskReport[] EMPTY_TASK_REPORTS = new TaskReport[0]; 612 613 /** 614 * Get the information of the current state of the map tasks of a job. 615 * 616 * @param jobId the job to query. 617 * @return the list of all of the map tips. 618 * @throws IOException 619 */ 620 public TaskReport[] getMapTaskReports(JobID jobId) throws IOException { 621 return getTaskReports(jobId, TaskType.MAP); 622 } 623 624 private TaskReport[] getTaskReports(JobID jobId, TaskType type) throws IOException { 625 try { 626 Job j = cluster.getJob(jobId); 627 if(j == null) { 628 return EMPTY_TASK_REPORTS; 629 } 630 return TaskReport.downgradeArray(j.getTaskReports(type)); 631 } catch (InterruptedException ie) { 632 throw new IOException(ie); 633 } 634 } 635 636 /**@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}*/ 637 @Deprecated 638 public TaskReport[] getMapTaskReports(String jobId) throws IOException { 639 return getMapTaskReports(JobID.forName(jobId)); 640 } 641 642 /** 643 * Get the information of the current state of the reduce tasks of a job. 644 * 645 * @param jobId the job to query. 646 * @return the list of all of the reduce tips. 647 * @throws IOException 648 */ 649 public TaskReport[] getReduceTaskReports(JobID jobId) throws IOException { 650 return getTaskReports(jobId, TaskType.REDUCE); 651 } 652 653 /** 654 * Get the information of the current state of the cleanup tasks of a job. 655 * 656 * @param jobId the job to query. 657 * @return the list of all of the cleanup tips. 658 * @throws IOException 659 */ 660 public TaskReport[] getCleanupTaskReports(JobID jobId) throws IOException { 661 return getTaskReports(jobId, TaskType.JOB_CLEANUP); 662 } 663 664 /** 665 * Get the information of the current state of the setup tasks of a job. 666 * 667 * @param jobId the job to query. 668 * @return the list of all of the setup tips. 669 * @throws IOException 670 */ 671 public TaskReport[] getSetupTaskReports(JobID jobId) throws IOException { 672 return getTaskReports(jobId, TaskType.JOB_SETUP); 673 } 674 675 676 /**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ 677 @Deprecated 678 public TaskReport[] getReduceTaskReports(String jobId) throws IOException { 679 return getReduceTaskReports(JobID.forName(jobId)); 680 } 681 682 /** 683 * Display the information about a job's tasks, of a particular type and 684 * in a particular state 685 * 686 * @param jobId the ID of the job 687 * @param type the type of the task (map/reduce/setup/cleanup) 688 * @param state the state of the task 689 * (pending/running/completed/failed/killed) 690 */ 691 public void displayTasks(JobID jobId, String type, String state) 692 throws IOException { 693 try { 694 super.displayTasks(cluster.getJob(jobId), type, state); 695 } catch (InterruptedException ie) { 696 throw new IOException(ie); 697 } 698 } 699 700 /** 701 * Get status information about the Map-Reduce cluster. 702 * 703 * @return the status information about the Map-Reduce cluster as an object 704 * of {@link ClusterStatus}. 705 * @throws IOException 706 */ 707 public ClusterStatus getClusterStatus() throws IOException { 708 try { 709 ClusterMetrics metrics = cluster.getClusterStatus(); 710 return new ClusterStatus(metrics.getTaskTrackerCount(), 711 metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(), 712 metrics.getOccupiedMapSlots(), 713 metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(), 714 metrics.getReduceSlotCapacity(), 715 cluster.getJobTrackerStatus(), 716 metrics.getDecommissionedTaskTrackerCount()); 717 } catch (InterruptedException ie) { 718 throw new IOException(ie); 719 } 720 } 721 722 private Collection<String> arrayToStringList(TaskTrackerInfo[] objs) { 723 Collection<String> list = new ArrayList<String>(); 724 for (TaskTrackerInfo info: objs) { 725 list.add(info.getTaskTrackerName()); 726 } 727 return list; 728 } 729 730 private Collection<BlackListInfo> arrayToBlackListInfo(TaskTrackerInfo[] objs) { 731 Collection<BlackListInfo> list = new ArrayList<BlackListInfo>(); 732 for (TaskTrackerInfo info: objs) { 733 BlackListInfo binfo = new BlackListInfo(); 734 binfo.setTrackerName(info.getTaskTrackerName()); 735 binfo.setReasonForBlackListing(info.getReasonForBlacklist()); 736 binfo.setBlackListReport(info.getBlacklistReport()); 737 list.add(binfo); 738 } 739 return list; 740 } 741 742 /** 743 * Get status information about the Map-Reduce cluster. 744 * 745 * @param detailed if true then get a detailed status including the 746 * tracker names 747 * @return the status information about the Map-Reduce cluster as an object 748 * of {@link ClusterStatus}. 749 * @throws IOException 750 */ 751 public ClusterStatus getClusterStatus(boolean detailed) throws IOException { 752 try { 753 ClusterMetrics metrics = cluster.getClusterStatus(); 754 return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()), 755 arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()), 756 cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(), 757 metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(), 758 metrics.getReduceSlotCapacity(), 759 cluster.getJobTrackerStatus()); 760 } catch (InterruptedException ie) { 761 throw new IOException(ie); 762 } 763 } 764 765 766 /** 767 * Get the jobs that are not completed and not failed. 768 * 769 * @return array of {@link JobStatus} for the running/to-be-run jobs. 770 * @throws IOException 771 */ 772 public JobStatus[] jobsToComplete() throws IOException { 773 List<JobStatus> stats = new ArrayList<JobStatus>(); 774 for (JobStatus stat : getAllJobs()) { 775 if (!stat.isJobComplete()) { 776 stats.add(stat); 777 } 778 } 779 return stats.toArray(new JobStatus[0]); 780 } 781 782 /** 783 * Get the jobs that are submitted. 784 * 785 * @return array of {@link JobStatus} for the submitted jobs. 786 * @throws IOException 787 */ 788 public JobStatus[] getAllJobs() throws IOException { 789 try { 790 org.apache.hadoop.mapreduce.JobStatus[] jobs = cluster.getAllJobStatuses(); 791 JobStatus[] stats = new JobStatus[jobs.length]; 792 for (int i = 0; i < jobs.length; i++) { 793 stats[i] = JobStatus.downgrade(jobs[i]); 794 } 795 return stats; 796 } catch (InterruptedException ie) { 797 throw new IOException(ie); 798 } 799 } 800 801 /** 802 * Utility that submits a job, then polls for progress until the job is 803 * complete. 804 * 805 * @param job the job configuration. 806 * @throws IOException if the job fails 807 */ 808 public static RunningJob runJob(JobConf job) throws IOException { 809 JobClient jc = new JobClient(job); 810 RunningJob rj = jc.submitJob(job); 811 try { 812 if (!jc.monitorAndPrintJob(job, rj)) { 813 throw new IOException("Job failed!"); 814 } 815 } catch (InterruptedException ie) { 816 Thread.currentThread().interrupt(); 817 } 818 return rj; 819 } 820 821 /** 822 * Monitor a job and print status in real-time as progress is made and tasks 823 * fail. 824 * @param conf the job's configuration 825 * @param job the job to track 826 * @return true if the job succeeded 827 * @throws IOException if communication to the JobTracker fails 828 */ 829 public boolean monitorAndPrintJob(JobConf conf, 830 RunningJob job 831 ) throws IOException, InterruptedException { 832 return ((NetworkedJob)job).monitorAndPrintJob(); 833 } 834 835 static String getTaskLogURL(TaskAttemptID taskId, String baseUrl) { 836 return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); 837 } 838 839 static Configuration getConfiguration(String jobTrackerSpec) 840 { 841 Configuration conf = new Configuration(); 842 if (jobTrackerSpec != null) { 843 if (jobTrackerSpec.indexOf(":") >= 0) { 844 conf.set("mapred.job.tracker", jobTrackerSpec); 845 } else { 846 String classpathFile = "hadoop-" + jobTrackerSpec + ".xml"; 847 URL validate = conf.getResource(classpathFile); 848 if (validate == null) { 849 throw new RuntimeException(classpathFile + " not found on CLASSPATH"); 850 } 851 conf.addResource(classpathFile); 852 } 853 } 854 return conf; 855 } 856 857 /** 858 * Sets the output filter for tasks. only those tasks are printed whose 859 * output matches the filter. 860 * @param newValue task filter. 861 */ 862 @Deprecated 863 public void setTaskOutputFilter(TaskStatusFilter newValue){ 864 this.taskOutputFilter = newValue; 865 } 866 867 /** 868 * Get the task output filter out of the JobConf. 869 * 870 * @param job the JobConf to examine. 871 * @return the filter level. 872 */ 873 public static TaskStatusFilter getTaskOutputFilter(JobConf job) { 874 return TaskStatusFilter.valueOf(job.get("jobclient.output.filter", 875 "FAILED")); 876 } 877 878 /** 879 * Modify the JobConf to set the task output filter. 880 * 881 * @param job the JobConf to modify. 882 * @param newValue the value to set. 883 */ 884 public static void setTaskOutputFilter(JobConf job, 885 TaskStatusFilter newValue) { 886 job.set("jobclient.output.filter", newValue.toString()); 887 } 888 889 /** 890 * Returns task output filter. 891 * @return task filter. 892 */ 893 @Deprecated 894 public TaskStatusFilter getTaskOutputFilter(){ 895 return this.taskOutputFilter; 896 } 897 898 protected long getCounter(org.apache.hadoop.mapreduce.Counters cntrs, 899 String counterGroupName, String counterName) throws IOException { 900 Counters counters = Counters.downgrade(cntrs); 901 return counters.findCounter(counterGroupName, counterName).getValue(); 902 } 903 904 /** 905 * Get status information about the max available Maps in the cluster. 906 * 907 * @return the max available Maps in the cluster 908 * @throws IOException 909 */ 910 public int getDefaultMaps() throws IOException { 911 try { 912 return cluster.getClusterStatus().getMapSlotCapacity(); 913 } catch (InterruptedException ie) { 914 throw new IOException(ie); 915 } 916 } 917 918 /** 919 * Get status information about the max available Reduces in the cluster. 920 * 921 * @return the max available Reduces in the cluster 922 * @throws IOException 923 */ 924 public int getDefaultReduces() throws IOException { 925 try { 926 return cluster.getClusterStatus().getReduceSlotCapacity(); 927 } catch (InterruptedException ie) { 928 throw new IOException(ie); 929 } 930 } 931 932 /** 933 * Grab the jobtracker system directory path where job-specific files are to be placed. 934 * 935 * @return the system directory where job-specific files are to be placed. 936 */ 937 public Path getSystemDir() { 938 try { 939 return cluster.getSystemDir(); 940 } catch (IOException ioe) { 941 return null; 942 } catch (InterruptedException ie) { 943 return null; 944 } 945 } 946 947 private JobQueueInfo[] getJobQueueInfoArray(QueueInfo[] queues) 948 throws IOException { 949 JobQueueInfo[] ret = new JobQueueInfo[queues.length]; 950 for (int i = 0; i < queues.length; i++) { 951 ret[i] = new JobQueueInfo(queues[i]); 952 } 953 return ret; 954 } 955 956 /** 957 * Returns an array of queue information objects about root level queues 958 * configured 959 * 960 * @return the array of root level JobQueueInfo objects 961 * @throws IOException 962 */ 963 public JobQueueInfo[] getRootQueues() throws IOException { 964 try { 965 return getJobQueueInfoArray(cluster.getRootQueues()); 966 } catch (InterruptedException ie) { 967 throw new IOException(ie); 968 } 969 } 970 971 /** 972 * Returns an array of queue information objects about immediate children 973 * of queue queueName. 974 * 975 * @param queueName 976 * @return the array of immediate children JobQueueInfo objects 977 * @throws IOException 978 */ 979 public JobQueueInfo[] getChildQueues(String queueName) throws IOException { 980 try { 981 return getJobQueueInfoArray(cluster.getChildQueues(queueName)); 982 } catch (InterruptedException ie) { 983 throw new IOException(ie); 984 } 985 } 986 987 /** 988 * Return an array of queue information objects about all the Job Queues 989 * configured. 990 * 991 * @return Array of JobQueueInfo objects 992 * @throws IOException 993 */ 994 public JobQueueInfo[] getQueues() throws IOException { 995 try { 996 return getJobQueueInfoArray(cluster.getQueues()); 997 } catch (InterruptedException ie) { 998 throw new IOException(ie); 999 } 1000 } 1001 1002 /** 1003 * Gets all the jobs which were added to particular Job Queue 1004 * 1005 * @param queueName name of the Job Queue 1006 * @return Array of jobs present in the job queue 1007 * @throws IOException 1008 */ 1009 1010 public JobStatus[] getJobsFromQueue(String queueName) throws IOException { 1011 try { 1012 QueueInfo queue = cluster.getQueue(queueName); 1013 if (queue == null) { 1014 return null; 1015 } 1016 org.apache.hadoop.mapreduce.JobStatus[] stats = 1017 queue.getJobStatuses(); 1018 JobStatus[] ret = new JobStatus[stats.length]; 1019 for (int i = 0 ; i < stats.length; i++ ) { 1020 ret[i] = JobStatus.downgrade(stats[i]); 1021 } 1022 return ret; 1023 } catch (InterruptedException ie) { 1024 throw new IOException(ie); 1025 } 1026 } 1027 1028 /** 1029 * Gets the queue information associated to a particular Job Queue 1030 * 1031 * @param queueName name of the job queue. 1032 * @return Queue information associated to particular queue. 1033 * @throws IOException 1034 */ 1035 public JobQueueInfo getQueueInfo(String queueName) throws IOException { 1036 try { 1037 QueueInfo queueInfo = cluster.getQueue(queueName); 1038 if (queueInfo != null) { 1039 return new JobQueueInfo(queueInfo); 1040 } 1041 return null; 1042 } catch (InterruptedException ie) { 1043 throw new IOException(ie); 1044 } 1045 } 1046 1047 /** 1048 * Gets the Queue ACLs for current user 1049 * @return array of QueueAclsInfo object for current user. 1050 * @throws IOException 1051 */ 1052 public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException { 1053 try { 1054 org.apache.hadoop.mapreduce.QueueAclsInfo[] acls = 1055 cluster.getQueueAclsForCurrentUser(); 1056 QueueAclsInfo[] ret = new QueueAclsInfo[acls.length]; 1057 for (int i = 0 ; i < acls.length; i++ ) { 1058 ret[i] = QueueAclsInfo.downgrade(acls[i]); 1059 } 1060 return ret; 1061 } catch (InterruptedException ie) { 1062 throw new IOException(ie); 1063 } 1064 } 1065 1066 /** 1067 * Get a delegation token for the user from the JobTracker. 1068 * @param renewer the user who can renew the token 1069 * @return the new token 1070 * @throws IOException 1071 */ 1072 public Token<DelegationTokenIdentifier> 1073 getDelegationToken(Text renewer) throws IOException, InterruptedException { 1074 return cluster.getDelegationToken(renewer); 1075 } 1076 1077 /** 1078 * Renew a delegation token 1079 * @param token the token to renew 1080 * @return true if the renewal went well 1081 * @throws InvalidToken 1082 * @throws IOException 1083 * @deprecated Use {@link Token#renew} instead 1084 */ 1085 public long renewDelegationToken(Token<DelegationTokenIdentifier> token 1086 ) throws InvalidToken, IOException, 1087 InterruptedException { 1088 return token.renew(getConf()); 1089 } 1090 1091 /** 1092 * Cancel a delegation token from the JobTracker 1093 * @param token the token to cancel 1094 * @throws IOException 1095 * @deprecated Use {@link Token#cancel} instead 1096 */ 1097 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token 1098 ) throws InvalidToken, IOException, 1099 InterruptedException { 1100 token.cancel(getConf()); 1101 } 1102 1103 /** 1104 */ 1105 public static void main(String argv[]) throws Exception { 1106 int res = ToolRunner.run(new JobClient(), argv); 1107 System.exit(res); 1108 } 1109 } 1110