001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.mapred;
019    
020    import java.io.FileNotFoundException;
021    import java.io.IOException;
022    import java.net.InetSocketAddress;
023    import java.net.URL;
024    import java.util.ArrayList;
025    import java.util.Collection;
026    import java.util.List;
027    
028    import org.apache.hadoop.classification.InterfaceAudience;
029    import org.apache.hadoop.classification.InterfaceStability;
030    import org.apache.hadoop.conf.Configuration;
031    import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo;
032    import org.apache.hadoop.mapreduce.Cluster;
033    import org.apache.hadoop.mapreduce.ClusterMetrics;
034    import org.apache.hadoop.mapreduce.Job;
035    import org.apache.hadoop.mapreduce.QueueInfo;
036    import org.apache.hadoop.mapreduce.TaskTrackerInfo;
037    import org.apache.hadoop.mapreduce.TaskType;
038    import org.apache.hadoop.mapreduce.filecache.DistributedCache;
039    import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
040    import org.apache.hadoop.mapreduce.tools.CLI;
041    import org.apache.hadoop.mapreduce.util.ConfigUtil;
042    import org.apache.hadoop.fs.FileSystem;
043    import org.apache.hadoop.fs.Path;
044    import org.apache.hadoop.io.Text;
045    import org.apache.hadoop.security.token.Token;
046    import org.apache.hadoop.security.token.TokenRenewer;
047    import org.apache.hadoop.security.token.SecretManager.InvalidToken;
048    import org.apache.hadoop.util.Tool;
049    import org.apache.hadoop.util.ToolRunner;
050    
051    /**
052     * <code>JobClient</code> is the primary interface for the user-job to interact
053     * with the cluster.
054     * 
055     * <code>JobClient</code> provides facilities to submit jobs, track their 
056     * progress, access component-tasks' reports/logs, get the Map-Reduce cluster
057     * status information etc.
058     * 
059     * <p>The job submission process involves:
060     * <ol>
061     *   <li>
062     *   Checking the input and output specifications of the job.
063     *   </li>
064     *   <li>
065     *   Computing the {@link InputSplit}s for the job.
066     *   </li>
067     *   <li>
068     *   Setup the requisite accounting information for the {@link DistributedCache} 
069     *   of the job, if necessary.
070     *   </li>
071     *   <li>
072     *   Copying the job's jar and configuration to the map-reduce system directory 
073     *   on the distributed file-system. 
074     *   </li>
075     *   <li>
076     *   Submitting the job to the cluster and optionally monitoring
077     *   it's status.
078     *   </li>
079     * </ol></p>
080     *  
081     * Normally the user creates the application, describes various facets of the
082     * job via {@link JobConf} and then uses the <code>JobClient</code> to submit 
083     * the job and monitor its progress.
084     * 
085     * <p>Here is an example on how to use <code>JobClient</code>:</p>
086     * <p><blockquote><pre>
087     *     // Create a new JobConf
088     *     JobConf job = new JobConf(new Configuration(), MyJob.class);
089     *     
090     *     // Specify various job-specific parameters     
091     *     job.setJobName("myjob");
092     *     
093     *     job.setInputPath(new Path("in"));
094     *     job.setOutputPath(new Path("out"));
095     *     
096     *     job.setMapperClass(MyJob.MyMapper.class);
097     *     job.setReducerClass(MyJob.MyReducer.class);
098     *
099     *     // Submit the job, then poll for progress until the job is complete
100     *     JobClient.runJob(job);
101     * </pre></blockquote></p>
102     * 
103     * <h4 id="JobControl">Job Control</h4>
104     * 
105     * <p>At times clients would chain map-reduce jobs to accomplish complex tasks 
106     * which cannot be done via a single map-reduce job. This is fairly easy since 
107     * the output of the job, typically, goes to distributed file-system and that 
108     * can be used as the input for the next job.</p>
109     * 
110     * <p>However, this also means that the onus on ensuring jobs are complete 
111     * (success/failure) lies squarely on the clients. In such situations the 
112     * various job-control options are:
113     * <ol>
114     *   <li>
115     *   {@link #runJob(JobConf)} : submits the job and returns only after 
116     *   the job has completed.
117     *   </li>
118     *   <li>
119     *   {@link #submitJob(JobConf)} : only submits the job, then poll the 
120     *   returned handle to the {@link RunningJob} to query status and make 
121     *   scheduling decisions.
122     *   </li>
123     *   <li>
124     *   {@link JobConf#setJobEndNotificationURI(String)} : setup a notification
125     *   on job-completion, thus avoiding polling.
126     *   </li>
127     * </ol></p>
128     * 
129     * @see JobConf
130     * @see ClusterStatus
131     * @see Tool
132     * @see DistributedCache
133     * @deprecated Use {@link Job} and {@link Cluster} instead
134     */
135    @Deprecated
136    @InterfaceAudience.Public
137    @InterfaceStability.Stable
138    public class JobClient extends CLI {
139      public static enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL }
140      private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; 
141    
142      static{
143        ConfigUtil.loadResources();
144      }
145    
146      /**
147       * A NetworkedJob is an implementation of RunningJob.  It holds
148       * a JobProfile object to provide some info, and interacts with the
149       * remote service to provide certain functionality.
150       */
151      static class NetworkedJob implements RunningJob {
152        Job job;
153        /**
154         * We store a JobProfile and a timestamp for when we last
155         * acquired the job profile.  If the job is null, then we cannot
156         * perform any of the tasks.  The job might be null if the cluster
157         * has completely forgotten about the job.  (eg, 24 hours after the
158         * job completes.)
159         */
160        public NetworkedJob(JobStatus status, Cluster cluster) throws IOException {
161          job = Job.getInstance(cluster, status, new JobConf(status.getJobFile()));
162        }
163    
164        public NetworkedJob(Job job) throws IOException {
165          this.job = job;
166        }
167    
168        public Configuration getConfiguration() {
169          return job.getConfiguration();
170        }
171    
172        /**
173         * An identifier for the job
174         */
175        public JobID getID() {
176          return JobID.downgrade(job.getJobID());
177        }
178        
179        /** @deprecated This method is deprecated and will be removed. Applications should 
180         * rather use {@link #getID()}.*/
181        @Deprecated
182        public String getJobID() {
183          return getID().toString();
184        }
185        
186        /**
187         * The user-specified job name
188         */
189        public String getJobName() {
190          return job.getJobName();
191        }
192    
193        /**
194         * The name of the job file
195         */
196        public String getJobFile() {
197          return job.getJobFile();
198        }
199    
200        /**
201         * A URL where the job's status can be seen
202         */
203        public String getTrackingURL() {
204          return job.getTrackingURL();
205        }
206    
207        /**
208         * A float between 0.0 and 1.0, indicating the % of map work
209         * completed.
210         */
211        public float mapProgress() throws IOException {
212          try {
213            return job.mapProgress();
214          } catch (InterruptedException ie) {
215            throw new IOException(ie);
216          }
217        }
218    
219        /**
220         * A float between 0.0 and 1.0, indicating the % of reduce work
221         * completed.
222         */
223        public float reduceProgress() throws IOException {
224          try {
225            return job.reduceProgress();
226          } catch (InterruptedException ie) {
227            throw new IOException(ie);
228          }
229        }
230    
231        /**
232         * A float between 0.0 and 1.0, indicating the % of cleanup work
233         * completed.
234         */
235        public float cleanupProgress() throws IOException {
236          try {
237            return job.cleanupProgress();
238          } catch (InterruptedException ie) {
239            throw new IOException(ie);
240          }
241        }
242    
243        /**
244         * A float between 0.0 and 1.0, indicating the % of setup work
245         * completed.
246         */
247        public float setupProgress() throws IOException {
248          try {
249            return job.setupProgress();
250          } catch (InterruptedException ie) {
251            throw new IOException(ie);
252          }
253        }
254    
255        /**
256         * Returns immediately whether the whole job is done yet or not.
257         */
258        public synchronized boolean isComplete() throws IOException {
259          try {
260            return job.isComplete();
261          } catch (InterruptedException ie) {
262            throw new IOException(ie);
263          }
264        }
265    
266        /**
267         * True iff job completed successfully.
268         */
269        public synchronized boolean isSuccessful() throws IOException {
270          try {
271            return job.isSuccessful();
272          } catch (InterruptedException ie) {
273            throw new IOException(ie);
274          }
275        }
276    
277        /**
278         * Blocks until the job is finished
279         */
280        public void waitForCompletion() throws IOException {
281          try {
282            job.waitForCompletion(false);
283          } catch (InterruptedException ie) {
284            throw new IOException(ie);
285          } catch (ClassNotFoundException ce) {
286            throw new IOException(ce);
287          }
288        }
289    
290        /**
291         * Tells the service to get the state of the current job.
292         */
293        public synchronized int getJobState() throws IOException {
294          try {
295            return job.getJobState().getValue();
296          } catch (InterruptedException ie) {
297            throw new IOException(ie);
298          }
299        }
300        
301        /**
302         * Tells the service to terminate the current job.
303         */
304        public synchronized void killJob() throws IOException {
305          try {
306            job.killJob();
307          } catch (InterruptedException ie) {
308            throw new IOException(ie);
309          }
310        }
311       
312        
313        /** Set the priority of the job.
314        * @param priority new priority of the job. 
315        */
316        public synchronized void setJobPriority(String priority) 
317                                                    throws IOException {
318          try {
319            job.setPriority(
320              org.apache.hadoop.mapreduce.JobPriority.valueOf(priority));
321          } catch (InterruptedException ie) {
322            throw new IOException(ie);
323          }
324        }
325        
326        /**
327         * Kill indicated task attempt.
328         * @param taskId the id of the task to kill.
329         * @param shouldFail if true the task is failed and added to failed tasks list, otherwise
330         * it is just killed, w/o affecting job failure status.
331         */
332        public synchronized void killTask(TaskAttemptID taskId,
333            boolean shouldFail) throws IOException {
334          try {
335            if (shouldFail) {
336              job.failTask(taskId);
337            } else {
338              job.killTask(taskId);
339            }
340          } catch (InterruptedException ie) {
341            throw new IOException(ie);
342          }
343        }
344    
345        /** @deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}*/
346        @Deprecated
347        public synchronized void killTask(String taskId, boolean shouldFail) throws IOException {
348          killTask(TaskAttemptID.forName(taskId), shouldFail);
349        }
350        
351        /**
352         * Fetch task completion events from cluster for this job. 
353         */
354        public synchronized TaskCompletionEvent[] getTaskCompletionEvents(
355            int startFrom) throws IOException {
356          try {
357            org.apache.hadoop.mapreduce.TaskCompletionEvent[] acls = 
358              job.getTaskCompletionEvents(startFrom, 10);
359            TaskCompletionEvent[] ret = new TaskCompletionEvent[acls.length];
360            for (int i = 0 ; i < acls.length; i++ ) {
361              ret[i] = TaskCompletionEvent.downgrade(acls[i]);
362            }
363            return ret;
364          } catch (InterruptedException ie) {
365            throw new IOException(ie);
366          }
367        }
368    
369        /**
370         * Dump stats to screen
371         */
372        @Override
373        public String toString() {
374          return job.toString();
375        }
376            
377        /**
378         * Returns the counters for this job
379         */
380        public Counters getCounters() throws IOException {
381          try { 
382            Counters result = null;
383            org.apache.hadoop.mapreduce.Counters temp = job.getCounters();
384            if(temp != null) {
385              result = Counters.downgrade(temp);
386            }
387            return result;
388          } catch (InterruptedException ie) {
389            throw new IOException(ie);
390          }
391        }
392        
393        @Override
394        public String[] getTaskDiagnostics(TaskAttemptID id) throws IOException {
395          try { 
396            return job.getTaskDiagnostics(id);
397          } catch (InterruptedException ie) {
398            throw new IOException(ie);
399          }
400        }
401    
402        public String getHistoryUrl() throws IOException {
403          try {
404            return job.getHistoryUrl();
405          } catch (InterruptedException ie) {
406            throw new IOException(ie);
407          }
408        }
409    
410        public boolean isRetired() throws IOException {
411          try {
412            return job.isRetired();
413          } catch (InterruptedException ie) {
414            throw new IOException(ie);
415          }
416        }
417        
418        boolean monitorAndPrintJob() throws IOException, InterruptedException {
419          return job.monitorAndPrintJob();
420        }
421      }
422    
423      Cluster cluster;
424      
425      /**
426       * Create a job client.
427       */
428      public JobClient() {
429      }
430        
431      /**
432       * Build a job client with the given {@link JobConf}, and connect to the 
433       * default cluster
434       * 
435       * @param conf the job configuration.
436       * @throws IOException
437       */
438      public JobClient(JobConf conf) throws IOException {
439        init(conf);
440      }
441    
442      /**
443       * Build a job client with the given {@link Configuration}, 
444       * and connect to the default cluster
445       * 
446       * @param conf the configuration.
447       * @throws IOException
448       */
449      public JobClient(Configuration conf) throws IOException {
450        init(new JobConf(conf));
451      }
452    
453      /**
454       * Connect to the default cluster
455       * @param conf the job configuration.
456       * @throws IOException
457       */
458      public void init(JobConf conf) throws IOException {
459        setConf(conf);
460        cluster = new Cluster(conf);
461      }
462    
463      @InterfaceAudience.Private
464      public static class Renewer extends TokenRenewer {
465    
466        @Override
467        public boolean handleKind(Text kind) {
468          return DelegationTokenIdentifier.MAPREDUCE_DELEGATION_KIND.equals(kind);
469        }
470    
471        @SuppressWarnings("unchecked")
472        @Override
473        public long renew(Token<?> token, Configuration conf
474                          ) throws IOException, InterruptedException {
475          return new Cluster(conf).
476            renewDelegationToken((Token<DelegationTokenIdentifier>) token);
477        }
478    
479        @SuppressWarnings("unchecked")
480        @Override
481        public void cancel(Token<?> token, Configuration conf
482                           ) throws IOException, InterruptedException {
483          new Cluster(conf).
484            cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
485        }
486    
487        @Override
488        public boolean isManaged(Token<?> token) throws IOException {
489          return true;
490        }
491        
492      }
493    
494      /**
495       * Build a job client, connect to the indicated job tracker.
496       * 
497       * @param jobTrackAddr the job tracker to connect to.
498       * @param conf configuration.
499       */
500      public JobClient(InetSocketAddress jobTrackAddr, 
501                       Configuration conf) throws IOException {
502        cluster = new Cluster(jobTrackAddr, conf);
503      }
504    
505      /**
506       * Close the <code>JobClient</code>.
507       */
508      public synchronized void close() throws IOException {
509        cluster.close();
510      }
511    
512      /**
513       * Get a filesystem handle.  We need this to prepare jobs
514       * for submission to the MapReduce system.
515       * 
516       * @return the filesystem handle.
517       */
518      public synchronized FileSystem getFs() throws IOException {
519        try { 
520          return cluster.getFileSystem();
521        } catch (InterruptedException ie) {
522          throw new IOException(ie);
523        }
524      }
525      
526      /**
527       * Get a handle to the Cluster
528       */
529      public Cluster getClusterHandle() {
530        return cluster;
531      }
532      
533      /**
534       * Submit a job to the MR system.
535       * 
536       * This returns a handle to the {@link RunningJob} which can be used to track
537       * the running-job.
538       * 
539       * @param jobFile the job configuration.
540       * @return a handle to the {@link RunningJob} which can be used to track the
541       *         running-job.
542       * @throws FileNotFoundException
543       * @throws InvalidJobConfException
544       * @throws IOException
545       */
546      public RunningJob submitJob(String jobFile) throws FileNotFoundException, 
547                                                         InvalidJobConfException, 
548                                                         IOException {
549        // Load in the submitted job details
550        JobConf job = new JobConf(jobFile);
551        return submitJob(job);
552      }
553        
554      /**
555       * Submit a job to the MR system.
556       * This returns a handle to the {@link RunningJob} which can be used to track
557       * the running-job.
558       * 
559       * @param conf the job configuration.
560       * @return a handle to the {@link RunningJob} which can be used to track the
561       *         running-job.
562       * @throws FileNotFoundException
563       * @throws IOException
564       */
565      public RunningJob submitJob(JobConf conf) throws FileNotFoundException,
566                                                      IOException {
567        try {
568          conf.setBooleanIfUnset("mapred.mapper.new-api", false);
569          conf.setBooleanIfUnset("mapred.reducer.new-api", false);
570          Job job = Job.getInstance(conf);
571          job.submit();
572          return new NetworkedJob(job);
573        } catch (InterruptedException ie) {
574          throw new IOException("interrupted", ie);
575        } catch (ClassNotFoundException cnfe) {
576          throw new IOException("class not found", cnfe);
577        }
578      }
579    
580      /**
581       * Get an {@link RunningJob} object to track an ongoing job.  Returns
582       * null if the id does not correspond to any known job.
583       * 
584       * @param jobid the jobid of the job.
585       * @return the {@link RunningJob} handle to track the job, null if the 
586       *         <code>jobid</code> doesn't correspond to any known job.
587       * @throws IOException
588       */
589      public RunningJob getJob(JobID jobid) throws IOException {
590        try {
591          Job job = cluster.getJob(jobid);
592          if (job != null) {
593            JobStatus status = JobStatus.downgrade(job.getStatus());
594            if (status != null) {
595              return new NetworkedJob(status, cluster);
596            } 
597          }
598        } catch (InterruptedException ie) {
599          throw new IOException(ie);
600        }
601        return null;
602      }
603    
604      /**@deprecated Applications should rather use {@link #getJob(JobID)}. 
605       */
606      @Deprecated
607      public RunningJob getJob(String jobid) throws IOException {
608        return getJob(JobID.forName(jobid));
609      }
610      
611      private static final TaskReport[] EMPTY_TASK_REPORTS = new TaskReport[0];
612      
613      /**
614       * Get the information of the current state of the map tasks of a job.
615       * 
616       * @param jobId the job to query.
617       * @return the list of all of the map tips.
618       * @throws IOException
619       */
620      public TaskReport[] getMapTaskReports(JobID jobId) throws IOException {
621        return getTaskReports(jobId, TaskType.MAP);
622      }
623      
624      private TaskReport[] getTaskReports(JobID jobId, TaskType type) throws IOException {
625        try {
626          Job j = cluster.getJob(jobId);
627          if(j == null) {
628            return EMPTY_TASK_REPORTS;
629          }
630          return TaskReport.downgradeArray(j.getTaskReports(type));
631        } catch (InterruptedException ie) {
632          throw new IOException(ie);
633        }
634      }
635      
636      /**@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}*/
637      @Deprecated
638      public TaskReport[] getMapTaskReports(String jobId) throws IOException {
639        return getMapTaskReports(JobID.forName(jobId));
640      }
641      
642      /**
643       * Get the information of the current state of the reduce tasks of a job.
644       * 
645       * @param jobId the job to query.
646       * @return the list of all of the reduce tips.
647       * @throws IOException
648       */    
649      public TaskReport[] getReduceTaskReports(JobID jobId) throws IOException {
650        return getTaskReports(jobId, TaskType.REDUCE);
651      }
652    
653      /**
654       * Get the information of the current state of the cleanup tasks of a job.
655       * 
656       * @param jobId the job to query.
657       * @return the list of all of the cleanup tips.
658       * @throws IOException
659       */    
660      public TaskReport[] getCleanupTaskReports(JobID jobId) throws IOException {
661        return getTaskReports(jobId, TaskType.JOB_CLEANUP);
662      }
663    
664      /**
665       * Get the information of the current state of the setup tasks of a job.
666       * 
667       * @param jobId the job to query.
668       * @return the list of all of the setup tips.
669       * @throws IOException
670       */    
671      public TaskReport[] getSetupTaskReports(JobID jobId) throws IOException {
672        return getTaskReports(jobId, TaskType.JOB_SETUP);
673      }
674    
675      
676      /**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/
677      @Deprecated
678      public TaskReport[] getReduceTaskReports(String jobId) throws IOException {
679        return getReduceTaskReports(JobID.forName(jobId));
680      }
681      
682      /**
683       * Display the information about a job's tasks, of a particular type and
684       * in a particular state
685       * 
686       * @param jobId the ID of the job
687       * @param type the type of the task (map/reduce/setup/cleanup)
688       * @param state the state of the task 
689       * (pending/running/completed/failed/killed)
690       */
691      public void displayTasks(JobID jobId, String type, String state) 
692      throws IOException {
693        try {
694          super.displayTasks(cluster.getJob(jobId), type, state);
695        } catch (InterruptedException ie) {
696          throw new IOException(ie);
697        }
698      }
699      
700      /**
701       * Get status information about the Map-Reduce cluster.
702       *  
703       * @return the status information about the Map-Reduce cluster as an object
704       *         of {@link ClusterStatus}.
705       * @throws IOException
706       */
707      public ClusterStatus getClusterStatus() throws IOException {
708        try {
709          ClusterMetrics metrics = cluster.getClusterStatus();
710          return new ClusterStatus(metrics.getTaskTrackerCount(),
711            metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
712            metrics.getOccupiedMapSlots(),
713            metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
714            metrics.getReduceSlotCapacity(),
715            cluster.getJobTrackerStatus(),
716            metrics.getDecommissionedTaskTrackerCount());
717        } catch (InterruptedException ie) {
718          throw new IOException(ie);
719        }
720      }
721    
722      private  Collection<String> arrayToStringList(TaskTrackerInfo[] objs) {
723        Collection<String> list = new ArrayList<String>();
724        for (TaskTrackerInfo info: objs) {
725          list.add(info.getTaskTrackerName());
726        }
727        return list;
728      }
729    
730      private  Collection<BlackListInfo> arrayToBlackListInfo(TaskTrackerInfo[] objs) {
731        Collection<BlackListInfo> list = new ArrayList<BlackListInfo>();
732        for (TaskTrackerInfo info: objs) {
733          BlackListInfo binfo = new BlackListInfo();
734          binfo.setTrackerName(info.getTaskTrackerName());
735          binfo.setReasonForBlackListing(info.getReasonForBlacklist());
736          binfo.setBlackListReport(info.getBlacklistReport());
737          list.add(binfo);
738        }
739        return list;
740      }
741    
742      /**
743       * Get status information about the Map-Reduce cluster.
744       *  
745       * @param  detailed if true then get a detailed status including the
746       *         tracker names
747       * @return the status information about the Map-Reduce cluster as an object
748       *         of {@link ClusterStatus}.
749       * @throws IOException
750       */
751      public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
752        try {
753          ClusterMetrics metrics = cluster.getClusterStatus();
754          return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
755            arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
756            cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
757            metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
758            metrics.getReduceSlotCapacity(), 
759            cluster.getJobTrackerStatus());
760        } catch (InterruptedException ie) {
761          throw new IOException(ie);
762        }
763      }
764        
765    
766      /** 
767       * Get the jobs that are not completed and not failed.
768       * 
769       * @return array of {@link JobStatus} for the running/to-be-run jobs.
770       * @throws IOException
771       */
772      public JobStatus[] jobsToComplete() throws IOException {
773        List<JobStatus> stats = new ArrayList<JobStatus>();
774        for (JobStatus stat : getAllJobs()) {
775          if (!stat.isJobComplete()) {
776            stats.add(stat);
777          }
778        }
779        return stats.toArray(new JobStatus[0]);
780      }
781    
782      /** 
783       * Get the jobs that are submitted.
784       * 
785       * @return array of {@link JobStatus} for the submitted jobs.
786       * @throws IOException
787       */
788      public JobStatus[] getAllJobs() throws IOException {
789        try {
790          org.apache.hadoop.mapreduce.JobStatus[] jobs = cluster.getAllJobStatuses();
791          JobStatus[] stats = new JobStatus[jobs.length];
792          for (int i = 0; i < jobs.length; i++) {
793            stats[i] = JobStatus.downgrade(jobs[i]);
794          }
795          return stats;
796        } catch (InterruptedException ie) {
797          throw new IOException(ie);
798        }
799      }
800      
801      /** 
802       * Utility that submits a job, then polls for progress until the job is
803       * complete.
804       * 
805       * @param job the job configuration.
806       * @throws IOException if the job fails
807       */
808      public static RunningJob runJob(JobConf job) throws IOException {
809        JobClient jc = new JobClient(job);
810        RunningJob rj = jc.submitJob(job);
811        try {
812          if (!jc.monitorAndPrintJob(job, rj)) {
813            throw new IOException("Job failed!");
814          }
815        } catch (InterruptedException ie) {
816          Thread.currentThread().interrupt();
817        }
818        return rj;
819      }
820      
821      /**
822       * Monitor a job and print status in real-time as progress is made and tasks 
823       * fail.
824       * @param conf the job's configuration
825       * @param job the job to track
826       * @return true if the job succeeded
827       * @throws IOException if communication to the JobTracker fails
828       */
829      public boolean monitorAndPrintJob(JobConf conf, 
830                                        RunningJob job
831      ) throws IOException, InterruptedException {
832        return ((NetworkedJob)job).monitorAndPrintJob();
833      }
834    
835      static String getTaskLogURL(TaskAttemptID taskId, String baseUrl) {
836        return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); 
837      }
838      
839      static Configuration getConfiguration(String jobTrackerSpec)
840      {
841        Configuration conf = new Configuration();
842        if (jobTrackerSpec != null) {        
843          if (jobTrackerSpec.indexOf(":") >= 0) {
844            conf.set("mapred.job.tracker", jobTrackerSpec);
845          } else {
846            String classpathFile = "hadoop-" + jobTrackerSpec + ".xml";
847            URL validate = conf.getResource(classpathFile);
848            if (validate == null) {
849              throw new RuntimeException(classpathFile + " not found on CLASSPATH");
850            }
851            conf.addResource(classpathFile);
852          }
853        }
854        return conf;
855      }
856    
857      /**
858       * Sets the output filter for tasks. only those tasks are printed whose
859       * output matches the filter. 
860       * @param newValue task filter.
861       */
862      @Deprecated
863      public void setTaskOutputFilter(TaskStatusFilter newValue){
864        this.taskOutputFilter = newValue;
865      }
866        
867      /**
868       * Get the task output filter out of the JobConf.
869       * 
870       * @param job the JobConf to examine.
871       * @return the filter level.
872       */
873      public static TaskStatusFilter getTaskOutputFilter(JobConf job) {
874        return TaskStatusFilter.valueOf(job.get("jobclient.output.filter", 
875                                                "FAILED"));
876      }
877        
878      /**
879       * Modify the JobConf to set the task output filter.
880       * 
881       * @param job the JobConf to modify.
882       * @param newValue the value to set.
883       */
884      public static void setTaskOutputFilter(JobConf job, 
885                                             TaskStatusFilter newValue) {
886        job.set("jobclient.output.filter", newValue.toString());
887      }
888        
889      /**
890       * Returns task output filter.
891       * @return task filter. 
892       */
893      @Deprecated
894      public TaskStatusFilter getTaskOutputFilter(){
895        return this.taskOutputFilter; 
896      }
897    
898      protected long getCounter(org.apache.hadoop.mapreduce.Counters cntrs,
899          String counterGroupName, String counterName) throws IOException {
900        Counters counters = Counters.downgrade(cntrs);
901        return counters.findCounter(counterGroupName, counterName).getValue();
902      }
903    
904      /**
905       * Get status information about the max available Maps in the cluster.
906       *  
907       * @return the max available Maps in the cluster
908       * @throws IOException
909       */
910      public int getDefaultMaps() throws IOException {
911        try {
912          return cluster.getClusterStatus().getMapSlotCapacity();
913        } catch (InterruptedException ie) {
914          throw new IOException(ie);
915        }
916      }
917    
918      /**
919       * Get status information about the max available Reduces in the cluster.
920       *  
921       * @return the max available Reduces in the cluster
922       * @throws IOException
923       */
924      public int getDefaultReduces() throws IOException {
925        try {
926          return cluster.getClusterStatus().getReduceSlotCapacity();
927        } catch (InterruptedException ie) {
928          throw new IOException(ie);
929        }
930      }
931    
932      /**
933       * Grab the jobtracker system directory path where job-specific files are to be placed.
934       * 
935       * @return the system directory where job-specific files are to be placed.
936       */
937      public Path getSystemDir() {
938        try {
939          return cluster.getSystemDir();
940        } catch (IOException ioe) {
941          return null;
942        } catch (InterruptedException ie) {
943          return null;
944        }
945      }
946    
947      private JobQueueInfo[] getJobQueueInfoArray(QueueInfo[] queues) 
948      throws IOException {
949        JobQueueInfo[] ret = new JobQueueInfo[queues.length];
950        for (int i = 0; i < queues.length; i++) {
951          ret[i] = new JobQueueInfo(queues[i]);
952        }
953        return ret;
954      }
955    
956      /**
957       * Returns an array of queue information objects about root level queues
958       * configured
959       *
960       * @return the array of root level JobQueueInfo objects
961       * @throws IOException
962       */
963      public JobQueueInfo[] getRootQueues() throws IOException {
964        try {
965          return getJobQueueInfoArray(cluster.getRootQueues());
966        } catch (InterruptedException ie) {
967          throw new IOException(ie);
968        }
969      }
970    
971      /**
972       * Returns an array of queue information objects about immediate children
973       * of queue queueName.
974       * 
975       * @param queueName
976       * @return the array of immediate children JobQueueInfo objects
977       * @throws IOException
978       */
979      public JobQueueInfo[] getChildQueues(String queueName) throws IOException {
980        try {
981          return getJobQueueInfoArray(cluster.getChildQueues(queueName));
982        } catch (InterruptedException ie) {
983          throw new IOException(ie);
984        }
985      }
986      
987      /**
988       * Return an array of queue information objects about all the Job Queues
989       * configured.
990       * 
991       * @return Array of JobQueueInfo objects
992       * @throws IOException
993       */
994      public JobQueueInfo[] getQueues() throws IOException {
995        try {
996          return getJobQueueInfoArray(cluster.getQueues());
997        } catch (InterruptedException ie) {
998          throw new IOException(ie);
999        }
1000      }
1001      
1002      /**
1003       * Gets all the jobs which were added to particular Job Queue
1004       * 
1005       * @param queueName name of the Job Queue
1006       * @return Array of jobs present in the job queue
1007       * @throws IOException
1008       */
1009      
1010      public JobStatus[] getJobsFromQueue(String queueName) throws IOException {
1011        try {
1012          QueueInfo queue = cluster.getQueue(queueName);
1013          if (queue == null) {
1014            return null;
1015          }
1016          org.apache.hadoop.mapreduce.JobStatus[] stats = 
1017            queue.getJobStatuses();
1018          JobStatus[] ret = new JobStatus[stats.length];
1019          for (int i = 0 ; i < stats.length; i++ ) {
1020            ret[i] = JobStatus.downgrade(stats[i]);
1021          }
1022          return ret;
1023        } catch (InterruptedException ie) {
1024          throw new IOException(ie);
1025        }
1026      }
1027      
1028      /**
1029       * Gets the queue information associated to a particular Job Queue
1030       * 
1031       * @param queueName name of the job queue.
1032       * @return Queue information associated to particular queue.
1033       * @throws IOException
1034       */
1035      public JobQueueInfo getQueueInfo(String queueName) throws IOException {
1036        try {
1037          QueueInfo queueInfo = cluster.getQueue(queueName);
1038          if (queueInfo != null) {
1039            return new JobQueueInfo(queueInfo);
1040          }
1041          return null;
1042        } catch (InterruptedException ie) {
1043          throw new IOException(ie);
1044        }
1045      }
1046      
1047      /**
1048       * Gets the Queue ACLs for current user
1049       * @return array of QueueAclsInfo object for current user.
1050       * @throws IOException
1051       */
1052      public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException {
1053        try {
1054          org.apache.hadoop.mapreduce.QueueAclsInfo[] acls = 
1055            cluster.getQueueAclsForCurrentUser();
1056          QueueAclsInfo[] ret = new QueueAclsInfo[acls.length];
1057          for (int i = 0 ; i < acls.length; i++ ) {
1058            ret[i] = QueueAclsInfo.downgrade(acls[i]);
1059          }
1060          return ret;
1061        } catch (InterruptedException ie) {
1062          throw new IOException(ie);
1063        }
1064      }
1065    
1066      /**
1067       * Get a delegation token for the user from the JobTracker.
1068       * @param renewer the user who can renew the token
1069       * @return the new token
1070       * @throws IOException
1071       */
1072      public Token<DelegationTokenIdentifier> 
1073        getDelegationToken(Text renewer) throws IOException, InterruptedException {
1074        return cluster.getDelegationToken(renewer);
1075      }
1076    
1077      /**
1078       * Renew a delegation token
1079       * @param token the token to renew
1080       * @return true if the renewal went well
1081       * @throws InvalidToken
1082       * @throws IOException
1083       * @deprecated Use {@link Token#renew} instead
1084       */
1085      public long renewDelegationToken(Token<DelegationTokenIdentifier> token
1086                                       ) throws InvalidToken, IOException, 
1087                                                InterruptedException {
1088        return token.renew(getConf());
1089      }
1090    
1091      /**
1092       * Cancel a delegation token from the JobTracker
1093       * @param token the token to cancel
1094       * @throws IOException
1095       * @deprecated Use {@link Token#cancel} instead
1096       */
1097      public void cancelDelegationToken(Token<DelegationTokenIdentifier> token
1098                                        ) throws InvalidToken, IOException, 
1099                                                 InterruptedException {
1100        token.cancel(getConf());
1101      }
1102    
1103      /**
1104       */
1105      public static void main(String argv[]) throws Exception {
1106        int res = ToolRunner.run(new JobClient(), argv);
1107        System.exit(res);
1108      }
1109    }
1110