001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.hadoop;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.hadoop.fs.Path;
019 import org.apache.hadoop.fs.FileSystem;
020 import org.apache.hadoop.mapred.JobClient;
021 import org.apache.hadoop.mapred.JobConf;
022 import org.apache.hadoop.mapred.RunningJob;
023 import org.apache.hadoop.security.UserGroupInformation;
024
025 import java.util.Map;
026 import java.util.Properties;
027 import java.io.ByteArrayOutputStream;
028 import java.io.IOException;
029 import java.io.FileOutputStream;
030 import java.io.OutputStream;
031 import java.io.File;
032
033 public class MapReduceMain extends LauncherMain {
034
035 public static void main(String[] args) throws Exception {
036 run(MapReduceMain.class, args);
037 }
038
039 protected void run(String[] args) throws Exception {
040 System.out.println();
041 System.out.println("Oozie Map-Reduce action configuration");
042 System.out.println("=======================");
043
044 // loading action conf prepared by Oozie
045 Configuration actionConf = new Configuration(false);
046 actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
047
048 System.out.println("Oozie Map-Reduce Configuration: ");
049 System.out.println("------------------------");
050 ByteArrayOutputStream baos = new ByteArrayOutputStream();
051 actionConf.writeXml(baos);
052 baos.close();
053 System.out.println(new String(baos.toByteArray()));
054 System.out.println("------------------------");
055 System.out.println();
056
057 System.out.println("Submitting Oozie action Map-Reduce job");
058 System.out.println();
059 // submitting job
060 RunningJob runningJob = submitJob(actionConf);
061
062 // propagating job id back to Oozie
063 String jobId = runningJob.getID().toString();
064 Properties props = new Properties();
065 props.setProperty("id", jobId);
066 File idFile = new File(System.getProperty("oozie.action.newId.properties"));
067 OutputStream os = new FileOutputStream(idFile);
068 props.store(os, "");
069 os.close();
070
071 System.out.println("=======================");
072 System.out.println();
073 }
074
075 protected void addActionConf(JobConf jobConf, Configuration actionConf) {
076 for (Map.Entry<String, String> entry : actionConf) {
077 jobConf.set(entry.getKey(), entry.getValue());
078 }
079 }
080
081 protected RunningJob submitJob(Configuration actionConf) throws Exception {
082 JobConf jobConf = new JobConf();
083 addActionConf(jobConf, actionConf);
084
085 // propagate delegation related props from launcher job to MR job
086 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
087 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
088 }
089 JobClient jobClient = null;
090 RunningJob runJob = null;
091 boolean exception = false;
092 try {
093 jobClient = createJobClient(jobConf);
094 runJob = jobClient.submitJob(jobConf);
095 }
096 catch (Exception ex) {
097 exception = true;
098 throw ex;
099 }
100 finally {
101 try {
102 if (jobClient != null) {
103 jobClient.close();
104 }
105 }
106 catch (Exception ex) {
107 if (exception) {
108 System.out.println("JobClient Error: " + ex);
109 }
110 else {
111 throw ex;
112 }
113 }
114 }
115 return runJob;
116 }
117
118 @SuppressWarnings("unchecked")
119 protected JobClient createJobClient(JobConf jobConf) throws IOException {
120 return new JobClient(jobConf);
121 }
122
123 // allows any character in the value, the conf.setStrings() does not allow
124 // commas
125 public static void setStrings(Configuration conf, String key, String[] values) {
126 if (values != null) {
127 conf.setInt(key + ".size", values.length);
128 for (int i = 0; i < values.length; i++) {
129 conf.set(key + "." + i, values[i]);
130 }
131 }
132 }
133
134 public static String[] getStrings(Configuration conf, String key) {
135 String[] values = new String[conf.getInt(key + ".size", 0)];
136 for (int i = 0; i < values.length; i++) {
137 values[i] = conf.get(key + "." + i);
138 }
139 return values;
140 }
141
142 }