001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.hadoop;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.hadoop.mapred.JobClient;
019 import org.apache.hadoop.mapred.JobConf;
020 import org.apache.hadoop.mapred.RunningJob;
021
022 public class StreamingMain extends MapReduceMain {
023
024 public static void main(String[] args) throws Exception {
025 run(StreamingMain.class, args);
026 }
027
028 protected RunningJob submitJob(Configuration actionConf) throws Exception {
029 JobConf jobConf = new JobConf();
030
031 jobConf.set("mapred.mapper.class", "org.apache.hadoop.streaming.PipeMapper");
032 jobConf.set("mapred.reducer.class", "org.apache.hadoop.streaming.PipeReducer");
033 jobConf.set("mapred.map.runner.class", "org.apache.hadoop.streaming.PipeMapRunner");
034
035 jobConf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat");
036 jobConf.set("mapred.output.format.class", "org.apache.hadoop.mapred.TextOutputFormat");
037 jobConf.set("mapred.output.value.class", "org.apache.hadoop.io.Text");
038 jobConf.set("mapred.output.key.class", "org.apache.hadoop.io.Text");
039
040 jobConf.set("mapred.create.symlink", "yes");
041 jobConf.set("mapred.used.genericoptionsparser", "true");
042
043 jobConf.set("stream.addenvironment", "");
044
045 String value = actionConf.get("oozie.streaming.mapper");
046 if (value != null) {
047 jobConf.set("stream.map.streamprocessor", value);
048 }
049 value = actionConf.get("oozie.streaming.reducer");
050 if (value != null) {
051 jobConf.set("stream.reduce.streamprocessor", value);
052 }
053 value = actionConf.get("oozie.streaming.record-reader");
054 if (value != null) {
055 jobConf.set("stream.recordreader.class", value);
056 }
057 String[] values = getStrings(actionConf, "oozie.streaming.record-reader-mapping");
058 for (String s : values) {
059 String[] kv = s.split("=");
060 jobConf.set("stream.recordreader." + kv[0], kv[1]);
061 }
062 values = getStrings(actionConf, "oozie.streaming.env");
063 value = jobConf.get("stream.addenvironment", "");
064 if (value.length() > 0) {
065 value = value + " ";
066 }
067 for (String s : values) {
068 value = value + s + " ";
069 }
070 jobConf.set("stream.addenvironment", value);
071
072 addActionConf(jobConf, actionConf);
073
074 // propagate delegation related props from launcher job to MR job
075 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
076 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
077 }
078
079 JobClient jobClient = null;
080 RunningJob runJob = null;
081 boolean exception = false;
082 try {
083 jobClient = createJobClient(jobConf);
084 runJob = jobClient.submitJob(jobConf);
085 }
086 catch (Exception ex) {
087 exception = true;
088 throw ex;
089 }
090 finally {
091 try {
092 if (jobClient != null) {
093 jobClient.close();
094 }
095 }
096 catch (Exception ex) {
097 if (exception) {
098 System.out.println("JobClient Error: " + ex);
099 }
100 else {
101 throw ex;
102 }
103 }
104 }
105 return runJob;
106 }
107
108 public static void setStreaming(Configuration conf, String mapper, String reducer, String recordReader,
109 String[] recordReaderMapping, String[] env) {
110 if (mapper != null) {
111 conf.set("oozie.streaming.mapper", mapper);
112 }
113 if (reducer != null) {
114 conf.set("oozie.streaming.reducer", reducer);
115 }
116 if (recordReader != null) {
117 conf.set("oozie.streaming.record-reader", recordReader);
118 }
119 setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping);
120 setStrings(conf, "oozie.streaming.env", env);
121 }
122
123 }