001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.action.hadoop;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.hadoop.mapred.JobConf;
019 import org.apache.hadoop.mapred.RunningJob;
020 import org.apache.hadoop.mapred.pipes.Submitter;
021 import org.apache.hadoop.filecache.DistributedCache;
022
023 public class PipesMain extends MapReduceMain {
024
025 public static void main(String[] args) throws Exception {
026 run(PipesMain.class, args);
027 }
028
029 protected RunningJob submitJob(Configuration actionConf) throws Exception {
030 JobConf jobConf = new JobConf();
031
032 String value = actionConf.get("oozie.pipes.map");
033 if (value != null) {
034 jobConf.setBoolean("hadoop.pipes.java.mapper", true);
035 jobConf.set("mapred.mapper.class", value);
036 }
037 value = actionConf.get("oozie.pipes.reduce");
038 if (value != null) {
039 jobConf.setBoolean("hadoop.pipes.java.reducer", true);
040 jobConf.set("mapred.reducer.class", value);
041 }
042 value = actionConf.get("oozie.pipes.inputformat");
043 if (value != null) {
044 jobConf.setBoolean("hadoop.pipes.java.recordreader", true);
045 jobConf.set("mapred.input.format.class", value);
046 }
047 value = actionConf.get("oozie.pipes.partitioner");
048 if (value != null) {
049 jobConf.set("mapred.partitioner.class", value);
050 }
051 value = actionConf.get("oozie.pipes.writer");
052 if (value != null) {
053 jobConf.setBoolean("hadoop.pipes.java.recordwriter", true);
054 jobConf.set("mapred.output.format.class", value);
055 }
056 value = actionConf.get("oozie.pipes.program");
057 if (value != null) {
058 jobConf.set("hadoop.pipes.executable", value);
059 if (value.contains("#")) {
060 DistributedCache.createSymlink(jobConf);
061 }
062 }
063
064 addActionConf(jobConf, actionConf);
065
066 //propagate delegation related props from launcher job to MR job
067 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
068 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
069 }
070
071 return Submitter.jobSubmit(jobConf);
072 }
073
074 public static void setPipes(Configuration conf, String map, String reduce, String inputFormat, String partitioner,
075 String writer, String program) {
076 if (map != null) {
077 conf.set("oozie.pipes.map", map);
078 }
079 if (reduce != null) {
080 conf.set("oozie.pipes.reduce", reduce);
081 }
082 if (inputFormat != null) {
083 conf.set("oozie.pipes.inputformat", inputFormat);
084 }
085 if (partitioner != null) {
086 conf.set("oozie.pipes.partitioner", partitioner);
087 }
088 if (writer != null) {
089 conf.set("oozie.pipes.writer", writer);
090 }
091 if (program != null) {
092 conf.set("oozie.pipes.program", program);
093 }
094 }
095
096 }