001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred.lib.aggregate; 020 021 import java.io.IOException; 022 import java.util.ArrayList; 023 024 import org.apache.hadoop.classification.InterfaceAudience; 025 import org.apache.hadoop.classification.InterfaceStability; 026 import org.apache.hadoop.conf.Configuration; 027 import org.apache.hadoop.fs.Path; 028 import org.apache.hadoop.io.Text; 029 import org.apache.hadoop.mapred.FileInputFormat; 030 import org.apache.hadoop.mapred.FileOutputFormat; 031 import org.apache.hadoop.mapred.InputFormat; 032 import org.apache.hadoop.mapred.JobClient; 033 import org.apache.hadoop.mapred.JobConf; 034 import org.apache.hadoop.mapred.SequenceFileInputFormat; 035 import org.apache.hadoop.mapred.TextInputFormat; 036 import org.apache.hadoop.mapred.TextOutputFormat; 037 import org.apache.hadoop.mapred.jobcontrol.Job; 038 import org.apache.hadoop.mapred.jobcontrol.JobControl; 039 import org.apache.hadoop.util.GenericOptionsParser; 040 041 /** 042 * This is the main class for creating a map/reduce job using Aggregate 043 * framework. The Aggregate is a specialization of map/reduce framework, 044 * specilizing for performing various simple aggregations. 045 * 046 * Generally speaking, in order to implement an application using Map/Reduce 047 * model, the developer is to implement Map and Reduce functions (and possibly 048 * combine function). However, a lot of applications related to counting and 049 * statistics computing have very similar characteristics. Aggregate abstracts 050 * out the general patterns of these functions and implementing those patterns. 051 * In particular, the package provides generic mapper/redducer/combiner classes, 052 * and a set of built-in value aggregators, and a generic utility class that 053 * helps user create map/reduce jobs using the generic class. The built-in 054 * aggregators include: 055 * 056 * sum over numeric values count the number of distinct values compute the 057 * histogram of values compute the minimum, maximum, media,average, standard 058 * deviation of numeric values 059 * 060 * The developer using Aggregate will need only to provide a plugin class 061 * conforming to the following interface: 062 * 063 * public interface ValueAggregatorDescriptor { public ArrayList<Entry> 064 * generateKeyValPairs(Object key, Object value); public void 065 * configure(JobConfjob); } 066 * 067 * The package also provides a base class, ValueAggregatorBaseDescriptor, 068 * implementing the above interface. The user can extend the base class and 069 * implement generateKeyValPairs accordingly. 070 * 071 * The primary work of generateKeyValPairs is to emit one or more key/value 072 * pairs based on the input key/value pair. The key in an output key/value pair 073 * encode two pieces of information: aggregation type and aggregation id. The 074 * value will be aggregated onto the aggregation id according the aggregation 075 * type. 076 * 077 * This class offers a function to generate a map/reduce job using Aggregate 078 * framework. The function takes the following parameters: input directory spec 079 * input format (text or sequence file) output directory a file specifying the 080 * user plugin class 081 * 082 * @deprecated Use 083 * {@link org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob} instead 084 */ 085 @Deprecated 086 @InterfaceAudience.Public 087 @InterfaceStability.Stable 088 public class ValueAggregatorJob { 089 090 public static JobControl createValueAggregatorJobs(String args[] 091 , Class<? extends ValueAggregatorDescriptor>[] descriptors) throws IOException { 092 093 JobControl theControl = new JobControl("ValueAggregatorJobs"); 094 ArrayList<Job> dependingJobs = new ArrayList<Job>(); 095 JobConf aJobConf = createValueAggregatorJob(args); 096 if(descriptors != null) 097 setAggregatorDescriptors(aJobConf, descriptors); 098 Job aJob = new Job(aJobConf, dependingJobs); 099 theControl.addJob(aJob); 100 return theControl; 101 } 102 103 public static JobControl createValueAggregatorJobs(String args[]) throws IOException { 104 return createValueAggregatorJobs(args, null); 105 } 106 107 /** 108 * Create an Aggregate based map/reduce job. 109 * 110 * @param args the arguments used for job creation. Generic hadoop 111 * arguments are accepted. 112 * @return a JobConf object ready for submission. 113 * 114 * @throws IOException 115 * @see GenericOptionsParser 116 */ 117 public static JobConf createValueAggregatorJob(String args[]) 118 throws IOException { 119 120 Configuration conf = new Configuration(); 121 122 GenericOptionsParser genericParser 123 = new GenericOptionsParser(conf, args); 124 args = genericParser.getRemainingArgs(); 125 126 if (args.length < 2) { 127 System.out.println("usage: inputDirs outDir " 128 + "[numOfReducer [textinputformat|seq [specfile [jobName]]]]"); 129 GenericOptionsParser.printGenericCommandUsage(System.out); 130 System.exit(1); 131 } 132 String inputDir = args[0]; 133 String outputDir = args[1]; 134 int numOfReducers = 1; 135 if (args.length > 2) { 136 numOfReducers = Integer.parseInt(args[2]); 137 } 138 139 Class<? extends InputFormat> theInputFormat = 140 TextInputFormat.class; 141 if (args.length > 3 && 142 args[3].compareToIgnoreCase("textinputformat") == 0) { 143 theInputFormat = TextInputFormat.class; 144 } else { 145 theInputFormat = SequenceFileInputFormat.class; 146 } 147 148 Path specFile = null; 149 150 if (args.length > 4) { 151 specFile = new Path(args[4]); 152 } 153 154 String jobName = ""; 155 156 if (args.length > 5) { 157 jobName = args[5]; 158 } 159 160 JobConf theJob = new JobConf(conf); 161 if (specFile != null) { 162 theJob.addResource(specFile); 163 } 164 String userJarFile = theJob.get("user.jar.file"); 165 if (userJarFile == null) { 166 theJob.setJarByClass(ValueAggregator.class); 167 } else { 168 theJob.setJar(userJarFile); 169 } 170 theJob.setJobName("ValueAggregatorJob: " + jobName); 171 172 FileInputFormat.addInputPaths(theJob, inputDir); 173 174 theJob.setInputFormat(theInputFormat); 175 176 theJob.setMapperClass(ValueAggregatorMapper.class); 177 FileOutputFormat.setOutputPath(theJob, new Path(outputDir)); 178 theJob.setOutputFormat(TextOutputFormat.class); 179 theJob.setMapOutputKeyClass(Text.class); 180 theJob.setMapOutputValueClass(Text.class); 181 theJob.setOutputKeyClass(Text.class); 182 theJob.setOutputValueClass(Text.class); 183 theJob.setReducerClass(ValueAggregatorReducer.class); 184 theJob.setCombinerClass(ValueAggregatorCombiner.class); 185 theJob.setNumMapTasks(1); 186 theJob.setNumReduceTasks(numOfReducers); 187 return theJob; 188 } 189 190 public static JobConf createValueAggregatorJob(String args[] 191 , Class<? extends ValueAggregatorDescriptor>[] descriptors) 192 throws IOException { 193 JobConf job = createValueAggregatorJob(args); 194 setAggregatorDescriptors(job, descriptors); 195 return job; 196 } 197 198 public static void setAggregatorDescriptors(JobConf job 199 , Class<? extends ValueAggregatorDescriptor>[] descriptors) { 200 job.setInt("aggregator.descriptor.num", descriptors.length); 201 //specify the aggregator descriptors 202 for(int i=0; i< descriptors.length; i++) { 203 job.set("aggregator.descriptor." + i, "UserDefined," + descriptors[i].getName()); 204 } 205 } 206 207 /** 208 * create and run an Aggregate based map/reduce job. 209 * 210 * @param args the arguments used for job creation 211 * @throws IOException 212 */ 213 public static void main(String args[]) throws IOException { 214 JobConf job = ValueAggregatorJob.createValueAggregatorJob(args); 215 JobClient.runJob(job); 216 } 217 }