001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred.lib; 020 021 import java.io.IOException; 022 023 import org.apache.hadoop.classification.InterfaceAudience; 024 import org.apache.hadoop.classification.InterfaceStability; 025 import org.apache.hadoop.fs.FileSystem; 026 import org.apache.hadoop.mapred.JobConf; 027 import org.apache.hadoop.mapred.OutputFormat; 028 import org.apache.hadoop.mapred.RecordWriter; 029 import org.apache.hadoop.mapred.Reporter; 030 import org.apache.hadoop.util.Progressable; 031 import org.apache.hadoop.util.ReflectionUtils; 032 033 /** 034 * A Convenience class that creates output lazily. 035 * @deprecated Use 036 * {@link org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat} instead. 037 */ 038 @Deprecated 039 @InterfaceAudience.Public 040 @InterfaceStability.Stable 041 public class LazyOutputFormat<K, V> extends FilterOutputFormat<K, V> { 042 /** 043 * Set the underlying output format for LazyOutputFormat. 044 * @param job the {@link JobConf} to modify 045 * @param theClass the underlying class 046 */ 047 @SuppressWarnings("unchecked") 048 public static void setOutputFormatClass(JobConf job, 049 Class<? extends OutputFormat> theClass) { 050 job.setOutputFormat(LazyOutputFormat.class); 051 job.setClass("mapreduce.output.lazyoutputformat.outputformat", theClass, OutputFormat.class); 052 } 053 054 @Override 055 public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, 056 String name, Progressable progress) throws IOException { 057 if (baseOut == null) { 058 getBaseOutputFormat(job); 059 } 060 return new LazyRecordWriter<K, V>(job, baseOut, name, progress); 061 } 062 063 @Override 064 public void checkOutputSpecs(FileSystem ignored, JobConf job) 065 throws IOException { 066 if (baseOut == null) { 067 getBaseOutputFormat(job); 068 } 069 super.checkOutputSpecs(ignored, job); 070 } 071 072 @SuppressWarnings("unchecked") 073 private void getBaseOutputFormat(JobConf job) throws IOException { 074 baseOut = ReflectionUtils.newInstance( 075 job.getClass("mapreduce.output.lazyoutputformat.outputformat", null, OutputFormat.class), 076 job); 077 if (baseOut == null) { 078 throw new IOException("Ouput format not set for LazyOutputFormat"); 079 } 080 } 081 082 /** 083 * <code>LazyRecordWriter</code> is a convenience 084 * class that works with LazyOutputFormat. 085 */ 086 087 private static class LazyRecordWriter<K,V> extends FilterRecordWriter<K,V> { 088 089 final OutputFormat of; 090 final String name; 091 final Progressable progress; 092 final JobConf job; 093 094 public LazyRecordWriter(JobConf job, OutputFormat of, String name, 095 Progressable progress) throws IOException { 096 this.of = of; 097 this.job = job; 098 this.name = name; 099 this.progress = progress; 100 } 101 102 @Override 103 public void close(Reporter reporter) throws IOException { 104 if (rawWriter != null) { 105 rawWriter.close(reporter); 106 } 107 } 108 109 @Override 110 public void write(K key, V value) throws IOException { 111 if (rawWriter == null) { 112 createRecordWriter(); 113 } 114 super.write(key, value); 115 } 116 117 @SuppressWarnings("unchecked") 118 private void createRecordWriter() throws IOException { 119 FileSystem fs = FileSystem.get(job); 120 rawWriter = of.getRecordWriter(fs, job, name, progress); 121 } 122 } 123 }