001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapreduce.lib.input;
020    
021    import java.io.IOException;
022    
023    import org.apache.hadoop.classification.InterfaceAudience;
024    import org.apache.hadoop.classification.InterfaceStability;
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.hadoop.io.Text;
027    import org.apache.hadoop.mapreduce.InputSplit;
028    import org.apache.hadoop.mapreduce.RecordReader;
029    import org.apache.hadoop.mapreduce.TaskAttemptContext;
030    
031    /**
032     * This class treats a line in the input as a key/value pair separated by a 
033     * separator character. The separator can be specified in config file 
034     * under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
035     * separator is the tab character ('\t').
036     */
037    @InterfaceAudience.Public
038    @InterfaceStability.Stable
039    public class KeyValueLineRecordReader extends RecordReader<Text, Text> {
040      public static final String KEY_VALUE_SEPERATOR = 
041        "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
042      
043      private final LineRecordReader lineRecordReader;
044    
045      private byte separator = (byte) '\t';
046    
047      private Text innerValue;
048    
049      private Text key;
050      
051      private Text value;
052      
053      public KeyValueLineRecordReader(Configuration conf)
054        throws IOException {
055        
056        lineRecordReader = new LineRecordReader();
057        String sepStr = conf.get(KEY_VALUE_SEPERATOR, "\t");
058        this.separator = (byte) sepStr.charAt(0);
059      }
060    
061      public void initialize(InputSplit genericSplit,
062          TaskAttemptContext context) throws IOException {
063        lineRecordReader.initialize(genericSplit, context);
064      }
065      
066      public static int findSeparator(byte[] utf, int start, int length, 
067          byte sep) {
068        for (int i = start; i < (start + length); i++) {
069          if (utf[i] == sep) {
070            return i;
071          }
072        }
073        return -1;
074      }
075    
076      public static void setKeyValue(Text key, Text value, byte[] line,
077          int lineLen, int pos) {
078        if (pos == -1) {
079          key.set(line, 0, lineLen);
080          value.set("");
081        } else {
082          key.set(line, 0, pos);
083          value.set(line, pos + 1, lineLen - pos - 1);
084        }
085      }
086      /** Read key/value pair in a line. */
087      public synchronized boolean nextKeyValue()
088        throws IOException {
089        byte[] line = null;
090        int lineLen = -1;
091        if (lineRecordReader.nextKeyValue()) {
092          innerValue = lineRecordReader.getCurrentValue();
093          line = innerValue.getBytes();
094          lineLen = innerValue.getLength();
095        } else {
096          return false;
097        }
098        if (line == null)
099          return false;
100        if (key == null) {
101          key = new Text();
102        }
103        if (value == null) {
104          value = new Text();
105        }
106        int pos = findSeparator(line, 0, lineLen, this.separator);
107        setKeyValue(key, value, line, lineLen, pos);
108        return true;
109      }
110      
111      public Text getCurrentKey() {
112        return key;
113      }
114    
115      public Text getCurrentValue() {
116        return value;
117      }
118    
119      public float getProgress() throws IOException {
120        return lineRecordReader.getProgress();
121      }
122      
123      public synchronized void close() throws IOException { 
124        lineRecordReader.close();
125      }
126    }