ml.shifu.guagua.yarn.GuaguaSequenceRecordReader Maven / Gradle / Ivy

Go to download
/*
 * Copyright [2013-2014] PayPal Software Foundation
 *  
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *  
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ml.shifu.guagua.yarn;

import java.io.IOException;

import ml.shifu.guagua.io.GuaguaFileSplit;
import ml.shifu.guagua.io.GuaguaRecordReader;
import ml.shifu.guagua.util.ReflectionUtils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.SequenceFileRecordReader;

/**
 * A reader read HDFS sequence file key by key. The sequence key and value types are both {@link BytesWritable}.
 * 
 * 
 * Copy some code from {@link org.apache.hadoop.mapred.SequenceFileRecordReader} but to support
 * {@link GuaguaRecordReader} interface.
 * 
 * 

 * If use default constructor, user should also call {@link #initialize(GuaguaFileSplit)} like in below:
 * 
 * 
 * this.setRecordReader(new SequenceFileRecordReader(Text.class, Text,class));
 * this.getRecordReader().initialize(fileSplit);
 * 
 * 
 * or directly use other constructors:
 * 
 *  * this.setRecordReader(new SequenceFileRecordReader(fileSplit, Text.class, Text,class));
 * 
 * 
 * @deprecated use {@link ml.shifu.guagua.hadoop.io.GuaguaSequenceRecordReader}
 */
@Deprecated
public class GuaguaSequenceRecordReader implements
        GuaguaRecordReader, GuaguaWritableAdapter> {

    private SequenceFileRecordReader sequenceReader;

    private Configuration conf;

    private GuaguaWritableAdapter key = null;
    private GuaguaWritableAdapter value = null;

    private Class keyClass;

    private Class valueClass;

    public GuaguaSequenceRecordReader(Class keyClass, Class valueClass)
            throws IOException {
        this(null, keyClass, valueClass);
    }

    public GuaguaSequenceRecordReader(GuaguaFileSplit split, Class keyClass,
            Class valueClass) throws IOException {
        this(new Configuration(), split, keyClass, valueClass);
    }

    public GuaguaSequenceRecordReader(Configuration conf, GuaguaFileSplit split, Class keyClass,
            Class valueClass) throws IOException {
        this.conf = conf;
        this.keyClass = keyClass;
        this.valueClass = valueClass;
        if(split != null) {
            initialize(split);
        }
    }

    /**
     * Return the progress within the input split
     * 
     * @return 0.0 to 1.0 of the input byte range
     */
    public float getProgress() throws IOException {
        return sequenceReader.getProgress();
    }

    @Override
    public void initialize(GuaguaFileSplit split) throws IOException {
        FileSplit fileSplit = new FileSplit(new Path(split.getPath()), split.getOffset(), split.getLength(),
                (String[]) null);
        this.sequenceReader = new SequenceFileRecordReader(conf, fileSplit);
    }

    @SuppressWarnings({ "rawtypes", "unchecked" })
    @Override
    public boolean nextKeyValue() throws IOException {
        if(key == null) {
            key = new GuaguaWritableAdapter(((Writable) ReflectionUtils.newInstance(this.keyClass)));
        }
        if(value == null) {
            value = new GuaguaWritableAdapter(((Writable) ReflectionUtils.newInstance(this.valueClass)));
        }
        return this.sequenceReader.next(key.getWritable(), value.getWritable());
    }

    @Override
    public GuaguaWritableAdapter getCurrentKey() {
        return key;
    }

    @Override
    public GuaguaWritableAdapter getCurrentValue() {
        return value;
    }

    @Override
    public synchronized void close() throws IOException {
        if(sequenceReader != null) {
            sequenceReader.close();
        }
    }

}