com.twitter.elephantbird.mapreduce.io.ProtobufBlockReader Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of elephant-bird-core Show documentation

Core utilities.

There is a newer version: 4.17

package com.twitter.elephantbird.mapreduce.io;

import java.io.IOException;
import java.io.InputStream;

import com.google.protobuf.Message;
import com.twitter.elephantbird.util.TypeRef;

import org.apache.hadoop.io.BytesWritable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/* A class to read blocks of protobuf data of type M.  To use, just instantiate
 * with an InputStream and a TypeRef, call readProtobuf until it returns false, and
 * then close the protobuf.  For example,
 * 
 * TypeRef personRef = new TypeRef();
 * ProtobufBlockReader reader = new ProtobufBlockReader(
 *     new FileInputStream("person_data"), personRef);
 * ProtobufWritable writable = new ProtobufWritable(personRef);
 * while (reader.readProtobuf(writable)) {
 *   Person person = writable.get();
 *   // do something with the protobuf.
 * }
 * reader.close();
 * 
 *
 * See the ProtobufBlockWriter for how to write data files like "person_data" above.
 */

public class ProtobufBlockReader extends BinaryBlockReader {
  private static final Logger LOG = LoggerFactory.getLogger(ProtobufBlockReader.class);

  public ProtobufBlockReader(InputStream in, TypeRef typeRef) {
    super(in, ProtobufConverter.newInstance(typeRef));
    LOG.info("ProtobufReader, my typeClass is " + typeRef.getRawClass());
  }

  // for backward compatibility :

  public boolean readProtobuf(ProtobufWritable message) throws IOException {
    return readNext(message);
  }

  public boolean readProtobufBytes(BytesWritable message) throws IOException {
    return readNextProtoBytes(message);
  }
}