ucar.nc2.stream.NcStreamReader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright 2009-2012 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */
package ucar.nc2.stream;

import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFileSubclass;
import ucar.nc2.Structure;
import ucar.ma2.*;

import java.io.InputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

import com.google.protobuf.InvalidProtocolBufferException;
import ucar.nc2.constants.CDM;

/**
 * Read an ncStream InputStream into a NetcdfFile.
 * Used by CdmRemote
 *
 * @author caron
 * @since Feb 7, 2009
 */
public class NcStreamReader {

  private static final boolean debug = false;

  public NetcdfFile readStream(InputStream is, NetcdfFile ncfile) throws IOException {
    byte[] b = new byte[4];
    NcStream.readFully(is, b);

    // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER
    if (test(b, NcStream.MAGIC_START)) {
      if (!readAndTest(is, NcStream.MAGIC_HEADER))
        throw new IOException("Data corrupted on "+ncfile.getLocation());

    } else {
      if (!test(b, NcStream.MAGIC_HEADER))
        throw new IOException("Data corrupted on "+ncfile.getLocation());
    }

    // header
    int msize = NcStream.readVInt(is);
    byte[] m = new byte[msize];
    NcStream.readFully(is, m);

    if (debug) {
      System.out.println("READ header len= " + msize);
    }

    NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);
    ncfile = proto2nc(proto, ncfile);
    if (debug) System.out.printf("  proto= %s%n", proto);

    // LOOK why doesnt this work ?
    //CodedInputStream cis = CodedInputStream.newInstance(is);
    //cis.setSizeLimit(msize);
    //NcStreamProto.Stream proto = NcStreamProto.Stream.parseFrom(cis);

    while (is.available() > 0) {
      readData(is, ncfile);
    }

    return ncfile;

  }

  static class DataResult {
    String varNameFullEsc;
    Array data;

    DataResult(String varName, Array data) {
      this.varNameFullEsc = varName;
      this.data = data;
    }
  }

  /**
   * Read the result of a data request. Only one variable at a time.
   * @param is read from input stream
   * @param ncfile need the metadata from here to interpret structure data
   * @return DataResult
   * @throws IOException on read error
   */
  public DataResult readData(InputStream is, NetcdfFile ncfile) throws IOException {
    if (!readAndTest(is, NcStream.MAGIC_DATA))
      throw new IOException("Data transfer corrupted on "+ncfile.getLocation());

    int psize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data message len= " + psize);
    byte[] dp = new byte[psize];
    NcStream.readFully(is, dp);
    NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
    // if (debug) System.out.println(" readData proto = " + dproto);

    DataType dataType = NcStream.decodeDataType(dproto.getDataType());
    Section section = (dataType == DataType.SEQUENCE) ? new Section() : NcStream.decodeSection(dproto.getSection());

    // special cases
    if (dataType == DataType.STRING) {
      Array data = Array.factory(dataType, section.getShape());
      IndexIterator ii = data.getIndexIterator();
      while(ii.hasNext()) {
        int slen = NcStream.readVInt(is);
        byte[] sb = new byte[slen];
        NcStream.readFully(is, sb);
        ii.setObjectNext( new String(sb, CDM.utf8Charset));
      }
      return new DataResult(dproto.getVarName(), data);

    } else if (dataType == DataType.OPAQUE) {
      Array data = Array.factory(dataType, section.getShape());
      IndexIterator ii = data.getIndexIterator();
      while(ii.hasNext()) {
        int slen = NcStream.readVInt(is);
        byte[] sb = new byte[slen];
        NcStream.readFully(is, sb);
        ii.setObjectNext( ByteBuffer.wrap(sb));
      }
      return new DataResult(dproto.getVarName(), data);

    } 

    // otherwise read data message
    int dsize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data len= " + dsize);
    byte[] datab = new byte[dsize];
    NcStream.readFully(is, datab);

    if (dataType == DataType.STRUCTURE) {
      Structure s = (Structure) ncfile.findVariable(dproto.getVarName());
      StructureMembers members = s.makeStructureMembers();

      if (dproto.getVersion() == 0) {
        ArrayStructureBB.setOffsets(members); // not setting heap objects for version 0
        ArrayStructureBB data = new ArrayStructureBB(members, section.getShape(), ByteBuffer.wrap(datab), 0);
        return new DataResult(dproto.getVarName(), data);

      } else { // version > 0 uses a NcStreamProto.StructureData message
        ArrayStructureBB data = NcStream.decodeArrayStructure(members, section.getShape(), datab);
        return new DataResult(dproto.getVarName(), data);
      }

    } else {
      Array data = Array.factory(dataType, section.getShape(), ByteBuffer.wrap(datab));
      return new DataResult(dproto.getVarName(), data);
    }
  }

  public StructureDataIterator getStructureIterator(InputStream is, NetcdfFile ncfile) throws IOException {
    if (!readAndTest(is, NcStream.MAGIC_DATA))
      throw new IOException("Data transfer corrupted on "+ncfile.getLocation());

    int psize = NcStream.readVInt(is);
    if (debug) System.out.println("  readData data message len= " + psize);
    byte[] dp = new byte[psize];
    NcStream.readFully(is, dp);
    NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
    // if (debug) System.out.println(" readData proto = " + dproto);

    Structure s = (Structure) ncfile.findVariable(dproto.getVarName());
    StructureMembers members = s.makeStructureMembers();
    ArrayStructureBB.setOffsets(members);

    return new StreamDataIterator(is, members);
  }

  private class StreamDataIterator implements StructureDataIterator {
    private InputStream is;
    private StructureMembers members;
    private StructureData curr = null;
    private int count = 0;

    StreamDataIterator(InputStream is, StructureMembers members) {
      this.is = is;
      this.members = members;
    }

    @Override
    public boolean hasNext() throws IOException {
      readNext();
      return (curr != null);
    }

    @Override
    public StructureData next() throws IOException {
      count++;
      return curr;
    }

    private void readNext() throws IOException {
      byte[] b = new byte[4];
      NcStream.readFully(is, b);

     // starts with MAGIC_START, MAGIC_HEADER or just MAGIC_HEADER
     if (test(b, NcStream.MAGIC_VDATA)) {
       int dsize = NcStream.readVInt(is);
       byte[] datab = new byte[dsize];
       NcStream.readFully(is, datab);
       curr = NcStream.decodeStructureData(members, datab);
       // System.out.printf("StreamDataIterator read sdata size= %d%n", dsize);

     } else if (test(b, NcStream.MAGIC_VEND)) {
        curr = null;

     } else {
       throw new IllegalStateException("bad stream");
     }
    }

    @Override
    public void setBufferSize(int bytes) {
    }

    @Override
    public StructureDataIterator reset() {
      return (count == 0) && (is != null) ? this : null;
    }

    @Override
    public int getCurrentRecno() {
      return count;
    }

    @Override
    public void finish() {
      if (is != null) {
        try {
          is.close();
          is = null;
        } catch (IOException ioe) {
          System.out.printf("NcStreamReader: Error closing input stream.");
        }
      }
    }
  }


  private boolean readAndTest(InputStream is, byte[] test) throws IOException {
    byte[] b = new byte[test.length];
    NcStream.readFully(is, b);

    if (b.length != test.length) return false;
    for (int i = 0; i < b.length; i++)
      if (b[i] != test[i]) return false;
    return true;
  }

  private boolean test(byte[] b, byte[] test) throws IOException {
    if (b.length != test.length) return false;
    for (int i = 0; i < b.length; i++)
      if (b[i] != test[i]) return false;
    return true;
  }

  /////////////////////////////////////////////////////////////////////

  public NetcdfFile proto2nc(NcStreamProto.Header proto, NetcdfFile ncfile) throws InvalidProtocolBufferException {
    if (ncfile == null)
      ncfile = new NetcdfFileSubclass(); // not used i think
    ncfile.setLocation(proto.getLocation());
    if (proto.hasId()) ncfile.setId(proto.getId());
    if (proto.hasTitle()) ncfile.setTitle(proto.getTitle());

    NcStreamProto.Group root = proto.getRoot();
    NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
    ncfile.finish();
    return ncfile;
  }

}