ucar.nc2.stream.NcStreamIosp Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
There is a newer version: 4.3.22
package ucar.nc2.stream;

import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.util.CancelTask;
import ucar.unidata.io.RandomAccessFile;
import ucar.ma2.*;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

/**
 * Read ncStream file (raf), into a NetcdfFile.
 */
public class NcStreamIosp extends AbstractIOServiceProvider {
  private static final boolean debug = false;

  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    raf.seek(0);
    if (!readAndTest(raf, NcStream.MAGIC_START)) return false; // must start with these 4 bytes
    byte[] b = new byte[4];
    raf.read(b);
    return test(b, NcStream.MAGIC_HEADER) || test(b, NcStream.MAGIC_DATA); // immed followed by one of these
  }

  public String getFileTypeId() {
    return "ncstream";
  }

  public String getFileTypeDescription() {
    return "netCDF streaming protocol";
  }

  //////////////////////////////////////////////////////////////////////

  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    this.raf = raf;
    raf.seek(0);
    assert readAndTest(raf, NcStream.MAGIC_START);

    // assume for the moment its always starts with one header message
    assert readAndTest(raf, NcStream.MAGIC_HEADER);

    int msize = readVInt(raf);
    if (debug) System.out.println("READ header len= " + msize);

    byte[] m = new byte[msize];
    raf.read(m);
    NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);

    NcStreamProto.Group root = proto.getRoot();
    NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
    ncfile.finish();

    // LOOK why doesnt this work ?
    //CodedInputStream cis = CodedInputStream.newInstance(is);
    //cis.setSizeLimit(msize);
    //NcStreamProto.Stream proto = NcStreamProto.Stream.parseFrom(cis);

    while (!raf.isAtEndOfFile()) {
      assert readAndTest(raf, NcStream.MAGIC_DATA);

      int psize = readVInt(raf);
      if (debug) System.out.println(" dproto len= " + psize);
      byte[] dp = new byte[psize];
      raf.read(dp);
      NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
      if (debug) System.out.println(" dproto = " + dproto);

      int dsize = readVInt(raf);
      if (debug) System.out.println(" data len= " + dsize);

      DataSection dataSection = new DataSection();
      dataSection.size = dsize;
      dataSection.filePos = raf.getFilePointer();
      dataSection.section = NcStream.decodeSection(dproto.getSection());

      Variable v = ncfile.getRootGroup().findVariable(dproto.getVarName());
      v.setSPobject(dataSection);

      raf.skipBytes(dsize);
    }
  }


  private class DataSection {
    int size;
    long filePos;
    Section section;

    @Override
    public String toString() {
      return "DataSection{" +
              "size=" + size +
              ", filePos=" + filePos +
              ", section=" + section +
              '}';
    }
  }

  public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
    DataSection dataSection = (DataSection) v.getSPobject();

    raf.seek(dataSection.filePos);
    byte[] data = new byte[dataSection.size];
    raf.read(data);

    Array dataArray = Array.factory(v.getDataType(), v.getShape(), ByteBuffer.wrap(data));
    return dataArray.section(section.getRanges());
  }

  private int readVInt(RandomAccessFile raf) throws IOException {
    byte b = (byte) raf.read();
    int i = b & 0x7F;
    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
      b = (byte) raf.read();
      i |= (b & 0x7F) << shift;
    }
    return i;
  }

  private boolean readAndTest(RandomAccessFile raf, byte[] test) throws IOException {
    byte[] b = new byte[test.length];
    raf.read(b);
    return test(b, test);
  }

  private boolean test(byte[] bread, byte[] test) throws IOException {
    if (bread.length != test.length) return false;
    for (int i = 0; i < bread.length; i++)
      if (bread[i] != test[i]) return false;
    return true;
  }

  ///////////////////////////////////////////////////////////////////////////////
 // lower level interface for debugging

  public List open(RandomAccessFile raf, NetcdfFile ncfile) throws IOException {
    this.raf = raf;
    raf.seek(0);
    assert readAndTest(raf, NcStream.MAGIC_START);

    // assume for the moment its always starts with one header message
    assert readAndTest(raf, NcStream.MAGIC_HEADER);

    int msize = readVInt(raf);
    if (debug) System.out.println("READ header len= " + msize);

    byte[] m = new byte[msize];
    raf.read(m);
    NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);

    NcStreamProto.Group root = proto.getRoot();
    NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
    ncfile.finish();

    List ncm = new ArrayList();
    ncm.add(new NcsMess(msize, proto));

    while (!raf.isAtEndOfFile()) {
      assert readAndTest(raf, NcStream.MAGIC_DATA);

      int psize = readVInt(raf);
      if (debug) System.out.println(" dproto len= " + psize);
      byte[] dp = new byte[psize];
      raf.read(dp);
      NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
      if (debug) System.out.println(" dproto = " + dproto);
      ncm.add(new NcsMess(psize, dproto));

      int dsize = readVInt(raf);
      if (debug) System.out.println(" data len= " + dsize);

      DataSection dataSection = new DataSection();
      dataSection.size = dsize;
      dataSection.filePos = raf.getFilePointer();
      dataSection.section = NcStream.decodeSection(dproto.getSection());

      ncm.add(new NcsMess(dsize, dataSection));

      Variable v = ncfile.getRootGroup().findVariable(dproto.getVarName());
      v.setSPobject(dataSection);

      raf.skipBytes(dsize);
    }

    return ncm;
  }

  public class NcsMess {
    public int len;
    public Object what;
    public NcsMess(int len, Object what) {
      this.len = len;
      this.what = what;
    }
  }

}