All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • ucar.nc2.stream.NcStream Maven / Gradle / Ivy

    Go to download

    The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.

    There is a newer version: 4.3.22
    Show newest version
    package ucar.nc2.stream;

import ucar.nc2.*;
import ucar.ma2.*;

import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.io.OutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import ucar.nc2.iosp.IospHelper;

/**
 * Defines the ncstream format, along with ncStream.proto.
 * 
     * To regenerate ncStreamProto.java from ncStream.proto:
 cd c:/dev/tds4.2/thredds/cdm/src/main/java
 protoc --proto_path=. --java_out=. ucar/nc2/stream/ncStream.proto
 * 
    
 * @see "http://www.unidata.ucar.edu/software/netcdf-java/stream/NcStream.html"
 * @see "http://www.unidata.ucar.edu/software/netcdf-java/stream/NcstreamGrammer.html"
 */
public class NcStream {
  //  must start with this "CDFS"
  static public final byte[] MAGIC_START = new byte[]{0x43, 0x44, 0x46, 0x53};

  static public final byte[] MAGIC_HEADER = new byte[]{(byte) 0xad, (byte) 0xec, (byte) 0xce, (byte) 0xda};
  static public final byte[] MAGIC_DATA =   new byte[]{(byte) 0xab, (byte) 0xec, (byte) 0xce, (byte) 0xba};
  static public final byte[] MAGIC_VDATA =  new byte[]{(byte) 0xab, (byte) 0xef, (byte) 0xfe, (byte) 0xba};    
  static public final byte[] MAGIC_VEND =  new byte[]{(byte) 0xed, (byte) 0xef, (byte) 0xfe, (byte) 0xda};

  static public final byte[] MAGIC_ERR = new byte[]{(byte) 0xab, (byte) 0xad, (byte) 0xba, (byte) 0xda};
  static public final byte[] MAGIC_END = new byte[]{(byte) 0xed, (byte) 0xed, (byte) 0xde, (byte) 0xde};

  static NcStreamProto.Group.Builder encodeGroup(Group g, int sizeToCache) throws IOException {
    NcStreamProto.Group.Builder groupBuilder = NcStreamProto.Group.newBuilder();
    groupBuilder.setName(g.getShortName());

    for (Dimension dim : g.getDimensions())
      groupBuilder.addDims(NcStream.encodeDim(dim));

    for (Attribute att : g.getAttributes())
      groupBuilder.addAtts(NcStream.encodeAtt(att));

    for (EnumTypedef enumType : g.getEnumTypedefs())
      groupBuilder.addEnumTypes(NcStream.encodeEnumTypedef(enumType));

    for (Variable var : g.getVariables()) {
      if (var instanceof Structure)
        groupBuilder.addStructs(NcStream.encodeStructure((Structure) var));
      else
        groupBuilder.addVars(NcStream.encodeVar(var, sizeToCache));
    }

    for (Group ng : g.getGroups())
      groupBuilder.addGroups( encodeGroup(ng, sizeToCache));

    return groupBuilder;
  }


  static NcStreamProto.Attribute.Builder encodeAtt(Attribute att) {
    NcStreamProto.Attribute.Builder attBuilder = NcStreamProto.Attribute.newBuilder();
    attBuilder.setName(att.getName());
    attBuilder.setType(encodeAttributeType(att.getDataType()));
    attBuilder.setLen(att.getLength());

    // values
    if (att.getLength() > 0) {
      if (att.isString()) {
        for (int i=0; i map = enumType.getMap();
    NcStreamProto.EnumTypedef.EnumType.Builder b2 = NcStreamProto.EnumTypedef.EnumType.newBuilder();
    for (int code : map.keySet()) {
      b2.clear();
      b2.setCode(code);
      b2.setValue(map.get(code));
      builder.addMap(b2);
    }
    return builder;
   }

  static NcStreamProto.Variable.Builder encodeVar(Variable var, int sizeToCache) throws IOException {
    NcStreamProto.Variable.Builder builder = NcStreamProto.Variable.newBuilder();
    builder.setName(var.getShortName());
    builder.setDataType(encodeDataType(var.getDataType()));
    if (var.isUnsigned())
      builder.setUnsigned(true);
    if (var.getDataType().isEnum()) {
      EnumTypedef enumType = var.getEnumTypedef();
      if (enumType != null)
        builder.setEnumType(enumType.getName());
    }

    for (Dimension dim : var.getDimensions()) {
      builder.addShape(encodeDim(dim));
    }

    for (Attribute att : var.getAttributes()) {
      builder.addAtts(encodeAtt(att));
    }

    // put small amounts of data in header "immediate mode"
    if (var.isCaching() && var.getDataType().isNumeric()) {
      if (var.isCoordinateVariable() || var.getSize() * var.getElementSize() < sizeToCache) {
        Array data = var.read();
        ByteBuffer bb = data.getDataAsByteBuffer();
        builder.setData(ByteString.copyFrom(bb.array()));
      }
    }

    return builder;
  }

  static NcStreamProto.Structure.Builder encodeStructure(Structure s) throws IOException {
    NcStreamProto.Structure.Builder builder = NcStreamProto.Structure.newBuilder();
    builder.setName(s.getShortName());
    builder.setDataType(encodeDataType(s.getDataType()));

    for (Dimension dim : s.getDimensions())
      builder.addShape(encodeDim(dim));

    for (Attribute att : s.getAttributes())
      builder.addAtts(encodeAtt(att));

    for (Variable v : s.getVariables()) {
      if (v instanceof Structure)
        builder.addStructs(NcStream.encodeStructure((Structure) v));
      else
        builder.addVars(NcStream.encodeVar(v, -1));
    }

    return builder;
  }

  static public NcStreamProto.Error encodeErrorMessage(String message) {
    NcStreamProto.Error.Builder builder = NcStreamProto.Error.newBuilder();
    builder.setMessage(message);
    return builder.build();
  }

  static NcStreamProto.Data encodeDataProto(Variable var, Section section) {
    NcStreamProto.Data.Builder builder = NcStreamProto.Data.newBuilder();
    builder.setVarName(var.getName());
    builder.setDataType(encodeDataType(var.getDataType()));
    builder.setSection(encodeSection(section));
    builder.setVersion(1);
    return builder.build();
  }

  static public NcStreamProto.Section encodeSection(Section section) {
    NcStreamProto.Section.Builder sbuilder = NcStreamProto.Section.newBuilder();
    for (Range r : section.getRanges()) {
      NcStreamProto.Range.Builder rbuilder = NcStreamProto.Range.newBuilder();
      rbuilder.setSize(r.length());
      sbuilder.addRange(rbuilder);
    }
    return sbuilder.build();
  }

  public static long encodeArrayStructure(ArrayStructure as, OutputStream os) throws java.io.IOException {
    long size = 0;

    ArrayStructureBB dataBB = IospHelper.makeArrayBB(as);
    List ss = new ArrayList();
    List heap = dataBB.getHeap();
    List count = new ArrayList();
    if (heap != null) {
      for (Object ho : heap) {
        if (ho instanceof String) {
          count.add(1);
          ss.add((String) ho);
        } else if (ho instanceof String[]) {
          String[] hos = (String[]) ho;
          count.add(hos.length);
          for (String s : hos)
            ss.add(s);
        }
      }
    }

    ByteBuffer bb = dataBB.getByteBuffer();
    NcStreamProto.StructureData proto = NcStream.encodeStructureDataProto(bb.array(), count, ss);
    byte[] datab = proto.toByteArray();
    size += NcStream.writeVInt(os, datab.length); // proto len
    os.write(datab); // proto
    size += datab.length;
    // System.out.printf("encodeArrayStructure write sdata size= %d%n", datab.length);

    return size;
  }

  static NcStreamProto.StructureData encodeStructureDataProto(byte[] fixed, List count, List ss) {
    NcStreamProto.StructureData.Builder builder = NcStreamProto.StructureData.newBuilder();
    builder.setData(ByteString.copyFrom(fixed));
    for (Integer c : count)
      builder.addHeapCount(c);
    for (String s : ss)
      builder.addSdata(s);
    return builder.build();
  }

  public static ArrayStructureBB decodeArrayStructure(StructureMembers sm, int shape[], byte[] proto) throws java.io.IOException {
    NcStreamProto.StructureData.Builder builder = NcStreamProto.StructureData.newBuilder();
    builder.mergeFrom(proto);
    long size = 0;

    ByteBuffer bb = ByteBuffer.wrap(builder.getData().toByteArray());
    ArrayStructureBB dataBB = new ArrayStructureBB(sm, shape, bb, 0);

    List ss = builder.getSdataList();
    List count = builder.getHeapCountList();

    int scount = 0;
    for (Integer c : count) {
      if (c == 1) {
        dataBB.addObjectToHeap(ss.get(scount++));
      } else {
        String[] hos = new String[c];
        for (int i=0; i ss = builder.getSdataList();
    List count = builder.getHeapCountList();

    int scount = 0;
    for (Integer c : count) {
      if (c == 1) {
        dataBB.addObjectToHeap(ss.get(scount++));
      } else {
        String[] hos = new String[c];
        for (int i=0; i list = enumType.getMapList();
    Map map = new HashMap( 2 * list.size());
    for (NcStreamProto.EnumTypedef.EnumType et : list) {
      map.put(et.getCode(), et.getValue());
    }
    return new EnumTypedef( enumType.getName(), map);
   }

  static Attribute decodeAtt(NcStreamProto.Attribute attp) {
    int len = attp.getLen();
    if (len == 0) // deal with empty attribute
      return new Attribute(attp.getName(), decodeAttributeType(attp.getType()));

    DataType dt = decodeAttributeType(attp.getType());

    if (dt == DataType.STRING) {
      int lenp = attp.getSdataCount();
      if (lenp != len)
        System.out.println("HEY lenp != len");
      if (lenp == 1)
        return new Attribute(attp.getName(), attp.getSdata(0));
      else {
        Array data = Array.factory(dt, new int[] {lenp});
        for (int i=0; i dims = new ArrayList(6);
    for (ucar.nc2.stream.NcStreamProto.Dimension dim : var.getShapeList()) {
      if (dim.getIsPrivate())
        dims.add(new Dimension(dim.getName(), (int) dim.getLength(), false, dim.getIsUnlimited(), dim.getIsVlen()));
      else {
        Dimension d = g.findDimension(dim.getName());
        if (d == null)
          throw new IllegalStateException("Can find shared dimension "+dim.getName());
        dims.add(d);
      }
    }
    ncvar.setDimensions(dims);

    for (ucar.nc2.stream.NcStreamProto.Attribute att : var.getAttsList())
      ncvar.addAttribute(decodeAtt(att));

    if (var.getUnsigned())
      ncvar.addAttribute(new Attribute("_Unsigned", "true"));

    if (var.hasData()) {
      // LOOK may mess with ability to change var size later.
      ByteBuffer bb = ByteBuffer.wrap(var.getData().toByteArray());
      Array data = Array.factory(varType, ncvar.getShape(), bb);
      ncvar.setCachedData(data, false);
      if (false) System.out.println(" read cached data for "+ncvar.getName()+" nbytes= "+bb.limit());
    }

    return ncvar;
  }

  static Structure decodeStructure(NetcdfFile ncfile, Group g, Structure parent, NcStreamProto.Structure s) {
    Structure ncvar = (s.getDataType() == ucar.nc2.stream.NcStreamProto.DataType.SEQUENCE) ?
        new Sequence(ncfile, g, parent, s.getName()) : new Structure(ncfile, g, parent, s.getName());

    ncvar.setDataType(decodeDataType(s.getDataType()));

    List dims = new ArrayList(6);
    for (ucar.nc2.stream.NcStreamProto.Dimension dim : s.getShapeList()) {
      if (dim.getIsPrivate() || dim.getIsVlen())
        dims.add(new Dimension(dim.getName(), (int) dim.getLength(), false, dim.getIsUnlimited(), dim.getIsVlen()));
      else {
        Dimension d = g.findDimension(dim.getName());
        if (d == null)
          throw new IllegalStateException("Can find shared dimension "+dim.getName());
        dims.add(d);
      }
    }
    ncvar.setDimensions(dims);

    for (ucar.nc2.stream.NcStreamProto.Attribute att : s.getAttsList())
      ncvar.addAttribute(decodeAtt(att));

    for (ucar.nc2.stream.NcStreamProto.Variable vp : s.getVarsList())
      ncvar.addMemberVariable(decodeVar(ncfile, g, ncvar, vp));

    for (NcStreamProto.Structure sp : s.getStructsList())
      ncvar.addMemberVariable(decodeStructure(ncfile, g, ncvar, sp));

    return ncvar;
  }

  static public Section decodeSection(NcStreamProto.Section proto) {
    Section section = new Section();

    for (ucar.nc2.stream.NcStreamProto.Range pr : proto.getRangeList()) {
      try {
        section.appendRange((int) pr.getStart(), (int) (pr.getStart() + pr.getSize() - 1));
      } catch (InvalidRangeException e) {
        throw new RuntimeException(e);
      }
    }
    return section;
  }

  ////////////////////////////////////////////////////////////////

  static ucar.nc2.stream.NcStreamProto.Attribute.Type encodeAttributeType(DataType dtype) {
    switch (dtype) {
      case CHAR:
      case STRING:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.STRING;
      case BYTE:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.BYTE;
      case SHORT:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.SHORT;
      case INT:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.INT;
      case LONG:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.LONG;
      case FLOAT:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.FLOAT;
      case DOUBLE:
        return ucar.nc2.stream.NcStreamProto.Attribute.Type.DOUBLE;
    }
    throw new IllegalStateException("illegal att type " + dtype);
  }

  static public ucar.nc2.stream.NcStreamProto.DataType encodeDataType(DataType dtype) {
    switch (dtype) {
      case CHAR:
        return ucar.nc2.stream.NcStreamProto.DataType.CHAR;
      case BYTE:
        return ucar.nc2.stream.NcStreamProto.DataType.BYTE;
      case SHORT:
        return ucar.nc2.stream.NcStreamProto.DataType.SHORT;
      case INT:
        return ucar.nc2.stream.NcStreamProto.DataType.INT;
      case LONG:
        return ucar.nc2.stream.NcStreamProto.DataType.LONG;
      case FLOAT:
        return ucar.nc2.stream.NcStreamProto.DataType.FLOAT;
      case DOUBLE:
        return ucar.nc2.stream.NcStreamProto.DataType.DOUBLE;
      case STRING:
        return ucar.nc2.stream.NcStreamProto.DataType.STRING;
      case STRUCTURE:
        return ucar.nc2.stream.NcStreamProto.DataType.STRUCTURE;
      case SEQUENCE:
        return ucar.nc2.stream.NcStreamProto.DataType.SEQUENCE;
      case ENUM1:
        return ucar.nc2.stream.NcStreamProto.DataType.ENUM1;
      case ENUM2:
        return ucar.nc2.stream.NcStreamProto.DataType.ENUM2;
      case ENUM4:
        return ucar.nc2.stream.NcStreamProto.DataType.ENUM4;
      case OPAQUE:
        return ucar.nc2.stream.NcStreamProto.DataType.OPAQUE;
    }
    throw new IllegalStateException("illegal data type " + dtype);
  }

  static DataType decodeAttributeType(ucar.nc2.stream.NcStreamProto.Attribute.Type dtype) {
    switch (dtype) {
      case STRING:
        return DataType.STRING;
      case BYTE:
        return DataType.BYTE;
      case SHORT:
        return DataType.SHORT;
      case INT:
        return DataType.INT;
      case LONG:
        return DataType.LONG;
      case FLOAT:
        return DataType.FLOAT;
      case DOUBLE:
        return DataType.DOUBLE;
    }
    throw new IllegalStateException("illegal att type " + dtype);
  }

  static public DataType decodeDataType(ucar.nc2.stream.NcStreamProto.DataType dtype) {
    switch (dtype) {
      case CHAR:
        return DataType.CHAR;
      case BYTE:
        return DataType.BYTE;
      case SHORT:
        return DataType.SHORT;
      case INT:
        return DataType.INT;
      case LONG:
        return DataType.LONG;
      case FLOAT:
        return DataType.FLOAT;
      case DOUBLE:
        return DataType.DOUBLE;
      case STRING:
        return DataType.STRING;
      case STRUCTURE:
        return DataType.STRUCTURE;
      case SEQUENCE:
        return DataType.SEQUENCE;
      case ENUM1:
         return DataType.ENUM1;
      case ENUM2:
         return DataType.ENUM2;
      case ENUM4:
         return DataType.ENUM4;
       case OPAQUE:
        return DataType.OPAQUE;
    }
    throw new IllegalStateException("illegal data type " + dtype);
  }

}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy