ucar.nc2.iosp.hdf5.H5header Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright 1998-2012 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package ucar.nc2.iosp.hdf5;

import ucar.nc2.constants.CDM;
import ucar.nc2.util.Misc;
import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.*;
import ucar.nc2.EnumTypedef;
import ucar.nc2.iosp.netcdf4.Nc4;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.nc2.iosp.Layout;
import ucar.nc2.iosp.LayoutRegular;
import ucar.ma2.*;

import java.util.*;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.*;

/**
 * Read all of the metadata of an HD5 file.
 *
 * @author caron
 */

/* Implementation notes
 * any field called address is actually reletive to the base address.
 * any field called filePos or dataPos is a byte offset within the file.
 */
  /*
   * it appears theres no sure fire way to tell if the file was written by netcdf4 library
   *  1) if one of the the NETCF4-XXX atts are set
   *  2) dimension scales:
   *     1) all dimensions have a dimension scale
   *     2) they all have the same length as the dimension
   *     3) all variables' dimensions have a dimension scale
   */
public class H5header {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H5header.class);

  // special attribute names in HDF5
  static public final String HDF5_CLASS  = "CLASS";
  static public final String HDF5_DIMENSION_LIST  = "DIMENSION_LIST";
  static public final String HDF5_DIMENSION_SCALE  = "DIMENSION_SCALE";
  static public final String HDF5_DIMENSION_LABELS  = "DIMENSION_LABELS";
  static public final String HDF5_DIMENSION_NAME  = "NAME";
  static public final String HDF5_REFERENCE_LIST  = "REFERENCE_LIST";

  // debugging
  static private boolean debugEnum = false, debugVlen = false;
  static private boolean debug1 = false, debugDetail = false, debugPos = false, debugHeap = false, debugV = false;
  static private boolean debugGroupBtree = false, debugDataBtree = false, debugDataChunk = false, debugBtree2 = false;
  static private boolean debugContinueMessage = false, debugTracker = false, debugSoftLink = false, debugSymbolTable = false;
  static private boolean warnings = false, debugReference = false, debugRegionReference = false, debugCreationOrder = false, debugFractalHeap = false;
  static private boolean debugDimensionScales = false;

  static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debug1 = debugFlag.isSet("H5header/header");
    debugBtree2 = debugFlag.isSet("H5header/btree2");
    debugContinueMessage = debugFlag.isSet("H5header/continueMessage");
    debugDetail = debugFlag.isSet("H5header/headerDetails");
    debugDataBtree = debugFlag.isSet("H5header/dataBtree");
    debugDataChunk = debugFlag.isSet("H5header/dataChunk");
    debugGroupBtree = debugFlag.isSet("H5header/groupBtree");
    debugFractalHeap = debugFlag.isSet("H5header/fractalHeap");
    debugHeap = debugFlag.isSet("H5header/Heap");
    debugPos = debugFlag.isSet("H5header/filePos");
    debugReference = debugFlag.isSet("H5header/reference");
    debugSoftLink = debugFlag.isSet("H5header/softLink");
    debugSymbolTable = debugFlag.isSet("H5header/symbolTable");
    debugTracker = debugFlag.isSet("H5header/memTracker");
    debugV = debugFlag.isSet("H5header/Variable");
  }

  static private final byte[] head = {(byte) 0x89, 'H', 'D', 'F', '\r', '\n', 0x1a, '\n'};
  static private final String hdf5magic = new String(head);
  static private final long maxHeaderPos = 500000; // header's gotta be within this
  static private boolean transformReference = true;

  static public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException {
    long filePos = 0;
    long size = raf.length();
    byte[] b = new byte[8];

    // search forward for the header
    while ((filePos < size) && (filePos < maxHeaderPos)) {
      raf.seek(filePos);
      raf.read(b);
      String magic = new String(b);
      if (magic.equals(hdf5magic))
        return true;
      filePos = (filePos == 0) ? 512 : 2 * filePos;
    }

    return false;
  }

  ////////////////////////////////////////////////////////////////////////////////

  RandomAccessFile raf;
  ucar.nc2.NetcdfFile ncfile;
  private H5iosp h5iosp;

  private long baseAddress;
  byte sizeOffsets, sizeLengths;
  boolean isOffsetLong, isLengthLong;

  /* Cant always tell if written with netcdf library. if all dimensions have coordinate variables, eg:
    Q:/cdmUnitTest/formats/netcdf4/ncom_relo_fukushima_1km_tmp_2011040800_t000.nc4
   */
  private boolean isNetcdf4 = false;
  //Map dimIds = null; // if isNetcdf4 and all dimension scales have _Netcdf4Dimid attribute

  private H5Group rootGroup;
  private Map symlinkMap = new HashMap(200);
  private Map addressMap = new HashMap(200);
  private Map heapMap = new HashMap();
  private java.text.SimpleDateFormat hdfDateParser;

  private java.io.PrintStream debugOut = System.out;
  private MemTracker memTracker;

  H5header(RandomAccessFile myRaf, ucar.nc2.NetcdfFile ncfile, H5iosp h5iosp) {
    this.ncfile = ncfile;
    this.raf = myRaf;
    this.h5iosp = h5iosp;
  }

  public byte getSizeOffsets() {
    return sizeOffsets;
  }

  boolean isNetcdf4() {
    return isNetcdf4;
  }

  public void read(java.io.PrintStream debugPS) throws IOException {
    if (debugPS != null)
      debugOut = debugPS;

    long actualSize = raf.length();
    memTracker = new MemTracker(actualSize);

    // find the superblock - no limits on how far in
    boolean ok = false;
    long filePos = 0;
    byte[] b = new byte[8];
    while ((filePos < actualSize)) {
      raf.seek(filePos);
      raf.read(b);
      String magic = new String(b);
      if (magic.equals(hdf5magic)) {
        ok = true;
        break;
      }
      filePos = (filePos == 0) ? 512 : 2 * filePos;
    }
    if (!ok) throw new IOException("Not a netCDF4/HDF5 file ");
    if (debug1) debugOut.println("H5header 0pened file to read:'" + raf.getLocation() +"' size= " + actualSize);
    // now we are positioned right after the header

    // header information is in le byte order
    raf.order(RandomAccessFile.LITTLE_ENDIAN);

    long superblockStart = raf.getFilePointer() - 8;
    memTracker.add("header", 0, superblockStart);

    // superblock version
    byte versionSB = raf.readByte();

    if (versionSB < 2) {
      readSuperBlock1(superblockStart, versionSB);
    } else if (versionSB == 2) {
      readSuperBlock2(superblockStart);
    } else {
      throw new IOException("Unknown superblock version= " + versionSB);
    }

    // now look for symbolic links
    replaceSymbolicLinks(rootGroup);

    // recursively run through all the dataObjects and add them to the ncfile
    boolean allSharedDimensions =  makeNetcdfGroup(ncfile.getRootGroup(), rootGroup);
    if (allSharedDimensions) isNetcdf4 = true;

    /*if (debugReference) {
     System.out.println("DataObjects");
     for (DataObject ob : addressMap.values())
       System.out.println("  " + ob.name + " address= " + ob.address + " filePos= " + getFileOffset(ob.address));
   } */
    if (debugTracker) memTracker.report();
  }

  private void readSuperBlock1(long superblockStart, byte versionSB) throws IOException {
    byte versionFSS, versionGroup, versionSHMF;
    short btreeLeafNodeSize, btreeInternalNodeSize;
    int fileFlags;

    long heapAddress;
    long eofAddress;
    long driverBlockAddress;

    versionFSS = raf.readByte();
    versionGroup = raf.readByte();
    raf.readByte(); // skip 1 byte
    versionSHMF = raf.readByte();
    if (debugDetail)
      debugOut.println(" versionSB= " + versionSB + " versionFSS= " + versionFSS + " versionGroup= " + versionGroup +
          " versionSHMF= " + versionSHMF);

    sizeOffsets = raf.readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = raf.readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) debugOut.println(" sizeOffsets= " + sizeOffsets + " sizeLengths= " + sizeLengths);
    if (debugDetail) debugOut.println(" isLengthLong= " + isLengthLong + " isOffsetLong= " + isOffsetLong);

    raf.read(); // skip 1 byte
    //debugOut.println(" position="+mapBuffer.position());

    btreeLeafNodeSize = raf.readShort();
    btreeInternalNodeSize = raf.readShort();
    if (debugDetail)
      debugOut.println(" btreeLeafNodeSize= " + btreeLeafNodeSize + " btreeInternalNodeSize= " + btreeInternalNodeSize);

    //debugOut.println(" position="+mapBuffer.position());

    fileFlags = raf.readInt();
    if (debugDetail) debugOut.println(" fileFlags= 0x" + Integer.toHexString(fileFlags));

    if (versionSB == 1) {
      short storageInternalNodeSize = raf.readShort();
      raf.skipBytes(2);
    }

    baseAddress = readOffset();
    heapAddress = readOffset();
    eofAddress = readOffset();
    driverBlockAddress = readOffset();

    if (baseAddress != superblockStart) {
      baseAddress = superblockStart;
      eofAddress += superblockStart;
      if (debugDetail) debugOut.println(" baseAddress set to superblockStart");
    }

    if (debugDetail) {
      debugOut.println(" baseAddress= 0x" + Long.toHexString(baseAddress));
      debugOut.println(" global free space heap Address= 0x" + Long.toHexString(heapAddress));
      debugOut.println(" eof Address=" + eofAddress);
      debugOut.println(" raf length= " + raf.length());
      debugOut.println(" driver BlockAddress= 0x" + Long.toHexString(driverBlockAddress));
      debugOut.println();
    }
    memTracker.add("superblock", superblockStart, raf.getFilePointer());

    // look for file truncation
    long fileSize = raf.length();
    if (fileSize < eofAddress)
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize+"%nlocation= "+raf.getLocation());

    // next comes the root objext's SymbolTableEntry
    SymbolTableEntry rootEntry = new SymbolTableEntry(raf.getFilePointer());

    // extract the root group object, recursively read all objects
    long rootObjectAddress = rootEntry.getObjectAddress();
    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);

    /* LOOK what is this crap ??
    if (rootGroup.group == null) {
      // if the root object doesnt have a group message, check if the rootEntry is cache type 2
      if (rootEntry.btreeAddress != 0) {
        rootGroup.group = new GroupOld(null, "", rootEntry.btreeAddress, rootEntry.nameHeapAddress);
      } else {
        throw new IllegalStateException("root object not a group");
      }
    } */
  }

  private void readSuperBlock2(long superblockStart) throws IOException {
    sizeOffsets = raf.readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = raf.readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) debugOut.println(" sizeOffsets= " + sizeOffsets + " sizeLengths= " + sizeLengths);
    if (debugDetail) debugOut.println(" isLengthLong= " + isLengthLong + " isOffsetLong= " + isOffsetLong);

    byte fileFlags = raf.readByte();
    if (debugDetail) debugOut.println(" fileFlags= 0x" + Integer.toHexString(fileFlags));

    baseAddress = readOffset();
    long extensionAddress = readOffset();
    long eofAddress = readOffset();
    long rootObjectAddress = readOffset();
    int checksum = raf.readInt();

    if (debugDetail) {
      debugOut.println(" baseAddress= 0x" + Long.toHexString(baseAddress));
      debugOut.println(" extensionAddress= 0x" + Long.toHexString(extensionAddress));
      debugOut.println(" eof Address=" + eofAddress);
      debugOut.println(" rootObjectAddress= 0x" + Long.toHexString(rootObjectAddress));
      debugOut.println();
    }

    memTracker.add("superblock", superblockStart, raf.getFilePointer());

    if (baseAddress != superblockStart) {
      baseAddress = superblockStart;
      eofAddress += superblockStart;
      if (debugDetail) debugOut.println(" baseAddress set to superblockStart");
    }

    // look for file truncation
    long fileSize = raf.length();
    if (fileSize < eofAddress)
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize);

    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);
  }

  private void replaceSymbolicLinks(H5Group group) {
    if (group == null) return;

    List objList = group.nestedObjects;
    int count = 0;
    while (count < objList.size()) {
      DataObjectFacade dof = objList.get(count);

      if (dof.group != null) {           // group - recurse
        replaceSymbolicLinks(dof.group);

      } else if (dof.linkName != null) { // symbolic links
        DataObjectFacade link = symlinkMap.get(dof.linkName);
        if (link == null) {
          log.error(" WARNING Didnt find symbolic link=" + dof.linkName + " from " + dof.name);
          objList.remove(count);
          continue;
        }

        // dont allow loops
        if (link.group != null) {
          if (group.isChildOf(link.group)) {
            log.error(" ERROR Symbolic Link loop found =" + dof.linkName);
            objList.remove(count);
            continue;
          }
        }

        // dont allow in the same group. better would be to replicate the group with the new name
        if (dof.parent == link.parent) {
          objList.remove(dof);
          count--; // negate the incr
        } else  // replace
          objList.set(count, link);

        if (debugSoftLink) debugOut.println("  Found symbolic link=" + dof.linkName);
      }

      count++;
    }
  }

  ///////////////////////////////////////////////////////////////
  // construct netcdf objects

  private boolean makeNetcdfGroup(ucar.nc2.Group ncGroup, H5Group h5group) throws IOException {
    boolean allHaveSharedDimensions = true;
    if (h5group == null) return allHaveSharedDimensions; // ??

  /* 6/21/2013 new algorithm for dimensions.
    1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in order
    2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
    3. use DIMENSION_LIST to assign dimensions to data variables.
  */

    //  1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in order
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.isVariable)
        findDimensionScales(ncGroup, h5group, facade);
    }

    // 2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.is2DCoordinate)
        findDimensionScales2D(h5group, facade);
    }

    // old way
    /* deal with multidim dimension scales - ugh!
    if (dimIds != null) {
      for (DataObjectFacade dimscale : dimIds.values()) {
        if (dimscale.dobj.mds.ndims > 1) {
          StringBuilder sbuff = new StringBuilder();
          Attribute att = dimscale.netcdf4CoordinatesAtt;
          for (int i=0; i= 0) ? name.substring(pos + 1) : name;
            sbuff.append(dimName);
            sbuff.append(" ");
          }
          dimscale.dimList = sbuff.toString();
        }
      }
    }

    // deal with multidim dimension scales part two - double ugh!
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.isVariable && facade.netcdf4CoordinatesAtt != null && facade.dimList.equals("%REDO%")) {
        Formatter f = new Formatter();
        for (int i=0 ;i fatts = filterAttributes(h5group.facade.dobj.attributes);
    for (MessageAttribute matt : fatts) {
      try {
        makeAttributes(null, matt, ncGroup.getAttributes());
      } catch (InvalidRangeException e) {
        throw new IOException(e.getMessage());
      }
    }

    // add system attributes
    processSystemAttributes(h5group.facade.dobj.messages, ncGroup.getAttributes());
    return allHaveSharedDimensions;
  }

  /////////////////////////
  /* from http://www.unidata.ucar.edu/software/netcdf/docs/netcdf.html#NetCDF_002d4-Format
  C.3.7 Attributes

  Attributes in HDF5 and netCDF-4 correspond very closely. Each attribute in an HDF5 file is represented as an attribute
  in the netCDF-4 file, with the exception of the attributes below, which are ignored by the netCDF-4 API.

  _Netcdf4Coordinates An integer array containing the dimension IDs of a variable which is a multi-dimensional coordinate variable.
  _nc3_strict         When this (scalar, H5T_NATIVE_INT) attribute exists in the root group of the HDF5 file, the netCDF API will enforce
                      the netCDF classic model on the data file.
  REFERENCE_LIST      This attribute is created and maintained by the HDF5 dimension scale API.
  CLASS               This attribute is created and maintained by the HDF5 dimension scale API.
  DIMENSION_LIST      This attribute is created and maintained by the HDF5 dimension scale API.
  NAME                This attribute is created and maintained by the HDF5 dimension scale API.

----------
  from dim_scales_wk9 - Nunes.ppt

  Attribute named "CLASS" with the value "DIMENSION_SCALE"
  Optional attribute named "NAME"
  Attribute references to any associated Dataset

-------------
  from http://www.unidata.ucar.edu/mailing_lists/archives/netcdfgroup/2008/msg00093.html

  Then comes the part you will have to do for your datasets. You open the data
  dataset, get an ID, DID variable here, open the latitude dataset, get its ID,
  DSID variable here, and "link" the 2 with this call

  if (H5DSattach_scale(did,dsid,DIM0) < 0)

  what this function does is to associated the dataset DSID (latitude) with the
  *dimension* specified by the parameter DIM0 (0, in this case, the first
  dimension of the 2D array) of the dataset DID

  If you open HDF Explorer and expand the attributes of the "data" dataset you
  will see an attribute called DIMENSION_LIST.
  This is done by this function. It is an array that contains 2 HDF5 references,
  one for the latitude dataset, other for the longitude)

  If you expand the "lat" dataset , you will see that it contains an attribute
  called REFERENCE_LIST. It is a compound type that contains
  1)      a reference to my "data" dataset
  2)      the index of the data dataset this scale is to be associated with (0
  for the lat, 1 for the lon)
  */

  // find the Dimension Scale objects, turn them into shared dimensions
  // always has attribute CLASS = "DIMENSION_SCALE"
  // note that we dont bother looking at their REFERENCE_LIST
  private void findDimensionScales(ucar.nc2.Group g, H5Group h5group, DataObjectFacade facade) throws IOException {
    Iterator iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      if (matt.name.equals(HDF5_CLASS)) {
        Attribute att = makeAttribute(matt);
        String val = att.getStringValue();
        if (val.equals(HDF5_DIMENSION_SCALE)) {

           // create a dimension - always use the first dataspace length
          facade.dimList = addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0], facade.dobj.mds.maxLength[0] == -1);
          facade.hasNetcdfDimensions = true;
          if (! h5iosp.includeOriginalAttributes) iter.remove();

          if (facade.dobj.mds.ndims > 1)
            facade.is2DCoordinate = true;

          /* old way
          findNetcdf4DimidAttribute(facade);
          if (facade.dobj.mds.ndims == 1) { // 1D dimension scale
            // create a dimension
            facade.dimList = addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0], facade.dobj.mds.maxLength[0] == -1);
            if (! h5iosp.includeOriginalAttributes) iter.remove();
            if (debugDimensionScales)
              System.out.printf("Found dimScale %s for group '%s' matt=%s %n", facade.dimList, g.getFullName(), matt);
          } else {  // multiD dimension scale
            int dimIndex = findCoordinateDimensionIndex(facade, h5group);
            addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[dimIndex], facade.dobj.mds.maxLength[dimIndex] == -1);
            if (! h5iosp.includeOriginalAttributes) iter.remove();
            if (debugDimensionScales)
              System.out.printf("Found multidim dimScale %s for group '%s' matt=%s %n", facade.dimList, g.getFullName(), matt);
          }
          */

        }
      }
    }
  }

  private void findDimensionScales2D(H5Group h5group, DataObjectFacade facade) throws IOException {
    int[] lens = facade.dobj.mds.dimLength;
    if (lens.length > 2) {
      log.error("DIMENSION_LIST: dimension scale > 2 = {}", facade.getName());
      return;
    }

    // first dimension is itself
    String name = facade.getName();
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    StringBuilder sbuff = new StringBuilder();
    sbuff.append(dimName);
    sbuff.append(" ");

    // second dimension is really an anonymous dimension, ironically now we go through amazing hoops to keep it shared
    // 1. use dimids if they exist
    // 2. if length matches and unique, use it
    // 3. if no length matches or multiple matches, then use anonymous

    int want_len = lens[1]; // second dimension
    Dimension match = null;
    boolean unique = true;
    for (Dimension d : h5group.dimList) {
      if  (d.getLength() == want_len) {
        if (match == null) match = d;
        else unique = false;
      }
    }
    if (match != null && unique) {
      sbuff.append(match.getShortName()); // 2. if length matches and unique, use it

    } else {
      if (match == null) { // 3. if no length matches or multiple matches, then use anonymous
        log.error("DIMENSION_LIST: dimension scale {} has second dimension {} but no match", facade.getName(), want_len);
        sbuff.append(Integer.toString(want_len));
      } else {
        log.warn("DIMENSION_LIST: dimension scale {} has second dimension {} but multiple matches", facade.getName(), want_len);
        sbuff.append(Integer.toString(want_len));
      }
    }

    facade.dimList = sbuff.toString();
  }

  /* private void findNetcdf4DimidAttribute(DataObjectFacade facade) throws IOException {
    for (MessageAttribute matt : facade.dobj.attributes) {
      if (matt.name.equals(Nc4.NETCDF4_DIMID)) {
        if (dimIds == null) dimIds = new HashMap();
        Attribute att_dimid = makeAttribute(matt);
        Integer dimid = (Integer) att_dimid.getNumericValue();
        dimIds.put(dimid, facade);
        return;
      }
    }
    if (dimIds != null) // supposed to all have them
      log.warn("Missing "+Nc4.NETCDF4_DIMID+" attribute on "+facade.getName());
  } */


  /* the case of multidimensional dimension scale. We need to identify which index to use as the dimension length.
     the pattern is, eg:
      _Netcdf4Coordinates = 6, 4
      _Netcdf4Dimid = 6
  *
  private int findCoordinateDimensionIndex(DataObjectFacade facade, H5Group h5group) throws IOException {
    Attribute att_coord = null;
    Attribute att_dimid = null;
    for (MessageAttribute matt : facade.dobj.attributes) {
      if (matt.name.equals(Nc4.NETCDF4_COORDINATES))
        att_coord = makeAttribute(matt);
      if (matt.name.equals(Nc4.NETCDF4_DIMID))
        att_dimid = makeAttribute(matt);
    }
    if (att_coord != null && att_dimid != null) {
      facade.netcdf4CoordinatesAtt = att_coord;
      Integer want = (Integer) att_dimid.getNumericValue();
      for (int i=0; i iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      // find the dimensions - set length to maximum
      // DIMENSION_LIST contains, for each dimension, a list of references to Dimension Scales
      if (matt.name.equals(HDF5_DIMENSION_LIST)) { // references : may extend the dimension length
        Attribute att = makeAttribute(matt);       // this reads in the data
        if (att == null) {
          log.error("DIMENSION_LIST: failed to read on variable {}", facade.getName());

        } else if (att.getLength() !=  facade.dobj.mds.dimLength.length) { // some attempts to writing hdf5 directly fail here
          log.error("DIMENSION_LIST: must have same number of dimension scales as dimensions att={} on variable {}", att, facade.getName());

        } else {
          StringBuilder sbuff = new StringBuilder();
          for (int i = 0; i < att.getLength(); i++) {
            String name = att.getStringValue(i);
            String dimName = extendDimension(g, h5group, name, facade.dobj.mds.dimLength[i]);
            sbuff.append(dimName).append(" ");
          }
          facade.dimList = sbuff.toString();
          facade.hasNetcdfDimensions = true;
          if (debugDimensionScales) System.out.printf("Found dimList '%s' for group '%s' matt=%s %n", facade.dimList, g.getFullName(), matt);
          if (! h5iosp.includeOriginalAttributes) iter.remove();
        }

      } else if (matt.name.equals(HDF5_DIMENSION_NAME)) {
        Attribute att = makeAttribute(matt);
        String val = att.getStringValue();
        if (val.startsWith("This is a netCDF dimension but not a netCDF variable")) {
          facade.isVariable = false;
          isNetcdf4 = true;
        }
        if (! h5iosp.includeOriginalAttributes) iter.remove();
        if (debugDimensionScales) System.out.printf("Found %s %n", val);

      } else if (matt.name.equals(HDF5_REFERENCE_LIST))
        if (! h5iosp.includeOriginalAttributes) iter.remove();
    }
    boolean result =  facade.hasNetcdfDimensions || facade.dobj.mds.dimLength.length == 0;
    //if (!result)
    //  System.out.println("HEY");
    return result;

  }

  // add a dimension, return its name
  private String addDimension(ucar.nc2.Group g, H5Group h5group, String name, int length, boolean isUnlimited) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    //if (d == null)
    //  d = g.findDimension(dimName); // then look in parent groups  LOOK

    if (d == null) { // create if not found
      d = new Dimension(dimName, length, true, isUnlimited, false);
      d.setGroup(g);
      h5group.dimMap.put(dimName, d);
      h5group.dimList.add(d);
      if (debugDimensionScales) debugOut.println("addDimension name=" + name + " dim= " + d + " to group " + g);

    } else { // check has correct length
      if (d.getLength() != length)
        throw new IllegalStateException("addDimension: DimScale has different length than dimension it references dimScale="+dimName);
    }

    return d.getShortName();
  }

  // look for unlimited dimensions without dimension scale - must get length from the variable
  private String extendDimension(ucar.nc2.Group g, H5Group h5group, String name, int length) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    if (d == null)
      d = g.findDimension(dimName); // then look in parent groups

    if (d != null) {
      if (d.isUnlimited() && (length > d.getLength()))
        d.setLength(length);

      if (!d.isUnlimited() && (length != d.getLength())) {
        throw new IllegalStateException("extendDimension: DimScale has different length than dimension it references dimScale="+dimName);
      }
      return d.getShortName();
    }

    return dimName;
  }

  private void createDimensions(ucar.nc2.Group g, H5Group h5group) throws IOException {
    for (Dimension d : h5group.dimList) {
      g.addDimension(d);
    }
  }

  private List filterAttributes(List attList) {
    List result = new ArrayList(attList.size());
    for  (MessageAttribute matt : attList) {
      if (matt.name.equals(Nc4.NETCDF4_COORDINATES) || matt.name.equals(Nc4.NETCDF4_DIMID) || matt.name.equals(Nc4.NETCDF4_STRICT)) {
        isNetcdf4 = true;
      } else  {
        result.add(matt);
      }
    }
    return result;
  }

  /**
   * Create Attribute objects from the MessageAttribute and add to list
   *
   * @param s       if attribute for a Structure, then deconstruct and add to member variables
   * @param matt    attribute message
   * @param attList add Attribute to this list
   * @throws IOException on io error
   * @throws ucar.ma2.InvalidRangeException on shape error
   */
  private void makeAttributes(Structure s, MessageAttribute matt, List attList) throws IOException, InvalidRangeException {
    MessageDatatype mdt = matt.mdt;

    if (mdt.type == 6) { // structure
      Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
      ArrayStructure attData = (ArrayStructure) readAttributeData(matt, vinfo, DataType.STRUCTURE);

      if (null == s) {
        // flatten and add to list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          Array memberData = attData.extractMemberArray(sm);
          attList.add(new Attribute(matt.name+"."+sm.getName(), memberData));
        }

      } else {  // assign seperate attribute for each member
        StructureMembers smember = attData.getStructureMembers();
        for (Variable v : s.getVariables()) {
          StructureMembers.Member sm = smember.findMember(v.getShortName());
          if (null != sm) {
            Array memberData = attData.extractMemberArray(sm);
            v.addAttribute(new Attribute(matt.name, memberData));
          }
        }

        // look for unassigned membres, add to the list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          if (s.findVariable(sm.getName()) == null) {
            Array memberData = attData.extractMemberArray(sm);
            attList.add(new Attribute(matt.name+"."+sm.getName(), memberData));
          }
        }
      }

    } else {
      // make a single attribute
      attList.add( makeAttribute(matt));
    }

    // reading attribute values might change byte order during a read
    // put back to little endian for further header processing
    raf.order(RandomAccessFile.LITTLE_ENDIAN);
  }

  private Attribute makeAttribute(MessageAttribute matt) throws IOException {
    Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
    DataType dtype = vinfo.getNCDataType();

    // check for empty attribute case
    if (matt.mds.type == 2) {
      if (dtype == DataType.CHAR)
        return new Attribute(matt.name, ""); // empty char considered to be a 0 length string
      else
        return new Attribute(matt.name, dtype);
    }

    Array attData = null;
    try {
      attData = readAttributeData(matt, vinfo, dtype);
      attData.setUnsigned(matt.mdt.unsigned);
      
    } catch (InvalidRangeException e) {
      log.error("failed to read Attribute "+matt.name, e);
      return null;
    }

    Attribute result;
    if (attData.getElementType() == Array.class) { // vlen LOOK
      List