• Home
• org.lasersonlab.thredds
• cdm
• 5.0.0
• source code
• H5header.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

ucar.nc2.iosp.hdf5.H5header Maven / Gradle / Ivy

/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.hdf5;

import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.NCheader;
import ucar.nc2.util.Misc;
import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.*;
import ucar.nc2.EnumTypedef;
import ucar.nc2.iosp.netcdf4.Nc4;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.nc2.iosp.Layout;
import ucar.nc2.iosp.LayoutRegular;
import ucar.ma2.*;

import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.*;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.*;

/**
 * Read all of the metadata of an HD5 file.
 *
 * @author caron
 */

/* Implementation notes
 * any field called address is actually reletive to the base address.
 * any field called filePos or dataPos is a byte offset within the file.
 */
  /*
   * it appears theres no sure fire way to tell if the file was written by netcdf4 library
   *  1) if one of the the NETCF4-XXX atts are set
   *  2) dimension scales:
   *     1) all dimensions have a dimension scale
   *     2) they all have the same length as the dimension
   *     3) all variables' dimensions have a dimension scale
   */
public class H5header extends NCheader
{
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H5header.class);

  // special attribute names in HDF5
  static public final String HDF5_CLASS            = "CLASS";
  static public final String HDF5_DIMENSION_LIST   = "DIMENSION_LIST";
  static public final String HDF5_DIMENSION_SCALE  = "DIMENSION_SCALE";
  static public final String HDF5_DIMENSION_LABELS = "DIMENSION_LABELS";
  static public final String HDF5_DIMENSION_NAME   = "NAME";
  static public final String HDF5_REFERENCE_LIST   = "REFERENCE_LIST";

  // debugging
  static private boolean debugEnum = false, debugVlen = false;
  static private boolean debug1 = false, debugDetail = false, debugPos = false, debugHeap = false, debugV = false;
  static private boolean debugGroupBtree = false, debugDataBtree = false, debugBtree2 = false;
  static private boolean debugContinueMessage = false, debugTracker = false, debugSoftLink = false, debugHardLink = false, debugSymbolTable = false;
  static private boolean warnings = true, debugReference = false, debugRegionReference = false, debugCreationOrder = false, debugStructure = false;
  static private boolean debugDimensionScales = false;

  static public void setWarnings(boolean warn) {
    warnings = warn;
  }

  static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debug1           = debugFlag.isSet("H5header/header");
    debugBtree2      = debugFlag.isSet("H5header/btree2");
    debugContinueMessage = debugFlag.isSet("H5header/continueMessage");
    debugDetail      = debugFlag.isSet("H5header/headerDetails");
    debugDataBtree   = debugFlag.isSet("H5header/dataBtree");
    debugGroupBtree  = debugFlag.isSet("H5header/groupBtree");
    debugHeap        = debugFlag.isSet("H5header/Heap");
    debugPos         = debugFlag.isSet("H5header/filePos");
    debugReference   = debugFlag.isSet("H5header/reference");
    debugSoftLink    = debugFlag.isSet("H5header/softLink");
    debugHardLink    = debugFlag.isSet("H5header/hardLink");
    debugSymbolTable = debugFlag.isSet("H5header/symbolTable");
    debugTracker     = debugFlag.isSet("H5header/memTracker");
    debugV           = debugFlag.isSet("H5header/Variable");
    debugStructure   = debugFlag.isSet("H5header/structure");
  }

  static private final byte[] head = {(byte) 0x89, 'H', 'D', 'F', '\r', '\n', 0x1a, '\n'};
  static private final String hdf5magic = new String(head, CDM.utf8Charset);
  static private final long maxHeaderPos = 50000; // header's gotta be within this
  static private final boolean transformReference = true;

  static public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException {
    switch (checkFileType(raf)) {
    case NC_FORMAT_NETCDF4:
	return true;
    default: break;
    }
    return false;
  }

  ////////////////////////////////////////////////////////////////////////////////

  RandomAccessFile raf;
  ucar.nc2.NetcdfFile ncfile;
  private H5iosp h5iosp;

  private long baseAddress;
  byte sizeOffsets, sizeLengths;
  boolean isOffsetLong, isLengthLong;
  // boolean alreadyWarnNdimZero;

  /* Cant always tell if written with netcdf library. if all dimensions have coordinate variables, eg:
    Q:/cdmUnitTest/formats/netcdf4/ncom_relo_fukushima_1km_tmp_2011040800_t000.nc4
   */
  private boolean isNetcdf4 = false;
  //Map dimIds = null; // if isNetcdf4 and all dimension scales have _Netcdf4Dimid attribute

  private H5Group rootGroup;
  private Map symlinkMap = new HashMap<>(200);
  private Map addressMap = new HashMap<>(200);
  private Map heapMap = new HashMap<>();
  private java.text.SimpleDateFormat hdfDateParser;

  private java.io.PrintWriter debugOut = null;
  private MemTracker memTracker;

  H5header(RandomAccessFile myRaf, ucar.nc2.NetcdfFile ncfile, H5iosp h5iosp) {
    this.ncfile = ncfile;
    this.raf = myRaf;
    this.h5iosp = h5iosp;
  }

  public byte getSizeOffsets() {
    return sizeOffsets;
  }

  boolean isNetcdf4() {
    return isNetcdf4;
  }

  public void read(java.io.PrintWriter debugPS) throws IOException {
    if (debugPS != null)
      debugOut = debugPS;
    else if (debug1 || debugContinueMessage || debugCreationOrder || debugDetail || debugDimensionScales || debugGroupBtree || debugHardLink || debugHeap ||
            debugPos || debugReference || debugTracker || debugV  || debugSoftLink || warnings)
      debugOut = new PrintWriter( new OutputStreamWriter(System.out, CDM.utf8Charset));

    long actualSize = raf.length();

    if (debugTracker) memTracker = new MemTracker(actualSize);

    // find the superblock - no limits on how far in
    boolean ok = false;
    long filePos = 0;
    while ((filePos < actualSize-8)) {
      raf.seek(filePos);
      String magic = raf.readString(8);
      if (magic.equals(hdf5magic)) {
        ok = true;
        break;
      }
      filePos = (filePos == 0) ? 512 : 2 * filePos;
    }
    if (!ok) {
        throw new IOException("Not a netCDF4/HDF5 file ");
    }
    if (debug1) {
        log.debug("H5header opened file to read:'{}' size= {}", raf.getLocation(), actualSize);
    }
    // now we are positioned right after the header

    // header information is in le byte order
    raf.order(RandomAccessFile.LITTLE_ENDIAN);

    long superblockStart = raf.getFilePointer() - 8;
    if (debugTracker) memTracker.add("header", 0, superblockStart);

    // superblock version
    byte versionSB = raf.readByte();

    if (versionSB < 2) {
      readSuperBlock1(superblockStart, versionSB);
    } else if (versionSB == 2) {
      readSuperBlock2(superblockStart);
    } else {
      throw new IOException("Unknown superblock version= " + versionSB);
    }

    // now look for symbolic links LOOK this doesnt work; probably remove 10/27/14 jc
    replaceSymbolicLinks(rootGroup);

    // recursively run through all the dataObjects and add them to the ncfile
    boolean allSharedDimensions = makeNetcdfGroup(ncfile.getRootGroup(), rootGroup);
    if (allSharedDimensions) isNetcdf4 = true;

    /*if (debugReference) {
     log.debug("DataObjects");
     for (DataObject ob : addressMap.values())
       log.debug("  " + ob.name + " address= " + ob.address + " filePos= " + getFileOffset(ob.address));
   } */
    if (debugTracker ) {
      Formatter f= new Formatter();
      memTracker.report(f);
      log.debug(f.toString());
    }
    debugOut = null;
  }

  private void readSuperBlock1(long superblockStart, byte versionSB) throws IOException {
    byte versionFSS, versionGroup, versionSHMF;
    short btreeLeafNodeSize, btreeInternalNodeSize;
    int fileFlags;

    long heapAddress;
    long eofAddress;
    long driverBlockAddress;

    versionFSS = raf.readByte();
    versionGroup = raf.readByte();
    raf.readByte(); // skip 1 byte
    versionSHMF = raf.readByte();
    if (debugDetail) {
      log.debug(" versionSB= " + versionSB + " versionFSS= " + versionFSS + " versionGroup= " + versionGroup +
              " versionSHMF= " + versionSHMF);
    }

    sizeOffsets = raf.readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = raf.readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) {
        log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths);
        log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong);
    }

    raf.read(); // skip 1 byte
    //log.debug(" position="+mapBuffer.position());

    btreeLeafNodeSize = raf.readShort();
    btreeInternalNodeSize = raf.readShort();
    if (debugDetail) {
        log.debug(" btreeLeafNodeSize= {} btreeInternalNodeSize= {}", btreeLeafNodeSize, btreeInternalNodeSize);
    }
    //log.debug(" position="+mapBuffer.position());

    fileFlags = raf.readInt();
    if (debugDetail) {
        log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags));
    }

    if (versionSB == 1) {
      short storageInternalNodeSize = raf.readShort();
      raf.skipBytes(2);
    }

    baseAddress = readOffset();
    heapAddress = readOffset();
    eofAddress = readOffset();
    driverBlockAddress = readOffset();

    if (baseAddress != superblockStart) {
        baseAddress = superblockStart;
        eofAddress += superblockStart;
        if (debugDetail) {
            log.debug(" baseAddress set to superblockStart");
        }
    }

    if (debugDetail) {
      log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress));
      log.debug(" global free space heap Address= 0x{}", Long.toHexString(heapAddress));
      log.debug(" eof Address={}", eofAddress);
      log.debug(" raf length= {}", raf.length());
      log.debug(" driver BlockAddress= 0x{}", Long.toHexString(driverBlockAddress));
      log.debug("");
    }
    if (debugTracker) memTracker.add("superblock", superblockStart, raf.getFilePointer());

    // look for file truncation
    long fileSize = raf.length();
    if (fileSize < eofAddress)
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= " + raf.getLocation());

    // next comes the root objext's SymbolTableEntry
    SymbolTableEntry rootEntry = new SymbolTableEntry(raf.getFilePointer());

    // extract the root group object, recursively read all objects
    long rootObjectAddress = rootEntry.getObjectAddress();
    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);

    /* LOOK what is this crap ??
    if (rootGroup.group == null) {
      // if the root object doesnt have a group message, check if the rootEntry is cache type 2
      if (rootEntry.btreeAddress != 0) {
        rootGroup.group = new GroupOld(null, "", rootEntry.btreeAddress, rootEntry.nameHeapAddress);
      } else {
        throw new IllegalStateException("root object not a group");
      }
    } */
  }

  private void readSuperBlock2(long superblockStart) throws IOException {
    sizeOffsets = raf.readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = raf.readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) {
        log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths);
        log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong);
    }

    byte fileFlags = raf.readByte();
    if (debugDetail) {
        log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags));
    }

    baseAddress = readOffset();
    long extensionAddress = readOffset();
    long eofAddress = readOffset();
    long rootObjectAddress = readOffset();
    int checksum = raf.readInt();

    if (debugDetail) {
      log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress));
      log.debug(" extensionAddress= 0x{}", Long.toHexString(extensionAddress));
      log.debug(" eof Address={}", eofAddress);
      log.debug(" rootObjectAddress= 0x{}", Long.toHexString(rootObjectAddress));
      log.debug("");
    }

    if (debugTracker) memTracker.add("superblock", superblockStart, raf.getFilePointer());

    if (baseAddress != superblockStart) {
        baseAddress = superblockStart;
        eofAddress += superblockStart;
        if (debugDetail) {
            log.debug(" baseAddress set to superblockStart");
        }
    }

    // look for file truncation
    long fileSize = raf.length();
    if (fileSize < eofAddress) {
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize);
    }

    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);
  }

  private void replaceSymbolicLinks(H5Group group) {
    if (group == null) return;

    List objList = group.nestedObjects;
    int count = 0;
    while (count < objList.size()) {
      DataObjectFacade dof = objList.get(count);

      if (dof.group != null) {           // group - recurse
        replaceSymbolicLinks(dof.group);

      } else if (dof.linkName != null) { // symbolic links
        DataObjectFacade link = symlinkMap.get(dof.linkName);
        if (link == null) {
          log.warn(" WARNING Didnt find symbolic link={} from {}", dof.linkName, dof.name);
          objList.remove(count);
          continue;
        }

        // dont allow loops
        if (link.group != null) {
          if (group.isChildOf(link.group)) {
            log.warn(" ERROR Symbolic Link loop found ={}", dof.linkName);
            objList.remove(count);
            continue;
          }
        }

        // dont allow in the same group. better would be to replicate the group with the new name
        if (dof.parent == link.parent) {
          objList.remove(dof);
          count--; // negate the incr
        } else  // replace
          objList.set(count, link);

        if (debugSoftLink) {
            log.debug("  Found symbolic link={}", dof.linkName);
        }
      }

      count++;
    }
  }

  ///////////////////////////////////////////////////////////////
  // construct netcdf objects

  private boolean makeNetcdfGroup(ucar.nc2.Group ncGroup, H5Group h5group) throws IOException {
    // if (h5group == null) return true; // ??

  /* 6/21/2013 new algorithm for dimensions.
    1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in order
    2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
    3. use DIMENSION_LIST to assign dimensions to data variables.
  */

    //  1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in order
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.isVariable)
        findDimensionScales(ncGroup, h5group, facade);
    }

    // 2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.is2DCoordinate)
        findDimensionScales2D(h5group, facade);
    }

    // old way
    /* deal with multidim dimension scales - ugh!
    if (dimIds != null) {
      for (DataObjectFacade dimscale : dimIds.values()) {
        if (dimscale.dobj.mds.ndims > 1) {
          StringBuilder sbuff = new StringBuilder();
          Attribute att = dimscale.netcdf4CoordinatesAtt;
          for (int i=0; i= 0) ? name.substring(pos + 1) : name;
            sbuff.append(dimName);
            sbuff.append(" ");
          }
          dimscale.dimList = sbuff.toString();
        }
      }
    }

    // deal with multidim dimension scales part two - double ugh!
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.isVariable && facade.netcdf4CoordinatesAtt != null && facade.dimList.equals("%REDO%")) {
        Formatter f = new Formatter();
        for (int i=0 ;i fatts = filterAttributes(h5group.facade.dobj.attributes);
    for (MessageAttribute matt : fatts) {
      try {
        makeAttributes(null, matt, ncGroup);
      } catch (InvalidRangeException e) {
        throw new IOException(e.getMessage());
      }
    }

    // add system attributes
    processSystemAttributes(h5group.facade.dobj.messages, ncGroup);
    return allHaveSharedDimensions;
  }

  /////////////////////////
  /* from http://www.unidata.ucar.edu/software/netcdf/docs/netcdf.html#NetCDF_002d4-Format
  C.3.7 Attributes

  Attributes in HDF5 and netCDF-4 correspond very closely. Each attribute in an HDF5 file is represented as an attribute
  in the netCDF-4 file, with the exception of the attributes below, which are ignored by the netCDF-4 API.

  _Netcdf4Coordinates An integer array containing the dimension IDs of a variable which is a multi-dimensional coordinate variable.
  _nc3_strict         When this (scalar, H5T_NATIVE_INT) attribute exists in the root group of the HDF5 file, the netCDF API will enforce
                      the netCDF classic model on the data file.
  REFERENCE_LIST      This attribute is created and maintained by the HDF5 dimension scale API.
  CLASS               This attribute is created and maintained by the HDF5 dimension scale API.
  DIMENSION_LIST      This attribute is created and maintained by the HDF5 dimension scale API.
  NAME                This attribute is created and maintained by the HDF5 dimension scale API.

----------
  from dim_scales_wk9 - Nunes.ppt

  Attribute named "CLASS" with the value "DIMENSION_SCALE"
  Optional attribute named "NAME"
  Attribute references to any associated Dataset

-------------
  from http://www.unidata.ucar.edu/mailing_lists/archives/netcdfgroup/2008/msg00093.html

  Then comes the part you will have to do for your datasets. You open the data
  dataset, get an ID, DID variable here, open the latitude dataset, get its ID,
  DSID variable here, and "link" the 2 with this call

  if (H5DSattach_scale(did,dsid,DIM0) < 0)

  what this function does is to associated the dataset DSID (latitude) with the
  *dimension* specified by the parameter DIM0 (0, in this case, the first
  dimension of the 2D array) of the dataset DID

  If you open HDF Explorer and expand the attributes of the "data" dataset you
  will see an attribute called DIMENSION_LIST.
  This is done by this function. It is an array that contains 2 HDF5 references,
  one for the latitude dataset, other for the longitude)

  If you expand the "lat" dataset , you will see that it contains an attribute
  called REFERENCE_LIST. It is a compound type that contains
  1)      a reference to my "data" dataset
  2)      the index of the data dataset this scale is to be associated with (0
  for the lat, 1 for the lon)
  */

  // find the Dimension Scale objects, turn them into shared dimensions
  // always has attribute CLASS = "DIMENSION_SCALE"
  // note that we dont bother looking at their REFERENCE_LIST
  private void findDimensionScales(ucar.nc2.Group g, H5Group h5group, DataObjectFacade facade) throws IOException {
    Iterator iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      if (matt.name.equals(HDF5_CLASS)) {
        Attribute att = makeAttribute(matt);
        if (att == null) throw new IllegalStateException();
        String val = att.getStringValue();
        if (val.equals(HDF5_DIMENSION_SCALE) && facade.dobj.mds.ndims > 0) {

          // create a dimension - always use the first dataspace length
          facade.dimList = addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0], facade.dobj.mds.maxLength[0] == -1);
          facade.hasNetcdfDimensions = true;
          if (!h5iosp.includeOriginalAttributes) iter.remove();

          if (facade.dobj.mds.ndims > 1)
            facade.is2DCoordinate = true;

          /* old way
          findNetcdf4DimidAttribute(facade);
          if (facade.dobj.mds.ndims == 1) { // 1D dimension scale
            // create a dimension
            facade.dimList = addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0], facade.dobj.mds.maxLength[0] == -1);
            if (! h5iosp.includeOriginalAttributes) iter.remove();
            if (debugDimensionScales)
              System.out.printf("Found dimScale %s for group '%s' matt=%s %n", facade.dimList, g.getFullName(), matt);
          } else {  // multiD dimension scale
            int dimIndex = findCoordinateDimensionIndex(facade, h5group);
            addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[dimIndex], facade.dobj.mds.maxLength[dimIndex] == -1);
            if (! h5iosp.includeOriginalAttributes) iter.remove();
            if (debugDimensionScales)
              System.out.printf("Found multidim dimScale %s for group '%s' matt=%s %n", facade.dimList, g.getFullName(), matt);
          }
          */

        }
      }
    }
  }

  private void findDimensionScales2D(H5Group h5group, DataObjectFacade facade) throws IOException {
    int[] lens = facade.dobj.mds.dimLength;
    if (lens.length > 2) {
      log.warn("DIMENSION_LIST: dimension scale > 2 = {}", facade.getName());
      return;
    }

    // first dimension is itself
    String name = facade.getName();
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    StringBuilder sbuff = new StringBuilder();
    sbuff.append(dimName);
    sbuff.append(" ");

    // second dimension is really an anonymous dimension, ironically now we go through amazing hoops to keep it shared
    // 1. use dimids if they exist
    // 2. if length matches and unique, use it
    // 3. if no length matches or multiple matches, then use anonymous

    int want_len = lens[1]; // second dimension
    Dimension match = null;
    boolean unique = true;
    for (Dimension d : h5group.dimList) {
      if (d.getLength() == want_len) {
        if (match == null) match = d;
        else unique = false;
      }
    }
    if (match != null && unique) {
      sbuff.append(match.getShortName()); // 2. if length matches and unique, use it

    } else {
      if (match == null) { // 3. if no length matches or multiple matches, then use anonymous
        log.warn("DIMENSION_LIST: dimension scale {} has second dimension {} but no match", facade.getName(), want_len);
        sbuff.append(Integer.toString(want_len));
      } else {
        log.warn("DIMENSION_LIST: dimension scale {} has second dimension {} but multiple matches", facade.getName(), want_len);
        sbuff.append(Integer.toString(want_len));
      }
    }

    facade.dimList = sbuff.toString();
  }

  /* private void findNetcdf4DimidAttribute(DataObjectFacade facade) throws IOException {
    for (MessageAttribute matt : facade.dobj.attributes) {
      if (matt.name.equals(Nc4.NETCDF4_DIMID)) {
        if (dimIds == null) dimIds = new HashMap();
        Attribute att_dimid = makeAttribute(matt);
        Integer dimid = (Integer) att_dimid.getNumericValue();
        dimIds.put(dimid, facade);
        return;
      }
    }
    if (dimIds != null) // supposed to all have them
      log.warn("Missing "+Nc4.NETCDF4_DIMID+" attribute on "+facade.getName());
  } */


  /* the case of multidimensional dimension scale. We need to identify which index to use as the dimension length.
     the pattern is, eg:
      _Netcdf4Coordinates = 6, 4
      _Netcdf4Dimid = 6
  *
  private int findCoordinateDimensionIndex(DataObjectFacade facade, H5Group h5group) throws IOException {
    Attribute att_coord = null;
    Attribute att_dimid = null;
    for (MessageAttribute matt : facade.dobj.attributes) {
      if (matt.name.equals(Nc4.NETCDF4_COORDINATES))
        att_coord = makeAttribute(matt);
      if (matt.name.equals(Nc4.NETCDF4_DIMID))
        att_dimid = makeAttribute(matt);
    }
    if (att_coord != null && att_dimid != null) {
      facade.netcdf4CoordinatesAtt = att_coord;
      Integer want = (Integer) att_dimid.getNumericValue();
      for (int i=0; i iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      // find the dimensions - set length to maximum
      // DIMENSION_LIST contains, for each dimension, a list of references to Dimension Scales
      if (matt.name.equals(HDF5_DIMENSION_LIST)) { // references : may extend the dimension length
        Attribute att = makeAttribute(matt);       // this reads in the data
        if (att == null) {
          log.warn("DIMENSION_LIST: failed to read on variable {}", facade.getName());

        } else if (att.getLength() != facade.dobj.mds.dimLength.length) { // some attempts to writing hdf5 directly fail here
          log.warn("DIMENSION_LIST: must have same number of dimension scales as dimensions att={} on variable {}", att, facade.getName());

        } else {
          StringBuilder sbuff = new StringBuilder();
          for (int i = 0; i < att.getLength(); i++) {
            String name = att.getStringValue(i);
            String dimName = extendDimension(g, h5group, name, facade.dobj.mds.dimLength[i]);
            sbuff.append(dimName).append(" ");
          }
          facade.dimList = sbuff.toString();
          facade.hasNetcdfDimensions = true;
          if (debugDimensionScales) {
            log.debug("Found dimList '{}' for group '{}' matt={}", facade.dimList, g.getFullName(), matt);
          }
          if (!h5iosp.includeOriginalAttributes) iter.remove();
        }

      } else if (matt.name.equals(HDF5_DIMENSION_NAME)) {
        Attribute att = makeAttribute(matt);
        if (att == null) throw new IllegalStateException();
        String val = att.getStringValue();
        if (val.startsWith("This is a netCDF dimension but not a netCDF variable")) {
          facade.isVariable = false;
          isNetcdf4 = true;
        }
        if (!h5iosp.includeOriginalAttributes) iter.remove();
        if (debugDimensionScales) {
            log.debug("Found {}", val);
        }

      } else if (matt.name.equals(HDF5_REFERENCE_LIST))
        if (!h5iosp.includeOriginalAttributes) iter.remove();
    }
    return facade.hasNetcdfDimensions || facade.dobj.mds.dimLength.length == 0;

  }

  // add a dimension, return its name
  private String addDimension(ucar.nc2.Group g, H5Group h5group, String name, int length, boolean isUnlimited) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    //if (d == null)
    //  d = g.findDimension(dimName); // then look in parent groups  LOOK

    if (d == null) { // create if not found
      d = new Dimension(dimName, length, true, isUnlimited, false);
      d.setGroup(g);
      h5group.dimMap.put(dimName, d);
      h5group.dimList.add(d);
        if (debugDimensionScales) {
            log.debug("addDimension name=" + name + " dim= " + d + " to group " + g);
        }

    } else { // check has correct length
      if (d.getLength() != length)
        throw new IllegalStateException("addDimension: DimScale has different length than dimension it references dimScale=" + dimName);
    }

    return d.getShortName();
  }

  // look for unlimited dimensions without dimension scale - must get length from the variable
  private String extendDimension(ucar.nc2.Group g, H5Group h5group, String name, int length) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    if (d == null)
      d = g.findDimension(dimName); // then look in parent groups

    if (d != null) {
      if (d.isUnlimited() && (length > d.getLength()))
        d.setLength(length);

      if (!d.isUnlimited() && (length != d.getLength())) {
        throw new IllegalStateException("extendDimension: DimScale has different length than dimension it references dimScale=" + dimName);
      }
      return d.getShortName();
    }

    return dimName;
  }

  private void createDimensions(ucar.nc2.Group g, H5Group h5group) throws IOException {
    for (Dimension d : h5group.dimList) {
      g.addDimension(d);
    }
  }

  private List filterAttributes(List attList) {
    List result = new ArrayList<>(attList.size());
    for (MessageAttribute matt : attList) {
      if (matt.name.equals(Nc4.NETCDF4_COORDINATES) || matt.name.equals(Nc4.NETCDF4_DIMID) || matt.name.equals(Nc4.NETCDF4_STRICT)) {
        isNetcdf4 = true;
      } else {
        result.add(matt);
      }
    }
    return result;
  }

  /**
   * Create Attribute objects from the MessageAttribute and add to list
   *
   * @param s            if attribute for a Structure, then deconstruct and add to member variables
   * @param matt         attribute message
   * @param attContainer add Attribute to this
   * @throws IOException                    on io error
   * @throws ucar.ma2.InvalidRangeException on shape error
   */
  private void makeAttributes(Structure s, MessageAttribute matt, AttributeContainer attContainer) throws IOException, InvalidRangeException {
    MessageDatatype mdt = matt.mdt;

    if (mdt.type == 6) { // structure
      Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
      ArrayStructure attData = (ArrayStructure) readAttributeData(matt, vinfo, DataType.STRUCTURE);

      if (null == s) {
        // flatten and add to list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          Array memberData = attData.extractMemberArray(sm);
          attContainer.addAttribute(new Attribute(matt.name + "." + sm.getName(), memberData));
        }

      } else if (matt.name.equals(CDM.FIELD_ATTS)) {
          // flatten and add to list
          for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
            String memberName = sm.getName();
            int pos = memberName.indexOf(":");
            if (pos < 0) continue; // LOOK
            String fldName = memberName.substring(0,pos);
            String attName = memberName.substring(pos+1);
            Array memberData = attData.extractMemberArray(sm);
            Variable v = s.findVariable(fldName);
            if (v == null) continue; // LOOK
            v.addAttribute(new Attribute(attName, memberData));
          }

      } else {  // assign separate attribute for each member
        StructureMembers attMembers = attData.getStructureMembers();
        for (Variable v : s.getVariables()) {
          StructureMembers.Member sm = attMembers.findMember(v.getShortName()); // does the compound attribute have a member with same name as nested variable ?
          if (null != sm) {
            Array memberData = attData.extractMemberArray(sm);                 // if so, add the att to the member variable, using the name of the compound attribute
            v.addAttribute(new Attribute(matt.name, memberData));              // LOOK want to check for missing values....
          }
        }

        // look for unassigned members, add to the list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          if (s.findVariable(sm.getName()) == null) {
            Array memberData = attData.extractMemberArray(sm);
            attContainer.addAttribute(new Attribute(matt.name + "." + sm.getName(), memberData));
          }
        }
      }

    } else {
      // make a single attribute
      Attribute att = makeAttribute(matt);
      if (att != null)
        attContainer.addAttribute(att);
    }

    // reading attribute values might change byte order during a read
    // put back to little endian for further header processing
    raf.order(RandomAccessFile.LITTLE_ENDIAN);
  }

  private Attribute makeAttribute(MessageAttribute matt) throws IOException {
    Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
    DataType dtype = vinfo.getNCDataType();

    // check for empty attribute case
    if (matt.mds.type == 2) {
      if (dtype == DataType.CHAR)
        return new Attribute(matt.name, ""); // empty char considered to be a 0 length string
      else
        return new Attribute(matt.name, dtype);
    }

    //if (matt.name.equals("DIMENSION_LIST"))
     // System.out.printf("HEY%n");

    Array attData;
    try {
      attData = readAttributeData(matt, vinfo, dtype);

    } catch (InvalidRangeException e) {
      log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + raf.getLocation());
      return null;
    }

    Attribute result;
    if (attData.isVlen()) {
      List