ucar.nc2.iosp.hdf5.H5header Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of org.gecko.ucar.netcdf Show documentation
Re-packaged Unidata netCDF
The newest version!
/*
 * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.iosp.hdf5;

import java.nio.charset.StandardCharsets;
import ucar.nc2.constants.CDM;
import ucar.nc2.iosp.NCheader;
import ucar.nc2.util.Misc;
import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.*;
import ucar.nc2.iosp.netcdf4.Nc4;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.nc2.iosp.Layout;
import ucar.nc2.iosp.LayoutRegular;
import ucar.ma2.*;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.*;
import java.io.IOException;
import java.nio.*;

/**
 * Read all of the metadata of an HD5 file.
 *
 * @author caron
 */

/*
 * Implementation notes
 * any field called address is actually relative to the base address.
 * any field called filePos or dataPos is a byte offset within the file.
 */
/*
 * it appears theres no sure fire way to tell if the file was written by netcdf4 library
 * 1) if one of the the NETCF4-XXX atts are set
 * 2) dimension scales:
 * 1) all dimensions have a dimension scale
 * 2) they all have the same length as the dimension
 * 3) all variables' dimensions have a dimension scale
 */
public class H5header extends NCheader implements H5headerIF {
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H5header.class);

  // special attribute names in HDF5
  public static final String HDF5_CLASS = "CLASS";
  public static final String HDF5_DIMENSION_LIST = "DIMENSION_LIST";
  public static final String HDF5_DIMENSION_SCALE = "DIMENSION_SCALE";
  public static final String HDF5_DIMENSION_LABELS = "DIMENSION_LABELS";
  public static final String HDF5_DIMENSION_NAME = "NAME";
  public static final String HDF5_REFERENCE_LIST = "REFERENCE_LIST";

  // debugging
  private static boolean debugEnum, debugVlen;
  private static boolean debug1, debugDetail, debugPos, debugHeap, debugV;
  private static boolean debugGroupBtree, debugDataBtree, debugBtree2;
  private static boolean debugContinueMessage, debugTracker, debugSoftLink, debugHardLink, debugSymbolTable;
  private static boolean warnings = true, debugReference, debugRegionReference, debugCreationOrder, debugStructure;
  private static boolean debugDimensionScales;

  // NULL string value, following netCDF-C, set to NIL
  private static final String NULL_STRING_VALUE = "NIL";

  public static void setWarnings(boolean warn) {
    warnings = warn;
  }

  public static void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
    debug1 = debugFlag.isSet("H5header/header");
    debugBtree2 = debugFlag.isSet("H5header/btree2");
    debugContinueMessage = debugFlag.isSet("H5header/continueMessage");
    debugDetail = debugFlag.isSet("H5header/headerDetails");
    debugDataBtree = debugFlag.isSet("H5header/dataBtree");
    debugGroupBtree = debugFlag.isSet("H5header/groupBtree");
    debugHeap = debugFlag.isSet("H5header/Heap");
    debugPos = debugFlag.isSet("H5header/filePos");
    debugReference = debugFlag.isSet("H5header/reference");
    debugSoftLink = debugFlag.isSet("H5header/softLink");
    debugHardLink = debugFlag.isSet("H5header/hardLink");
    debugSymbolTable = debugFlag.isSet("H5header/symbolTable");
    debugTracker = debugFlag.isSet("H5header/memTracker");
    debugV = debugFlag.isSet("H5header/Variable");
    debugStructure = debugFlag.isSet("H5header/structure");
  }

  private static final byte[] head = {(byte) 0x89, 'H', 'D', 'F', '\r', '\n', 0x1a, '\n'};
  private static final String hdf5magic = new String(head, StandardCharsets.UTF_8);
  private static final long maxHeaderPos = 50000; // header's gotta be within this
  private static final boolean transformReference = true;

  public static boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException {
    return checkFileType(raf) == NC_FORMAT_NETCDF4;
  }

  ////////////////////////////////////////////////////////////////////////////////

  ucar.nc2.NetcdfFile ncfile;
  private H5iosp h5iosp;

  private long baseAddress;
  byte sizeOffsets, sizeLengths;
  boolean isOffsetLong, isLengthLong;
  // boolean alreadyWarnNdimZero;

  /*
   * Cant always tell if written with netcdf library. if all dimensions have coordinate variables, eg:
   * Q:/cdmUnitTest/formats/netcdf4/ncom_relo_fukushima_1km_tmp_2011040800_t000.nc4
   */
  private boolean isNetcdf4;
  // Map dimIds = null; // if isNetcdf4 and all dimension scales have _Netcdf4Dimid attribute

  private H5Group rootGroup;
  private Map symlinkMap = new HashMap<>(200);
  private Map addressMap = new HashMap<>(200);
  private Map heapMap = new HashMap<>();
  private java.text.SimpleDateFormat hdfDateParser;

  private java.io.PrintWriter debugOut;
  private MemTracker memTracker;

  H5header(ucar.nc2.NetcdfFile ncfile, H5iosp h5iosp) {
    this.ncfile = ncfile;
    this.h5iosp = h5iosp;
  }

  @Override
  public byte getSizeOffsets() {
    return sizeOffsets;
  }

  @Override
  public byte getSizeLengths() {
    return sizeLengths;
  }

  boolean isNetcdf4() {
    return isNetcdf4;
  }

  boolean isClassic() {
    return false; // TODO
  }

  public void read(java.io.PrintWriter debugPS) throws IOException {
    if (debugPS != null) {
      debugOut = debugPS;
    } else if (debug1 || debugContinueMessage || debugCreationOrder || debugDetail || debugDimensionScales
        || debugGroupBtree || debugHardLink || debugHeap || debugPos || debugReference || debugTracker || debugV
        || debugSoftLink || warnings) {
      debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
    }

    long actualSize = getRandomAccessFile().length();

    if (debugTracker)
      memTracker = new MemTracker(actualSize);

    // find the superblock - no limits on how far in
    boolean ok = false;
    long filePos = 0;
    while ((filePos < actualSize - 8)) {
      getRandomAccessFile().seek(filePos);
      String magic = getRandomAccessFile().readString(8);
      if (magic.equals(hdf5magic)) {
        ok = true;
        break;
      }
      filePos = (filePos == 0) ? 512 : 2 * filePos;
    }
    if (!ok) {
      throw new IOException("Not a netCDF4/HDF5 file ");
    }
    if (debug1) {
      log.debug("H5header opened file to read:'{}' size= {}", getRandomAccessFile().getLocation(), actualSize);
    }
    // now we are positioned right after the header

    // header information is in le byte order
    getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN);

    long superblockStart = getRandomAccessFile().getFilePointer() - 8;
    if (debugTracker)
      memTracker.add("header", 0, superblockStart);

    // superblock version
    byte versionSB = getRandomAccessFile().readByte();

    if (versionSB < 2) {
      readSuperBlock1(superblockStart, versionSB);
    } else if (versionSB == 2) {
      readSuperBlock2(superblockStart);
    } else {
      throw new IOException("Unknown superblock version= " + versionSB);
    }

    // now look for symbolic links LOOK this doesnt work; probably remove 10/27/14 jc
    replaceSymbolicLinks(rootGroup);

    // recursively run through all the dataObjects and add them to the ncfile
    boolean allSharedDimensions = makeNetcdfGroup(ncfile.getRootGroup(), rootGroup);
    if (allSharedDimensions)
      isNetcdf4 = true;

    if (debugTracker) {
      Formatter f = new Formatter();
      memTracker.report(f);
      log.debug(f.toString());
    }

    debugOut = null;
  }

  private void readSuperBlock1(long superblockStart, byte versionSB) throws IOException {
    byte versionFSS, versionGroup, versionSHMF;
    short btreeLeafNodeSize, btreeInternalNodeSize;
    int fileFlags;

    long heapAddress;
    long eofAddress;
    long driverBlockAddress;

    versionFSS = getRandomAccessFile().readByte();
    versionGroup = getRandomAccessFile().readByte();
    getRandomAccessFile().readByte(); // skip 1 byte
    versionSHMF = getRandomAccessFile().readByte();
    if (debugDetail) {
      log.debug(" versionSB= " + versionSB + " versionFSS= " + versionFSS + " versionGroup= " + versionGroup
          + " versionSHMF= " + versionSHMF);
    }

    sizeOffsets = getRandomAccessFile().readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = getRandomAccessFile().readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) {
      log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths);
      log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong);
    }

    getRandomAccessFile().read(); // skip 1 byte
    // log.debug(" position="+mapBuffer.position());

    btreeLeafNodeSize = getRandomAccessFile().readShort();
    btreeInternalNodeSize = getRandomAccessFile().readShort();
    if (debugDetail) {
      log.debug(" btreeLeafNodeSize= {} btreeInternalNodeSize= {}", btreeLeafNodeSize, btreeInternalNodeSize);
    }
    // log.debug(" position="+mapBuffer.position());

    fileFlags = getRandomAccessFile().readInt();
    if (debugDetail) {
      log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags));
    }

    if (versionSB == 1) {
      short storageInternalNodeSize = getRandomAccessFile().readShort();
      getRandomAccessFile().skipBytes(2);
    }

    baseAddress = readOffset();
    heapAddress = readOffset();
    eofAddress = readOffset();
    driverBlockAddress = readOffset();

    if (baseAddress != superblockStart) {
      baseAddress = superblockStart;
      eofAddress += superblockStart;
      if (debugDetail) {
        log.debug(" baseAddress set to superblockStart");
      }
    }

    if (debugDetail) {
      log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress));
      log.debug(" global free space heap Address= 0x{}", Long.toHexString(heapAddress));
      log.debug(" eof Address={}", eofAddress);
      log.debug(" raf length= {}", getRandomAccessFile().length());
      log.debug(" driver BlockAddress= 0x{}", Long.toHexString(driverBlockAddress));
      log.debug("");
    }
    if (debugTracker)
      memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer());

    // look for file truncation
    long fileSize = getRandomAccessFile().length();
    if (fileSize < eofAddress)
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= "
          + getRandomAccessFile().getLocation());

    // next comes the root objext's SymbolTableEntry
    SymbolTableEntry rootEntry = new SymbolTableEntry(getRandomAccessFile().getFilePointer());

    // extract the root group object, recursively read all objects
    long rootObjectAddress = rootEntry.getObjectAddress();
    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);

    /*
     * LOOK what is this crap ??
     * if (rootGroup.group == null) {
     * // if the root object doesnt have a group message, check if the rootEntry is cache type 2
     * if (rootEntry.btreeAddress != 0) {
     * rootGroup.group = new GroupOld(null, "", rootEntry.btreeAddress, rootEntry.nameHeapAddress);
     * } else {
     * throw new IllegalStateException("root object not a group");
     * }
     * }
     */
  }

  private void readSuperBlock2(long superblockStart) throws IOException {
    sizeOffsets = getRandomAccessFile().readByte();
    isOffsetLong = (sizeOffsets == 8);

    sizeLengths = getRandomAccessFile().readByte();
    isLengthLong = (sizeLengths == 8);
    if (debugDetail) {
      log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths);
      log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong);
    }

    byte fileFlags = getRandomAccessFile().readByte();
    if (debugDetail) {
      log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags));
    }

    baseAddress = readOffset();
    long extensionAddress = readOffset();
    long eofAddress = readOffset();
    long rootObjectAddress = readOffset();
    int checksum = getRandomAccessFile().readInt();

    if (debugDetail) {
      log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress));
      log.debug(" extensionAddress= 0x{}", Long.toHexString(extensionAddress));
      log.debug(" eof Address={}", eofAddress);
      log.debug(" rootObjectAddress= 0x{}", Long.toHexString(rootObjectAddress));
      log.debug("");
    }

    if (debugTracker)
      memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer());

    if (baseAddress != superblockStart) {
      baseAddress = superblockStart;
      eofAddress += superblockStart;
      if (debugDetail) {
        log.debug(" baseAddress set to superblockStart");
      }
    }

    // look for file truncation
    long fileSize = getRandomAccessFile().length();
    if (fileSize < eofAddress) {
      throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize);
    }

    DataObjectFacade f = new DataObjectFacade(null, "", rootObjectAddress);
    rootGroup = new H5Group(f);
  }

  private void replaceSymbolicLinks(H5Group group) {
    if (group == null)
      return;

    List objList = group.nestedObjects;
    int count = 0;
    while (count < objList.size()) {
      DataObjectFacade dof = objList.get(count);

      if (dof.group != null) { // group - recurse
        replaceSymbolicLinks(dof.group);

      } else if (dof.linkName != null) { // symbolic links
        DataObjectFacade link = symlinkMap.get(dof.linkName);
        if (link == null) {
          log.warn(" WARNING Didnt find symbolic link={} from {}", dof.linkName, dof.name);
          objList.remove(count);
          continue;
        }

        // dont allow loops
        if (link.group != null) {
          if (group.isChildOf(link.group)) {
            log.warn(" ERROR Symbolic Link loop found ={}", dof.linkName);
            objList.remove(count);
            continue;
          }
        }

        // dont allow in the same group. better would be to replicate the group with the new name
        if (dof.parent == link.parent) {
          objList.remove(dof);
          count--; // negate the incr
        } else // replace
          objList.set(count, link);

        if (debugSoftLink) {
          log.debug("  Found symbolic link={}", dof.linkName);
        }
      }

      count++;
    }
  }

  ///////////////////////////////////////////////////////////////
  // construct netcdf objects

  private boolean makeNetcdfGroup(ucar.nc2.Group ncGroup, H5Group h5group) throws IOException {
    // if (h5group == null) return true; // ??

    /*
     * 6/21/2013 new algorithm for dimensions.
     * 1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in
     * order
     * 2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple
     * match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
     * 3. use DIMENSION_LIST to assign dimensions to data variables.
     */

    // 1. find all objects with all CLASS = "DIMENSION_SCALE", make into a dimension. use shape(0) as length. keep in
    // order
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.isVariable)
        findDimensionScales(ncGroup, h5group, facade);
    }

    // 2. if also a variable (NAME != "This is a ...") then first dim = itself, second matches length, if multiple
    // match, use :_Netcdf4Coordinates = 0, 3 and order of dimensions.
    for (DataObjectFacade facade : h5group.nestedObjects) {
      if (facade.is2DCoordinate)
        findDimensionScales2D(h5group, facade);
    }

    // old way
    /*
     * deal with multidim dimension scales - ugh!
     * if (dimIds != null) {
     * for (DataObjectFacade dimscale : dimIds.values()) {
     * if (dimscale.dobj.mds.ndims > 1) {
     * StringBuilder sbuff = new StringBuilder();
     * Attribute att = dimscale.netcdf4CoordinatesAtt;
     * for (int i=0; i= 0) ? name.substring(pos + 1) : name;
     * sbuff.append(dimName);
     * sbuff.append(" ");
     * }
     * dimscale.dimList = sbuff.toString();
     * }
     * }
     * }
     * 
     * // deal with multidim dimension scales part two - double ugh!
     * for (DataObjectFacade facade : h5group.nestedObjects) {
     * if (facade.isVariable && facade.netcdf4CoordinatesAtt != null && facade.dimList.equals("%REDO%")) {
     * Formatter f = new Formatter();
     * for (int i=0 ;i fatts = filterAttributes(h5group.facade.dobj.attributes);
    for (MessageAttribute matt : fatts) {
      try {
        makeAttributes(null, matt, ncGroup);
      } catch (InvalidRangeException e) {
        throw new IOException(e.getMessage());
      }
    }

    // add system attributes
    processSystemAttributes(h5group.facade.dobj.messages, ncGroup);
    return allHaveSharedDimensions;
  }

  /////////////////////////
  /*
   * from https://www.unidata.ucar.edu/software/netcdf/docs/netcdf.html#NetCDF_002d4-Format
   * C.3.7 Attributes
   * 
   * Attributes in HDF5 and netCDF-4 correspond very closely. Each attribute in an HDF5 file is represented as an
   * attribute
   * in the netCDF-4 file, with the exception of the attributes below, which are ignored by the netCDF-4 API.
   * 
   * _Netcdf4Coordinates An integer array containing the dimension IDs of a variable which is a multi-dimensional
   * coordinate variable.
   * _nc3_strict When this (scalar, H5T_NATIVE_INT) attribute exists in the root group of the HDF5 file, the netCDF API
   * will enforce
   * the netCDF classic model on the data file.
   * REFERENCE_LIST This attribute is created and maintained by the HDF5 dimension scale API.
   * CLASS This attribute is created and maintained by the HDF5 dimension scale API.
   * DIMENSION_LIST This attribute is created and maintained by the HDF5 dimension scale API.
   * NAME This attribute is created and maintained by the HDF5 dimension scale API.
   * 
   * ----------
   * from dim_scales_wk9 - Nunes.ppt
   * 
   * Attribute named "CLASS" with the value "DIMENSION_SCALE"
   * Optional attribute named "NAME"
   * Attribute references to any associated Dataset
   * 
   * -------------
   * from https://www.unidata.ucar.edu/mailing_lists/archives/netcdfgroup/2008/msg00093.html
   * 
   * Then comes the part you will have to do for your datasets. You open the data
   * dataset, get an ID, DID variable here, open the latitude dataset, get its ID,
   * DSID variable here, and "link" the 2 with this call
   * 
   * if (H5DSattach_scale(did,dsid,DIM0) < 0)
   * 
   * what this function does is to associated the dataset DSID (latitude) with the
   * dimension* specified by the parameter DIM0 (0, in this case, the first
   * dimension of the 2D array) of the dataset DID
   * 
   * If you open HDF Explorer and expand the attributes of the "data" dataset you
   * will see an attribute called DIMENSION_LIST.
   * This is done by this function. It is an array that contains 2 HDF5 references,
   * one for the latitude dataset, other for the longitude)
   * 
   * If you expand the "lat" dataset , you will see that it contains an attribute
   * called REFERENCE_LIST. It is a compound type that contains
   * 1) a reference to my "data" dataset
   * 2) the index of the data dataset this scale is to be associated with (0
   * for the lat, 1 for the lon)
   */

  // find the Dimension Scale objects, turn them into shared dimensions
  // always has attribute CLASS = "DIMENSION_SCALE"
  // note that we dont bother looking at their REFERENCE_LIST
  private void findDimensionScales(ucar.nc2.Group g, H5Group h5group, DataObjectFacade facade) throws IOException {
    Iterator iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      if (matt.name.equals(HDF5_CLASS)) {
        Attribute att = makeAttribute(matt);
        if (att == null)
          throw new IllegalStateException();
        String val = att.getStringValue();
        if (val.equals(HDF5_DIMENSION_SCALE) && facade.dobj.mds.ndims > 0) {

          // create a dimension - always use the first dataspace length
          facade.dimList =
              addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0], facade.dobj.mds.maxLength[0] == -1);
          facade.hasNetcdfDimensions = true;
          if (!h5iosp.includeOriginalAttributes)
            iter.remove();

          if (facade.dobj.mds.ndims > 1)
            facade.is2DCoordinate = true;

          /*
           * old way
           * findNetcdf4DimidAttribute(facade);
           * if (facade.dobj.mds.ndims == 1) { // 1D dimension scale
           * // create a dimension
           * facade.dimList = addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[0],
           * facade.dobj.mds.maxLength[0] == -1);
           * if (! h5iosp.includeOriginalAttributes) iter.remove();
           * } else { // multiD dimension scale
           * int dimIndex = findCoordinateDimensionIndex(facade, h5group);
           * addDimension(g, h5group, facade.name, facade.dobj.mds.dimLength[dimIndex],
           * facade.dobj.mds.maxLength[dimIndex] == -1);
           * if (!h5iosp.includeOriginalAttributes) iter.remove();
           * }
           */

        }
      }
    }
  }

  private void findDimensionScales2D(H5Group h5group, DataObjectFacade facade) {
    int[] lens = facade.dobj.mds.dimLength;
    if (lens.length > 2) {
      log.warn("DIMENSION_LIST: dimension scale > 2 = {}", facade.getName());
      return;
    }

    // first dimension is itself
    String name = facade.getName();
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    StringBuilder sbuff = new StringBuilder();
    sbuff.append(dimName);
    sbuff.append(" ");

    // second dimension is really an anonymous dimension, ironically now we go through amazing hoops to keep it shared
    // 1. use dimids if they exist
    // 2. if length matches and unique, use it
    // 3. if no length matches or multiple matches, then use anonymous

    int want_len = lens[1]; // second dimension
    Dimension match = null;
    boolean unique = true;
    for (Dimension d : h5group.dimList) {
      if (d.getLength() == want_len) {
        if (match == null)
          match = d;
        else
          unique = false;
      }
    }
    if (match != null && unique) {
      sbuff.append(match.getShortName()); // 2. if length matches and unique, use it

    } else {
      if (match == null) { // 3. if no length matches or multiple matches, then use anonymous
        log.warn("DIMENSION_LIST: dimension scale {} has second dimension {} but no match", facade.getName(), want_len);
        sbuff.append(want_len);
      } else {
        log.warn("DIMENSION_LIST: dimension scale {} has second dimension {} but multiple matches", facade.getName(),
            want_len);
        sbuff.append(want_len);
      }
    }

    facade.dimList = sbuff.toString();
  }

  /*
   * private void findNetcdf4DimidAttribute(DataObjectFacade facade) throws IOException {
   * for (MessageAttribute matt : facade.dobj.attributes) {
   * if (matt.name.equals(Nc4.NETCDF4_DIMID)) {
   * if (dimIds == null) dimIds = new HashMap();
   * Attribute att_dimid = makeAttribute(matt);
   * Integer dimid = (Integer) att_dimid.getNumericValue();
   * dimIds.put(dimid, facade);
   * return;
   * }
   * }
   * if (dimIds != null) // supposed to all have them
   * log.warn("Missing "+Nc4.NETCDF4_DIMID+" attribute on "+facade.getName());
   * }
   */


  /*
   * the case of multidimensional dimension scale. We need to identify which index to use as the dimension length.
   * the pattern is, eg:
   * _Netcdf4Coordinates = 6, 4
   * _Netcdf4Dimid = 6
   *
   * private int findCoordinateDimensionIndex(DataObjectFacade facade, H5Group h5group) throws IOException {
   * Attribute att_coord = null;
   * Attribute att_dimid = null;
   * for (MessageAttribute matt : facade.dobj.attributes) {
   * if (matt.name.equals(Nc4.NETCDF4_COORDINATES))
   * att_coord = makeAttribute(matt);
   * if (matt.name.equals(Nc4.NETCDF4_DIMID))
   * att_dimid = makeAttribute(matt);
   * }
   * if (att_coord != null && att_dimid != null) {
   * facade.netcdf4CoordinatesAtt = att_coord;
   * Integer want = (Integer) att_dimid.getNumericValue();
   * for (int i=0; i iter = facade.dobj.attributes.iterator();
    while (iter.hasNext()) {
      MessageAttribute matt = iter.next();
      // find the dimensions - set length to maximum
      // DIMENSION_LIST contains, for each dimension, a list of references to Dimension Scales
      switch (matt.name) {
        case HDF5_DIMENSION_LIST: { // references : may extend the dimension length
          Attribute att = makeAttribute(matt); // this reads in the data

          if (att == null) {
            log.warn("DIMENSION_LIST: failed to read on variable {}", facade.getName());

          } else if (att.getLength() != facade.dobj.mds.dimLength.length) { // some attempts to writing hdf5 directly
                                                                            // fail here
            log.warn("DIMENSION_LIST: must have same number of dimension scales as dimensions att={} on variable {}",
                att, facade.getName());

          } else {
            StringBuilder sbuff = new StringBuilder();
            for (int i = 0; i < att.getLength(); i++) {
              String name = att.getStringValue(i);
              String dimName = extendDimension(g, h5group, name, facade.dobj.mds.dimLength[i]);
              sbuff.append(dimName).append(" ");
            }
            facade.dimList = sbuff.toString();
            facade.hasNetcdfDimensions = true;
            if (debugDimensionScales) {
              log.debug("Found dimList '{}' for group '{}' matt={}", facade.dimList, g.getFullName(), matt);
            }
            if (!h5iosp.includeOriginalAttributes)
              iter.remove();
          }

          break;
        }
        case HDF5_DIMENSION_NAME: {
          Attribute att = makeAttribute(matt);
          if (att == null)
            throw new IllegalStateException();
          String val = att.getStringValue();
          if (val.startsWith("This is a netCDF dimension but not a netCDF variable")) {
            facade.isVariable = false;
            isNetcdf4 = true;
          }
          if (!h5iosp.includeOriginalAttributes)
            iter.remove();
          if (debugDimensionScales) {
            log.debug("Found {}", val);
          }

          break;
        }
        case HDF5_REFERENCE_LIST:
          if (!h5iosp.includeOriginalAttributes)
            iter.remove();
          break;
      }
    }
    return facade.hasNetcdfDimensions || facade.dobj.mds.dimLength.length == 0;

  }

  // add a dimension, return its name
  private String addDimension(ucar.nc2.Group g, H5Group h5group, String name, int length, boolean isUnlimited) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    // if (d == null)
    // d = g.findDimension(dimName); // then look in parent groups LOOK

    if (d == null) { // create if not found
      d = new Dimension(dimName, length, true, isUnlimited, false);
      d.setGroup(g);
      h5group.dimMap.put(dimName, d);
      h5group.dimList.add(d);
      if (debugDimensionScales) {
        log.debug("addDimension name=" + name + " dim= " + d + " to group " + g);
      }

    } else { // check has correct length
      if (d.getLength() != length)
        throw new IllegalStateException(
            "addDimension: DimScale has different length than dimension it references dimScale=" + dimName);
    }

    return d.getShortName();
  }

  // look for unlimited dimensions without dimension scale - must get length from the variable
  private String extendDimension(ucar.nc2.Group g, H5Group h5group, String name, int length) {
    int pos = name.lastIndexOf('/');
    String dimName = (pos >= 0) ? name.substring(pos + 1) : name;

    Dimension d = h5group.dimMap.get(dimName); // first look in current group
    if (d == null)
      d = g.findDimension(dimName); // then look in parent groups

    if (d != null) {
      if (d.isUnlimited() && (length > d.getLength()))
        d.setLength(length);

      if (!d.isUnlimited() && (length != d.getLength())) {
        throw new IllegalStateException(
            "extendDimension: DimScale has different length than dimension it references dimScale=" + dimName);
      }
      return d.getShortName();
    }

    return dimName;
  }

  private void createDimensions(ucar.nc2.Group g, H5Group h5group) {
    for (Dimension d : h5group.dimList) {
      g.addDimension(d);
    }
  }

  private List filterAttributes(List attList) {
    List result = new ArrayList<>(attList.size());
    for (MessageAttribute matt : attList) {
      if (matt.name.equals(Nc4.NETCDF4_COORDINATES) || matt.name.equals(Nc4.NETCDF4_DIMID)
          || matt.name.equals(Nc4.NETCDF4_STRICT)) {
        isNetcdf4 = true;
      } else {
        result.add(matt);
      }
    }
    return result;
  }

  /**
   * Create Attribute objects from the MessageAttribute and add to list
   *
   * @param s if attribute for a Structure, then deconstruct and add to member variables
   * @param matt attribute message
   * @param attContainer add Attribute to this
   * @throws IOException on io error
   * @throws ucar.ma2.InvalidRangeException on shape error
   */
  private void makeAttributes(Structure s, MessageAttribute matt, AttributeContainer attContainer)
      throws IOException, InvalidRangeException {
    MessageDatatype mdt = matt.mdt;

    if (mdt.type == 6) { // structure
      Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
      ArrayStructure attData = (ArrayStructure) readAttributeData(matt, vinfo, DataType.STRUCTURE);

      if (null == s) {
        // flatten and add to list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          Array memberData = attData.extractMemberArray(sm);
          attContainer.addAttribute(new Attribute(matt.name + "." + sm.getName(), memberData));
        }

      } else if (matt.name.equals(CDM.FIELD_ATTS)) {
        // flatten and add to list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          String memberName = sm.getName();
          int pos = memberName.indexOf(":");
          if (pos < 0)
            continue; // LOOK
          String fldName = memberName.substring(0, pos);
          String attName = memberName.substring(pos + 1);
          Array memberData = attData.extractMemberArray(sm);
          Variable v = s.findVariable(fldName);
          if (v == null)
            continue; // LOOK
          v.addAttribute(new Attribute(attName, memberData));
        }

      } else { // assign separate attribute for each member
        StructureMembers attMembers = attData.getStructureMembers();
        for (Variable v : s.getVariables()) {
          StructureMembers.Member sm = attMembers.findMember(v.getShortName()); // does the compound attribute have a
                                                                                // member with same name as nested
                                                                                // variable ?
          if (null != sm) {
            Array memberData = attData.extractMemberArray(sm); // if so, add the att to the member variable, using the
                                                               // name of the compound attribute
            v.addAttribute(new Attribute(matt.name, memberData)); // LOOK want to check for missing values....
          }
        }

        // look for unassigned members, add to the list
        for (StructureMembers.Member sm : attData.getStructureMembers().getMembers()) {
          if (s.findVariable(sm.getName()) == null) {
            Array memberData = attData.extractMemberArray(sm);
            attContainer.addAttribute(new Attribute(matt.name + "." + sm.getName(), memberData));
          }
        }
      }

    } else {
      // make a single attribute
      Attribute att = makeAttribute(matt);
      if (att != null)
        attContainer.addAttribute(att);
    }

    // reading attribute values might change byte order during a read
    // put back to little endian for further header processing
    getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN);
  }

  private Attribute makeAttribute(MessageAttribute matt) throws IOException {
    Vinfo vinfo = new Vinfo(matt.mdt, matt.mds, matt.dataPos);
    DataType dtype = vinfo.getNCDataType();

    // check for empty attribute case
    if (matt.mds.type == 2) {
      if (dtype == DataType.CHAR)
        return new Attribute(matt.name, DataType.STRING); // empty char considered to be a null string attr
      else
        return new Attribute(matt.name, dtype);
    }

    Array attData;
    try {
      attData = readAttributeData(matt, vinfo, dtype);

    } catch (InvalidRangeException e) {
      log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + getRandomAccessFile().getLocation());
      return null;
    }

    Attribute result;
    if (attData.isVlen()) {
      List