ucar.nc2.dataset.StructureDS Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright 1998-2014 University Corporation for Atmospheric Research/Unidata
 *
 *   Portions of this software were developed by the Unidata Program at the
 *   University Corporation for Atmospheric Research.
 *
 *   Access and use of this software shall impose the following obligations
 *   and understandings on the user. The user is granted the right, without
 *   any fee or cost, to use, copy, modify, alter, enhance and distribute
 *   this software, and any derivative works thereof, and its supporting
 *   documentation for any purpose whatsoever, provided that this entire
 *   notice appears in all copies of the software, derivative works and
 *   supporting documentation.  Further, UCAR requests that the user credit
 *   UCAR/Unidata in any publications that result from the use of this
 *   software or in any product that includes this software. The names UCAR
 *   and/or Unidata, however, may not be used in any advertising or publicity
 *   to endorse or promote any products or commercial entity unless specific
 *   written permission is obtained from UCAR/Unidata. The user also
 *   understands that UCAR/Unidata is not obligated to provide the user with
 *   any support, consulting, training or assistance of any kind with regard
 *   to the use, operation and performance of this software nor to provide
 *   the user with any updates, revisions, new versions or "bug fixes."
 *
 *   THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 *   IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 *   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 *   DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 *   INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 *   FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 *   NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 *   WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package ucar.nc2.dataset;

import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.CancelTask;
import ucar.ma2.*;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

/**
 * An "enhanced" Structure.
 *
 * @author john caron
 * @see NetcdfDataset
 */

public class StructureDS extends ucar.nc2.Structure implements VariableEnhanced {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(StructureDS.class);

  private EnhancementsImpl proxy; // API relies that this cant be null

  protected Structure orgVar; // wrap this Variable
  private String orgName; // in case Variable wwas renamed, and we need the original name for aggregation

  protected StructureDS(NetcdfFile ncfile, Group group, String shortName) {
    super(ncfile, group, null, shortName);
    this.proxy = new EnhancementsImpl(this);
  }

  /**
   * Constructor when theres no underlying variable. You better set the values too!
   *
   * @param ds              the containing NetcdfDataset.
   * @param group           the containing group; if null, use rootGroup
   * @param parentStructure parent Structure, may be null
   * @param shortName       variable shortName, must be unique within the Group
   * @param dims            list of dimension names, space delimited
   * @param units           unit string (may be null)
   * @param desc            description (may be null)
   */
  public StructureDS(NetcdfDataset ds, Group group, Structure parentStructure, String shortName,
                     String dims, String units, String desc) {

    super(ds, group, parentStructure, shortName);
    setDimensions(dims);
    this.proxy = new EnhancementsImpl(this, units, desc);

    if (units != null)
      addAttribute(new Attribute(CDM.UNITS, units));
    if (desc != null)
      addAttribute(new Attribute(CDM.LONG_NAME, desc));
  }

  /**
   * Create a StructureDS thats wraps a Structure
   *
   * @param g      parent group
   * @param orgVar original Structure
   */
  public StructureDS(Group g, ucar.nc2.Structure orgVar) { // , boolean reparent) {
    super(orgVar);
    setParentGroup(g);
    this.orgVar = orgVar;
    this.proxy = new EnhancementsImpl(this);

    // dont share cache, iosp : all IO is delegated
    this.ncfile = null;
    this.spiObject = null;
    createNewCache();

    if (orgVar instanceof StructureDS)
      return;

    // all member variables must be wrapped, reparented
    List newList = new ArrayList<>(members.size());
    for (Variable v : members) {
      Variable newVar = convertVariable(g, v);
      newVar.setParentStructure(this);
      newList.add(newVar);
    }
    setMemberVariables(newList);
  }

  private Variable convertVariable(Group g, Variable v) {
    Variable newVar;
    if (v instanceof Sequence) {
      newVar = new SequenceDS(g, (Sequence) v);
    } else if (v instanceof Structure) {
      newVar = new StructureDS(g, (Structure) v);
    } else {
      newVar = new VariableDS(g, v, false); // enhancement done later
    }
    return newVar;
  }

  /**
   * Wrap the given Structure, making it into a StructureDS.
   * Delegate data reading to the original variable.
   * Does not share cache, iosp.
   * This is for NcML explicit mode
   *
   * @param ds        the containing NetcdfDataset.
   * @param group     the containing group; may not be null
   * @param parent    parent Structure, may be null
   * @param shortName variable shortName, must be unique within the Group
   * @param orgVar    the original Structure to wrap.
   */
  public StructureDS(NetcdfDataset ds, Group group, Structure parent, String shortName, Structure orgVar) {
    super(ds, group, parent, shortName);

    // dont share cache, iosp : all IO is delegated
    // this.ncfile = null;
    this.spiObject = null;
    createNewCache();

    this.orgVar = orgVar;
    this.proxy = new EnhancementsImpl(this);
  }

  // for section and slice and select

  @Override
  protected Variable copy() {
    return new StructureDS(getParentGroup(), this);
  }

  // copy() doesnt work because convert gets called twice

  @Override
  public Structure select(List memberNames) {
    StructureDS result = new StructureDS(getParentGroup(), orgVar);
    List members = new ArrayList<>();
    for (String name : memberNames) {
      Variable m = findVariable(name);
      if (null != m) members.add(m);
    }
    result.setMemberVariables(members);
    result.isSubset = true;
    return result;
  }

  /**
   * A StructureDS may wrap another Structure.
   *
   * @return original Structure or null
   */
  public ucar.nc2.Variable getOriginalVariable() {
    return orgVar;
  }

  /**
   * Set the Structure to wrap.
   *
   * @param orgVar original Variable, must be a Structure
   */
  public void setOriginalVariable(ucar.nc2.Variable orgVar) {
    if (!(orgVar instanceof Structure))
      throw new IllegalArgumentException("StructureDS must wrap a Structure; name=" + orgVar.getFullName());
    this.orgVar = (Structure) orgVar;
  }

  /**
   * When this wraps another Variable, get the original Variable's DataType.
   *
   * @return original Variable's DataType
   */
  public DataType getOriginalDataType() {
    return DataType.STRUCTURE;
  }

  /**
   * When this wraps another Variable, get the original Variable's DataType.
   *
   * @return original Variable's DataType
   */
  public String getOriginalName() {
    return orgName;
  }

  @Override
  public String setName(String newName) {
    this.orgName = getShortName();
    setShortName(newName);
    return newName;
  }

  // regular Variables.

  @Override
  public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException {
    Array result;

    if (hasCachedData())
      result = super.reallyRead(client, cancelTask);
    else if (orgVar != null)
      result = orgVar.read();
    else {
      throw new IllegalStateException("StructureDS has no way to get data");
      //Object data = smProxy.getFillValue(getDataType());
      //return Array.factoryConstant(dataType.getPrimitiveClassType(), getShape(), data);
    }

    return convert(result, null);
  }

  // section of regular Variable

  @Override
  public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
    if (section.computeSize() == getSize())
      return _read();

    Array result;
    if (hasCachedData())
      result = super.reallyRead(client, section, cancelTask);
    else if (orgVar != null)
      result = orgVar.read(section);
    else {
      throw new IllegalStateException("StructureDS has no way to get data");
      //Object data = smProxy.getFillValue(getDataType());
      //return Array.factoryConstant(dataType.getPrimitiveClassType(), section.getShape(), data);
    }

    // do any needed conversions (scale/offset, enums, etc)
    return convert(result, section);
  }

  ///////////////////////////////////////

  // is conversion needed?

  private boolean convertNeeded(StructureMembers smData) {

    for (Variable v : getVariables()) {

      if (v instanceof VariableDS) {
        VariableDS vds = (VariableDS) v;
        if (vds.needConvert())
          return true;
      } else if (v instanceof StructureDS) {
        StructureDS nested = (StructureDS) v;
        if (nested.convertNeeded(null))
          return true;
      }

      // a variable with no data in the underlying smData
      if ((smData != null) && !varHasData(v, smData))
        return true;
    }

    return false;
  }

  // possible things needed:
  //   1) scale/offset/enum conversion
  //   2) name, info change
  //   3) variable with cached data added to StructureDS through NcML

  protected ArrayStructure convert(Array data, Section section) throws IOException {
    ArrayStructure orgAS = (ArrayStructure) data;
    if (!convertNeeded(orgAS.getStructureMembers())) {
      // name, info change only
      convertMemberInfo(orgAS.getStructureMembers());
      return orgAS;
    }

    // LOOK! converting to ArrayStructureMA
    // do any scale/offset/enum conversions
    ArrayStructure newAS = ArrayStructureMA.factoryMA(orgAS);
    for (StructureMembers.Member m : newAS.getMembers()) {
      VariableEnhanced v2 = (VariableEnhanced) findVariable(m.getName());
      if ((v2 == null) && (orgVar != null)) // these are from orgVar - may have been renamed
        v2 = findVariableFromOrgName(m.getName());
      if (v2 == null) continue;

      if (v2 instanceof VariableDS) {
        VariableDS vds = (VariableDS) v2;
        if (vds.needConvert()) {
          Array mdata = newAS.extractMemberArray(m);
          mdata = vds.convert(mdata);
          newAS.setMemberArray(m, mdata);
        }

      } else if (v2 instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v2;
        if (innerStruct.convertNeeded(null)) {

          if (innerStruct.getDataType() == DataType.SEQUENCE) {
            ArrayObject.D1 seqArray = (ArrayObject.D1) newAS.extractMemberArray(m);
            ArrayObject.D1 newSeq = new ArrayObject.D1(ArraySequence.class, (int) seqArray.getSize());
            m.setDataArray(newSeq); // put back into member array

            // wrap each Sequence
            for (int i = 0; i < seqArray.getSize(); i++) {
              ArraySequence innerSeq = (ArraySequence) seqArray.get(i); // get old ArraySequence
              newSeq.set(i, new SequenceConverter(innerStruct, innerSeq)); // wrap in converter
            }

            // non-Sequence Structures
          } else {
            Array mdata = newAS.extractMemberArray(m);
            mdata = innerStruct.convert(mdata, null);
            newAS.setMemberArray(m, mdata);
          }

        }

        // always convert the inner StructureMembers
        innerStruct.convertMemberInfo(m.getStructureMembers());
      }
    }

    StructureMembers sm = newAS.getStructureMembers();
    convertMemberInfo(sm);

    // check for variables that have been added by NcML
    for (Variable v : getVariables()) {
      if (!varHasData(v, sm)) {
        try {
          Variable completeVar = getParentGroup().findVariable(v.getShortName()); // LOOK BAD
          Array mdata = completeVar.read(section);
          StructureMembers.Member m = sm.addMember(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType(), v.getShape());
          newAS.setMemberArray(m, mdata);
        } catch (InvalidRangeException e) {
          throw new IOException(e.getMessage());
        }
      }
    }

    return newAS;
  }

  /* convert original structureData to one that conforms to this Structure */

  protected StructureData convert(StructureData orgData, int recno) throws IOException {
    if (!convertNeeded(orgData.getStructureMembers())) {
      // name, info change only
      convertMemberInfo(orgData.getStructureMembers());
      return orgData;
    }

    // otherwise we create a new StructureData and convert to it. expensive
    StructureMembers smResult = new StructureMembers(orgData.getStructureMembers());
    StructureDataW result = new StructureDataW(smResult);

    for (StructureMembers.Member m : orgData.getMembers()) {
      VariableEnhanced v2 = (VariableEnhanced) findVariable(m.getName());
      if ((v2 == null) && (orgVar != null))     // why ?
        v2 = findVariableFromOrgName(m.getName());
      if (v2 == null) {
        findVariableFromOrgName(m.getName());  // debug
        // log.warn("StructureDataDS.convert Cant find member " + m.getName());
        continue;
      }
      StructureMembers.Member mResult = smResult.findMember(m.getName());

      if (v2 instanceof VariableDS) {
        VariableDS vds = (VariableDS) v2;
        Array mdata = orgData.getArray(m);

        if (vds.needConvert())
          mdata = vds.convert(mdata);

        result.setMemberData(mResult, mdata);
      }

      // recurse into sub-structures
      if (v2 instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v2;
        // if (innerStruct.convertNeeded(null)) {

        if (innerStruct.getDataType() == DataType.SEQUENCE) {
          Array a = orgData.getArray(m);

          if (a instanceof ArrayObject.D1) { // LOOK when does this happen vs ArraySequence?
            ArrayObject.D1 seqArray = (ArrayObject.D1) a;
            ArrayObject.D1 newSeq = new ArrayObject.D1(ArraySequence.class, (int) seqArray.getSize());
            mResult.setDataArray(newSeq); // put into result member array

            for (int i = 0; i < seqArray.getSize(); i++) {
              ArraySequence innerSeq = (ArraySequence) seqArray.get(i); // get old ArraySequence
              newSeq.set(i, new SequenceConverter(innerStruct, innerSeq)); // wrap in converter
            }

          } else {
            ArraySequence seqArray = (ArraySequence) a;
            result.setMemberData(mResult, new SequenceConverter(innerStruct, seqArray)); // wrap in converter
          }

          // non-Sequence Structures
        } else {
          Array mdata = orgData.getArray(m);
          mdata = innerStruct.convert(mdata, null);
          result.setMemberData(mResult, mdata);
        }
        //}

        // always convert the inner StructureMembers
        innerStruct.convertMemberInfo(mResult.getStructureMembers());
      }
    }

    StructureMembers sm = result.getStructureMembers();
    convertMemberInfo(sm);

    // check for variables that have been added by NcML
    for (Variable v : getVariables()) {
      if (!varHasData(v, sm)) {
        try {
          Variable completeVar = getParentGroup().findVariable(v.getShortName()); // LOOK BAD
          Array mdata = completeVar.read(new Section().appendRange(recno, recno));
          StructureMembers.Member m = sm.addMember(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType(), v.getShape());
          result.setMemberData(m, mdata);
        } catch (InvalidRangeException e) {
          throw new IOException(e.getMessage());
        }
      }
    }

    return result;
  }

  // the wrapper StructureMembers must be converted to correspond to the wrapper Structure

  private void convertMemberInfo(StructureMembers wrapperSm) {
    for (StructureMembers.Member m : wrapperSm.getMembers()) {
      Variable v = findVariable(m.getName());
      if ((v == null) && (orgVar != null)) // may have been renamed
        v = (Variable) findVariableFromOrgName(m.getName());

      if (v != null) { // a section will have missing variables LOOK wrapperSm probably wrong in that case
        //  log.error("Cant find " + m.getName());
        //else
        m.setVariableInfo(v.getShortName(), v.getDescription(), v.getUnitsString(), v.getDataType());
      }

      // nested structures
      if (v instanceof StructureDS) {
        StructureDS innerStruct = (StructureDS) v;
        innerStruct.convertMemberInfo(m.getStructureMembers());
      }

    }
  }

  // look for the top variable that has an orgVar with the wanted orgName

  private VariableEnhanced findVariableFromOrgName(String orgName) {
    for (Variable vTop : getVariables()) {
      Variable v = vTop;
      while (v instanceof VariableEnhanced) {
        VariableEnhanced ve = (VariableEnhanced) v;
        if ((ve.getOriginalName() != null) && (ve.getOriginalName().equals(orgName)))
          return (VariableEnhanced) vTop;
        v = ve.getOriginalVariable();
      }
    }
    return null;
  }

  // verify that the variable has data in the data array

  private boolean varHasData(Variable v, StructureMembers sm) {
    if (sm.findMember(v.getShortName()) != null) return true;
    while (v instanceof VariableEnhanced) {
      VariableEnhanced ve = (VariableEnhanced) v;
      if (sm.findMember(ve.getOriginalName()) != null) return true;
      v = ve.getOriginalVariable();
    }
    return false;
  }

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////

  private static class SequenceConverter extends ArraySequence {
    StructureDS orgStruct;
    ArraySequence orgSeq;

    SequenceConverter(StructureDS orgStruct, ArraySequence orgSeq) {
      super(orgSeq.getStructureMembers(), orgSeq.getShape());
      this.orgStruct = orgStruct;
      this.orgSeq = orgSeq;
      this.nelems = orgSeq.getStructureDataCount();

      // copy and convert the members
      members = new StructureMembers(orgSeq.getStructureMembers());
      orgStruct.convertMemberInfo(members);
    }

    @Override
    public StructureDataIterator getStructureDataIterator() { // throws java.io.IOException {
      return new StructureDataConverter(orgStruct, orgSeq.getStructureDataIterator());
    }
  }

  private static class StructureDataConverter implements StructureDataIterator {
    private StructureDataIterator orgIter;
    private StructureDS newStruct;
    private int count = 0;

    StructureDataConverter(StructureDS newStruct, StructureDataIterator orgIter) {
      this.newStruct = newStruct;
      this.orgIter = orgIter;
    }

    @Override
    public boolean hasNext() throws IOException {
      return orgIter.hasNext();
    }

    @Override
    public StructureData next() throws IOException {
      StructureData sdata = orgIter.next();
      return newStruct.convert(sdata, count++);
    }

    @Override
    public void setBufferSize(int bytes) {
      orgIter.setBufferSize(bytes);
    }

    @Override
    public StructureDataIterator reset() {
      orgIter = orgIter.reset();
      return (orgIter == null) ? null : this;
    }

    @Override
    public int getCurrentRecno() {
      return orgIter.getCurrentRecno();
    }

    @Override
    public void finish() {
      orgIter.finish();
    }
  }

  /* public StructureDataIterator getStructureIterator(int bufferSize) throws java.io.IOException {
    StructureDataIterator iter = orgVar.getStructureIterator(bufferSize);
    return new StructureDataConverter(this, iter);
  } */

  /* public StructureDataIterator getStructureIterator(int bufferSize) throws java.io.IOException {
     //StructureDataIterator iter = ncfile.getStructureIterator(this, bufferSize);
     //return (iter != null) ? iter : new Structure.Iterator(bufferSize);
     return new Structure.Iterator(bufferSize);
   } */

  /* Experimental - convert entire original array at once, to avoid one-by-one StructureData conversion.
private class Iterator implements StructureDataIterator {
  private int count = 0;
  private int recnum = (int) getSize();
  private int readStart = 0;
  private int readCount = 0;
  private int readAtaTime;
  private ArrayStructure as = null;
  private Index ii = null;

  protected Iterator(int bufferSize) {
    setBufferSize( bufferSize);
    reset();
  }

  public boolean hasNext() { return count < recnum; }

  public StructureDataIterator reset() {
    count = 0;
    readStart = 0;
    readCount = 0;
    ii = Index.factory(shape); // convert to nD index

    return this;
  }

  public StructureData next() throws IOException {
    if (count >= readStart) {
      if (getRank() == 1) readNextRank1();
      else readNextGeneralRank();
    }

    count++;
    return as.getStructureData( readCount++);
  }

  private void readNextRank1() throws IOException {
    int left = Math.min(recnum, readStart+readAtaTime); // dont go over recnum
    int need = left - readStart; // how many to read this time
    try {
      // System.out.println(" read start= "+readStart+" count= "+need);
      as = readStructure( readStart, need);

    } catch (InvalidRangeException e) {
      log.error("Structure.Iterator.readNext() ",e);
      throw new IllegalStateException("Structure.Iterator.readNext() ",e);
    } // cant happen

    readStart += need;
    readCount = 0;
  }

  private void readNextGeneralRank() throws IOException {
   ii.setCurrentCounter(index);
   int[] origin = ii.getCurrentCounter();
   section = new Section();
   for (int i=0;i 0) return; // too late
    int structureSize = calcStructureSize();
    if (bytes <= 0)
      bytes = defaultBufferSize;
    readAtaTime = Math.max( 10, bytes / structureSize);
  }

}  */

  ///////////////////////////////////////////////////////////

  /**
   * DO NOT USE DIRECTLY. public by accident.
   * recalc any enhancement info
   */
  public void enhance(Set mode) {
    for (Variable v : getVariables()) {
      VariableEnhanced ve = (VariableEnhanced) v;
      ve.enhance(mode);
    }
  }

  public void clearCoordinateSystems() {
    this.proxy = new EnhancementsImpl(this, getUnitsString(), getDescription());
  }

  public void addCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
    proxy.addCoordinateSystem(p0);
  }

  public void removeCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
    proxy.removeCoordinateSystem(p0);
  }

  public java.util.List getCoordinateSystems() {
    return proxy.getCoordinateSystems();
  }

  public java.lang.String getDescription() {
    return proxy.getDescription();
  }

  public java.lang.String getUnitsString() {
    return proxy.getUnitsString();
  }

  public void setUnitsString(String units) {
    proxy.setUnitsString(units);
  }

}