ucar.nc2.dataset.VariableDS Maven / Gradle / Ivy
The newest version!
/*
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
* See LICENSE for license information.
*/
package ucar.nc2.dataset;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.util.CancelTask;
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
import java.util.Formatter;
import java.util.Set;
/**
* An wrapper around a Variable, creating an "enhanced" Variable.
* The original Variable is used for the I/O.
* There are several distinct uses:
*
* - 1) handle scale/offset/missing values/enum conversion; this can change DataType and data values
*
- 2) container for coordinate system information
*
- 3) NcML modifications to underlying Variable
*
*
* @author caron
* @see NetcdfDataset
*/
public class VariableDS extends ucar.nc2.Variable implements VariableEnhanced, EnhanceScaleMissing {
private EnhancementsImpl enhanceProxy;
private EnhanceScaleMissingImpl scaleMissingProxy;
private EnumSet enhanceMode;
private boolean needScaleOffsetMissing = false;
private boolean needEnumConversion = false;
private boolean needUnsignedConversion = false;
protected Variable orgVar; // wrap this Variable : use it for the I/O
protected DataType orgDataType; // keep separate for the case where there is no orgVar.
protected String orgName; // in case Variable was renamed, and we need to keep track of the original name
/**
* Constructor when there's no underlying variable.
* You must also set the values by doing one of:
* - set the values with setCachedData()
*
- set a proxy reader with setProxyReader()
*
* Otherwise, it is assumed to have constant values (using the fill value)
*
* @param ds the containing dataset
* @param group the containing group
* @param parentStructure the containing Structure (may be null)
* @param shortName the (short) name
* @param dataType the data type
* @param dims list of dimension names, these must already exist in the Group; empty String = scalar
* @param units String value of units, may be null
* @param desc String value of description, may be null
*/
public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String shortName,
DataType dataType, String dims, String units, String desc) {
super(ds, group, parentStructure, shortName);
setDataType(dataType);
setDimensions(dims);
// this.orgDataType = dataType;
if (dataType == DataType.STRUCTURE)
throw new IllegalArgumentException("VariableDS must not wrap a Structure; name=" + shortName);
if (units != null)
addAttribute(new Attribute(CDM.UNITS, units));
if (desc != null)
addAttribute(new Attribute(CDM.LONG_NAME, desc));
this.enhanceProxy = new EnhancementsImpl(this, units, desc);
this.scaleMissingProxy = new EnhanceScaleMissingImpl(); // gets replaced later, in enhance()
}
/**
* Make a new VariableDS, delegate data reading to the original variable, but otherwise
* dont take any info from it. This is used by NcML explicit mode.
*
* @param group the containing group; may not be null
* @param parent parent Structure, may be null
* @param shortName variable shortName, must be unique within the Group
* @param orgVar the original Variable to wrap. The original Variable is not modified.
* Must not be a Structure, use StructureDS instead.
*/
public VariableDS(Group group, Structure parent, String shortName, Variable orgVar) {
//coverity[FORWARD_NULL]
super(null, group, parent, shortName);
setDimensions(getDimensionsString()); // reset the dimensions
if (orgVar instanceof Structure)
throw new IllegalArgumentException("VariableDS must not wrap a Structure; name=" + orgVar.getFullName());
// dont share cache, iosp : all IO is delegated
this.ncfile = null;
this.spiObject = null;
createNewCache();
this.orgVar = orgVar;
this.orgDataType = orgVar.getDataType();
this.enhanceProxy = new EnhancementsImpl(this);
this.scaleMissingProxy = new EnhanceScaleMissingImpl();
}
/**
* Wrap the given Variable, making it into a VariableDS.
* Delegate data reading to the original variable.
* Take all metadata from original variable.
* Does not share cache, iosp.
*
* @param g logical container, if null use orgVar's group
* @param orgVar the original Variable to wrap. The original Variable is not modified.
* Must not be a Structure, use StructureDS instead.
* @param enhance if true, use NetcdfDataset.defaultEnhanceMode to define what enhancements are made.
* Note that this can change DataType and data values.
* You can also call enhance() later. If orgVar is VariableDS, then enhance is inherited from there,
* and this parameter is ignored.
*/
public VariableDS(Group g, Variable orgVar, boolean enhance) {
super(orgVar);
if (g != null) setParentGroup(g); // otherwise super() sets group; this affects the long name and the dimensions.
setDimensions(getDimensionsString()); // reset the dimensions
if (orgVar instanceof Structure)
throw new IllegalArgumentException("VariableDS must not wrap a Structure; name=" + orgVar.getFullName());
// dont share cache, iosp : all IO is delegated
this.ncfile = null;
this.spiObject = null;
createNewCache();
this.orgVar = orgVar;
this.orgDataType = orgVar.getDataType();
this.enhanceProxy = new EnhancementsImpl(this);
if (enhance) {
enhance(NetcdfDataset.getDefaultEnhanceMode());
} else {
this.scaleMissingProxy = new EnhanceScaleMissingImpl();
}
}
/**
* Copy constructor, for subclasses.
* Used by copy() and CoordinateAxis
* Share everything except the coord systems.
*
* @param vds copy from here.
* @param isCopy called from copy()
*/
protected VariableDS(VariableDS vds, boolean isCopy) {
super(vds);
this.orgVar = vds;
this.orgDataType = vds.orgDataType;
this.orgName = vds.orgName;
this.scaleMissingProxy = vds.scaleMissingProxy;
this.enhanceProxy = new EnhancementsImpl(this); //decouple coordinate systems
this.enhanceMode = vds.enhanceMode;
if (isCopy) { // enhancement done in orgVar
//this.needScaleOffsetMissing = vds.needScaleOffsetMissing;
//this.needEnumConversion = vds.needEnumConversion;
} else {
createNewCache(); // dont share cache unless its a copy
}
}
@Override
public NetcdfFile getNetcdfFile() {
return group.getNetcdfFile();
}
// for section and slice
@Override
protected Variable copy() {
return new VariableDS(this, true);
}
/**
* Remove coordinate system info.
*/
public void clearCoordinateSystems() {
this.enhanceProxy = new EnhancementsImpl(this, getUnitsString(), getDescription());
}
/**
* DO NOT USE DIRECTLY. public by accident.
* Calculate scale/offset/missing value info. This may change the DataType.
*/
public void enhance(Set mode) {
this.enhanceMode = EnumSet.copyOf(mode);
boolean alreadyScaleOffsetMissing = false;
boolean alreadyEnumConversion = false;
// see if underlying variable has enhancements already applied
if (orgVar != null && orgVar instanceof VariableDS) {
VariableDS orgVarDS = (VariableDS) orgVar;
EnumSet orgEnhanceMode = orgVarDS.getEnhanceMode();
if (orgEnhanceMode != null) {
if (orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing)) {
alreadyScaleOffsetMissing = true;
this.enhanceMode.add(NetcdfDataset.Enhance.ScaleMissing); // Note: promote the enhancement to the wrapped variable
}
if (orgEnhanceMode.contains(NetcdfDataset.Enhance.ConvertEnums)) {
alreadyEnumConversion = true;
this.enhanceMode.add(NetcdfDataset.Enhance.ConvertEnums); // Note: promote the enhancement to the wrapped variable
}
}
}
// do we need to calculate the ScaleMissing ?
if (!alreadyScaleOffsetMissing && (dataType.isNumeric() || dataType == DataType.CHAR) &&
mode.contains(NetcdfDataset.Enhance.ScaleMissing) || mode.contains(NetcdfDataset.Enhance.ScaleMissingDefer)) {
this.scaleMissingProxy = new EnhanceScaleMissingImpl(this);
// promote the data type if ScaleMissing is set
if (mode.contains(NetcdfDataset.Enhance.ScaleMissing) &&
scaleMissingProxy.hasScaleOffset() && (scaleMissingProxy.getConvertedDataType() != getDataType())) {
setDataType(scaleMissingProxy.getConvertedDataType());
removeAttributeIgnoreCase(CDM.UNSIGNED);
}
// do we need to actually convert data ?
needScaleOffsetMissing = mode.contains(NetcdfDataset.Enhance.ScaleMissing) &&
(scaleMissingProxy.hasScaleOffset() || scaleMissingProxy.getUseNaNs());
}
// do we need to do enum conversion ?
if (!alreadyEnumConversion && mode.contains(NetcdfDataset.Enhance.ConvertEnums) && dataType.isEnum()) {
this.needEnumConversion = true;
// LOOK promote data type to STRING ????
setDataType(DataType.STRING);
removeAttributeIgnoreCase(CDM.UNSIGNED);
}
// do we need to do unsigned conversion ?
Variable ultimateOrigVar = this;
while (ultimateOrigVar instanceof VariableDS && ((VariableDS) ultimateOrigVar).getOriginalVariable() != null) {
ultimateOrigVar = ((VariableDS) ultimateOrigVar).getOriginalVariable();
}
if (ultimateOrigVar == this) {
needUnsignedConversion = false;
} else {
needUnsignedConversion = this.getDataType().isUnsigned() && !ultimateOrigVar.getDataType().isUnsigned();
}
}
boolean needConvert() {
if (needScaleOffsetMissing || needEnumConversion || needUnsignedConversion) return true;
if ((orgVar != null) && (orgVar instanceof VariableDS)) return ((VariableDS) orgVar).needConvert();
return false;
}
Array convert(Array data) {
if (needScaleOffsetMissing)
return convertScaleOffsetMissing(data);
else if (needEnumConversion)
return convertEnums(data);
else if (needUnsignedConversion)
return convertUnsigned(data);
if ((orgVar != null) && (orgVar instanceof VariableDS))
return ((VariableDS) orgVar).convert(data);
return data;
}
/**
* Get the enhancement mode
*
* @return the enhancement mode
*/
public EnumSet getEnhanceMode() {
return enhanceMode;
}
// Enhancements interface
public void addCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
enhanceProxy.addCoordinateSystem(p0);
}
public void removeCoordinateSystem(ucar.nc2.dataset.CoordinateSystem p0) {
enhanceProxy.removeCoordinateSystem(p0);
}
public java.util.List getCoordinateSystems() {
return enhanceProxy.getCoordinateSystems();
}
public java.lang.String getDescription() {
return enhanceProxy.getDescription();
}
public java.lang.String getUnitsString() {
return enhanceProxy.getUnitsString();
}
public void setUnitsString(String units) {
enhanceProxy.setUnitsString(units);
}
// EnhanceScaleMissing interface
public Array convertScaleOffsetMissing(Array data) {
return scaleMissingProxy.convertScaleOffsetMissing(data);
}
public double getValidMax() {
return scaleMissingProxy.getValidMax();
}
public double getValidMin() {
return scaleMissingProxy.getValidMin();
}
public double getFillValue() {
return scaleMissingProxy.getFillValue();
}
public double[] getMissingValues() {
return scaleMissingProxy.getMissingValues();
}
public boolean hasFillValue() {
return scaleMissingProxy.hasFillValue();
}
public boolean hasInvalidData() {
return scaleMissingProxy.hasInvalidData();
}
public boolean hasMissing() {
return scaleMissingProxy.hasMissing();
}
public boolean hasMissingValue() {
return scaleMissingProxy.hasMissingValue();
}
public boolean hasScaleOffset() {
return scaleMissingProxy.hasScaleOffset();
}
public boolean isFillValue(double p0) {
return scaleMissingProxy.isFillValue(p0);
}
public boolean isInvalidData(double p0) {
return scaleMissingProxy.isInvalidData(p0);
}
public boolean isMissing(double val) {
return scaleMissingProxy.isMissing(val);
}
public boolean isMissingFast(double val) {
return scaleMissingProxy.isMissingFast(val);
}
public boolean isMissingValue(double p0) {
return scaleMissingProxy.isMissingValue(p0);
}
public void setFillValueIsMissing(boolean p0) {
scaleMissingProxy.setFillValueIsMissing(p0);
}
public void setInvalidDataIsMissing(boolean p0) {
scaleMissingProxy.setInvalidDataIsMissing(p0);
}
public void setMissingDataIsMissing(boolean p0) {
scaleMissingProxy.setMissingDataIsMissing(p0);
}
public void setUseNaNs(boolean useNaNs) {
scaleMissingProxy.setUseNaNs(useNaNs);
}
public boolean getUseNaNs() {
return scaleMissingProxy.getUseNaNs();
}
public double convertScaleOffsetMissing(byte value) {
return scaleMissingProxy.convertScaleOffsetMissing(value);
}
public double convertScaleOffsetMissing(short value) {
return scaleMissingProxy.convertScaleOffsetMissing(value);
}
public double convertScaleOffsetMissing(int value) {
return scaleMissingProxy.convertScaleOffsetMissing(value);
}
public double convertScaleOffsetMissing(long value) {
return scaleMissingProxy.convertScaleOffsetMissing(value);
}
public double convertScaleOffsetMissing(double value) {
return scaleMissingProxy.convertScaleOffsetMissing(value);
}
/**
* A VariableDS usually wraps another Variable.
*
* @return original Variable or null
*/
public ucar.nc2.Variable getOriginalVariable() {
return orgVar;
}
/**
* Set the Variable to wrap. Used by NcML explicit mode.
*
* @param orgVar original Variable, must not be a Structure
*/
public void setOriginalVariable(ucar.nc2.Variable orgVar) {
if (orgVar instanceof Structure)
throw new IllegalArgumentException("VariableDS must not wrap a Structure; name=" + orgVar.getFullName());
this.orgVar = orgVar;
}
/**
* When this wraps another Variable, get the original Variable's DataType.
*
* @return original Variable's DataType, or current data type if it doesnt wrap anothe rvariable
*/
public DataType getOriginalDataType() {
return orgDataType != null ? orgDataType : getDataType();
}
/**
* When this wraps another Variable, get the original Variable's name.
*
* @return original Variable's name
*/
public String getOriginalName() {
return orgName;
}
public String lookupEnumString(int val) {
if (dataType.isEnum())
return super.lookupEnumString(val);
return orgVar.lookupEnumString(val);
}
@Override
public String setName(String newName) {
this.orgName = getShortName();
super.setShortName(newName);
return newName;
}
public String toStringDebug() {
return (orgVar != null) ? orgVar.toStringDebug() : "";
}
@Override
public String getDatasetLocation() {
String result = super.getDatasetLocation();
if (result != null) return result;
if (orgVar != null) return orgVar.getDatasetLocation();
return null;
}
public boolean hasCachedDataRecurse() {
if (super.hasCachedData()) return true;
if ((orgVar != null) && orgVar.hasCachedData()) return true;
return false;
}
@Override
public void setCaching(boolean caching) {
if (caching && orgVar != null) orgVar.setCaching(true); // propagate down only if true
}
@Override
protected Array _read() throws IOException {
Array result;
// check if already cached - caching in VariableDS only done explicitly by app
if (hasCachedData())
result = super._read();
else
result = proxyReader.reallyRead(this, null);
if (needScaleOffsetMissing)
return convertScaleOffsetMissing(result);
else if (needEnumConversion)
return convertEnums(result);
else if (needUnsignedConversion)
return convertUnsigned(result);
else
return result;
}
// do not call directly
@Override
public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException {
if (orgVar == null)
return getMissingDataArray(shape);
return orgVar.read();
}
// section of regular Variable
@Override
protected Array _read(Section section) throws IOException, InvalidRangeException {
// really a full read
if ((null == section) || section.computeSize() == getSize())
return _read();
Array result;
if (hasCachedData())
result = super._read(section);
else
result = proxyReader.reallyRead(this, section, null);
if (needScaleOffsetMissing)
return convertScaleOffsetMissing(result);
else if (needEnumConversion)
return convertEnums(result);
else if (needUnsignedConversion)
return convertUnsigned(result);
else
return result;
}
// do not call directly
@Override
public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException {
// see if its really a full read
if ((null == section) || section.computeSize() == getSize())
return reallyRead(client, cancelTask);
if (orgVar == null)
return getMissingDataArray(section.getShape());
return orgVar.read(section);
}
@Override
public long readToStream(Section section, OutputStream out) throws IOException, InvalidRangeException {
if (orgVar == null)
return super.readToStream(section, out);
return orgVar.readToStream(section, out);
}
/**
* Return Array with missing data
*
* @param shape of this shape
* @return Array with given shape
*/
public Array getMissingDataArray(int[] shape) {
Object data = scaleMissingProxy.getFillValue(getDataType());
return Array.factoryConstant(dataType, shape, data);
}
/**
* public for debugging
*
* @param f put info here
*/
public void showScaleMissingProxy(Formatter f) {
f.format("use NaNs = %s%n", scaleMissingProxy.getUseNaNs());
f.format("has missing = %s%n", scaleMissingProxy.hasMissing());
if (scaleMissingProxy.hasMissing()) {
if (scaleMissingProxy.hasMissingValue()) {
f.format(" missing value(s) = ");
for (double d : scaleMissingProxy.getMissingValues())
f.format(" %f", d);
f.format("%n");
}
if (scaleMissingProxy.hasFillValue())
f.format(" fillValue = %f%n", scaleMissingProxy.getFillValue());
if (scaleMissingProxy.hasInvalidData())
f.format(" valid min/max = [%f,%f]%n", scaleMissingProxy.getValidMin(), scaleMissingProxy.getValidMax());
}
Object mv = scaleMissingProxy.getFillValue(getDataType());
String mvs = (mv instanceof String) ? (String) mv : java.lang.reflect.Array.get(mv, 0).toString();
f.format("FillValue or default = %s%n", mvs);
f.format("%nhas scale/offset = %s%n", scaleMissingProxy.hasScaleOffset());
if (scaleMissingProxy.hasScaleOffset()) {
double offset = scaleMissingProxy.convertScaleOffsetMissing(0.0);
double scale = scaleMissingProxy.convertScaleOffsetMissing(1.0) - offset;
f.format(" scale_factor = %f add_offset = %f%n", scale, offset);
}
f.format("original data type = %s%n", getDataType());
f.format("converted data type = %s%n", scaleMissingProxy.getConvertedDataType());
}
protected Array convertEnums(Array values) {
DataType dt = DataType.getType(values);
if (!dt.isNumeric())
throw new IllegalStateException("HEY !dt.isNumeric() "+dt);
Array result = Array.factory(DataType.STRING, values.getShape());
IndexIterator ii = result.getIndexIterator();
values.resetLocalIterator();
while (values.hasNext()) {
String sval = lookupEnumString(values.nextInt());
ii.setObjectNext(sval);
}
return result;
}
protected Array convertUnsigned(Array org) {
return Array.factory(
org.getDataType().withSignedness(DataType.Signedness.UNSIGNED), org.getShape(), org.getStorage());
}
}