ucar.nc2.dataset.EnhanceScaleMissingImpl Maven / Gradle / Ivy
The newest version!
/*
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
* See LICENSE for license information.
*/
package ucar.nc2.dataset;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.DataFormatType;
import ucar.nc2.iosp.netcdf3.N3iosp;
import ucar.nc2.util.Misc;
import java.util.EnumSet;
/**
* Implementation of EnhanceScaleMissing for missing data, unsigned, and scale/offset packed data.
*
* @author caron
* @see EnhanceScaleMissing
*/
class EnhanceScaleMissingImpl implements EnhanceScaleMissing {
// Default fill values, used unless _FillValue variable attribute is set.
// duplicated from N3iosp, used in getFillValue.
/* static private final byte NC_FILL_BYTE = -127;
static private final char NC_FILL_CHAR = 0;
static private final short NC_FILL_SHORT = (short) -32767;
static private final int NC_FILL_INT = -2147483647;
static private final float NC_FILL_FLOAT = 9.9692099683868690e+36f; /* near 15 * 2^119
static private final double NC_FILL_DOUBLE = 9.9692099683868690e+36; */
static private final boolean debug = false, debugRead = false, debugMissing = false;
private DataType convertedDataType = null;
private boolean useNaNs = false;
// defaults from NetcdfDataset modes
private boolean invalidDataIsMissing = NetcdfDataset.invalidDataIsMissing;
private boolean fillValueIsMissing = NetcdfDataset.fillValueIsMissing;
private boolean missingDataIsMissing = NetcdfDataset.missingDataIsMissing;
private boolean hasScaleOffset = false;
private double scale = 1.0, offset = 0.0;
private boolean hasValidRange = false, hasValidMin = false, hasValidMax = false;
private double valid_min = -Double.MAX_VALUE, valid_max = Double.MAX_VALUE;
private boolean hasFillValue = false;
private double fillValue; // LOOK making it double not really correct
private boolean hasMissingValue = false;
private double[] missingValue;
private boolean isUnsigned;
/**
* Constructor, when you dont want anything done.
*/
EnhanceScaleMissingImpl() {
}
/**
* Constructor, default values.
*
* @param forVar the Variable to decorate.
*/
EnhanceScaleMissingImpl(VariableDS forVar) {
this(forVar, NetcdfDataset.useNaNs, NetcdfDataset.fillValueIsMissing,
NetcdfDataset.invalidDataIsMissing, NetcdfDataset.missingDataIsMissing);
}
/**
* Constructor.
* If scale/offset attributes are found, remove them from the decorated variable.
*
* @param forVar the Variable to decorate.
* @param useNaNs pre-fill isMissing() data with NaNs
* @param fillValueIsMissing use _FillValue for isMissing()
* @param invalidDataIsMissing use valid_range for isMissing()
* @param missingDataIsMissing use missing_value for isMissing()
*/
EnhanceScaleMissingImpl(VariableDS forVar, boolean useNaNs, boolean fillValueIsMissing,
boolean invalidDataIsMissing, boolean missingDataIsMissing) {
this.fillValueIsMissing = fillValueIsMissing;
this.invalidDataIsMissing = invalidDataIsMissing;
this.missingDataIsMissing = missingDataIsMissing;
boolean isNetcdfIosp = false;
NetcdfFile ncfile = forVar.getNetcdfFile();
if (ncfile != null) {
String iosp = ncfile.getFileTypeId();
isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(iosp) ||
DataFormatType.NETCDF4.getDescription().equals(iosp);
}
// see if underlying variable has scale/offset already applied
Variable orgVar = forVar.getOriginalVariable();
if (orgVar instanceof VariableDS) {
VariableDS orgVarDS = (VariableDS) orgVar;
EnumSet orgEnhanceMode = orgVarDS.getEnhanceMode();
if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing))
return;
}
// deal with legacy use of attribute with Unsigned = true
Attribute unsignedAttrib = forVar.findAttributeIgnoreCase(CDM.UNSIGNED);
boolean isUnsignedSet = unsignedAttrib != null && unsignedAttrib.getStringValue().equalsIgnoreCase("true");
if (isUnsignedSet) {
forVar.setDataType(forVar.getDataType().withSignedness(DataType.Signedness.UNSIGNED));
}
this.isUnsigned = forVar.getDataType().isUnsigned();
this.convertedDataType = forVar.getDataType(); // only for netcDF !!??
DataType scaleType = null, missType = null, validType = null;
if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName());
Attribute att;
// scale and offset
if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) {
if (!att.isString()) {
scale = att.getNumericValue().doubleValue();
hasScaleOffset = true;
scaleType = att.getDataType();
forVar.remove(att);
if (debug) System.out.println("scale = " + scale + " type " + scaleType);
}
}
if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) {
if (!att.isString()) {
offset = att.getNumericValue().doubleValue();
hasScaleOffset = true;
DataType offType = att.getDataType();
if (rank(offType) > rank(scaleType))
scaleType = offType;
forVar.remove(att);
if (debug) System.out.println("offset = " + offset);
}
}
////// missing data : valid_range. assume here its in units of unpacked data. correct this below
Attribute validRangeAtt;
if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) {
if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) {
valid_min = validRangeAtt.getNumericValue(0).doubleValue();
valid_max = validRangeAtt.getNumericValue(1).doubleValue();
hasValidRange = true;
validType = validRangeAtt.getDataType();
if (hasScaleOffset) forVar.remove(validRangeAtt);
if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max);
}
}
Attribute validMinAtt = null, validMaxAtt = null;
if (!hasValidRange) {
if (null != (validMinAtt = forVar.findAttribute("valid_min"))) {
if (!validMinAtt.isString()) {
valid_min = validMinAtt.getNumericValue().doubleValue();
hasValidMin = true;
validType = validMinAtt.getDataType();
if (hasScaleOffset) forVar.remove(validMinAtt);
if (debug) System.out.println("valid_min = " + valid_min);
}
}
if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) {
if (!validMaxAtt.isString()) {
valid_max = validMaxAtt.getNumericValue().doubleValue();
hasValidMax = true;
DataType t = validMaxAtt.getDataType();
if (rank(t) > rank(validType))
validType = t;
if (hasScaleOffset) forVar.remove(validMaxAtt);
if (debug) System.out.println("valid_min = " + valid_max);
}
}
}
boolean hasValidData = hasValidMin || hasValidMax || hasValidRange;
if (hasValidMin && hasValidMax)
hasValidRange = true;
/// _FillValue
DataType fillType = null;
Attribute fillValueAtt = forVar.findAttribute(CDM.FILL_VALUE);
boolean fillAttOk = (null != fillValueAtt && !fillValueAtt.isString() && fillValueAtt.getLength() > 0);
if (fillAttOk || isNetcdfIosp) {
if (fillAttOk) {
fillValue = fillValueAtt.getNumericValue().doubleValue();
} else {
fillValue = N3iosp.getFillValueDefault(forVar.getDataType()).doubleValue();
}
fillValue = convertScaleOffsetMissing(fillValue);
hasFillValue = true;
fillType = (null != fillValueAtt) ? fillValueAtt.getDataType() : forVar.getDataType();
if (hasScaleOffset) forVar.remove(fillValueAtt);
}
/// missing_value
if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) {
if (att.isString()) {
String svalue = att.getStringValue();
if (forVar.getDataType() == DataType.CHAR) {
missingValue = new double[1];
if (svalue.length() == 0) missingValue[0] = 0;
else missingValue[0] = svalue.charAt(0);
missType = DataType.CHAR;
hasMissingValue = true;
} else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute
try {
missingValue = new double[1];
missingValue[0] = Double.parseDouble(svalue);
missType = att.getDataType();
hasMissingValue = true;
} catch (NumberFormatException ex) {
if (debug) System.out.println("String missing_value not parsable as double= " + att.getStringValue());
}
}
} else { // not a string
missingValue = getValueAsDouble(att);
missType = att.getDataType();
for (double mv : missingValue)
if (!Double.isNaN(mv))
hasMissingValue = true; // dont need to do anything if its already a NaN
}
if (hasScaleOffset) forVar.remove(att);
}
// missing
boolean hasMissing = (invalidDataIsMissing && hasValidData) ||
(fillValueIsMissing && hasFillValue) ||
(missingDataIsMissing && hasMissingValue);
/// assign convertedDataType if needed
if (hasScaleOffset) {
convertedDataType = forVar.getDataType();
if (hasMissing) { // with default fill, always has missing data unless explicitly turned off
// has missing data : must be float or double
if (rank(scaleType) > rank(convertedDataType))
convertedDataType = scaleType;
if (missingDataIsMissing && rank(missType) > rank(convertedDataType))
convertedDataType = missType;
if (fillValueIsMissing && rank(fillType) > rank(convertedDataType))
convertedDataType = fillType;
if (invalidDataIsMissing && rank(validType) > rank(convertedDataType))
convertedDataType = validType;
if (rank(convertedDataType) < rank(DataType.DOUBLE))
convertedDataType = DataType.FLOAT;
} else {
// no missing data; can use wider of data and scale
if (rank(scaleType) > rank(convertedDataType))
convertedDataType = scaleType;
}
if (debug) System.out.println("assign dataType = " + convertedDataType);
// validData may be external or internal
if (hasValidData) {
DataType orgType = forVar.getDataType();
// If valid_range is the same type as scale_factor (actually the wider of
// scale_factor and add_offset) and this is wider than the external data, then it
// will be interpreted as being in the units of the internal (unpacked) data.
// Otherwise it is in the units of the external (unpacked) data.
// we assumed unpacked data above, redo if its really packed data
if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) {
if (validRangeAtt != null) {
double[] values = getValueAsDouble(validRangeAtt);
valid_min = values[0];
valid_max = values[1];
} else {
if (validMinAtt != null) {
double[] values = getValueAsDouble(validMinAtt);
valid_min = values[0];
}
if (validMaxAtt != null) {
double[] values = getValueAsDouble(validMaxAtt);
valid_max = values[0];
}
}
}
}
}
if (hasMissing && ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT)))
this.useNaNs = useNaNs;
if (debug) System.out.println("this.useNaNs = " + this.useNaNs);
}
private double[] getValueAsDouble(Attribute att) {
int n = att.getLength();
double[] value = new double[n];
if (debugMissing) System.out.printf("missing_data: ");
for (int i = 0; i < n; i++) {
if (isUnsigned && att.getDataType() == DataType.BYTE) // LOOK BYTE or UBYTE ?
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).byteValue());
else if (isUnsigned && att.getDataType() == DataType.SHORT)
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).shortValue());
else if (isUnsigned && att.getDataType() == DataType.INT)
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).intValue());
else
value[i] = scale * att.getNumericValue(i).doubleValue() + offset;
if (debugMissing) System.out.print(" " + value[i]);
}
if (debugMissing) System.out.println();
return value;
}
private int rank(DataType c) {
if (c == null) return -1;
if (c.getPrimitiveClassType() == byte.class)
return 0;
else if (c.getPrimitiveClassType() == short.class)
return 1;
else if (c.getPrimitiveClassType() == int.class)
return 2;
else if (c.getPrimitiveClassType() == long.class)
return 3;
else if (c == DataType.FLOAT)
return 4;
else if (c == DataType.DOUBLE)
return 5;
else
return -1;
}
/**
* @return converted DataType, else null if hasScaleOffset is true.
*/
public DataType getConvertedDataType() {
return convertedDataType;
}
/**
* true if Variable has valid_range, valid_min or valid_max attributes
*/
public boolean hasInvalidData() {
return hasValidRange || hasValidMin || hasValidMax;
}
/**
* return the minimum value in the valid range
*/
public double getValidMin() {
return valid_min;
}
/**
* return the maximum value in the valid range
*/
public double getValidMax() {
return valid_max;
}
/**
* return true if val is outside the valid range
*/
public boolean isInvalidData(double val) {
// valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double.
// That potential loss of precision means that we cannot do the nearlyEquals() comparison with
// Misc.defaultMaxRelativeDiffDouble.
boolean greaterThanOrEqualToValidMin =
Misc.nearlyEquals(val, valid_min, Misc.defaultMaxRelativeDiffFloat) || val > valid_min;
boolean lessThanOrEqualToValidMax =
Misc.nearlyEquals(val, valid_max, Misc.defaultMaxRelativeDiffFloat) || val < valid_max;
if (hasValidRange)
return !greaterThanOrEqualToValidMin || !lessThanOrEqualToValidMax;
else if (hasValidMin)
return !greaterThanOrEqualToValidMin;
else if (hasValidMax)
return !lessThanOrEqualToValidMax;
return false;
}
/**
* true if Variable has _FillValue attribute
*/
public boolean hasFillValue() {
return hasFillValue;
}
/**
* return true if val equals the _FillValue
*/
public boolean isFillValue(double val) {
return hasFillValue && Misc.nearlyEquals(val, fillValue, Misc.defaultMaxRelativeDiffFloat);
}
public double getFillValue() {
return fillValue;
}
/**
* true if Variable data will be converted using scale and offet
*/
public boolean hasScaleOffset() {
return hasScaleOffset;
}
/**
* true if Variable has missing_value attribute
*/
public boolean hasMissingValue() {
return hasMissingValue;
}
/**
* return true if val equals a missing_value (low level)
*/
public boolean isMissingValue(double val) {
if (!hasMissingValue)
return false;
for (double aMissingValue : missingValue)
if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat))
return true;
return false;
}
public double[] getMissingValues() {
return missingValue;
}
/**
* set whether to use NaNs for missing values, for efficiency
*/
public void setUseNaNs(boolean useNaNs) {
this.useNaNs = useNaNs;
}
/**
* @return whether to use NaNs for missing values (for efficiency)
*/
public boolean getUseNaNs() {
return useNaNs;
}
/**
* set if _FillValue is considered isMissing(); better set in constructor if possible
*/
public void setFillValueIsMissing(boolean b) {
this.fillValueIsMissing = b;
}
/**
* set if valid_range is considered isMissing(); better set in constructor if possible
*/
public void setInvalidDataIsMissing(boolean b) {
this.invalidDataIsMissing = b;
}
/**
* set if missing_data is considered isMissing(); better set in constructor if possible
*/
public void setMissingDataIsMissing(boolean b) {
this.missingDataIsMissing = b;
}
/**
* true if Variable has missing data values
*/
public boolean hasMissing() {
return (invalidDataIsMissing && hasInvalidData()) ||
(fillValueIsMissing && hasFillValue()) ||
(missingDataIsMissing && hasMissingValue());
}
/**
* Is this a missing value ?
*
* @param val check this value
* @return true if missing
*/
public boolean isMissing(double val) {
if (Double.isNaN(val)) return true;
return hasMissing() && isMissing_(val);
}
/**
* Optimize "Is this a missing value"? Assumes NaNs have already been set if its missing.
*
* @param val check this value
* @return true if missing
*/
public boolean isMissingFast(double val) {
if (useNaNs) return Double.isNaN(val); // no need to check again
if (Double.isNaN(val)) return true;
return hasMissing() && isMissing_(val);
}
// find data values that match a missing value
/* assumes that hasMissing() == true
private final boolean isMissing_(double val) {
return (invalidDataIsMissing && isInvalidData(val)) ||
(fillValueIsMissing && isFillValue(val)) ||
(missingDataIsMissing && isMissingValue(val));
} */
private boolean isMissing_(double val) {
if (missingDataIsMissing && hasMissingValue && isMissingValue(val))
return true;
if (fillValueIsMissing && hasFillValue && isFillValue(val))
return true;
if (invalidDataIsMissing)
return isInvalidData(val);
return false;
}
/**
* Get FillValue. Check if set, use default value if not
*
* @param dt the variable datatype
* @return java primitive array of length 1, or a String.
*/
public Object getFillValue(DataType dt) {
DataType useType = convertedDataType == null ? dt : convertedDataType;
if (useType.getPrimitiveClassType() == byte.class) {
byte[] result = new byte[1];
result[0] = hasFillValue ? (byte) fillValue : (dt.isUnsigned() ? N3iosp.NC_FILL_UBYTE : N3iosp.NC_FILL_BYTE);
return result;
} else if (useType == DataType.BOOLEAN) {
boolean[] result = new boolean[1];
result[0] = false;
return result;
} else if (useType == DataType.CHAR) {
char[] result = new char[1];
result[0] = hasFillValue ? (char) fillValue : N3iosp.NC_FILL_CHAR;
return result;
} else if (useType.getPrimitiveClassType() == short.class) {
short[] result = new short[1];
result[0] = hasFillValue ? (short) fillValue : (dt.isUnsigned() ? N3iosp.NC_FILL_USHORT : N3iosp.NC_FILL_SHORT);
return result;
} else if (useType.getPrimitiveClassType() == int.class) {
int[] result = new int[1];
result[0] = hasFillValue ? (int) fillValue : (dt.isUnsigned() ? N3iosp.NC_FILL_UINT : N3iosp.NC_FILL_INT);
return result;
} else if (useType.getPrimitiveClassType() == long.class) {
long[] result = new long[1];
result[0] = hasFillValue ? (long) fillValue : (dt.isUnsigned() ? N3iosp.NC_FILL_UINT64 : N3iosp.NC_FILL_INT64);
return result;
} else if (useType == DataType.FLOAT) {
float[] result = new float[1];
result[0] = hasFillValue ? (float) fillValue : N3iosp.NC_FILL_FLOAT;
return result;
} else if (useType == DataType.DOUBLE) {
double[] result = new double[1];
result[0] = hasFillValue ? fillValue : N3iosp.NC_FILL_DOUBLE;
return result;
} else {
String[] result = new String[1];
result[0] = CDM.FILL_VALUE;
return result;
}
}
public double convertScaleOffsetMissing(byte valb) {
if (!hasScaleOffset)
return useNaNs && isMissing((double) valb) ? Double.NaN : (double) valb;
double convertedValue;
if (isUnsigned)
convertedValue = scale * DataType.unsignedByteToShort(valb) + offset;
else
convertedValue = scale * valb + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(short vals) {
if (!hasScaleOffset)
return useNaNs && isMissing((double) vals) ? Double.NaN : (double) vals;
double convertedValue;
if (isUnsigned)
convertedValue = scale * DataType.unsignedShortToInt(vals) + offset;
else
convertedValue = scale * vals + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(int vali) {
if (!hasScaleOffset)
return useNaNs && isMissing((double) vali) ? Double.NaN : (double) vali;
double convertedValue;
if (isUnsigned)
convertedValue = scale * DataType.unsignedIntToLong(vali) + offset;
else
convertedValue = scale * vali + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(long vall) {
if (!hasScaleOffset)
return useNaNs && isMissing((double) vall) ? Double.NaN : (double) vall;
double convertedValue = scale * vall + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(double value) {
if (!hasScaleOffset)
return useNaNs && isMissing(value) ? Double.NaN : value;
double convertedValue = scale * value + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public Array convertScaleOffsetMissing(Array data) {
if (hasScaleOffset())
data = convertScaleOffset(data);
else if (hasMissing() && getUseNaNs())
data = convertMissing(data);
return data;
}
/**
* Convert Data with scale and offset.
* Also translate missing data to NaNs if useNaNs = true.
*
* @param in data to convert
* @return converted data.
*/
private Array convertScaleOffset(Array in) {
if (!hasScaleOffset) return in;
if (debugRead) System.out.println("convertScaleOffset ");
Array out = Array.factory(convertedDataType, in.getShape());
IndexIterator iterIn = in.getIndexIterator();
IndexIterator iterOut = out.getIndexIterator();
if (isUnsigned && in.getElementType() == byte.class)
convertScaleOffsetUnsignedByte(iterIn, iterOut);
else if (isUnsigned && in.getElementType() == short.class)
convertScaleOffsetUnsignedShort(iterIn, iterOut);
else if (isUnsigned && in.getElementType() == int.class)
convertScaleOffsetUnsignedInt(iterIn, iterOut);
else {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
double val = scale * iterIn.getDoubleNext() + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
return out;
}
private void convertScaleOffsetUnsignedByte(IndexIterator iterIn, IndexIterator iterOut) {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
byte valb = iterIn.getByteNext();
double val = scale * DataType.unsignedByteToShort(valb) + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
private void convertScaleOffsetUnsignedShort(IndexIterator iterIn, IndexIterator iterOut) {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
short valb = iterIn.getShortNext();
double val = scale * DataType.unsignedShortToInt(valb) + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
private void convertScaleOffsetUnsignedInt(IndexIterator iterIn, IndexIterator iterOut) {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
int valb = iterIn.getIntNext();
double val = scale * DataType.unsignedIntToLong(valb) + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
/**
* Translate missing data to NaNs. Data must be DOUBLE or FLOAT
*
* @param in convert this array
* @return same array, with missing values replaced by NaNs
*/
private Array convertMissing(Array in) {
if (debugRead) System.out.println("convertMissing ");
IndexIterator iterIn = in.getIndexIterator();
if (in.getElementType() == double.class) {
while (iterIn.hasNext()) {
double val = iterIn.getDoubleNext();
if (isMissing_(val))
iterIn.setDoubleCurrent(Double.NaN);
}
} else if (in.getElementType() == float.class) {
while (iterIn.hasNext()) {
float val = iterIn.getFloatNext();
if (isMissing_(val))
iterIn.setFloatCurrent(Float.NaN);
}
}
return in;
}
/**
* Convert (in place) all values in the given array that are considered
* as "missing" to Float.NaN
*
* @param values input array
* @return input array, with missing values converted to NaNs.
*/
public float[] setMissingToNaN(float[] values) {
if (!hasMissing()) return values;
for (int i = 0; i < values.length; i++) {
if (isMissing_(values[i]))
values[i] = Float.NaN;
}
return values;
}
static public void main(String[] args) {
double d = Double.NaN;
float f = (float) d;
System.out.println(" f=" + f + " " + Float.isNaN(f) + " " + Double.isNaN((double) f));
}
}