ucar.nc2.dods.ConvertD2N Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
/*
* Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.dods;
import opendap.dap.*;
import java.io.IOException;
import java.util.Enumeration;
import java.util.Vector;
import java.util.List;
import java.util.ArrayList;
import ucar.ma2.*;
import ucar.nc2.Variable;
/**
* Convert Dods object tree to netcdf.
*
* @author caron
*/
public class ConvertD2N {
static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DODSNetcdfFile.class);
/* Difficult cases.
V is a top variable vs member of a structure (or group ??)
Dgrid.member request may or may not get returned inside a Structure.
Structures may have arbitrary nesting.
*/
/*
DodsV: a container for dods basetypes, so we can track stuff as we process it.
We make the DodsV tree follow the name heirarchy, by putting the darray object to the side.
The root is just a container, but the other nodes each represent a BaseType/Variable.
DDS has a vector of BaseTypes, which may be:
* scalar primitive (byte, int, float, String, etc)
* array of primitive: DArray with (empty) PrimitiveVector with "template" as the BaseType.
* DConstructor is a container for other BaseTypes, can call getVariables() to get them
- DStructure
- DGrid first Variable is the Array, rest are the maps
- DSequence
* array of DStructure: DArray with BaseTypePrimitiveVector whose template is a DStructure
DataDDS also contains the data:
o scalar primitive (byte, int, float, String, etc): getValue()
o array of primitive: DArray with PrimitiveVector (BytePrimitiveVector, FloatPrimitiveVector, etc): getValue(i)
Get internal java array with getInternalStorage().
o array of String: DArray with BaseTypePrimitiveVector containing String objects
o scalar DStructure: same as DDS
o scalar DGrid: same as DDS
o array of DStructure: DArray with BaseTypePrimitiveVector, whose values are DStructure
X array of DGrid, DSequence: (not allowd)
o DSequence: values = Vector (rows) containing Vector (fields)
*/
/**
* Convert a DataDDS into an Array for a Structure member variable.
*
* @param v must be a member of a structure
* @param section the requested variable section, as a List of type Range
* @param dataV the dataDDS has been parsed into this dodsV, this is the top variable containing v
* @param flatten if true, remove the StructureData "wrapper".
* @return the data as as Array
* @throws IOException on io error
* @throws DAP2Exception on bad things happening
*/
public Array convertNestedVariable(ucar.nc2.Variable v, List section, DodsV dataV, boolean flatten) throws IOException, DAP2Exception {
Array data = convertTopVariable(v, section, dataV);
if (flatten) {
ArrayStructure as = (ArrayStructure) data;
// make list of names
List names = new ArrayList();
Variable nested = v;
while (nested.isMemberOfStructure()) {
names.add( 0, nested.getShortName());
nested = nested.getParentStructure();
}
StructureMembers.Member m = findNested(as, names, v.getShortName());
Array mdata = (Array) m.getDataArray();
if (mdata instanceof ArraySequenceNested) {
// gotta unroll
ArraySequenceNested arraySeq = (ArraySequenceNested) mdata;
return arraySeq.flatten();
}
return mdata;
}
return data;
}
private StructureMembers.Member findNested(ArrayStructure as, List names, String want) {
String name = names.get(0);
StructureMembers sm = as.getStructureMembers();
StructureMembers.Member m = sm.findMember(name);
if (name.equals(want))
return m;
// keep going
ArrayStructure nested = (ArrayStructure) m.getDataArray();
names.remove(0);
return findNested( nested, names, want);
}
/**
* Convert a DataDDS into an Array for a top level variable, ie not a Structure member variable.
*
* @param v must be a top variable
* @param section the requested variable section
* @param dataV the dataDDS has been parsed into this dodsV
* @return the data as as Array
* @throws IOException on io error
* @throws DAP2Exception on bad
*/
public Array convertTopVariable(ucar.nc2.Variable v, List section, DodsV dataV) throws IOException, DAP2Exception {
Array data = convert(dataV);
// arrays
if ((dataV.darray != null) && (dataV.bt instanceof DString)) {
if (v.getDataType() == DataType.STRING)
return convertStringArray(data, v);
else if (v.getDataType() == DataType.CHAR)
return convertStringArrayToChar(dataV.darray, v, section);
else {
String mess = "DODSVariable convertArray String invalid dataType= "+v.getDataType();
logger.error(mess);
throw new IllegalArgumentException( mess);
}
}
if ((dataV.bt instanceof DString) && (v.getDataType() == DataType.CHAR)) {
// special case: convert String back to CHAR
return convertStringToChar(data, v);
}
return data;
/* else { // the DGrid case comes here also
// create the array, using DODS internal array so there's no copying
dods.dap.PrimitiveVector pv = dataV.darray.getPrimitiveVector();
Object storage = pv.getInternalStorage();
//storage = widenArray( pv, storage); // LOOK data conversion if needed
int[] shape = (section == null) ? v.getShape() : Range.getShape(section);
return Array.factory( v.getDataType().getPrimitiveClassType(), shape, storage);
} */
}
/**
* Convert a DataDDS into an Array
*
* @param dataV the dataDDS has been parsed into this dodsV
* @return the data as as Array
* @throws IOException on io error
* @throws DAP2Exception on bad
*/
public Array convert(DodsV dataV) throws IOException, DAP2Exception {
// scalars
if (dataV.darray == null) {
if (dataV.bt instanceof DStructure) {
ArrayStructure structArray = makeArrayStructure( dataV);
iconvertDataStructure( (DStructure) dataV.bt, structArray.getStructureMembers());
return structArray;
} else if (dataV.bt instanceof DGrid) {
throw new IllegalStateException( "DGrid without a darray");
} else if (dataV.bt instanceof DSequence) {
ArrayStructure structArray = makeArrayStructure( dataV);
iconvertDataSequenceArray( (DSequence) dataV.bt, structArray.getStructureMembers());
return structArray;
} else { // scalar
DataType dtype = DODSNetcdfFile.convertToNCType( dataV.elemType);
Array scalarData = Array.factory( dtype, new int[0]);
IndexIterator scalarIndex = scalarData.getIndexIterator();
iconvertDataPrimitiveScalar( dataV.bt, scalarIndex);
return scalarData;
}
}
// arrays
if (dataV.darray != null) {
if (dataV.bt instanceof DStructure) {
ArrayStructure structArray = makeArrayStructure( dataV);
iconvertDataStructureArray( dataV.darray, structArray.getStructureMembers());
return structArray;
} else if (dataV.bt instanceof DString) {
return convertStringArray(dataV.darray);
} else {
// the DGrid case comes here also
// create the array, using DODS internal array so there's no copying
opendap.dap.PrimitiveVector pv = dataV.darray.getPrimitiveVector();
Object storage = pv.getInternalStorage();
DataType dtype = DODSNetcdfFile.convertToNCType( dataV.elemType);
return Array.factory( dtype.getPrimitiveClassType(), makeShape( dataV.darray), storage);
}
}
String mess = "Unknown baseType "+dataV.bt.getClass().getName()+" name="+ dataV.getName();
logger.error(mess);
throw new IllegalStateException(mess);
}
private ArrayStructure makeArrayStructure(DodsV dataV) {
StructureMembers members = new StructureMembers( dataV.getNetcdfShortName());
for (DodsV dodsV : dataV.children) {
StructureMembers.Member m = members.addMember(dodsV.getNetcdfShortName(), null, null, dodsV.getDataType(), dodsV.getShape());
Array data;
if ((dodsV.bt instanceof DStructure) || (dodsV.bt instanceof DGrid)) {
data = makeArrayStructure(dodsV);
} else if (dodsV.bt instanceof DSequence) {
data = makeArrayNestedSequence(dodsV);
m.setShape(data.getShape()); // fix the shape based on the actual data LOOK
} else {
data = Array.factory(dodsV.getDataType(), dodsV.getShapeAll());
}
m.setDataArray(data);
m.setDataObject(data.getIndexIterator()); // for setting values
}
return new ArrayStructureMA( members, dataV.getShapeAll());
}
private ArrayStructure makeArrayNestedSequence(DodsV dataV) {
// make the members
StructureMembers members = new StructureMembers(dataV.getName());
for (DodsV dodsV : dataV.children) {
members.addMember(dodsV.getNetcdfShortName(), null, null, dodsV.getDataType(), dodsV.getShape());
}
// make the ArraySequence
// LOOK single nested
DSequence outerSeq = (DSequence) dataV.parent.bt;
int outerLength = outerSeq.getRowCount();
ArraySequenceNested aseq = new ArraySequenceNested( members, outerLength);
// tell it how long each one is
String name = dataV.getName();
for (int row=0; row < outerLength; row++) {
Vector dv = outerSeq.getRow(row);
for (int j = 0; j < dv.size(); j++) {
BaseType bt = (BaseType) dv.elementAt(j);
if (bt.getName().equals(name)) {
DSequence innerSeq = (DSequence) bt;
int innerLength = innerSeq.getRowCount();
aseq.setSequenceLength(row, innerLength);
}
}
}
aseq.finish();
// ArraySequence makes the inner data arrays; now make iterators for them
List memberList = members.getMembers();
for (StructureMembers.Member m : memberList) {
Array data = (Array) m.getDataArray();
m.setDataObject(data.getIndexIterator()); // for setting values
}
return aseq;
}
// dataV is an array of DStructure: DArray with BaseTypePrimitiveVector, whose values are DStructure
private void iconvertDataStructureArray(DVector darray, StructureMembers members) throws DAP2Exception {
List mlist = members.getMembers();
for (StructureMembers.Member member : mlist) {
// get the Array for this member
String name = member.getName();
IndexIterator ii = (IndexIterator) member.getDataObject();
// loop over each row, fill up the data
BaseTypePrimitiveVector pv = (BaseTypePrimitiveVector) darray.getPrimitiveVector();
for (int row = 0; row < pv.getLength(); row++) {
DStructure ds_data = (DStructure) pv.getValue(row);
BaseType member_data = ds_data.getVariable(name);
iconvertData(member_data, ii);
}
}
}
private void iconvertDataSequenceArray(DSequence dseq, StructureMembers members) throws DAP2Exception {
for (int row=0; row < dseq.getRowCount(); row++) {
Vector dv = dseq.getRow(row);
for (int j = 0; j < dv.size(); j++) {
BaseType member_data = (BaseType) dv.elementAt(j);
StructureMembers.Member member = members.findMember( member_data.getName());
IndexIterator ii = (IndexIterator) member.getDataObject();
iconvertData( member_data, ii);
}
}
}
private void iconvertDataStructure(DConstructor ds, StructureMembers members) throws DAP2Exception {
List mlist = members.getMembers();
for (StructureMembers.Member member : mlist) {
// get the Array for this member
IndexIterator ii = (IndexIterator) member.getDataObject();
// track down the corresponding DODS member
String name = member.getName();
String dodsName = name; // LOOK should be: findDodsName (name);
if (dodsName == null) {
throw new DAP2Exception("Cant find dodsName for member variable " + name);
}
BaseType bt = ds.getVariable(dodsName);
iconvertData(bt, ii); // recursively fill up this data array
}
}
private void iconvertData(opendap.dap.BaseType dodsVar, IndexIterator ii) throws DAP2Exception {
if (dodsVar instanceof DSequence) {
DSequence dseq = (DSequence) dodsVar;
StructureData sd = (StructureData) ii.getObjectNext();
iconvertDataSequenceArray(dseq, sd.getStructureMembers());
// this is the "new" 2.0 that (correctly) wraps a Grid array in a Structure
/* else if ((dodsVar instanceof DStructure) && (ncVar instanceof DODSGrid)){
DStructure ds = (DStructure) dodsVar;
try {
DArray da = (DArray) ds.getVariable(ncVar.getShortName());
return convertArray(da, ncVar);
} catch (NoSuchVariableException e) {
e.printStackTrace();
return null;
}
} */
/* else
/* else if (dodsVar instanceof DGrid) { // scalar grid
DGrid ds = (DGrid) dodsVar;
try {
DArray da = (DArray) ds.getVariable(getDODSshortName( ncVar));
return convertArray(da, ncVar);
} catch (NoSuchVariableException e) {
e.printStackTrace();
return null;
}
} */
} else if ((dodsVar instanceof DStructure) || (dodsVar instanceof DGrid)) { // nested scalar structure
DConstructor ds = (DConstructor) dodsVar;
StructureData sd = (StructureData) ii.getObjectNext();
iconvertDataStructure(ds, sd.getStructureMembers());
// arrays
} else if (dodsVar instanceof DVector) {
DVector da = (DVector) dodsVar;
BaseType bt = da.getPrimitiveVector().getTemplate();
if (bt instanceof DStructure) {
StructureData sd = (StructureData) ii.getObjectNext();
iconvertDataStructureArray(da, sd.getStructureMembers());
} else if (bt instanceof DGrid) {
throw new UnsupportedOperationException();
} else if (bt instanceof DSequence) {
throw new UnsupportedOperationException();
} else {
iconvertDataPrimitiveArray( da.getPrimitiveVector(), ii);
}
// primitive scalars
} else {
iconvertDataPrimitiveScalar(dodsVar, ii);
}
}
/* private Array convertArray(DArray da, Variable ncVar) {
BaseType elemType = da.getPrimitiveVector().getTemplate();
if (debugConvertData) System.out.println(" DArray type "+ elemType.getClass().getName());
if (elemType instanceof DStructure) { // array of structures LOOK no array of DGrid
Structure s = (Structure) ncVar;
StructureMembers members = makeStructureMembers((DStructure) elemType, s);
ArrayStructureW structArray = new ArrayStructureW( members, makeShape( da));
// populate it
IndexIterator ii = structArray.getIndexIterator();
BaseTypePrimitiveVector pv = (BaseTypePrimitiveVector) da.getPrimitiveVector();
for (int i=0; i