ucar.nc2.iosp.hdf5.H5iosp Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
The newest version!
/*
* Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.iosp.hdf5;
import thredds.catalog.DataFormatType;
import ucar.ma2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.time.CalendarDate;
import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.iosp.*;
import ucar.nc2.iosp.hdf4.HdfEos;
import ucar.nc2.iosp.hdf4.H4header;
import ucar.nc2.*;
import java.io.IOException;
import java.io.PrintStream;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Formatter;
/**
* HDF5 I/O
*
* @author caron
*/
public class H5iosp extends AbstractIOServiceProvider {
static public final String IOSP_MESSAGE_INCLUDE_ORIGINAL_ATTRIBUTES = "IncludeOrgAttributes";
static public final int VLEN_T_SIZE = 16; // Appears to be no way to compute on the fly.
static boolean debug = false;
static boolean debugPos = false;
static boolean debugHeap = false;
static boolean debugFilter = false;
static boolean debugRead = false;
static boolean debugFilterIndexer = false;
static boolean debugChunkIndexer = false;
static boolean debugVlen = false;
static boolean debugStructure = false;
static boolean skipEos = false;
static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H5iosp.class);
static public void setDebugFlags(ucar.nc2.util.DebugFlags debugFlag) {
debug = debugFlag.isSet("H5iosp/read");
debugPos = debugFlag.isSet("H5iosp/filePos");
debugHeap = debugFlag.isSet("H5iosp/Heap");
debugFilter = debugFlag.isSet("H5iosp/filter");
debugFilterIndexer = debugFlag.isSet("H5iosp/filterIndexer");
debugChunkIndexer = debugFlag.isSet("H5iosp/chunkIndexer");
debugVlen = debugFlag.isSet("H5iosp/vlen");
skipEos = debugFlag.isSet("HdfEos/turnOff");
H5header.setDebugFlags(debugFlag);
H4header.setDebugFlags(debugFlag);
}
public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException {
return H5header.isValidFile(raf);
}
public String getFileTypeId() {
if (isEos) return "HDF5-EOS";
if (headerParser.isNetcdf4()) return DataFormatType.NETCDF4.toString();
return DataFormatType.HDF5.toString();
}
public String getFileTypeDescription() {
return "Hierarchical Data Format, version 5";
}
public void getEosInfo(Formatter f) throws IOException {
NetcdfFile ncfile = headerParser.ncfile;
Group eosInfo = ncfile.getRootGroup().findGroup(HdfEos.HDF5_GROUP);
if (eosInfo != null) {
HdfEos.getEosInfo(ncfile, eosInfo, f);
} else {
f.format("Cant find GROUP '%s'", HdfEos.HDF5_GROUP);
}
}
//////////////////////////////////////////////////////////////////////////////////
//private RandomAccessFile raf;
private H5header headerParser;
private boolean isEos;
boolean includeOriginalAttributes = false;
/////////////////////////////////////////////////////////////////////////////
// reading
public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util.CancelTask cancelTask) throws IOException {
super.open(raf, ncfile, cancelTask);
headerParser = new H5header(this.raf, ncfile, this);
headerParser.read(null);
// check if its an HDF5-EOS file
Group eosInfo = ncfile.getRootGroup().findGroup(HdfEos.HDF5_GROUP);
if (eosInfo != null && !skipEos) {
isEos = HdfEos.amendFromODL(ncfile, eosInfo);
}
ncfile.finish();
}
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException {
H5header.Vinfo vinfo = (H5header.Vinfo) v2.getSPobject();
if (debugRead) System.out.printf("%s read %s%n", v2.getFullName(), section);
return readData(v2, vinfo.dataPos, section);
}
// all the work is here, so can be called recursively
private Array readData(ucar.nc2.Variable v2, long dataPos, Section wantSection) throws IOException, InvalidRangeException {
H5header.Vinfo vinfo = (H5header.Vinfo) v2.getSPobject();
DataType dataType = v2.getDataType();
Object data;
Layout layout;
if (vinfo.useFillValue) { // fill value only
Object pa = IospHelper.makePrimitiveArray((int) wantSection.computeSize(), dataType, vinfo.getFillValue());
if (dataType == DataType.CHAR)
pa = IospHelper.convertByteToChar((byte[]) pa);
return Array.factory(dataType.getPrimitiveClassType(), wantSection.getShape(), pa);
}
if (vinfo.mfp != null) { // filtered
if (debugFilter) System.out.println("read variable filtered " + v2.getFullName() + " vinfo = " + vinfo);
assert vinfo.isChunked;
ByteOrder bo = (vinfo.typeInfo.endian == 0) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
layout = new H5tiledLayoutBB(v2, wantSection, raf, vinfo.mfp.getFilters(), bo);
data = IospHelper.readDataFill((LayoutBB) layout, v2.getDataType(), vinfo.getFillValue());
} else { // normal case
if (debug) System.out.println("read variable " + v2.getFullName() + " vinfo = " + vinfo);
DataType readDtype = v2.getDataType();
int elemSize = v2.getElementSize();
Object fillValue = vinfo.getFillValue();
int endian = vinfo.typeInfo.endian;
// fill in the wantSection
wantSection = Section.fill(wantSection, v2.getShape());
if (vinfo.typeInfo.hdfType == 2) { // time
readDtype = vinfo.mdt.timeType;
elemSize = readDtype.getSize();
fillValue = vinfo.getFillValueDefault(readDtype);
} else if (vinfo.typeInfo.hdfType == 8) { // enum
H5header.TypeInfo baseInfo = vinfo.typeInfo.base;
readDtype = baseInfo.dataType;
elemSize = readDtype.getSize();
fillValue = vinfo.getFillValueDefault(readDtype);
endian = baseInfo.endian;
} else if (vinfo.typeInfo.hdfType == 9) { // vlen
elemSize = vinfo.typeInfo.byteSize;
endian = vinfo.typeInfo.endian;
//wantSection = wantSection.removeVlen(); // remove vlen dimension
}
if (vinfo.isChunked) {
layout = new H5tiledLayout((H5header.Vinfo) v2.getSPobject(), readDtype, wantSection);
} else {
layout = new LayoutRegular(dataPos, elemSize, v2.getShape(), wantSection);
}
data = readData(vinfo, v2, layout, readDtype, wantSection.getShape(), fillValue, endian);
}
if (data instanceof Array)
return (Array) data;
else if (dataType == DataType.STRUCTURE)
return convertStructure((Structure) v2, layout, wantSection.getShape(), (byte[]) data); // LOOK
else
return Array.factory(dataType.getPrimitiveClassType(), wantSection.getShape(), data);
}
/*
* Read data subset from file for a variable, return Array or java primitive array.
*
* @param v the variable to read.
* @param layout handles skipping around in the file.
* @param dataType dataType of the data to read
* @param shape the shape of the output
* @param fillValue fill value as a wrapped primitive
* @return primitive array or Array with data read in
* @throws java.io.IOException if read error
* @throws ucar.ma2.InvalidRangeException if invalid section
*/
private Object readData(H5header.Vinfo vinfo, Variable v, Layout layout, DataType dataType, int[] shape,
Object fillValue, int endian) throws java.io.IOException, InvalidRangeException {
H5header.TypeInfo typeInfo = vinfo.typeInfo;
// special processing
if (typeInfo.hdfType == 2) { // time
Object data = IospHelper.readDataFill(raf, layout, dataType, fillValue, endian, true);
Array timeArray = Array.factory(dataType.getPrimitiveClassType(), shape, data);
// now transform into an ISO Date String
String[] stringData = new String[(int) timeArray.getSize()];
int count = 0;
while (timeArray.hasNext()) {
long time = timeArray.nextLong();
stringData[count++] = CalendarDate.of(time).toString();
}
return Array.factory(String.class, shape, stringData);
}
if (typeInfo.hdfType == 8) { // enum
Object data = IospHelper.readDataFill(raf, layout, dataType, fillValue, endian);
return Array.factory(dataType.getPrimitiveClassType(), shape, data);
}
if ((typeInfo.hdfType == 9) && !typeInfo.isVString) { // vlen (not string)
DataType readType = dataType;
if (typeInfo.isVString) // string
readType = DataType.BYTE;
else if (typeInfo.base.hdfType == 7) // reference
readType = DataType.LONG;
// general case is to read an array of vlen objects
// each vlen generates an Array - so return ArrayObject of Array
//boolean scalar = false; // layout.getTotalNelems() == 1; // if scalar, return just the len Array // remove 12/25/10 jcaron
Array[] data = new Array[(int) layout.getTotalNelems()];
int count = 0;
while (layout.hasNext()) {
Layout.Chunk chunk = layout.next();
if (chunk == null) continue;
for (int i = 0; i < chunk.getNelems(); i++) {
long address = chunk.getSrcPos() + layout.getElemSize() * i;
Array vlenArray = headerParser.getHeapDataArray(address, readType, endian);
data[count++] = (typeInfo.base.hdfType == 7) ? convertReference(vlenArray) : vlenArray;
}
}
int prefixrank = 0;
for(int i=0;i= 0)
m.setDataObject(vm.typeInfo.endian == RandomAccessFile.LITTLE_ENDIAN ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN);
// vm.dataPos : offset since start of Structure
m.setDataParam((int) vm.dataPos);
// track if there is a heap
if (v2.getDataType() == DataType.STRING || v2.isVariableLength())
hasHeap = true;
// recurse
if (v2 instanceof Structure) {
Structure nested = (Structure) v2;
StructureMembers nestSm = nested.makeStructureMembers();
m.setStructureMembers(nestSm);
hasHeap |= convertStructure(nested, nestSm);
}
}
return hasHeap;
}
/*
public static int setOffsets(StructureMembers members) {
int offset = 0;
for (StructureMembers.Member m : members.getMembers()) {
m.setDataParam(offset);
offset += m.getSizeBytes();
// set inner offsets (starts again at 0)
if (m.getStructureMembers() != null)
setOffsets(m.getStructureMembers());
}
members.setStructureSize(offset);
return offset;
}
*/
void convertHeap(ArrayStructureBB asbb, int pos, StructureMembers sm) throws java.io.IOException, InvalidRangeException {
ByteBuffer bb = asbb.getByteBuffer();
for (StructureMembers.Member m : sm.getMembers()) {
if (m.getDataType() == DataType.STRING) {
m.setDataObject(ByteOrder.nativeOrder()); // the index is always written in "native order"
int size = m.getSize();
int destPos = pos + m.getDataParam();
String[] result = new String[size];
for (int i = 0; i < size; i++)
result[i] = headerParser.readHeapString(bb, destPos + i * 16); // 16 byte "heap ids" are in the ByteBuffer
int index = asbb.addObjectToHeap(result);
bb.order(ByteOrder.nativeOrder()); // the string index is always written in "native order"
bb.putInt(destPos, index); // overwrite with the index into the StringHeap
} else if (m.isVariableLength()) {
int startPos = pos + m.getDataParam();
bb.order(ByteOrder.LITTLE_ENDIAN);
ByteOrder bo = (ByteOrder) m.getDataObject();
int endian = bo.equals(ByteOrder.LITTLE_ENDIAN) ? RandomAccessFile.LITTLE_ENDIAN : RandomAccessFile.BIG_ENDIAN;
// Compute rank and size upto the first (and ideally last) VLEN
int[] fieldshape = m.getShape();
int prefixrank = 0;
int size = 1;
for(;prefixrank < fieldshape.length;prefixrank++) {
if(fieldshape[prefixrank] < 0) break;
size *= fieldshape[prefixrank];
}
assert size == m.getSize() : "Internal error: field size mismatch";
Array[] fieldarray = new Array[size]; // hold all the vlen instance data
// destPos will point to each vlen instance in turn
// assuming we have 'size' such instances in a row.
int destPos = startPos;
for(int i = 0;i < size;i++) {
// vlenarray extracts the i'th vlen contents (struct not supported).
Array vlenArray = headerParser.readHeapVlen(bb, destPos, m.getDataType(), endian);
fieldarray[i] = vlenArray;
destPos += VLEN_T_SIZE; // Apparentlly no way to compute VLEN_T_SIZE on the fly
}
Array result = null;
if(prefixrank == 0) // if scalar, return just the singleton vlen array
result = fieldarray[0];
else if(prefixrank == 1)
result = new ArrayObject(fieldarray[0].getClass(), new int[]{size}, fieldarray);
else {
// Otherwise create and fill in an n-dimensional Array Of Arrays
int[] newshape = new int[prefixrank];
System.arraycopy(fieldshape, 0, newshape, 0, prefixrank);
Array ndimarray = Array.factory(Array.class, newshape);
// Transfer the elements of data into the n-dim arrays
IndexIterator iter = ndimarray.getIndexIterator();
for(int i = 0;iter.hasNext();i++) {
iter.setObjectNext(fieldarray[i]);
}
result = ndimarray;
}
//Array vlenArray = headerParser.readHeapVlen(bb, destPos, m.getDataType(), endian);
int index = asbb.addObjectToHeap(result);
bb.order(ByteOrder.nativeOrder());
bb.putInt(startPos, index); // overwrite with the index into the Heap
}
}
}
/*
* Read data subset from file for a variable, create primitive array.
*
* @param layout handles skipping around in the file.
* @param dataType dataType of the variable
* @param shape the shape of the output
* @param fillValue fill value as a wrapped primitive
* @param endian byte order
* @return primitive array with data read in
* @throws java.io.IOException if read error
* @throws ucar.ma2.InvalidRangeException if invalid section
*/
Object readDataPrimitive(Layout layout, DataType dataType, int[] shape, Object fillValue, int endian, boolean convertChar) throws java.io.IOException, InvalidRangeException {
if (dataType == DataType.STRING) {
int size = (int) layout.getTotalNelems();
String[] sa = new String[size];
int count = 0;
while (layout.hasNext()) {
Layout.Chunk chunk = layout.next();
if (chunk == null) continue;
for (int i = 0; i < chunk.getNelems(); i++) { // 16 byte "heap ids"
sa[count++] = headerParser.readHeapString(chunk.getSrcPos() + layout.getElemSize() * i);
}
}
return sa;
}
if (dataType == DataType.OPAQUE) {
Array opArray = new ArrayObject(ByteBuffer.class, shape);
assert (new Section(shape).computeSize() == layout.getTotalNelems());
int count = 0;
while (layout.hasNext()) {
Layout.Chunk chunk = layout.next();
if (chunk == null) continue;
int recsize = layout.getElemSize();
for (int i = 0; i < chunk.getNelems(); i++) {
byte[] pa = new byte[recsize];
raf.seek(chunk.getSrcPos() + i * recsize);
raf.readFully(pa, 0, recsize);
opArray.setObject(count++, ByteBuffer.wrap(pa));
}
}
return opArray;
}
// normal case
return IospHelper.readDataFill(raf, layout, dataType, fillValue, endian, convertChar);
}
// old way
private StructureData readStructure(Structure s, ArrayStructureW asw, long dataPos) throws IOException, InvalidRangeException {
StructureDataW sdata = new StructureDataW(asw.getStructureMembers());
if (debug) System.out.println(" readStructure " + s.getFullName() + " dataPos = " + dataPos);
for (Variable v2 : s.getVariables()) {
H5header.Vinfo vinfo = (H5header.Vinfo) v2.getSPobject();
if (debug) System.out.println(" readStructureMember " + v2.getFullName() + " vinfo = " + vinfo);
Array dataArray = readData(v2, dataPos + vinfo.dataPos, v2.getShapeAsSection());
sdata.setMemberData(v2.getShortName(), dataArray);
}
return sdata;
}
//////////////////////////////////////////////////////////////////////////
// utilities
/**
* Close the file.
*
* @throws IOException on io error
*/
public void close() throws IOException {
if (raf != null) {
raf.close();
// log.warn("H5iosp.close called on "+myRaf.getLocation()+" for ncfile="+ncfile.hashCode()+" for iosp="+this.hashCode());
}
raf = null;
headerParser.close();
}
/**
* Debug info for this object.
*/
public String toStringDebug(Object o) {
if (o instanceof Variable) {
Variable v = (Variable) o;
H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
return vinfo.toString();
}
return null;
}
public String getDetailInfo() {
Formatter f = new Formatter();
ByteArrayOutputStream ff = new ByteArrayOutputStream(100 * 1000);
try {
NetcdfFile ncfile = new NetcdfFileSubclass();
H5header detailParser = new H5header(raf, ncfile, this);
detailParser.read(new PrintStream(ff, false, CDM.UTF8));
f.format("%s",super.getDetailInfo());
f.format("%s",ff.toString(CDM.UTF8));
} catch (IOException e) {
e.printStackTrace();
e.printStackTrace(new PrintStream(ff));
}
return f.toString();
}
// debug
public Object sendIospMessage(Object message) {
if (message.toString().equals(IOSP_MESSAGE_INCLUDE_ORIGINAL_ATTRIBUTES)) {
includeOriginalAttributes = true;
return null;
}
if (message.toString().equals("header"))
return headerParser;
if (message.toString().equals("headerEmpty")) {
NetcdfFile ncfile = new NetcdfFileSubclass();
return new H5header(raf, ncfile, this);
}
return super.sendIospMessage(message);
}
NetcdfFile getNetcdfFile() {
return headerParser.ncfile;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy