ucar.nc2.iosp.netcdf3.N3iosp Maven / Gradle / Ivy
The newest version!
/*
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
* See LICENSE for license information.
*/
package ucar.nc2.iosp.netcdf3;
import static ucar.nc2.NetcdfFile.IOSP_MESSAGE_GET_NETCDF_FILE_FORMAT;
import com.google.re2j.Matcher;
import com.google.re2j.Pattern;
import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.DataFormatType;
import ucar.nc2.iosp.*;
import ucar.nc2.write.NetcdfFileFormat;
import ucar.unidata.io.RandomAccessFile;
import java.io.File;
import java.io.IOException;
import java.nio.channels.WritableByteChannel;
import java.util.Formatter;
/**
* IOServiceProvider implementation abstract base class to read/write "version 3" netcdf files.
* AKA "file format version 1" files.
*
* @author caron
* @see N3raf concrete class
* @deprecated do not use
*/
@Deprecated
public abstract class N3iosp extends AbstractIOServiceProvider implements IOServiceProviderWriter {
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(N3iosp.class);
// Default fill values, used unless _FillValue variable attribute is set.
public static final byte NC_FILL_BYTE = -127;
public static final char NC_FILL_CHAR = (char) 0;
public static final short NC_FILL_SHORT = (short) -32767;
public static final int NC_FILL_INT = -2147483647;
public static final float NC_FILL_FLOAT = 9.9692099683868690e+36f; /* near 15 * 2^119 */
public static final double NC_FILL_DOUBLE = 9.9692099683868690e+36;
public static final byte NC_FILL_UBYTE = (byte) 255;
public static final short NC_FILL_USHORT = (short) 65535;
public static final int NC_FILL_UINT = (int) 4294967295L;
public static final long NC_FILL_INT64 = -9223372036854775806L; // 0x8000000000000002. Only bits 63 and 1 set.
// We want to use 18446744073709551614ULL here (see https://goo.gl/buBal9), but that's too big to fit into a
// signed long (Java doesn't have unsigned types). So, assign the hex string that WOULD correspond to that value
// *if it were treated as unsigned*. Java will treat it as signed and see "-2", but we don't much care.
public static final long NC_FILL_UINT64 = 0xfffffffffffffffeL;
public static final String NC_FILL_STRING = "";
public static Number getFillValueDefault(DataType dtype) {
if ((dtype == DataType.BYTE) || (dtype == DataType.ENUM1))
return N3iosp.NC_FILL_BYTE;
if (dtype == DataType.UBYTE)
return N3iosp.NC_FILL_UBYTE;
if (dtype == DataType.CHAR)
return (byte) 0;
if ((dtype == DataType.SHORT) || (dtype == DataType.ENUM2))
return N3iosp.NC_FILL_SHORT;
if (dtype == DataType.USHORT)
return N3iosp.NC_FILL_USHORT;
if ((dtype == DataType.INT) || (dtype == DataType.ENUM4))
return N3iosp.NC_FILL_INT;
if (dtype == DataType.UINT)
return N3iosp.NC_FILL_UINT;
if (dtype == DataType.LONG)
return N3iosp.NC_FILL_INT64;
if (dtype == DataType.ULONG)
return N3iosp.NC_FILL_UINT64;
if (dtype == DataType.FLOAT)
return N3iosp.NC_FILL_FLOAT;
if (dtype == DataType.DOUBLE)
return N3iosp.NC_FILL_DOUBLE;
return null;
}
/*
* CLASSIC
* The maximum size of a record in the classic format in versions 3.5.1 and earlier is 2^32 - 4 bytes.
* In versions 3.6.0 and later, there is no such restriction on total record size for the classic format
* or 64-bit offset format.
*
* If you don't use the unlimited dimension, only one variable can exceed 2 GiB in size, but it can be as
* large as the underlying file system permits. It must be the last variable in the dataset, and the offset
* to the beginning of this variable must be less than about 2 GiB.
*
* The limit is really 2^31 - 4. If you were to specify a variable size of 2^31 -3, for example, it would be
* rounded up to the nearest multiple of 4 bytes, which would be 2^31, which is larger than the largest
* signed integer, 2^31 - 1.
*
* If you use the unlimited dimension, record variables may exceed 2 GiB in size, as long as the offset of the
* start of each record variable within a record is less than 2 GiB - 4.
*/
/*
* LARGE FILE
* Assuming an operating system with Large File Support, the following restrictions apply to the netCDF 64-bit offset
* format.
*
* No fixed-size variable can require more than 2^32 - 4 bytes of storage for its data, unless it is the last
* fixed-size variable and there are no record variables. When there are no record variables, the last
* fixed-size variable can be any size supported by the file system, e.g. terabytes.
*
* A 64-bit offset format netCDF file can have up to 2^32 - 1 fixed sized variables, each under 4GiB in size.
* If there are no record variables in the file the last fixed variable can be any size.
*
* No record variable can require more than 2^32 - 4 bytes of storage for each record's worth of data,
* unless it is the last record variable. A 64-bit offset format netCDF file can have up to 2^32 - 1 records,
* of up to 2^32 - 1 variables, as long as the size of one record's data for each record variable except the
* last is less than 4 GiB - 4.
*
* Note also that all netCDF variables and records are padded to 4 byte boundaries.
*/
/**
* Each fixed-size variable and the data for one record's worth of a single record variable are limited
* to a little less than 4 GiB.
*/
public static final long MAX_VARSIZE = (long) 2 * Integer.MAX_VALUE - 2; // 4,294,967,292
/**
* The maximum number of records is 2^32-1.
*/
public static final int MAX_NUMRECS = Integer.MAX_VALUE;
// NetCDF File Format Type (defined in netcdf.h from the C library)
private static final String NC_FORMATX_NC3 = String.valueOf(NetcdfFileFormat.NETCDF3.version());
private static boolean syncExtendOnly;
/**
* Set a static property.
* Supported static properties:
*
* - syncExtendOnly = "true" : assume all file changes are syncExtend only.
*
*
* @param name property name
* @param value property value
*/
public static void setProperty(String name, String value) {
if (name.equalsIgnoreCase("syncExtendOnly"))
syncExtendOnly = value.equalsIgnoreCase("true");
}
/**
* Determine if the given name can be used for a NetCDF object, i.e. a Dimension, Attribute, or Variable.
* The allowed name syntax (in RE form) is:
*
*
* ([a-zA-Z0-9_]|{UTF8})([^\x00-\x1F\x7F/]|{UTF8})*
*
*
* where UTF8 represents a multi-byte UTF-8 encoding. Also, no trailing spaces are permitted in names. We do not
* allow '/' because HDF5 does not permit slashes in names as slash is used as a group separator. If UTF-8 is
* supported, then a multi-byte UTF-8 character can occur anywhere within an identifier.
*
* @param name the name to validate.
* @return {@code true} if the name is valid.
*/
// Implements "int NC_check_name(const char *name)" from NetCDF-C:
// https://github.com/Unidata/netcdf-c/blob/v4.3.3.1/libdispatch/dstring.c#L169
// Should match makeValidNetcdfObjectName()
public static boolean isValidNetcdfObjectName(String name) {
if (name == null || name.isEmpty()) { // Null and empty names disallowed
return false;
}
int cp = name.codePointAt(0);
// First char must be [a-z][A-Z][0-9]_ | UTF8
if (cp <= 0x7f) {
if (!('A' <= cp && cp <= 'Z') && !('a' <= cp && cp <= 'z') && !('0' <= cp && cp <= '9') && cp != '_') {
return false;
}
}
for (int i = 1; i < name.length(); ++i) {
cp = name.codePointAt(i);
// handle simple 0x00-0x7f characters here
if (cp <= 0x7f) {
if (cp < ' ' || cp > 0x7E || cp == '/') { // control char, DEL, or forward-slash
return false;
}
}
}
// trailing spaces disallowed
return cp > 0x7f || !Character.isWhitespace(cp);
}
/**
* Convert a name to a legal netcdf-3 name.
*
* @param name the name to convert.
* @return the converted name.
* @see #isValidNetcdfObjectName(String)
*/
public static String makeValidNetcdfObjectName(String name) {
StringBuilder sb = new StringBuilder(name);
while (sb.length() > 0) {
int cp = sb.codePointAt(0);
// First char must be [a-z][A-Z][0-9]_ | UTF8
if (cp <= 0x7f) {
if (!('A' <= cp && cp <= 'Z') && !('a' <= cp && cp <= 'z') && !('0' <= cp && cp <= '9') && cp != '_') {
sb.deleteCharAt(0);
continue;
}
}
break;
}
for (int pos = 1; pos < sb.length(); ++pos) {
int cp = sb.codePointAt(pos);
// handle simple 0x00-0x7F characters here
if (cp <= 0x7F) {
if (cp < ' ' || cp > 0x7E || cp == '/') { // control char, DEL, or forward-slash
sb.deleteCharAt(pos);
--pos;
}
}
}
while (sb.length() > 0) {
int cp = sb.codePointAt(sb.length() - 1);
if (cp <= 0x7f && Character.isWhitespace(cp)) {
sb.deleteCharAt(sb.length() - 1);
} else {
break;
}
}
if (sb.length() == 0) {
throw new IllegalArgumentException(String.format("Illegal NetCDF object name: '%s'", name));
}
return sb.toString();
}
/////////////////////////////////////////////////////////////////
// I think these go together - consider deprecated
/**
* @deprecated use makeValidNetcdfObjectName
*/
public static String makeValidNetcdf3ObjectName(String name) {
StringBuilder sb = new StringBuilder(name);
while (sb.length() > 0) {
char c = sb.charAt(0);
if (Character.isLetter(c) || (c == '_'))
break;
if (Character.isDigit(c)) {
sb.insert(0, 'N');
break;
}
sb.deleteCharAt(0);
}
int i = 1;
while (i < sb.length()) {
char c = sb.charAt(i);
if (c == ' ')
sb.setCharAt(i, '_');
else {
boolean ok = Character.isLetterOrDigit(c) || (c == '-') || (c == '_');
// || (c == '@') || (c == ':') || (c == '(') || (c == ')') || (c == '+') || (c == '.');
if (!ok) {
sb.delete(i, i + 1);
i--;
// sb.setCharAt(i, '-');
}
}
i++;
}
return sb.toString();
}
// static private final String special1 = "_\\.@\\+\\-";
// static private final String special2 = " ";
// static private final Pattern objectNamePattern = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_@\\.\\-\\+]*");
private static final Pattern objectNamePatternOld = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_@\\:\\(\\)\\.\\-\\+]*");
/**
* Determine if the given name can be used for a Dimension, Attribute, or Variable name.
* Should match makeValidNetcdf3ObjectName.
*
* @param name test this.
* @return true if valid name.
* @deprecated use isValidNetcdfObjectName
*/
public static boolean isValidNetcdf3ObjectName(String name) {
Matcher m = objectNamePatternOld.matcher(name);
return m.matches();
}
private static final Pattern objectNamePattern = Pattern.compile("[a-zA-Z0-9_][^\\x00-\\x1F\\x2F\\x7F]*");
/**
* Valid Netcdf Object name as a regular expression.
*
* @return regular expression pattern describing valid Netcdf Object names.
*/
public static Pattern getValidNetcdf3ObjectNamePattern() {
return objectNamePattern;
}
///////////////////////////////////////////////////////////////////////////////////////////
/**
* Convert a name to a legal netcdf name.
* From the user manual:
* "The names of dimensions, variables and attributes consist of arbitrary sequences of
* alphanumeric characters (as well as underscore '_' and hyphen '-'), beginning with a letter
* or underscore. (However names commencing with underscore are reserved for system use.)
* Case is significant in netCDF names."
*
* Algorithm:
*
* - leading character: if alpha or underscore, ok; if digit, prepend "N"; otherwise discard
*
- other characters: if space, change to underscore; other delete.
*
*
* @param name convert this name
* @return converted name
* @deprecated use makeValidNetcdfObjectName
*/
public static String createValidNetcdf3ObjectName(String name) {
StringBuilder sb = new StringBuilder(name);
// LOOK: could escape characters, as in DODS (%xx) ??
while (sb.length() > 0) {
char c = sb.charAt(0);
if (Character.isLetter(c) || (c == '_'))
break;
if (Character.isDigit(c)) {
sb.insert(0, 'N');
break;
}
sb.deleteCharAt(0);
}
int i = 1;
while (i < sb.length()) {
char c = sb.charAt(i);
if ((c == ' ') || (c == '/'))
sb.setCharAt(i, '_');
else {
boolean ok = Character.isLetterOrDigit(c) || (c == '-') || (c == '_') || (c == '@') || (c == ':') || (c == '(')
|| (c == ')') || (c == '+') || (c == '.');
if (!ok) {
sb.delete(i, i + 1);
i--;
// sb.setCharAt(i, '-');
}
}
i++;
}
return sb.toString();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
protected boolean readonly;
protected N3header header;
// protected int numrecs;
// protected long recsize;
protected long lastModified; // used by sync
// used for writing only
// protected long fileUsed = 0; // how much of the file is written to ?
// protected long recStart = 0; // where the record data starts
protected boolean debug, debugSize, debugSPIO, debugRecord, debugRead;
protected boolean showHeaderBytes;
@Override
public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws IOException {
return N3header.isValidFile(raf);
}
@Override
public String getDetailInfo() {
Formatter f = new Formatter();
f.format("%s", super.getDetailInfo());
try {
header.showDetail(f);
} catch (IOException e) {
return e.getMessage();
}
return f.toString();
}
// properties
protected boolean useRecordStructure;
//////////////////////////////////////////////////////////////////////////////////////
// read existing file
@Override
public void openForWriting(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile,
ucar.nc2.util.CancelTask cancelTask) throws IOException {
open(raf, ncfile, cancelTask);
}
@Override
public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile,
ucar.nc2.util.CancelTask cancelTask) throws IOException {
super.open(raf, ncfile, cancelTask);
String location = raf.getLocation();
if (!location.startsWith("http:")) {
File file = new File(location);
if (file.exists())
lastModified = file.lastModified();
}
// its a netcdf-3 file
raf.order(RandomAccessFile.BIG_ENDIAN);
header = new N3header();
header.read(raf, ncfile, null); // read header here
// numrecs = header.numrecs;
// recsize = header.recsize;
// recStart = header.recStart;
_open(raf);
ncfile.finish();
}
@Override
public void setFill(boolean fill) {
this.fill = fill;
}
/////////////////////////////////////////////////////////////////////////////
// data reading
@Override
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException {
if (v2 instanceof Structure)
return readRecordData((Structure) v2, section);
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
DataType dataType = v2.getDataType();
Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
: new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
if (layout.getTotalNelems() == 0) {
return Array.factory(dataType, section.getShape());
}
Object data = readData(layout, dataType);
return Array.factory(dataType, section.getShape(), data);
}
/**
* Read data from record structure. For N3, this is the only possible structure, and there can be no nesting.
* Read all variables for each record, put in ByteBuffer.
*
* @param s the record structure
* @param section the record range to read
* @return an ArrayStructure, with all the data read in.
* @throws IOException on error
*/
private ucar.ma2.Array readRecordData(ucar.nc2.Structure s, Section section) throws java.io.IOException {
// if (s.isSubset())
// return readRecordDataSubset(s, section);
// has to be 1D
Range recordRange = section.getRange(0);
// create the ArrayStructure
StructureMembers members = s.makeStructureMembers();
for (StructureMembers.Member m : members.getMembers()) {
Variable v2 = s.findVariable(m.getName());
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
m.setDataParam((int) (vinfo.begin - header.recStart));
}
// protect agains too large of reads
if (header.recsize > Integer.MAX_VALUE)
throw new IllegalArgumentException("Cant read records when recsize > " + Integer.MAX_VALUE);
long nrecs = section.computeSize();
if (nrecs * header.recsize > Integer.MAX_VALUE)
throw new IllegalArgumentException(
"Too large read: nrecs * recsize= " + (nrecs * header.recsize) + "bytes exceeds " + Integer.MAX_VALUE);
members.setStructureSize((int) header.recsize);
ArrayStructureBB structureArray = new ArrayStructureBB(members, new int[] {recordRange.length()});
// note dependency on raf; should probably defer to subclass
// loop over records
byte[] result = structureArray.getByteBuffer().array();
int count = 0;
for (int recnum : recordRange) {
if (debugRecord)
System.out.println(" read record " + recnum);
raf.seek(header.recStart + recnum * header.recsize); // where the record starts
if (recnum != header.numrecs - 1)
raf.readFully(result, (int) (count * header.recsize), (int) header.recsize);
else
raf.read(result, (int) (count * header.recsize), (int) header.recsize); // "wart" allows file to be one byte
// short. since its always padding, we
// allow
count++;
}
return structureArray;
}
/**
* Read data from record structure, that has been subsetted.
* Read one record at at time, put requested variable into ArrayStructureMA.
*
* @param s the record structure
* @param section the record range to read
* @return an ArrayStructure, with all the data read in.
*/
private ucar.ma2.Array readRecordDataSubset(ucar.nc2.Structure s, Section section) {
Range recordRange = section.getRange(0);
int nrecords = recordRange.length();
// create the ArrayStructureMA
StructureMembers members = s.makeStructureMembers();
for (StructureMembers.Member m : members.getMembers()) {
Variable v2 = s.findVariable(m.getName());
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
m.setDataParam((int) (vinfo.begin - header.recStart)); // offset from start of record
// construct the full shape
int rank = m.getShape().length;
int[] fullShape = new int[rank + 1];
fullShape[0] = nrecords; // the first dimension
System.arraycopy(m.getShape(), 0, fullShape, 1, rank); // the remaining dimensions
Array data = Array.factory(m.getDataType(), fullShape);
m.setDataArray(data);
m.setDataObject(data.getIndexIterator());
}
// LOOK this is all wrong - why using recsize ???
return null;
/*
* members.setStructureSize(recsize);
* ArrayStructureMA structureArray = new ArrayStructureMA(members, new int[]{nrecords});
*
* // note dependency on raf; should probably defer to subclass
* // loop over records
* byte[] record = new byte[ recsize];
* ByteBuffer bb = ByteBuffer.wrap(record);
* for (int recnum = recordRange.first(); recnum <= recordRange.last(); recnum += recordRange.stride()) {
* if (debugRecord) System.out.println(" readRecordDataSubset recno= " + recnum);
*
* // read one record
* raf.seek(recStart + recnum * recsize); // where the record starts
* if (recnum != numrecs - 1)
* raf.readFully(record, 0, recsize);
* else
* raf.read(record, 0, recsize); // "wart" allows file to be one byte short. since its always padding, we allow
*
* // transfer desired variable(s) to result array(s)
* for (StructureMembers.Member m : members.getMembers()) {
* IndexIterator dataIter = (IndexIterator) m.getDataObject();
* IospHelper.copyFromByteBuffer(bb, m, dataIter);
* }
* }
*
* return structureArray;
*/
}
private ucar.ma2.Array readNestedData(ucar.nc2.Variable v2, Section section)
throws java.io.IOException, ucar.ma2.InvalidRangeException {
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
DataType dataType = v2.getDataType();
// construct the full shape for use by RegularIndexer
int[] fullShape = new int[v2.getRank() + 1];
fullShape[0] = header.numrecs; // the first dimension
System.arraycopy(v2.getShape(), 0, fullShape, 1, v2.getRank()); // the remaining dimensions
Layout layout = new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, fullShape, section);
Object dataObject = readData(layout, dataType);
return Array.factory(dataType, section.getShape(), dataObject);
}
@Override
public long readToByteChannel(ucar.nc2.Variable v2, Section section, WritableByteChannel channel)
throws java.io.IOException, ucar.ma2.InvalidRangeException {
if (v2 instanceof Structure)
return readRecordData((Structure) v2, section, channel);
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
DataType dataType = v2.getDataType();
Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
: new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
return readData(layout, dataType, channel);
}
private long readRecordData(ucar.nc2.Structure s, Section section, WritableByteChannel out)
throws java.io.IOException {
long count = 0;
/*
* RegularIndexer index = new RegularIndexer( s.getShape(), recsize, recStart, section, recsize);
* while (index.hasNext()) {
* Indexer.Chunk chunk = index.next();
* count += raf.readBytes( out, chunk.getFilePos(), chunk.getNelems() * s.getElementSize());
* }
*/
// LOOK this is the OTW layout based on netcdf-3
// not sure this works but should give an idea of timing
Range recordRange = section.getRange(0);
/*
* int stride = recordRange.stride();
* if (stride == 1) {
* int first = recordRange.first();
* int n = recordRange.length();
* if (false) System.out.println(" read record " + first+" "+ n * header.recsize+" bytes ");
* return raf.readToByteChannel(out, header.recStart + first * header.recsize, n * header.recsize);
*
* } else {
*/
for (int recnum : recordRange) {
if (debugRecord)
System.out.println(" read record " + recnum);
raf.seek(header.recStart + recnum * header.recsize); // where the record starts
count += raf.readToByteChannel(out, header.recStart + recnum * header.recsize, header.recsize);
}
// }
return count;
}
//////////////////////////////////////////////////////////////////////////////////////
// create new file
protected boolean fill = true;
// protected HashMap dimHash = new HashMap(50);
@Override
public void create(String filename, ucar.nc2.NetcdfFile ncfile, int extra, long preallocateSize, boolean largeFile)
throws IOException {
this.ncfile = ncfile;
this.readonly = false;
// finish any structures
ncfile.finish();
raf = new ucar.unidata.io.RandomAccessFile(filename, "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
if (preallocateSize > 0) {
java.io.RandomAccessFile myRaf = raf.getRandomAccessFile();
myRaf.setLength(preallocateSize);
}
header = new N3header();
header.create(raf, ncfile, extra, largeFile, null);
// recsize = header.recsize; // record size
// recStart = header.recStart; // record variables start here
// fileUsed = headerParser.getMinLength(); // track what is actually used
_create(raf);
if (fill)
fillNonRecordVariables();
// else
// raf.setMinLength(recStart); // make sure file length is long enough, even if not written to.
}
@Override
public boolean rewriteHeader(boolean largeFile) throws IOException {
return header.rewriteHeader(largeFile, null);
}
//////////////////////////////////////////////////////////////////////////////////////
// write
@Override
public void writeData(Variable v2, Section section, Array values) throws java.io.IOException, InvalidRangeException {
N3header.Vinfo vinfo = (N3header.Vinfo) v2.getSPobject();
DataType dataType = v2.getDataType();
if (v2.isUnlimited()) {
Range firstRange = section.getRange(0);
setNumrecs(firstRange.last() + 1);
}
if (v2 instanceof Structure) {
if (!(values instanceof ArrayStructure))
throw new IllegalArgumentException("writeData for Structure: data must be ArrayStructure");
if (v2.getRank() == 0)
throw new IllegalArgumentException("writeData for Structure: must have rank > 0");
Dimension d = v2.getDimension(0);
if (!d.isUnlimited())
throw new IllegalArgumentException("writeData for Structure: must have unlimited dimension");
writeRecordData((Structure) v2, section, (ArrayStructure) values);
} else {
Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
: new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
writeData(values, layout, dataType);
}
}
@Override
public int appendStructureData(Structure s, StructureData sdata) throws IOException, InvalidRangeException {
int recnum = header.numrecs;
setNumrecs(recnum + 1);
writeRecordData(s, recnum, sdata);
return recnum;
}
private void writeRecordData(ucar.nc2.Structure s, Section section, ArrayStructure structureArray)
throws java.io.IOException, ucar.ma2.InvalidRangeException {
int countSrcRecnum = 0;
Range recordRange = section.getRange(0);
for (int recnum : recordRange) {
StructureData sdata = structureArray.getStructureData(countSrcRecnum);
writeRecordData(s, recnum, sdata);
countSrcRecnum++;
}
}
private void writeRecordData(ucar.nc2.Structure s, int recnum, StructureData sdata)
throws java.io.IOException, ucar.ma2.InvalidRangeException {
StructureMembers members = sdata.getStructureMembers();
// loop over members
for (Variable vm : s.getVariables()) {
StructureMembers.Member m = members.findMember(vm.getShortName());
if (null == m)
continue; // this means that the data is missing from the ArrayStructure
// convert String member data into CHAR data
Array data = sdata.getArray(m);
if (data instanceof ArrayObject && vm.getDataType() == DataType.CHAR && vm.getRank() > 0) {
int strlen = vm.getShape(vm.getRank() - 1);
data = ArrayChar.makeFromStringArray((ArrayObject) data, strlen); // turn it into an ArrayChar
}
// layout of the destination
N3header.Vinfo vinfo = (N3header.Vinfo) vm.getSPobject();
long begin = vinfo.begin + recnum * header.recsize; // this assumes unlimited dimension
Section memberSection = vm.getShapeAsSection();
Layout layout = new LayoutRegular(begin, vm.getElementSize(), vm.getShape(), memberSection);
try {
writeData(data, layout, vm.getDataType());
} catch (Exception e) {
log.error("Error writing member=" + vm.getShortName() + " in struct=" + s.getFullName(), e);
throw new IOException(e);
}
}
}
protected void setNumrecs(int n) throws IOException, InvalidRangeException {
if (n <= header.numrecs)
return;
int startRec = header.numrecs;
if (debugSize)
System.out.println("extend records to = " + n);
// fileUsed = recStart + recsize * n;
header.setNumrecs(n);
// this.numrecs = n;
// need to let unlimited dimension know of new shape
for (Dimension dim : ncfile.getDimensions()) {
if (dim.isUnlimited())
dim.setLength(n);
}
// need to let all unlimited variables know of new shape
for (Variable v : ncfile.getVariables()) {
if (v.isUnlimited()) {
v.resetShape();
v.setCachedData(null, false);
}
}
// extend file, handle filling
if (fill)
fillRecordVariables(startRec, n);
else
raf.setMinLength(header.calcFileSize());
}
/**
* Update the value of an existing attribute. Attribute is found by name, which must match exactly.
* You cannot make an attribute longer, or change the number of values.
* For strings: truncate if longer, zero fill if shorter. Strings are padded to 4 byte boundaries, ok to use padding
* if it exists.
* For numerics: must have same number of values.
*
* @param v2 variable, or null for fglobal attribute
* @param att replace with this value
*/
@Override
public void updateAttribute(ucar.nc2.Variable v2, Attribute att) throws IOException {
header.updateAttribute(v2, att);
}
/////////////////////////////////////////////////////////////
// fill buffer with fill value
protected void fillNonRecordVariables() throws IOException {
// run through each variable
for (Variable v : ncfile.getVariables()) {
if (v.isUnlimited())
continue;
try {
writeData(v, v.getShapeAsSection(), makeConstantArray(v));
} catch (InvalidRangeException e) {
e.printStackTrace(); // shouldnt happen
}
}
}
protected void fillRecordVariables(int recStart, int recEnd) throws IOException, InvalidRangeException {
// do each record completely, should be a bit more efficient
for (int i = recStart; i < recEnd; i++) { // do one record at a time
Range r = new Range(i, i);
// run through each variable
for (Variable v : ncfile.getVariables()) {
if (!v.isUnlimited() || (v instanceof Structure))
continue;
Section.Builder recordSection = Section.builder().appendRanges(v.getRanges());
recordSection.setRange(0, r);
writeData(v, recordSection.build(), makeConstantArray(v));
}
}
}
private Array makeConstantArray(Variable v) {
Class classType = v.getDataType().getPrimitiveClassType();
// int [] shape = v.getShape();
Attribute att = v.findAttribute(CDM.FILL_VALUE);
Object storage = null;
if (classType == double.class) {
double[] storageP = new double[1];
storageP[0] = (att == null) ? NC_FILL_DOUBLE : att.getNumericValue().doubleValue();
storage = storageP;
} else if (classType == float.class) {
float[] storageP = new float[1];
storageP[0] = (att == null) ? NC_FILL_FLOAT : att.getNumericValue().floatValue();
storage = storageP;
} else if (classType == int.class) {
int[] storageP = new int[1];
storageP[0] = (att == null) ? NC_FILL_INT : att.getNumericValue().intValue();
storage = storageP;
} else if (classType == short.class) {
short[] storageP = new short[1];
storageP[0] = (att == null) ? NC_FILL_SHORT : att.getNumericValue().shortValue();
storage = storageP;
} else if (classType == byte.class) {
byte[] storageP = new byte[1];
storageP[0] = (att == null) ? NC_FILL_BYTE : att.getNumericValue().byteValue();
storage = storageP;
} else if (classType == char.class) {
char[] storageP = new char[1];
storageP[0] = (att != null) && (!att.getStringValue().isEmpty()) ? att.getStringValue().charAt(0) : NC_FILL_CHAR;
storage = storageP;
}
return Array.factoryConstant(v.getDataType(), v.getShape(), storage);
}
//////////////////////////////////////////////////////////////////////////////////////////////
@Override
public boolean syncExtend() throws IOException {
boolean result = header.synchNumrecs();
if (result && log.isDebugEnabled())
log.debug(" N3iosp syncExtend " + raf.getLocation() + " numrecs =" + header.numrecs);
return result;
}
/*
* @Override
* public boolean sync() throws IOException {
* if (syncExtendOnly)
* return syncExtend();
*
* if (lastModified == 0) // ?? HttpRandomAccessFile
* return false;
*
* File file = new File(raf.getLocation());
* if (file.exists()) {
* long currentModified = file.lastModified();
* if (currentModified == lastModified)
* return false;
*
* // so things have been modified, heres where we need to reread the header !!
* ncfile.empty();
* open(raf, ncfile, null);
* if (log.isDebugEnabled())
* log.debug(" N3iosp resynced " + raf.getLocation() + " currentModified=" + currentModified + " lastModified= " +
* lastModified);
* return true;
* }
*
* // can this happen ?
* throw new IOException("File does not exist");
* }
*/
@Override
public void flush() throws java.io.IOException {
if (raf != null) {
raf.flush();
header.writeNumrecs();
raf.flush();
}
}
@Override
public void close() throws java.io.IOException {
if (raf != null) {
long size = header.calcFileSize();
raf.setMinLength(size);
raf.close();
}
raf = null;
}
@Override
public void reacquire() throws IOException {
super.reacquire();
header.raf = this.raf;
}
/**
* Debug info for this object.
*/
@Override
public String toStringDebug(Object o) {
return null;
}
@Override
public Object sendIospMessage(Object message) {
if (null == header)
return null;
if (message == NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE)
return header.makeRecordStructure();
else if (message == NetcdfFile.IOSP_MESSAGE_REMOVE_RECORD_STRUCTURE)
return header.removeRecordStructure();
if (message.equals(IOSP_MESSAGE_GET_NETCDF_FILE_FORMAT)) {
return header.useLongOffset ? NetcdfFileFormat.NETCDF3_64BIT_OFFSET : NetcdfFileFormat.NETCDF3;
}
return super.sendIospMessage(message);
}
@Override
public String getFileTypeId() {
return DataFormatType.NETCDF.getDescription();
}
@Override
public String getFileTypeDescription() {
return "NetCDF-3/CDM";
}
@Override
public String getFileTypeVersion() {
return NC_FORMATX_NC3;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
// stuff we need the subclass to implement
/**
* Read data subset from file for a variable, create primitive array.
*
* @param index handles skipping around in the file.
* @param dataType dataType of the variable
* @return primitive array with data read in
* @throws java.io.IOException on error
*/
protected abstract Object readData(Layout index, DataType dataType) throws IOException;
protected abstract long readData(Layout index, DataType dataType, WritableByteChannel out) throws IOException;
/**
* Write data subset to file for a variable, create primitive array.
*
* @param aa write data from this Array.
* @param index handles skipping around in the file.
* @param dataType dataType of the variable
* @throws java.io.IOException on error
*/
protected abstract void writeData(Array aa, Layout index, DataType dataType) throws IOException;
protected abstract void _open(ucar.unidata.io.RandomAccessFile raf);
protected abstract void _create(ucar.unidata.io.RandomAccessFile raf);
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy