All Downloads are FREE. Search and download functionalities are using the official Maven repository.

ucar.nc2.iosp.grads.GradsBinaryGridServiceProvider Maven / Gradle / Ivy

Go to download

The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.

The newest version!
/*
 * Copyright 1998-2011 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package ucar.nc2.iosp.grads;

import ucar.ma2.Array;
import ucar.ma2.ArrayDouble;
import ucar.ma2.DataType;
import ucar.ma2.IndexIterator;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;
import ucar.ma2.Section;

import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.constants.AxisType;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants.CF;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.util.CancelTask;

import ucar.unidata.io.RandomAccessFile;

import java.io.IOException;

import java.util.HashMap;
import java.util.List;


/**
 * IOSP for GrADS Binary data files.  This IOSP only handles the binary formatted grids,
 * most other GrADS data types can be read directly through other IOSPs.
 *
 * Notes jcaron
 *  Apparently we need the control file (.ctl), which then references the data file (.dat)
 *  Dont see any test data - added to cdmUnitTest/formats/grads
 *
 * @author Don Murray - CU/CIRES
 * @see "http://www.iges.org/grads/gadoc/descriptorfile.html"
 * @see "http://www.iges.org/grads/gadoc/aboutgriddeddata.html"
 */
public class GradsBinaryGridServiceProvider extends AbstractIOServiceProvider {

  /**
   * GrADS file reader
   */
  protected GradsDataDescriptorFile gradsDDF;

  /**
   * GrADS binary file
   */
  private RandomAccessFile dataFile;

  /**
   * the netCDF file
   */
  private NetcdfFile ncFile;

  /**
   * the size of the x dimension
   */
  private int sizeX = 0;

  /**
   * the size of the y dimension
   */
  private int sizeY = 0;

  /**
   * the number of xy header bytes
   */
  private int xyHeaderBytes = 0;

  /**
   * the sequential record bytes
   */
  private int sequentialRecordBytes = 0;

  /**
   * the number of file header bytes
   */
  private int fileHeaderBytes = 0;

  /**
   * the number of time header bytes
   */
  private int timeHeaderBytes = 0;

  /**
   * the number of time trailer bytes
   */
  private int timeTrailerBytes = 0;

  /**
   * The name for the ensemble varaible
   */
  private static final String ENS_VAR = "ensemble";

  /**
   * The name for the time variable
   */
  private static final String TIME_VAR = "time";

  /**
   * The name for the z dimension variable
   */
  private static final String Z_VAR = "level";

  /**
   * The name for the y dimension variable
   */
  private static final String Y_VAR = "latitude";

  /**
   * The name for the x dimension variable
   */
  private static final String X_VAR = "longitude";

  /**
   * The dimension names
   */
  private String[] dimNames = {GradsDataDescriptorFile.EDEF,
          GradsDataDescriptorFile.TDEF,
          GradsDataDescriptorFile.ZDEF,
          GradsDataDescriptorFile.YDEF,
          GradsDataDescriptorFile.XDEF};

  /**
   * The corresponding variable names
   */
  private String[] dimVarNames = {ENS_VAR, TIME_VAR, Z_VAR, Y_VAR, X_VAR,};

  /**
   * The word size in bytes
   */
  private int wordSize = 4;

  /**
   * Is this a valid file?  For this GrADS IOSP, the valid file must be:
   * 
    *
  • raw binary grid (not GRIB, netCDF, HDF, etc) *
  • not a cross section (x and y > 1) *
  • not an ensemble definded by EDEF/ENDEDEF (need examples) *
* * @param raf RandomAccessFile to check * @return true if a valid GrADS grid file of the type listed above * @throws IOException problem reading file */ public boolean isValidFile(RandomAccessFile raf) throws IOException { // need a fast failure for non GRADS files if (GradsDataDescriptorFile.failFast(raf)) return false; // we think its a GRADS file, but we have lots of restrictions on what we can handle try { gradsDDF = new GradsDataDescriptorFile(raf.getLocation()); GradsDimension x = gradsDDF.getXDimension(); GradsDimension y = gradsDDF.getYDimension(); //J- return gradsDDF.getDataType() == null && // only handle raw binary gradsDDF.getDataFile() != null && !gradsDDF.hasProjection() && // can't handle projections !gradsDDF.getVariables().isEmpty() && // must have valid entries !gradsDDF.getDimensions().isEmpty() && (x.getSize() > 1) && (y.getSize() > 1); // can't handle cross sections //J+ } catch (Exception ioe) { return false; } } /** * Get the file type id * * @return the file type id */ public String getFileTypeId() { return "GradsBinaryGrid"; } /** * Get the file type description * * @return the file type description */ public String getFileTypeDescription() { return "GrADS Binary Gridded Data"; } /** * Open the service provider for reading. * * @param raf file to read from * @param ncfile netCDF file we are writing to (memory) * @param cancelTask task for cancelling * @throws IOException problem reading file */ public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); this.ncFile = ncfile; // debugProj = true; if (gradsDDF == null) { gradsDDF = new GradsDataDescriptorFile(raf.getLocation()); } xyHeaderBytes = gradsDDF.getXYHeaderBytes(); fileHeaderBytes = gradsDDF.getFileHeaderBytes(); timeHeaderBytes = gradsDDF.getTimeHeaderBytes(); timeTrailerBytes = gradsDDF.getTimeTrailerBytes(); // get the first file so we can calculate the sequentialRecordBytes dataFile = getDataFile(0, 0); dataFile.order(getByteOrder()); // assume all files are the same as the first if (gradsDDF.isSequential()) { GradsDimension ensDim = gradsDDF.getEnsembleDimension(); int numens = ((ensDim != null) && !gradsDDF.isTemplate()) ? ensDim.getSize() : 1; GradsTimeDimension timeDim = gradsDDF.getTimeDimension(); int numtimes = 0; if (gradsDDF.isTemplate()) { int[] timesPerFile = gradsDDF.getTimeStepsPerFile(dataFile.getLocation()); numtimes = timesPerFile[0]; } else { numtimes = timeDim.getSize(); } int gridsPerTimeStep = gradsDDF.getGridsPerTimeStep(); int numrecords = numens * numtimes * gridsPerTimeStep; int xlen = gradsDDF.getXDimension().getSize(); int ylen = gradsDDF.getYDimension().getSize(); long fileSize = dataFile.length(); // calculate record indicator length long dataSize = fileHeaderBytes + (xlen * ylen * 4l + xyHeaderBytes) * numrecords; // add on the bytes for the time header/trailers dataSize += numtimes * (timeHeaderBytes + timeTrailerBytes); int leftovers = (int) (fileSize - dataSize); sequentialRecordBytes = (leftovers / numrecords) / 2; if (sequentialRecordBytes < 0) { throw new IOException("Incorrect sequential record byte size: " + sequentialRecordBytes); } } buildNCFile(); } /** * Get the byte order from the data descriptor file * * @return the byte order */ private int getByteOrder() { return (gradsDDF.isBigEndian()) ? RandomAccessFile.BIG_ENDIAN : RandomAccessFile.LITTLE_ENDIAN; } /** * Build the netCDF file * * @throws IOException problem reading the file */ protected void buildNCFile() throws IOException { ncFile.empty(); fillNCFile(); ncFile.finish(); //System.out.println(ncfile); } /** * Get the variable name for the given dimension * * @param dim the dimension * @return the variable name */ private String getVarName(GradsDimension dim) { for (int i = 0; i < dimNames.length; i++) { if (dim.getName().equalsIgnoreCase(dimNames[i])) { return dimVarNames[i]; } } return dim.getName(); } /** * Fill out the netCDF file * * @throws IOException problem reading or writing stuff */ private void fillNCFile() throws IOException { List vars = gradsDDF.getVariables(); List attrs = gradsDDF.getAttributes(); //TODO: ensembles List dims = gradsDDF.getDimensions(); Variable v; int numZ = 0; HashMap zDims = new HashMap(); for (GradsDimension dim : dims) { String name = getVarName(dim); int size = dim.getSize(); Dimension ncDim = new Dimension(name, size, true); ncFile.addDimension(null, ncDim); if (name.equals(ENS_VAR)) { v = new Variable(ncFile, null, null, name, DataType.STRING, name); v.addAttribute(new Attribute("standard_name", "ensemble")); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Ensemble.toString())); List names = gradsDDF.getEnsembleDimension().getEnsembleNames(); String[] nameArray = new String[names.size()]; for (int i = 0; i < nameArray.length; i++) { nameArray[i] = names.get(i); } Array dataArray = Array.factory(DataType.STRING, new int[]{nameArray.length}, nameArray); v.setCachedData(dataArray, false); } else { double[] vals = dim.getValues(); v = new Variable(ncFile, null, null, name, DataType.DOUBLE, name); v.addAttribute(new Attribute(CDM.UNITS, dim.getUnit())); if (name.equals(Y_VAR)) { v.addAttribute(new Attribute(CDM.LONG_NAME, "latitude")); v.addAttribute(new Attribute("standard_name", "latitude")); v.addAttribute(new Attribute("axis", "Y")); sizeY = dim.getSize(); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString())); } else if (name.equals(X_VAR)) { v.addAttribute(new Attribute(CDM.LONG_NAME, "longitude")); v.addAttribute(new Attribute("standard_name", "longitude")); v.addAttribute(new Attribute("axis", "X")); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString())); sizeX = dim.getSize(); } else if (name.equals(Z_VAR)) { numZ = size; zDims.put(name, ncDim); v.addAttribute(new Attribute(CDM.LONG_NAME, "level")); addZAttributes(dim, v); } else if (name.equals(TIME_VAR)) { v.addAttribute(new Attribute(CDM.LONG_NAME, "time")); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString())); } ArrayDouble.D1 varArray = new ArrayDouble.D1(size); for (int i = 0; i < vals.length; i++) { varArray.set(i, vals[i]); } v.setCachedData(varArray, false); } ncFile.addVariable(null, v); } if (numZ > 0) { GradsDimension zDim = gradsDDF.getZDimension(); double[] vals = zDim.getValues(); for (GradsVariable var : vars) { int nl = var.getNumLevels(); if ((nl > 0) && (nl != numZ)) { String name = Z_VAR + nl; if (zDims.get(name) == null) { Dimension ncDim = new Dimension(name, nl, true); ncFile.addDimension(null, ncDim); Variable vz = new Variable(ncFile, null, null, name, DataType.DOUBLE, name); vz.addAttribute(new Attribute(CDM.LONG_NAME, name)); vz.addAttribute(new Attribute(CDM.UNITS, zDim.getUnit())); addZAttributes(zDim, vz); ArrayDouble.D1 varArray = new ArrayDouble.D1(nl); for (int i = 0; i < nl; i++) { varArray.set(i, vals[i]); } vz.setCachedData(varArray, false); ncFile.addVariable(null, vz); zDims.put(name, ncDim); } } } } zDims = null; for (GradsVariable var : vars) { String coords = "latitude longitude"; int nl = var.getNumLevels(); if (nl > 0) { if (nl == numZ) { coords = "level " + coords; } else { coords = Z_VAR + nl + " " + coords; } } coords = "time " + coords; if (gradsDDF.getEnsembleDimension() != null) { coords = "ensemble " + coords; } v = new Variable(ncFile, null, null, var.getName(), DataType.FLOAT, coords); v.addAttribute(new Attribute(CDM.LONG_NAME, var.getDescription())); if (var.getUnitName() != null) { v.addAttribute(new Attribute(CDM.UNITS, var.getUnitName())); } v.addAttribute(new Attribute(CDM.FILL_VALUE, new Float(gradsDDF.getMissingValue()))); v.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(gradsDDF.getMissingValue()))); for (GradsAttribute attr : attrs) { if (attr.getVariable().equalsIgnoreCase(var.getName())) { // TODO: what to do about a UINT16/32 if (attr.getType().equalsIgnoreCase( GradsAttribute.STRING)) { v.addAttribute(new Attribute(attr.getName(), attr.getValue())); } else if (attr.getType().equalsIgnoreCase( GradsAttribute.BYTE)) { try { v.addAttribute(new Attribute(attr.getName(), new Byte(attr.getValue()))); } catch (NumberFormatException nfe) { } } else if (attr.getType().equalsIgnoreCase( GradsAttribute.INT16)) { try { v.addAttribute(new Attribute(attr.getName(), new Short(attr.getValue()))); } catch (NumberFormatException nfe) { } } else if (attr.getType().equalsIgnoreCase( GradsAttribute.INT32)) { try { v.addAttribute(new Attribute(attr.getName(), new Integer(attr.getValue()))); } catch (NumberFormatException nfe) { } } else if (attr.getType().equalsIgnoreCase( GradsAttribute.FLOAT32)) { try { v.addAttribute(new Attribute(attr.getName(), new Float(attr.getValue()))); } catch (NumberFormatException nfe) { } } else if (attr.getType().equalsIgnoreCase( GradsAttribute.FLOAT64)) { try { v.addAttribute(new Attribute(attr.getName(), new Double(attr.getValue()))); } catch (NumberFormatException nfe) { } } } } ncFile.addVariable(null, v); } // Global Attributes ncFile.addAttribute(null, new Attribute(CDM.CONVENTIONS, "CF-1.0")); ncFile.addAttribute( null, new Attribute( CDM.HISTORY, "Direct read of GrADS binary grid into NetCDF-Java 4 API")); String title = gradsDDF.getTitle(); if ((title != null) && !title.isEmpty()) { ncFile.addAttribute(null, new Attribute("title", title)); } for (GradsAttribute attr : attrs) { if (attr.getVariable().equalsIgnoreCase(GradsAttribute.GLOBAL)) { ncFile.addAttribute(null, new Attribute(attr.getName(), attr.getValue())); } } } /** * Add the appropriate attributes for a Z dimension * * @param zDim The GrADS Z dimension * @param v the variable to augment */ private void addZAttributes(GradsDimension zDim, Variable v) { if (zDim.getUnit().indexOf("Pa") >= 0) { v.addAttribute(new Attribute(CF.POSITIVE, CF.POSITIVE_DOWN)); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Pressure.toString())); } else { v.addAttribute(new Attribute(CF.POSITIVE, CF.POSITIVE_UP)); v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Height.toString())); } } /** * Read the grid * * @param index the index of the grid * @return the grid data * @throws IOException problem reading stuff */ private float[] readGrid(int index) throws IOException { //System.out.println("grid number: " + index); // NB: RandomAccessFile.skipBytes only takes an int. For files larger than // 2GB, that is problematic, so we use offset as a long long offset = 0; dataFile.seek(offset); // skip over the file header //dataFile.skipBytes(fileHeaderBytes); offset += fileHeaderBytes; // The full record structure of a Fortran sequential binary file is: // [Length] [Record 1 data] [Length] // [Length] [Record 2 data] [Length] // [Length] [Record 3 data] [Length] // ... // [End of file] // so we have to add 2*sequentialRecordBytes for each record we skip, offset += (sizeX * sizeY * wordSize + xyHeaderBytes + 2l * sequentialRecordBytes) * (long) index; //System.out.println("offset to grid = " + offset); // TODO: make sure this works - need an example int curTimeStep = index / gradsDDF.getGridsPerTimeStep(); // add time headers offset += (curTimeStep + 1) * timeHeaderBytes; // add time trailers offset += curTimeStep * timeTrailerBytes; // and then 1 sequentialRecordBytes for the record itself (+ xyHeader) offset += (xyHeaderBytes + sequentialRecordBytes); //dataFile.skipBytes(offset); dataFile.seek(offset); float[] data = new float[sizeX * sizeY]; dataFile.readFloat(data, 0, sizeX * sizeY); if (gradsDDF.isYReversed()) { int newLoc = 0; float[] temp = new float[sizeX * sizeY]; for (int y = sizeY - 1; y >= 0; y--) { for (int x = 0; x < sizeX; x++) { int oldLoc = y * sizeX + x; temp[newLoc++] = data[oldLoc]; } } data = temp; } return data; } /** * Close this IOSP and associated files * * @throws IOException problem closing files */ public void close() throws IOException { if (dataFile != null) { dataFile.close(); } dataFile = null; super.close(); } /** * Find the GradsVariable associated with the netCDF variable * * @param v2 the netCDF variable * @return the corresponding GradsVariable */ private GradsVariable findVar(Variable v2) { List vars = gradsDDF.getVariables(); String varName = v2.getFullName(); for (GradsVariable var : vars) { if (var.getName().equals(varName)) { return var; } } return null; // can't happen? } /** * Read the data for the variable * * @param v2 Variable to read * @param section section infomation * @return Array of data * @throws IOException problem reading from file * @throws InvalidRangeException invalid Range */ public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException { Array dataArray = Array.factory(DataType.FLOAT, section.getShape()); GradsVariable gradsVar = findVar(v2); if (gradsVar == null) throw new IOException(); // Canonical ordering is ens, time, level, lat, lon int rangeIdx = 0; Range ensRange = (gradsDDF.getEnsembleDimension() != null) ? section.getRange(rangeIdx++) : new Range(0, 0); Range timeRange = (section.getRank() > 2) ? section.getRange(rangeIdx++) : new Range(0, 0); Range levRange = (gradsVar.getNumLevels() > 0) ? section.getRange(rangeIdx++) : new Range(0, 0); Range yRange = section.getRange(rangeIdx++); Range xRange = section.getRange(rangeIdx); IndexIterator ii = dataArray.getIndexIterator(); // loop over ens for (int ensIdx = ensRange.first(); ensIdx <= ensRange.last(); ensIdx += ensRange.stride()) { //loop over time for (int timeIdx = timeRange.first(); timeIdx <= timeRange.last(); timeIdx += timeRange.stride()) { //loop over level for (int levelIdx = levRange.first(); levelIdx <= levRange.last(); levelIdx += levRange.stride()) { readXY(v2, ensIdx, timeIdx, levelIdx, yRange, xRange, ii); } } } return dataArray; } /** * read one YX array * * @param v2 variable to put the data into * @param ensIdx ensemble index * @param timeIdx time index * @param levIdx level index * @param yRange x range * @param xRange y range * @param ii index iterator * @throws IOException problem reading the file * @throws InvalidRangeException invalid range */ private void readXY(Variable v2, int ensIdx, int timeIdx, int levIdx, Range yRange, Range xRange, IndexIterator ii) throws IOException, InvalidRangeException { //System.out.println("ens: " + ensIdx + " , time = " + timeIdx // + ", lev = " + levIdx); dataFile = getDataFile(ensIdx, timeIdx); List vars = gradsDDF.getVariables(); // if it's an ensemble template, then all data is in this file int numEns = ((gradsDDF.getTemplateType() == GradsDataDescriptorFile.ENS_TEMPLATE) || (gradsDDF .getTemplateType() == GradsDataDescriptorFile .ENS_TIME_TEMPLATE)) ? 0 : ensIdx; // if it's a time template figure out how many previous times we should use int numTimes = gradsDDF.getTimeDimension().getSize(); if ((gradsDDF.getTemplateType() == GradsDataDescriptorFile .TIME_TEMPLATE) || (gradsDDF .getTemplateType() == GradsDataDescriptorFile .ENS_TIME_TEMPLATE)) { int[] tpf = gradsDDF.getTimeStepsPerFile(dataFile.getLocation()); numTimes = tpf[0]; timeIdx = (timeIdx - tpf[1]) % numTimes; } int gridNum = numEns * numTimes * gradsDDF.getGridsPerTimeStep(); // loop up to the last time in the last ensemble for (int t = 0; t < timeIdx; t++) { for (GradsVariable var : vars) { int numVLevels = var.getNumLevels(); if (numVLevels == 0) { numVLevels = 1; } for (int l = 0; l < numVLevels; l++) { gridNum++; } } } // loop up to the last level in the last time in the last ensemble for (GradsVariable var : vars) { int numVLevels = var.getNumLevels(); if (numVLevels == 0) { numVLevels = 1; } if (var.getName().equals(v2.getFullName())) { gridNum += levIdx; break; } for (int l = 0; l < numVLevels; l++) { gridNum++; } } // TODO: Flip grid if Y is reversed float[] data = readGrid(gridNum); // LOOK can improve with System.copy ?? for (int y = yRange.first(); y <= yRange.last(); y += yRange.stride()) { for (int x = xRange.first(); x <= xRange.last(); x += xRange.stride()) { int index = y * sizeX + x; ii.setFloatNext(data[index]); } } } /** * Get the data file to use (if this is a template) * * @param eIndex ensemble index * @param tIndex time index * @return the current file * @throws IOException couldn't open the current file */ private RandomAccessFile getDataFile(int eIndex, int tIndex) throws IOException { String dataFilePath = gradsDDF.getFileName(eIndex, tIndex); if (!gradsDDF.isTemplate()) { // we only have one file if (dataFile != null) { return dataFile; } } if (dataFile != null) { String path = dataFile.getLocation(); if (path.equals(dataFilePath)) { return dataFile; } else { dataFile.close(); } } dataFile = new RandomAccessFile(dataFilePath, "r"); dataFile.order(getByteOrder()); return dataFile; } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy