ucar.nc2.iosp.cinrad.Cinrad2VolumeScan Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
The newest version!
/*
*
* Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.iosp.cinrad;
import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.DiskCache;
import ucar.nc2.NetcdfFile;
import ucar.nc2.iosp.nexrad2.NexradStationDB;
import java.io.*;
import java.util.*;
import ucar.unidata.io.bzip2.CBZip2InputStream;
import ucar.unidata.io.bzip2.BZip2ReadException;
/**
* This class reads a CINRAD level II data file.
* It can handle NCDC archives (ARCHIVE2), as well as CRAFT/IDD compressed files (AR2V0001).
*
* Adapted with permission from the Java Iras software developed by David Priegnitz at NSSL.
*
* Documentation on Archive Level II data format can be found at:
*
* http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html
*
* @author caron
* @author David Priegnitz
*/
public class Cinrad2VolumeScan {
// data formats
static public final String ARCHIVE2 = "ARCHIVE2";
static public final String AR2V0001 = "AR2V0001";
static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Cinrad2VolumeScan.class);
////////////////////////////////////////////////////////////////////////////////////
// Data file
RandomAccessFile raf;
private String dataFormat = null; // ARCHIVE2 or AR2V0001
private String volumeNo = null; // volume number (1 to 999)
private int title_julianDay; // days since 1/1/70
private int title_msecs; // milliseconds since midnight
private String stationId; // 4 letter station assigned by ICAO
private NexradStationDB.Station station; // from lookup table, may be null
private Cinrad2Record first, last;
private int vcp = 0; // Volume coverage pattern
private int max_radials = 0;
private int min_radials = Integer.MAX_VALUE;
private int dopplarResolution;
private boolean hasDifferentDopplarResolutions;
// List of List of Cinrad2Record
private ArrayList reflectivityGroups, dopplerGroups;
private boolean showMessages = false, showData = false, debugScans = false, debugGroups2 = false, debugRadials = false;
Cinrad2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
boolean debug = log.isDebugEnabled();
if (debug)
log.debug("Cinrad2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.LITTLE_ENDIAN); //.BIG_ENDIAN);
// try to get it from the filename LOOK
String loc = raf.getLocation();
stationId = getStationID(loc);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
//stationId = raf.readString(4).trim(); // only in AR2V0001
if (debug) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
station = NexradStationDB.get("K"+ stationId);
dataFormat = "CINRAD-SA";
}
// if(station == null) {
// station = new NexradStationDB.Station();
// stationId = "CHGZ";
// station.id = "CHGZ";
// station.name = "CHINA, GuanZhou";
// station.lat = parseDegree("23:0:14");
// station.lon = parseDegree("113:21:18");
// station.elev = Double.parseDouble("180.3");
// dataFormat = "CINRAD-SA";
// }
//see if we have to uncompress
if (dataFormat.equals(AR2V0001)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf = null;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists()) {
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath(), debug);
try {
uraf.flush();
} catch (IOException e) {
uraf.close();
throw e;
}
if (debug) log.debug("flushed uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Cinrad2Record.FILE_HEADER_SIZE);
}
ArrayList reflectivity = new ArrayList();
ArrayList doppler = new ArrayList();
int recno = 0;
while (true) {
Cinrad2Record r = Cinrad2Record.factory(raf, recno++);
if (r == null) break;
// skip non-data messages
if (r.message_type != 1) {
if (showMessages) r.dumpMessage(System.out, null);
continue;
}
if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (!r.checkOk()) {
continue;
}
if (r.hasReflectData)
reflectivity.add(r);
if (r.hasDopplerData)
doppler.add(r);
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials) System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
reflectivityGroups = sortScans("reflect", reflectivity);
dopplerGroups = sortScans("doppler", doppler);
}
public String getStationID( String location) {
String stationID;
// posFirst: last '/' if it exists
int posFirst = location.lastIndexOf('/') + 1;
if (posFirst < 0) posFirst = 0;
stationID = location.substring(posFirst,posFirst+4);
return stationID;
}
private static double parseDegree( String s) {
StringTokenizer stoke = new StringTokenizer(s, ":");
String degS = stoke.nextToken();
String minS = stoke.nextToken();
String secS = stoke.nextToken();
try {
double deg = Double.parseDouble( degS);
double min = Double.parseDouble( minS);
double sec = Double.parseDouble( secS);
if (deg < 0)
return deg - min/60 - sec/3600;
else
return deg + min/60 + sec/3600;
} catch (NumberFormatException e) {
e.printStackTrace();
}
return 0.0;
}
private ArrayList sortScans(String name, List scans) {
// now group by elevation_num
HashMap groupHash = new HashMap(600);
for (int i = 0; i < scans.size(); i++) {
Cinrad2Record record = (Cinrad2Record) scans.get(i);
Integer groupNo = new Integer(record.elevation_num);
ArrayList group = (ArrayList) groupHash.get(groupNo);
if (null == group) {
group = new ArrayList();
groupHash.put(groupNo, group);
}
group.add(record);
}
// sort the groups by elevation_num
ArrayList groups = new ArrayList(groupHash.values());
Collections.sort(groups, new GroupComparator());
// use the maximum radials
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
testScan(name, group);
max_radials = Math.max(max_radials, group.size());
min_radials = Math.min(min_radials, group.size());
}
if (debugRadials) {
System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
Cinrad2Record lastr = (Cinrad2Record) group.get(0);
for (int j = 1; j < group.size(); j++) {
Cinrad2Record r = (Cinrad2Record) group.get(j);
if (r.data_msecs < lastr.data_msecs)
System.out.println(" out of order " + j);
lastr = r;
}
}
}
testVariable(name, groups);
if (debugScans) System.out.println("-----------------------------");
return groups;
}
public int getMaxRadials() {
return max_radials;
}
public int getMinRadials() {
return min_radials;
}
public int getDopplarResolution() {
return dopplarResolution;
}
public boolean hasDifferentDopplarResolutions() {
return hasDifferentDopplarResolutions;
}
// do we have same characteristics for all records in a scan?
private int MAX_RADIAL = 401;
private int[] radial = new int[MAX_RADIAL];
private boolean testScan(String name, ArrayList group) {
int datatype = name.equals("reflect") ? Cinrad2Record.REFLECTIVITY : Cinrad2Record.VELOCITY_HI;
Cinrad2Record first = (Cinrad2Record) group.get(0);
int n = group.size();
if (debugScans) {
boolean hasBoth = first.hasDopplerData && first.hasReflectData;
System.out.println(name + " " + first + " has " + n + " radials resolution= " + first.resolution + " has both = " + hasBoth);
}
boolean ok = true;
double sum = 0.0;
double sum2 = 0.0;
for (int i = 0; i < MAX_RADIAL; i++)
radial[i] = 0;
for (int i = 0; i < group.size(); i++) {
Cinrad2Record r = (Cinrad2Record) group.get(i);
/* this appears to be common - seems to be ok, we put missing values in
if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
"!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
ok = false;
} */
if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
log.warn(raf.getLocation() + " different gate size (" + r.getGateSize(datatype) + ") in record " + name + " " + r);
ok = false;
}
if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
log.warn(raf.getLocation() + " different gate start (" + r.getGateStart(datatype) + ") in record " + name + " " + r);
ok = false;
}
if (r.resolution != first.resolution) {
log.warn(raf.getLocation() + " different resolution (" + r.resolution + ") in record " + name + " " + r);
ok = false;
}
if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
log.info(raf.getLocation() + " radial out of range= " + r.radial_num + " in record " + name + " " + r);
continue;
}
if (radial[r.radial_num] > 0) {
log.warn(raf.getLocation() + " duplicate radial = " + r.radial_num + " in record " + name + " " + r);
ok = false;
}
radial[r.radial_num] = r.recno + 1;
sum += r.getElevation();
sum2 += r.getElevation() * r.getElevation();
// System.out.println(" elev="+r.getElevation()+" azi="+r.getAzimuth());
}
for (int i = 1; i < radial.length; i++) {
if (0 == radial[i]) {
if (n != (i - 1)) {
log.warn(" missing radial(s)");
ok = false;
}
break;
}
}
// double avg = sum / n;
// double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1)));
// System.out.println(" avg elev="+avg+" std.dev="+sd);
return ok;
}
// do we have same characteristics for all groups in a variable?
private boolean testVariable(String name, List scans) {
int datatype = name.equals("reflect") ? Cinrad2Record.REFLECTIVITY : Cinrad2Record.VELOCITY_HI;
if (scans.size() == 0) {
log.warn(" No data for = " + name);
return false;
}
boolean ok = true;
List firstScan = (List) scans.get(0);
Cinrad2Record firstRecord = (Cinrad2Record) firstScan.get(0);
dopplarResolution = firstRecord.resolution;
if (debugGroups2)
System.out.println("Group " + Cinrad2Record.getDatatypeName(datatype) + " ngates = " + firstRecord.getGateCount(datatype) +
" start = " + firstRecord.getGateStart(datatype) + " size = " + firstRecord.getGateSize(datatype));
for (int i = 1; i < scans.size(); i++) {
List scan = (List) scans.get(i);
Cinrad2Record record = (Cinrad2Record) scan.get(0);
if ((datatype == Cinrad2Record.VELOCITY_HI) && (record.resolution != firstRecord.resolution))
{ // do all velocity resolutions match ??
log.warn(name + " scan " + i + " diff resolutions = " + record.resolution + ", " + firstRecord.resolution +
" elev= " + record.elevation_num + " " + record.getElevation());
ok = false;
hasDifferentDopplarResolutions = true;
}
if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
log.warn(name + " scan " + i + " diff gates size = " + record.getGateSize(datatype) + " " + firstRecord.getGateSize(datatype) +
" elev= " + record.elevation_num + " " + record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(" ok gates size elev= " + record.elevation_num + " " + record.getElevation());
if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
log.warn(name + " scan " + i + " diff gates start = " + record.getGateStart(datatype) + " " + firstRecord.getGateStart(datatype) +
" elev= " + record.elevation_num + " " + record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(" ok gates start elev= " + record.elevation_num + " " + record.getElevation());
}
return ok;
}
/**
* Get Reflectivity Groups
* Groups are all the records for a variable and elevation_num;
*
* @return List of type List of type Cinrad2Record
*/
public List getReflectivityGroups() {
return reflectivityGroups;
}
/**
* Get Velocity Groups
* Groups are all the records for a variable and elevation_num;
*
* @return List of type List of type Cinrad2Record
*/
public List getVelocityGroups() {
return dopplerGroups;
}
private static class GroupComparator implements Comparator {
public int compare(Object o1, Object o2) {
List group1 = (List) o1;
List group2 = (List) o2;
Cinrad2Record record1 = (Cinrad2Record) group1.get(0);
Cinrad2Record record2 = (Cinrad2Record) group2.get(0);
//if (record1.elevation_num != record2.elevation_num)
return record1.elevation_num - record2.elevation_num;
//return record1.cut - record2.cut;
}
}
/**
* Get data format (ARCHIVE2, AR2V0001) for this file.
*/
public String getDataFormat() {
return dataFormat;
}
/**
* Get the starting Julian day for this volume
*
* @return days since 1/1/70.
*/
public int getTitleJulianDays() {
return title_julianDay;
}
/**
* Get the starting time in seconds since midnight.
*
* @return Generation time of data in milliseconds of day past midnight (UTC).
*/
public int getTitleMsecs() {
return title_msecs;
}
/**
* Get the Volume Coverage Pattern number for this data.
*
* @return VCP
* @see Cinrad2Record#getVolumeCoveragePatternName
*/
public int getVCP() {
return vcp;
}
/**
* Get the 4-char station ID for this data
*
* @return station ID (may be null)
*/
public String getStationId() {
return stationId;
}
public String getStationName() {
return station == null ? "unknown" : station.name;
}
public double getStationLatitude() {
return station == null ? 0.0 : station.lat;
}
public double getStationLongitude() {
return station == null ? 0.0 : station.lon;
}
public double getStationElevation() {
return station == null ? 0.0 : station.elev;
}
public Date getStartDate() {
return first.getDate();
}
public Date getEndDate() {
return last.getDate();
}
/**
* Write equivilent uncompressed version of the file.
*
* @param raf2 file to uncompress
* @param ufilename write to this file
* @return raf of uncompressed file
* @throws IOException
*/
private RandomAccessFile uncompress(RandomAccessFile raf2, String ufilename, boolean debug) throws IOException {
raf2.seek(0);
byte[] header = new byte[Cinrad2Record.FILE_HEADER_SIZE];
int bytesRead = raf2.read(header);
if (bytesRead != header.length)
{
throw new IOException("Error reading CINRAD header -- got " +
bytesRead + " rather than" + header.length);
}
RandomAccessFile dout2 = new RandomAccessFile(ufilename, "rw");
boolean eof = false;
int numCompBytes;
byte[] ubuff = new byte[40000];
byte[] obuff = new byte[40000];
try {
dout2.write(header);
CBZip2InputStream cbzip2 = new CBZip2InputStream();
while (!eof) {
try {
numCompBytes = raf2.readInt();
if (numCompBytes == -1) {
if (debug) log.debug(" done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
if (debug) log.debug(" got EOFException ");
break; // assume this is ok
}
if (debug) {
log.debug("reading compressed bytes " + numCompBytes + " input starts at " + raf2.getFilePointer() + "; output starts at " + dout2.getFilePointer());
}
/*
* For some stupid reason, the last block seems to
* have the number of bytes negated. So, we just
* assume that any negative number (other than -1)
* is the last block and go on our merry little way.
*/
if (numCompBytes < 0) {
if (debug) log.debug("last block?" + numCompBytes);
numCompBytes = -numCompBytes;
eof = true;
}
byte[] buf = new byte[numCompBytes];
raf2.readFully(buf);
ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);
//CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
cbzip2.setStream(bis);
int total = 0;
int nread;
/*
while ((nread = cbzip2.read(ubuff)) != -1) {
dout2.write(ubuff, 0, nread);
total += nread;
}
*/
try {
while ((nread = cbzip2.read(ubuff)) != -1) {
if (total + nread > obuff.length) {
byte[] temp = obuff;
obuff = new byte[temp.length * 2];
System.arraycopy(temp, 0, obuff, 0, temp.length);
}
System.arraycopy(ubuff, 0, obuff, total, nread);
total += nread;
}
if (obuff.length >= 0) dout2.write(obuff, 0, total);
} catch (BZip2ReadException ioe) {
log.debug("Cinrad2IOSP.uncompress ", ioe);
}
float nrecords = (float) (total / 2432.0);
if (debug)
log.debug(" unpacked " + total + " num bytes " + nrecords + " records; ouput ends at " + dout2.getFilePointer());
}
dout2.flush();
} catch (EOFException e) {
e.printStackTrace();
} catch (Exception e) {
dout2.close();
throw e;
}
return dout2;
}
// check if compressed file seems ok
static public long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
try (RandomAccessFile raf = new RandomAccessFile(ufilename, "r")) {
raf.order(RandomAccessFile.LITTLE_ENDIAN); //.BIG_ENDIAN);
raf.seek(0);
String test = raf.readString(8);
if (test.equals(Cinrad2VolumeScan.ARCHIVE2) || test.equals
(Cinrad2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
while (!eof) {
if (lookForHeader) {
test = raf.readString(8);
if (test.equals(Cinrad2VolumeScan.ARCHIVE2) || test.equals(Cinrad2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
return raf.getFilePointer();
} catch (EOFException e) {
e.printStackTrace();
}
return 0;
}
/**
* test
*/
public static void main2(String[] args) throws IOException {
File testDir = new File("/share/testdata/radar/problem");
File[] files = testDir.listFiles();
for (int i = 0; i < files.length; i++) {
File file = files[i];
if (!file.getPath().endsWith(".ar2v")) continue;
System.out.println(file.getPath() + " " + file.length());
long pos = testValid(file.getPath());
if (pos == file.length()) {
System.out.println("OK");
try {
NetcdfFile.open(file.getPath());
} catch (Throwable t) {
System.out.println("ERROR= " + t);
}
} else
System.out.println("NOT pos=" + pos);
System.out.println();
}
}
public static void main(String args[]) throws IOException {
NexradStationDB.init();
RandomAccessFile raf = new RandomAccessFile("R:/testdata/radar/nexrad/Cinrad2/problem/KCCX_20060627_1701", "r");
new Cinrad2VolumeScan(raf, null);
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy