ucar.nc2.stream.NcStreamIosp Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
The newest version!
/*
* Copyright 2009-2012 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package ucar.nc2.stream;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.util.CancelTask;
import ucar.nc2.util.IO;
import ucar.unidata.io.RandomAccessFile;
import ucar.ma2.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.List;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
/**
* IOSP to read ncStream file (RandomAccessFile), into a NetcdfFile.
*/
public class NcStreamIosp extends AbstractIOServiceProvider {
private static final boolean debug = false;
public boolean isValidFile(RandomAccessFile raf) throws IOException {
raf.seek(0);
if (!readAndTest(raf, NcStream.MAGIC_START)) return false; // must start with these 4 bytes
byte[] b = new byte[4];
raf.read(b);
return test(b, NcStream.MAGIC_HEADER) || test(b, NcStream.MAGIC_DATA); // immed followed by one of these
}
public String getFileTypeId() {
return "ncstream";
}
public String getFileTypeDescription() {
return "netCDF streaming protocol";
}
public int getVersion() {
return version;
}
//////////////////////////////////////////////////////////////////////
private int version;
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
openDebug(raf, ncfile, null);
}
/* public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask2) throws IOException {
try {
this.raf = raf;
raf.seek(0);
if (!readAndTest(raf, NcStream.MAGIC_START))
throw new IOException("Data corrupted on " + ncfile.getLocation());
// assume for the moment its always starts with one header message
if (!readAndTest(raf, NcStream.MAGIC_HEADER))
throw new IOException("Data corrupted on " + ncfile.getLocation());
int msize = readVInt(raf);
if (debug) System.out.printf("READ header len= %d%n", msize);
byte[] m = new byte[msize];
raf.read(m);
NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);
version = proto.getVersion();
NcStreamProto.Group root = proto.getRoot();
NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
ncfile.finish();
// LOOK why doesnt this work ?
//CodedInputStream cis = CodedInputStream.newInstance(is);
//cis.setSizeLimit(msize);
//NcStreamProto.Stream proto = NcStreamProto.Stream.parseFrom(cis);
while (!raf.isAtEndOfFile()) {
if (debug) System.out.printf("READ message at = %d%n", raf.getFilePointer());
byte[] b = new byte[4];
raf.read(b);
if (test(b, NcStream.MAGIC_END))
break;
else if (!test(b, NcStream.MAGIC_DATA))
throw new IllegalStateException("bad format");
int psize = readVInt(raf);
if (debug) System.out.println(" dproto len= " + psize);
byte[] dp = new byte[psize];
raf.read(dp);
NcStreamProto.Data pdata = NcStreamProto.Data.parseFrom(dp);
Variable v = ncfile.getRootGroup().findVariable(pdata.getVarName());
if (debug) System.out.printf(" dproto = %s for %s%n", pdata, v.getShortName());
List storage = (List) v.getSPobject(); // LOOK should be an in memory Rtree using section
if (storage == null) {
storage = new ArrayList();
v.setSPobject(storage);
}
if (!pdata.getVdata()) { // regular data
int dsize = readVInt(raf);
if (debug) System.out.println(" data len= " + dsize);
DataStorage dataStorage = new DataStorage();
dataStorage.size = dsize;
dataStorage.filePos = raf.getFilePointer();
dataStorage.section = NcStream.decodeSection(pdata.getSection());
dataStorage.isDeflate = pdata.getCompress() == NcStreamProto.Compress.DEFLATE;
storage.add(dataStorage);
raf.skipBytes(dsize);
} else {
DataStorage dataStorage = new DataStorage();
dataStorage.filePos = raf.getFilePointer();
int nelems = readVInt(raf);
int totalSize = 0;
for (int i = 0; i < nelems; i++) {
int dsize = readVInt(raf);
totalSize += dsize;
raf.skipBytes(dsize);
}
dataStorage.isVlen = true;
dataStorage.nelems = nelems;
dataStorage.size = totalSize;
dataStorage.section = NcStream.decodeSection(pdata.getSection());
storage.add(dataStorage);
}
}
} catch (Throwable t) {
throw new RuntimeException("NcStreamIosp: " + t.getMessage() + " on " + raf.getLocation(), t);
}
} */
private class DataStorage {
int size;
long filePos;
Section section;
boolean isVlen, isDeflate;
int nelems, uncompressedLen;
@Override
public String toString() {
return "size=" + size +
", filePos=" + filePos +
", section=" + section +
", nelems=" + nelems +
", isVlen=" + isVlen +
", isDeflate=" + isDeflate;
}
}
public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
List storage = (List) v.getSPobject();
ByteBuffer result = null;
for (DataStorage dataStorage : storage) {
//if (dataStorage.isVlen)
// return readVlenData(v, section, dataStorage);
if (dataStorage.section.intersects(section)) { // LOOK WRONG
raf.seek(dataStorage.filePos);
byte[] data = new byte[dataStorage.size];
raf.read(data);
if (dataStorage.isDeflate) {
ByteArrayInputStream bin = new ByteArrayInputStream(data);
InflaterInputStream in = new InflaterInputStream(bin);
ByteArrayOutputStream bout = new ByteArrayOutputStream(data.length * 7);
IO.copy(in, bout);
byte[] resultb = bout.toByteArray();
result = ByteBuffer.wrap(resultb); // look - an extra copy !! override ByteArrayOutputStream to fix
if (debug) System.out.printf(" uncompressedLen header=%d actual=%d%n", dataStorage.uncompressedLen , resultb.length);
//result.order(ByteOrder.LITTLE_ENDIAN); // LOOK
} else {
result = ByteBuffer.wrap(data);
}
}
}
return Array.factory(v.getDataType(), v.getShape(), result);
//return dataArray.sectionNoReduce(section.getRanges());
}
public Array readVlenData(Variable v, Section section, DataStorage dataStorage) throws IOException, InvalidRangeException {
raf.seek(dataStorage.filePos);
int nelems = readVInt(raf);
Object[] result = new Object[nelems];
for (int elem = 0; elem < nelems; elem++) {
int dsize = readVInt(raf);
byte[] data = new byte[dsize];
raf.read(data);
Array dataArray = Array.factory(v.getDataType(), null, ByteBuffer.wrap(data));
result[elem] = dataArray;
}
return new ArrayObject(result[0].getClass(), new int[]{nelems}, result);
//return dataArray.section(section.getRanges());
}
private int readVInt(RandomAccessFile raf) throws IOException {
byte b = (byte) raf.read();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = (byte) raf.read();
i |= (b & 0x7F) << shift;
}
return i;
}
private boolean readAndTest(RandomAccessFile raf, byte[] test) throws IOException {
byte[] b = new byte[test.length];
raf.read(b);
return test(b, test);
}
private boolean test(byte[] bread, byte[] test) throws IOException {
if (bread.length != test.length) return false;
for (int i = 0; i < bread.length; i++)
if (bread[i] != test[i]) return false;
return true;
}
///////////////////////////////////////////////////////////////////////////////
// lower level interface for debugging
// optionally read in all messages, return as List
public void openDebug(RandomAccessFile raf, NetcdfFile ncfile, List ncm) throws IOException {
this.raf = raf;
raf.seek(0);
long pos = raf.getFilePointer();
if (!readAndTest(raf, NcStream.MAGIC_START)) {
if (ncm != null) {
ncm.add(new NcsMess(pos, 0, "MAGIC_START missing - abort"));
return;
}
throw new IOException("Data corrupted on " + ncfile.getLocation());
}
if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_START"));
pos = raf.getFilePointer();
if (!readAndTest(raf, NcStream.MAGIC_HEADER)) {
if (ncm != null) {
ncm.add(new NcsMess(pos, 0, "MAGIC_HEADER missing - abort"));
return;
}
throw new IOException("Data corrupted on " + ncfile.getLocation());
}
if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_HEADER"));
// assume for the moment it always starts with one header message
int msize = readVInt(raf);
byte[] m = new byte[msize];
raf.read(m);
NcStreamProto.Header proto = NcStreamProto.Header.parseFrom(m);
if (ncm != null) ncm.add(new NcsMess(pos, msize, proto));
version = proto.getVersion();
NcStreamProto.Group root = proto.getRoot();
NcStream.readGroup(root, ncfile, ncfile.getRootGroup());
ncfile.finish();
while (!raf.isAtEndOfFile()) {
pos = raf.getFilePointer();
byte[] b = new byte[4];
raf.read(b);
if (test(b, NcStream.MAGIC_END)) {
if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_END"));
break;
}
if (test(b, NcStream.MAGIC_ERR)) {
int esize = readVInt(raf);
byte[] dp = new byte[esize];
raf.read(dp);
NcStreamProto.Error error = NcStreamProto.Error.parseFrom(dp);
if (ncm != null) ncm.add(new NcsMess(pos, esize, error.getMessage()));
break;
}
if (!test(b, NcStream.MAGIC_DATA)) {
if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_DATA missing - abort"));
break;
}
if (ncm != null) ncm.add(new NcsMess(pos, 4, "MAGIC_DATA".intern()));
// data messages
pos = raf.getFilePointer();
int psize = readVInt(raf);
byte[] dp = new byte[psize];
raf.read(dp);
NcStreamProto.Data dproto = NcStreamProto.Data.parseFrom(dp);
Variable v = ncfile.findVariable(dproto.getVarName());
if (v == null) {
System.out.printf(" ERR cant find var %s%n%s%n", dproto.getVarName(), dproto);
continue;
}
if (debug) System.out.printf(" dproto = %s for %s%n", dproto, v.getShortName());
if (ncm != null) ncm.add(new NcsMess(pos, psize, dproto));
List storage = (List) v.getSPobject(); // LOOK should be an in memory Rtree using section
if (storage == null) {
storage = new ArrayList();
v.setSPobject(storage);
}
if (!dproto.getVdata()) { // regular data
int dsize = readVInt(raf);
DataStorage dataStorage = new DataStorage();
dataStorage.size = dsize;
dataStorage.filePos = raf.getFilePointer();
dataStorage.section = NcStream.decodeSection(dproto.getSection());
dataStorage.nelems = (int) dataStorage.section.computeSize();
dataStorage.isDeflate = dproto.getCompress() == NcStreamProto.Compress.DEFLATE;
if (dataStorage.isDeflate)
dataStorage.uncompressedLen = dproto.getUncompressedSize();
if (ncm != null) ncm.add(new NcsMess(dataStorage.filePos, dsize, dataStorage));
storage.add(dataStorage);
raf.skipBytes(dsize);
} else { // LOOK WRONG
DataStorage dataStorage = new DataStorage();
dataStorage.filePos = raf.getFilePointer();
int nelems = readVInt(raf);
int totalSize = 0;
for (int i = 0; i < nelems; i++) {
int dsize = readVInt(raf);
totalSize += dsize;
raf.skipBytes(dsize);
}
dataStorage.isVlen = true;
dataStorage.nelems = nelems;
dataStorage.size = totalSize;
dataStorage.section = NcStream.decodeSection(dproto.getSection());
if (ncm != null) ncm.add(new NcsMess(dataStorage.filePos, totalSize, dataStorage));
storage.add(dataStorage);
}
}
}
public class NcsMess {
public long filePos;
public int len;
public int nelems;
public Object what;
public NcsMess(long filePos, int len, Object what) {
this.filePos = filePos;
this.len = len;
this.what = what;
if (what instanceof DataStorage)
this.nelems = ((DataStorage) what).nelems;
}
public String showDeflate() {
if (!(what instanceof NcStreamIosp.DataStorage))
return "Must select a NcStreamIosp.DataStorage object";
Formatter f = new Formatter();
try {
NcStreamIosp.DataStorage dataStorage = (NcStreamIosp.DataStorage) what;
raf.seek(dataStorage.filePos);
byte[] data = new byte[dataStorage.size];
raf.read(data);
ByteArrayInputStream bin = new ByteArrayInputStream(data);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DeflaterOutputStream dout = new DeflaterOutputStream(bout);
IO.copy(bin, dout);
dout.close();
int deflatedSize = bout.size();
float ratio = ((float)data.length) / deflatedSize;
f.format("Original size = %d bytes, deflated = %d bytes ratio = %f %n", data.length, deflatedSize, ratio);
return f.toString();
} catch (IOException e) {
e.printStackTrace();
return e.getMessage();
}
}
}
}