ucar.nc2.stream.NcStreamDataRow Maven / Gradle / Ivy
The newest version!
/*
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
* See LICENSE for license information.
*/
package ucar.nc2.stream;
import com.google.protobuf.ByteString;
import ucar.ma2.*;
import ucar.nc2.iosp.IospHelper;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* LOOK Not Done Yet
*
* @author caron
* @since 10/26/2015.
*/
public class NcStreamDataRow {
/*
message DataRow {
string fullName = 1;
DataType dataType = 2;
Section section = 3;
bool bigend = 4;
uint32 version = 5;
bool isVlen = 7;
uint32 nelems = 9;
// oneof
bytes primarray = 10; // rectangular, primitive array # <1>
repeated string stringdata = 11; // string dataType # <2>
ArrayStructureRow structdata = 12; // structure/seq dataType # <3>
repeated uint32 vlens = 13; // isVlen true # <4>
repeated bytes opaquedata = 14; // opaque dataType # <5>
}
primarray has nelems * sizeof(dataType) bytes, turn into multidim array of primitives with section info and bigend
stringdata has nelems strings, turn into multidim array of String with section info
structdata has nelems StructureData objects, turn into multidim array of StructureData with section info and bigend
vlens has section.size array lengths
opaquedata has nelems opaque objects, turn into multidim array of Opaque with section info
*/
public NcStreamProto.DataRow encodeData3(String name, boolean isVlen, Section section, Array data) {
NcStreamProto.DataRow.Builder builder = NcStreamProto.DataRow.newBuilder();
encodeData3(builder, name, isVlen, section, data);
return builder.build();
}
void encodeData3(NcStreamProto.DataRow.Builder builder, String name, boolean isVlen, Section section, Array data) {
DataType dataType = data.getDataType();
builder.setFullName(name);
builder.setDataType(NcStream.convertDataType(data.getDataType()));
builder.setBigend(ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN);
builder.setVersion(NcStream.ncstream_data_version);
if (!isVlen) {
builder.setNelems((int) data.getSize());
builder.setSection(NcStream.encodeSection(section));
}
if (isVlen) {
builder.setIsVlen(true);
encodeVlenData(builder, section, data);
} else if (dataType == DataType.STRING) {
if (data instanceof ArrayChar) { // is this possible ?
ArrayChar cdata =(ArrayChar) data;
for (String s:cdata)
builder.addStringdata(s);
Section ssection = section.removeLast();
builder.setSection(NcStream.encodeSection(ssection));
} else if (data instanceof ArrayObject) {
IndexIterator iter = data.getIndexIterator();
while (iter.hasNext())
builder.addStringdata( (String) iter.next());
} else {
throw new IllegalStateException("Unknown class for STRING ="+ data.getClass().getName());
}
} else if (dataType == DataType.OPAQUE) {
if (data instanceof ArrayObject) {
IndexIterator iter = data.getIndexIterator();
while (iter.hasNext()) {
ByteBuffer bb = (ByteBuffer) iter.next();
builder.addOpaquedata(ByteString.copyFrom(bb));
}
} else {
throw new IllegalStateException("Unknown class for OPAQUE ="+ data.getClass().getName());
}
} else if (dataType == DataType.STRUCTURE) {
builder.setStructdata( encodeStructureData(name, data));
} else if (dataType == DataType.SEQUENCE) {
throw new UnsupportedOperationException("Not implemented yet SEQUENCE ="+ data.getClass().getName());
} else { // normal case
int nbytes = (int) data.getSizeBytes();
ByteBuffer bb = ByteBuffer.allocate(nbytes);
copyArrayToBB(data, bb);
bb.flip();
builder.setPrimdata(ByteString.copyFrom(bb));
}
}
void encodeVlenData(NcStreamProto.DataRow.Builder builder, Section section, Array data) {
if (!(data instanceof ArrayObject))
throw new IllegalStateException("Unknown class for OPAQUE =" + data.getClass().getName());
IndexIterator iter = data.getIndexIterator();
int count = 0;
while (iter.hasNext()) {
Array varray = (Array) iter.next();
int vlensize = (int) varray.getSize();
builder.addVlens(vlensize);
count += vlensize;
}
builder.setNelems(count);
Section ssection = section.removeLast();
builder.setSection(NcStream.encodeSection(ssection));
assert section.computeSize() == count;
int nbytes = count * data.getDataType().getSize();
ByteBuffer bb = ByteBuffer.allocate(nbytes);
// copyArrayToBB(data, bb);
iter = data.getIndexIterator();
while (iter.hasNext()) {
Array varray = (Array) iter.next();
copyArrayToBB(varray, bb);
}
}
void copyArrayToBB(Array data, ByteBuffer out) {
Class classType = data.getElementType();
IndexIterator iterA = data.getIndexIterator();
if (classType == double.class) {
while (iterA.hasNext())
out.putDouble(iterA.getDoubleNext());
} else if (classType == float.class) {
while (iterA.hasNext())
out.putFloat(iterA.getFloatNext());
} else if (classType == long.class) {
while (iterA.hasNext())
out.putLong(iterA.getLongNext());
} else if (classType == int.class) {
while (iterA.hasNext())
out.putInt(iterA.getIntNext());
} else if (classType == short.class) {
while (iterA.hasNext())
out.putShort(iterA.getShortNext());
} else if (classType == char.class) {
byte[] pa = IospHelper.convertCharToByte((char[]) data.get1DJavaArray(DataType.CHAR));
out.put(pa, 0, pa.length);
} else if (classType == byte.class) {
while (iterA.hasNext())
out.put(iterA.getByteNext());
} else
throw new UnsupportedOperationException("Class type = " + classType.getName());
}
private class MemberData {
StructureMembers.Member member;
int nelems;
DataType dtype;
boolean isVlen;
ByteBuffer bb;
List stringList;
List opaqueList;
List vlenList;
List vlens;
public MemberData(StructureMembers.Member member, int nelems) {
this.member = member;
this.nelems = nelems;
this.dtype = member.getDataType();
this.isVlen = member.isVariableLength();
if (isVlen)
vlenList = new ArrayList<>(nelems);
else if (dtype == DataType.STRING)
stringList = new ArrayList<>(nelems * member.getSize());
else if (dtype == DataType.OPAQUE)
opaqueList = new ArrayList<>(nelems * member.getSize());
else if (dtype == DataType.STRUCTURE);
else
bb = ByteBuffer.allocate(nelems * member.getSizeBytes());
}
int completeVlens() {
vlens = new ArrayList<>(nelems);
int total = 0;
for (Array va : vlenList) {
vlens.add((int) va.getSize());
total += va.getSize();
}
bb = ByteBuffer.allocate(nelems * member.getSizeBytes());
for (Array va : vlenList) {
copyArrayToBB(va, bb);
}
return total;
}
}
/*
message Member {
string shortName = 1;
DataType dataType = 2;
repeated uint32 shape = 3; // or section?
bool isVlen = 4;
}
message ArrayStructureRow {
repeated Member members = 1;
uint64 nrows = 5; // number of rows in this message
uint32 rowLength = 6; // length in bytes of each row
bytes fixdata = 10; // fixed data
repeated string stringdata = 11; // string dataType
repeated bytes bytedata = 13; // opaque dataType and vlens
repeated ArrayStructureRow structdata = 14; // structure/seq dataType
}
*/
NcStreamProto.ArrayStructureRow.Builder encodeStructureData(String structName, Array data) {
assert data instanceof ArrayStructure;
ArrayStructure as = (ArrayStructure) data;
int nelems = (int) as.getSize();
List memberData = new ArrayList<>();
StructureMembers sm = as.getStructureMembers();
for (StructureMembers.Member m : sm.getMembers()) {
memberData.add( new MemberData(m, nelems));
}
// use most efficient form of data extraction
for (int recno=0; recno