com.javanut.pronghorn.pipe.stream.RingStreams Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of pronghorn-pipes Show documentation
Show all versions of pronghorn-pipes Show documentation
Ring buffer based queuing utility for applications that require high performance and/or a small
footprint. Well suited for embedded and stream based processing.
package com.javanut.pronghorn.pipe.stream;
import static com.javanut.pronghorn.pipe.Pipe.byteBackingArray;
import static com.javanut.pronghorn.pipe.Pipe.bytePosition;
import static com.javanut.pronghorn.pipe.Pipe.headPosition;
import static com.javanut.pronghorn.pipe.Pipe.tailPosition;
import static com.javanut.pronghorn.pipe.Pipe.takeRingByteLen;
import static com.javanut.pronghorn.pipe.Pipe.takeRingByteMetaData;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import com.javanut.pronghorn.pipe.Pipe;
import com.javanut.pronghorn.pipe.RawDataSchema;
@Deprecated //Consider using the newer DataInputBlobReader or DataOutputBlobWriter or the fieldWrite methods in PipeReader Pipe or PipeWriter
public class RingStreams {
/**
* Copies all bytes from the inputRing to the outputStream. Will continue to do this until the inputRing
* provides a byteArray reference with negative length.
*
* Upon exit the RingBuffer and OutputStream are NOT closed so this method can be called again if needed.
*
* For example the same connection can be left open for sending multiple files in sequence.
*
*
* @param inputRing
* @param outputStream
* @throws IOException
*/
public static void writeToOutputStream(Pipe inputRing, OutputStream outputStream) throws IOException {
long step = RawDataSchema.FROM.fragDataSize[0];
//this blind byte copy only works for this simple message type, it is not appropriate for other complex types
if (Pipe.from(inputRing) != RawDataSchema.FROM) {
throw new UnsupportedOperationException("This method can only be used with the very simple RAW_BYTES catalog of messages.");
}
//target is always 1 ahead of where we are then we step by step size, this lets us pick up the
//EOF message which is only length 2
long target = 2+tailPosition(inputRing);
//write to outputStream only when we have data on inputRing.
long headPosCache = headPosition(inputRing);
//NOTE: This can be made faster by looping and summing all the lengths to do one single copy to the output stream
// That change may however increase latency.
int byteMask = inputRing.blobMask;
int byteSize = byteMask+1;
while (true) {
//block until one more byteVector is ready.
long lastCheckedValue = headPosCache;
while ( lastCheckedValue < target) {
Pipe.spinWork(inputRing);//TODO: WARNING this may hang when using a single thread scheduler
lastCheckedValue = Pipe.headPosition(inputRing);
}
headPosCache = lastCheckedValue;
int msgId = Pipe.takeMsgIdx(inputRing);
if (msgId<0) { //exit logic
Pipe.releaseReadLock(inputRing);
break;
} else {
int meta = Pipe.takeByteArrayMetaData((Pipe) inputRing);//side effect, this moves the pointer.
int len = Pipe.takeByteArrayLength((Pipe) inputRing);
if (len>0) {
byte[] data = byteBackingArray(meta, inputRing);
int off = bytePosition(meta,inputRing,len)&byteMask;
int len1 = byteSize-off;
if (len1>=len) {
//simple add bytes
outputStream.write(data, off, len);
} else {
//rolled over the end of the buffer
outputStream.write(data, off, len1);
outputStream.write(data, 0, len-len1);
}
outputStream.flush();
}
Pipe.releaseReadLock(inputRing);
}
target += step;
}
}
/**
* Copies all bytes from the inputRing to each of the outputStreams. Will continue to do this until the inputRing
* provides a byteArray reference with negative length.
*
* Upon exit the RingBuffer and OutputStream are NOT closed so this method can be called again if needed.
*
* For example the same connection can be left open for sending multiple files in sequence.
*
*
* @param inputRing
* @param outputStreams the streams we want to write the data to.
* @throws IOException
*/
public static void writeToOutputStreams(Pipe inputRing, OutputStream... outputStreams) throws IOException {
long step = RawDataSchema.FROM.fragDataSize[0];
//this blind byte copy only works for this simple message type, it is not appropriate for other complex types
if (Pipe.from(inputRing) != RawDataSchema.FROM) {
throw new UnsupportedOperationException("This method can only be used with the very simple RAW_BYTES catalog of messages.");
}
//only need to look for 2 value then step forward by steps this lets us pick up the EOM message without hanging.
long target = 2+tailPosition(inputRing);
//write to outputStream only when we have data on inputRing.
long headPosCache = headPosition(inputRing);
//NOTE: This can be made faster by looping and summing all the lengths to do one single copy to the output stream
// That change may however increase latency.
while (true) {
//block until one more byteVector is ready.
long lastCheckedValue = headPosCache;
while ( lastCheckedValue < target) {
Pipe.spinWork(inputRing);//TODO: WARNING this may hang when using a single thread scheduler
lastCheckedValue = Pipe.headPosition(inputRing);
}
headPosCache = lastCheckedValue;
int msgId = Pipe.takeMsgIdx(inputRing);
if (msgId<0) { //exit logic
int bytesCount = Pipe.takeInt(inputRing);
assert(0==bytesCount);
Pipe.releaseReadLock(inputRing);
return;
} else {
int meta = Pipe.takeByteArrayMetaData((Pipe) inputRing);//side effect, this moves the pointer.
int len = Pipe.takeByteArrayLength((Pipe) inputRing);
int byteMask = inputRing.blobMask;
byte[] data = byteBackingArray(meta, inputRing);
int offset = bytePosition(meta,inputRing,len);
int adjustedOffset = offset & byteMask;
int adjustedEnd = (offset + len) & byteMask;
int adjustedLength = 1 + byteMask - adjustedOffset;
for(OutputStream os : outputStreams) {
if ( adjustedOffset > adjustedEnd) {
//rolled over the end of the buffer
os.write(data, adjustedOffset, adjustedLength);
os.write(data, 0, len - adjustedLength);
} else {
//simple add bytes
os.write(data, adjustedOffset, len);
}
os.flush();
}
Pipe.releaseReadLock(inputRing);
}
target += step;
}
}
/**
* Copies all bytes from the inputStream to the outputRing.
*
* Blocks as needed for the outputRing.
* Writes until the inputStream reaches EOF, this is signaled by a negative length from the call to read.
*
* @param inputStream
* @param outputRing
* @throws IOException
*/
@Deprecated
public static void readFromInputStream(InputStream inputStream, Pipe outputRing) throws IOException {
assert (Pipe.from(outputRing) == RawDataSchema.FROM);
int step = RawDataSchema.FROM.fragDataSize[0];
int fill = 1 + outputRing.slabMask - step;
int maxBlockSize = outputRing.maxVarLen;
long targetTailValue = headPosition(outputRing)-fill;
long tailPosCache = tailPosition(outputRing);
byte[] buffer = Pipe.blob((Pipe) outputRing);
int byteMask = outputRing.blobMask;
int position = Pipe.getWorkingBlobHeadPosition((Pipe) outputRing);
int size = 0;
try{
new Exception("this does not support wrapping of blob data and any usages should be changed over to the new stream apis int PipeReader, Pipe and PipeWriter").printStackTrace();
while ( (size=inputStream.read(buffer,position&byteMask,((position&byteMask) > ((position+maxBlockSize-1) & byteMask)) ? 1+byteMask-(position&byteMask) : maxBlockSize))>=0 ) {
if (size>0) {
//block until there is a slot to write into
long lastCheckedValue = tailPosCache;
while (null==Pipe.slab(outputRing) || lastCheckedValue < targetTailValue) {
Pipe.spinWork(outputRing);
lastCheckedValue = Pipe.tailPosition(outputRing);
}
tailPosCache = lastCheckedValue;///TODO:M Rewrite using RingBuffer.roomToLowLevelWrite(output, size)
targetTailValue += step;
Pipe.addMsgIdx(outputRing, 0);
Pipe.validateVarLength(outputRing, size);
Pipe.addBytePosAndLen(outputRing, position, size);
Pipe.addAndGetBlobWorkingHeadPosition(outputRing, size);
Pipe.confirmLowLevelWrite(outputRing, RawDataSchema.FROM.fragDataSize[0]);
Pipe.publishWrites(outputRing);
position += size;
} else {
Thread.yield();
}
}
} catch (IOException ioex) {
System.err.println("FAILURE detected at position: "+position+" last known sizes: "+size+" byteMask: "+outputRing.blobMask+
" rolloever "+((position&byteMask) >= ((position+maxBlockSize-1) & byteMask))+" "+(position&byteMask)+" > "+((position+maxBlockSize-1) & byteMask));
throw ioex;
}
}
/**
* copied data array into ring buffer. It blocks if needed and will split the array on ring buffer if needed.
*
* @param data
* @param output
* @param blockSize
*/
public static void writeBytesToRing(byte[] data, int dataOffset, int dataLength, Pipe output, int blockSize) {
assert (Pipe.from(output) == RawDataSchema.FROM);
int fill = 1 + output.slabMask - RawDataSchema.FROM.fragDataSize[0];
long tailPosCache = tailPosition(output);
int position = dataOffset; //position within the data array
int stop = dataOffset+dataLength;
while (position) inputRing);//side effect, this moves the pointer.
int len = Pipe.takeByteArrayLength((Pipe) inputRing);
byte[] data = byteBackingArray(meta, inputRing);
int offset = bytePosition(meta,inputRing,len);
if ((offset&byteMask) > ((offset+len-1) & byteMask)) {
//rolled over the end of the buffer
int len1 = 1+byteMask-(offset&byteMask);
visitor.visit(data, offset&byteMask, len1, 0, len-len1);
} else {
//simple add bytes
visitor.visit(data, offset&byteMask, len);
}
Pipe.confirmLowLevelRead(inputRing, RawDataSchema.FROM.fragDataSize[0]);
Pipe.releaseReadLock(inputRing);
}
target += step;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy