Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2010-2022 60East Technologies Inc., All Rights Reserved.
//
// This computer software is owned by 60East Technologies Inc. and is
// protected by U.S. copyright laws and other laws and by international
// treaties. This computer software is furnished by 60East Technologies
// Inc. pursuant to a written license agreement and may be used, copied,
// transmitted, and stored only in accordance with the terms of such
// license agreement and with the inclusion of the above copyright notice.
// This computer software or any other copies thereof may not be provided
// or otherwise made available to any other person.
//
// U.S. Government Restricted Rights. This computer software: (a) was
// developed at private expense and is in all respects the proprietary
// information of 60East Technologies Inc.; (b) was not developed with
// government funds; (c) is a trade secret of 60East Technologies Inc.
// for all purposes of the Freedom of Information Act; and (d) is a
// commercial item and thus, pursuant to Section 12.212 of the Federal
// Acquisition Regulations (FAR) and DFAR Supplement Section 227.7202,
// Government's use, duplication or disclosure of the computer software
// is subject to the restrictions set forth by 60East Technologies Inc..
//
////////////////////////////////////////////////////////////////////////////
package com.crankuptheamps.client;
import com.crankuptheamps.client.exception.CommandException;
import com.crankuptheamps.client.exception.TimedOutException;
import com.crankuptheamps.client.fields.Field;
import com.crankuptheamps.client.fields.IntegerField;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;
import com.crankuptheamps.client.exception.DisconnectedException;
import com.crankuptheamps.client.exception.StoreException;
/**
* BlockPublishStore is a base class for publish {@link Store} implementations. As messages are stored,
* space is allocated from a pre-created flat buffer. As messages are discarded, space in that buffer
* is marked "free" for future store operations. If messages are stored faster than they are published, the buffer
* is re-sized to create more capacity.
*/
public class BlockPublishStore implements Store
{
private int _blocksPerRealloc = 10000;
private volatile long _nextSequence = 1;
private final Lock lock= new ReentrantLock();
private final Condition _blocksFree = lock.newCondition();
private final Condition _messageReady = lock.newCondition();
private volatile boolean _resizing = false;
protected PublishStoreResizeHandler _resizeHandler = null;
/**
* A simple wrapper object around a byte array that allows a sub-range to be specified using its offset and length
* properties.
*/
public static class ByteSequence
{
public byte[] array;
public long offset, len;
public ByteSequence()
{
this.array = null;
this.offset = this.len = 0;
}
public ByteSequence(byte[] array, long offset, long len)
{
this.array = array;
this.offset = offset;
this.len = len;
}
}
/**
* Interface which is used to hold the BlockPublishStore
* buffer data.
*/
public interface Buffer extends AutoCloseable
{
public long getSize() throws IOException;
public void setSize(long newSize) throws IOException;
public long getPosition() throws IOException;
public void setPosition(long position) throws IOException;
public void putByte(byte b) throws IOException;
public byte getByte() throws IOException;
public void putInt(int i) throws IOException;
public void putInt(long position, int i) throws IOException;
public int getInt() throws IOException;
public int getInt(long position) throws IOException;
public void putLong(long l) throws IOException;
public void putLong(long position, long l) throws IOException;
public long getLong() throws IOException;
public void putBytes(ByteSequence bytes) throws IOException;
public void getBytes(ByteSequence outBytes) throws IOException;
public void zero(long offset, int length) throws IOException;
public void putBytes(byte[] buffer, long offset, long length) throws IOException;
public void getBytes(Field outField, int length) throws IOException;
}
static class Block
{
public int index;
public Block nextInChain;
public Block nextInList;
public long sequenceNumber;
public static final int SIZE = 2048; // finely honed.
public static final int BLOCK_HEADER_SIZE = 32;
public static final int BLOCK_DATA_SIZE = Block.SIZE - BLOCK_HEADER_SIZE;
public static final int CHAIN_HEADER = 32;
// Postion of the CRC value
public static final int BLOCK_CRC_POSITION = 16;
// Postion of the ready flag, non-zero is ready
public static final int BLOCK_READY_POSITION = 24;
public Block(int index) { this.index = index; }
public long getPosition() { return (long)SIZE * index; }
public String toString()
{
return String.format("block %d sequenceNumber %d", index, sequenceNumber);
}
}
// A buffer acts as a random access file -- mmap, memory only, and file implementations exist.
protected Buffer _buffer;
// This system uses intrusive lists. _freeList is the list of free blocks in the file.
// _usedList is the list of the used head blocks. Used non-head blocks are only kept
// alive by "chain" references from the head block.
// Used list is ordered where the start is the LEAST recently used.
volatile Block _freeList = null;
volatile Block _usedList = null;
volatile Block _endOfUsedList = null;
// The metadata format is currently:
// int size - unused
// int nextInChain - set to the int of the version number, such as 4000100 for 4.0.1.0
// long sequence - set to the index of the last discarded
Block _metadataBlock = null;
// Save this so we're not GCing stuff.
CRC32 _crc = null;
// We use this _argument thing since Java doesn't have output parameters, but we sort of need them.
ByteSequence _argument = new ByteSequence();
// We use this to hold everything we read from the buffer for replay
Message _message;
// When a message spans multiple blocks we need to reassemble it
// into one place before replaying. We use this buffer to do so,
// allocating on demand.
byte[] _readBuffer;
// Write assembly area
ArrayStoreBuffer _sb = null;
protected static final int AMPS_MIN_PUB_STORE_DISCARDED_VERSION = 4000100;
protected static final int METADATA_VERSION_LOCATION = 4;
protected static final int METADATA_LAST_DISCARDED_LOCATION = 8;
protected BlockPublishStore(Buffer buffer, int blocksPerRealloc, boolean isAFile)
{
this._blocksPerRealloc = blocksPerRealloc;
this._buffer = buffer;
if(isAFile)
{
this._crc = new CRC32();
this._sb = new ArrayStoreBuffer();
try
{
this._sb.setSize(Block.SIZE);
} catch (IOException e)
{
this._sb = null;
}
}
}
protected BlockPublishStore(Buffer buffer, int blocksPerRealloc)
{
this(buffer, blocksPerRealloc, false);
}
protected BlockPublishStore(Buffer buffer)
{
this(buffer, 10000, false);
}
public void store(Message m) throws StoreException
{
store(m, true);
}
public void close() throws Exception
{
_buffer.close();
}
private void flattenToFreeList(Block toRemove)
{
Block currentInChain = toRemove;
lock.lock();
try {
if (toRemove == _usedList) {
_usedList = _usedList.nextInList;
}
else {
Block prev = _usedList;
for (Block b = prev.nextInList; b!=null; b = b.nextInList) {
if (b == toRemove) {
prev.nextInList = b.nextInList;
break;
}
prev = prev.nextInList;
}
}
while(currentInChain != null)
{
Block nextInChain = currentInChain.nextInChain;
try {
_buffer.zero(currentInChain.getPosition(),
Block.BLOCK_HEADER_SIZE);
} catch (IOException ex) {
// Going to ignore this, since we here because of an error
}
currentInChain.sequenceNumber = 0;
currentInChain.nextInChain = null;
// and, flatten this chain onto the zeroed out list.
currentInChain.nextInList = _freeList;
_freeList = currentInChain;
currentInChain = nextInChain;
}
} finally { lock.unlock(); }
}
protected void store(Message m, boolean assignSequence)
throws StoreException
{
int operation = m.getCommand();
int flag = -1;
Field data = null;
if (operation == Message.Command.Publish ||
operation == Message.Command.DeltaPublish ||
(operation == Message.Command.SOWDelete && !m.isDataNull()))
{
data = m.getDataRaw();
if (operation == Message.Command.SOWDelete)
flag = Store.SOWDeleteByData;
} else if (!m.isFilterNull()) {
data = m.getFilterRaw();
flag = Store.SOWDeleteByFilter;
} else if (!m.isSowKeysNull()) {
data = m.getSowKeysRaw();
flag = Store.SOWDeleteByKeys;
} else if (!m.isBookmarkNull()) {
if (m.isOptionsNull() || !m.getOptions().contains("cancel")) {
data = m.getBookmarkRaw();
flag = Store.SOWDeleteByBookmark;
} else {
data = m.getBookmarkRaw();
flag = Store.SOWDeleteByBookmarkCancel;
}
} else {
throw new StoreException("Cannot store a Message with no data, filter, sow keys, or bookmark.");
}
Block current, next, first = null;
Field cmdId = m.getCommandIdRaw();
long commandIdRemaining = cmdId == null ? 0 : cmdId.length;
Field correlationId = m.getCorrelationIdRaw();
long correlationIdRemaining = correlationId == null ? 0 : correlationId.length;
Field expiration = m.getExpirationRaw();
long expirationRemaining = expiration == null ? 0 : expiration.length;
Field sowKey = m.getSowKeyRaw();
long sowKeyRemaining = sowKey == null ? 0 : sowKey.length;
Field topic = m.getTopicRaw();
long topicRemaining = topic.length;
long dataRemaining = data.length;
// Nothing to write if this is a Unknown command, which is a Queue ack
long totalRemaining = ((operation == Message.Command.Unknown) ? 0 :
(Block.CHAIN_HEADER + commandIdRemaining +
correlationIdRemaining + expirationRemaining +
sowKeyRemaining + topicRemaining + dataRemaining));
int lastBlockLength = ((operation == Message.Command.Unknown) ? 0 :
((int)(totalRemaining % Block.BLOCK_DATA_SIZE)));
int blocksToWrite = ((operation == Message.Command.Unknown) ? 0 :
((int)((totalRemaining / Block.BLOCK_DATA_SIZE) +
(lastBlockLength>0? 1: 0))));
long crcVal = 0;
try
{
// First block gets sequence assigned
long currentSequence = (assignSequence ? 0 : m.getSequence());
first = next = get(assignSequence, currentSequence);
if(first == null)
{
throw new StoreException("The store is full, and no blocks can be allocated.");
}
if (assignSequence) m.setSequence(first.sequenceNumber);
boolean loopComplete = false;
while(blocksToWrite > 0)
{
loopComplete = false;
current = next;
int bytesRemainingInBlock = Block.BLOCK_DATA_SIZE;
if(blocksToWrite > 1)
{
next = get(false, 0);
if(next == null)
{
flattenToFreeList(first);
throw new StoreException("The store is full, and no additional blocks can be allocated.");
}
} else
{
next = null;
}
current.nextInChain = next;
// write the block info
Buffer buffer = _sb == null ? _buffer : _sb;
lock.lock();
try
{
long start = _sb == null ? current.getPosition() : 0;
buffer.setPosition(start);
if(current == first)
{
buffer.putInt((int)totalRemaining);
}
else
{
buffer.putInt(next == null ? lastBlockLength : Block.BLOCK_DATA_SIZE);
}
buffer.putInt(next == null ? current.index : next.index);
buffer.putLong(current==first?m.getSequence():0);
if (_crc != null) /* we keep CRC off if we don't write to disk. */
{
_crc.reset();
if (cmdId != null && commandIdRemaining > 0)
_crc.update(cmdId.buffer,
(int) cmdId.position,
(int) commandIdRemaining);
if (correlationId != null && correlationIdRemaining > 0)
_crc.update(correlationId.buffer,
(int) correlationId.position,
(int) correlationIdRemaining);
if (expiration != null && expirationRemaining > 0)
_crc.update(expiration.buffer,
(int) expiration.position,
(int) expirationRemaining);
if (sowKey != null && sowKeyRemaining > 0)
_crc.update(sowKey.buffer,
(int) sowKey.position,
(int) sowKeyRemaining);
if (topicRemaining > 0)
_crc.update(topic.buffer,
(int) topic.position,
(int) topicRemaining);
if (dataRemaining > 0)
_crc.update(data.buffer,
(int) data.position,
(int) dataRemaining);
crcVal = _crc.getValue();
}
buffer.putLong(crcVal);
// now write data and topic and stuff.
buffer.setPosition(start + Block.BLOCK_HEADER_SIZE);
if(current == first)
{
buffer.putInt((int)operation);
buffer.putInt((int)commandIdRemaining);
buffer.putInt((int)correlationIdRemaining);
buffer.putInt((int)expirationRemaining);
buffer.putInt((int)sowKeyRemaining);
buffer.putInt((int)topicRemaining);
buffer.putInt(flag);
buffer.putInt((int)m.getAckTypeOutgoing());
bytesRemainingInBlock -= Block.CHAIN_HEADER;
}
long bytesToWrite = commandIdRemaining <= bytesRemainingInBlock ?
commandIdRemaining : bytesRemainingInBlock;
if (bytesToWrite > 0)
{
Field rawField = m.getCommandIdRaw();
buffer.putBytes(rawField.buffer,
rawField.position + rawField.length -
commandIdRemaining,
bytesToWrite);
bytesRemainingInBlock -= bytesToWrite;
commandIdRemaining -= bytesToWrite;
}
bytesToWrite = correlationIdRemaining <= bytesRemainingInBlock ?
correlationIdRemaining : bytesRemainingInBlock;
if (bytesToWrite > 0)
{
Field rawField = m.getCorrelationIdRaw();
buffer.putBytes(rawField.buffer,
rawField.position + rawField.length -
correlationIdRemaining,
bytesToWrite);
bytesRemainingInBlock -= bytesToWrite;
correlationIdRemaining -= bytesToWrite;
}
bytesToWrite = expirationRemaining <= bytesRemainingInBlock ?
expirationRemaining : bytesRemainingInBlock;
if (bytesToWrite > 0)
{
Field rawField = m.getExpirationRaw();
buffer.putBytes(rawField.buffer,
rawField.position + rawField.length -
expirationRemaining,
bytesToWrite);
bytesRemainingInBlock -= bytesToWrite;
expirationRemaining -= bytesToWrite;
}
bytesToWrite = sowKeyRemaining <= bytesRemainingInBlock ?
sowKeyRemaining : bytesRemainingInBlock;
if (bytesToWrite > 0)
{
Field rawField = m.getSowKeyRaw();
buffer.putBytes(rawField.buffer,
rawField.position + rawField.length -
sowKeyRemaining,
bytesToWrite);
bytesRemainingInBlock -= bytesToWrite;
sowKeyRemaining -= bytesToWrite;
}
bytesToWrite = topicRemaining <= bytesRemainingInBlock ?
topicRemaining : bytesRemainingInBlock;
if(bytesToWrite > 0)
{
Field rawField = m.getTopicRaw();
buffer.putBytes(rawField.buffer,
rawField.position + rawField.length -
topicRemaining,
bytesToWrite);
bytesRemainingInBlock -= bytesToWrite;
topicRemaining -= bytesToWrite;
}
bytesToWrite = dataRemaining <= bytesRemainingInBlock ?
dataRemaining : bytesRemainingInBlock;
if(bytesToWrite > 0)
{
buffer.putBytes(data.buffer,
data.position + data.length -
dataRemaining,
bytesToWrite);
dataRemaining -= bytesToWrite;
}
// If we've used sb as an assembly area, here's where we actually write to the file.
if(_sb != null)
{
_buffer.setPosition(current.getPosition());
_argument.array = _sb.getBuffer();
_argument.offset = 0;
_argument.len = _sb.getPosition();
_buffer.putBytes(_argument);
}
loopComplete = true;
}
finally {
lock.unlock();
if (!loopComplete) {
flattenToFreeList(first);
first = null;
}
}
--blocksToWrite;
}
// now, there should be no data or topic left to write.
assert(dataRemaining == 0 && topicRemaining == 0);
lock.lock();
try // Done, signal the ready flag
{
_buffer.putInt(first.getPosition() + Block.BLOCK_READY_POSITION,
1);
_messageReady.signalAll();
}
finally { lock.unlock(); }
} catch (IOException e) {
if (first != null) flattenToFreeList(first);
throw new StoreException(e);
}
}
public void discardUpTo(long index) throws StoreException
{
// If we're empty or already past this value, just return
if (index == 0 || _usedList == null ||
index <= _metadataBlock.sequenceNumber)
{
lock.lock();
try {
if (_getLastPersisted() < index)
{
_metadataBlock.sequenceNumber = index;
// Position plus size of two ints
_buffer.putLong(_metadataBlock.getPosition() +
METADATA_LAST_DISCARDED_LOCATION,
_metadataBlock.sequenceNumber);
}
if (_nextSequence <= index) _nextSequence = index + 1;
} catch (IOException ex) {
throw new StoreException("Error saving last discarded: " + ex,
ex);
} finally {
lock.unlock();
}
return;
}
_metadataBlock.sequenceNumber = index;
// First, find the beginning of where we should discard.
Block almostFreeList = detach(index);
// we no longer need to keep a lock. The only funny thing about our state is that
// there are "missing" entries -- some on _usedList, some on _freeList, and some
// on our (local) almostFreeList, about to be freed. We could be fancy
// and enqueue them for freeing, not going to do that.
// For each one of these guys on our list, flatten onto the
// new "zeroedOutList" and remember the last entry. We'll use
// that to efficiently prepend, below.
Block zeroedOutList = null;
Block lastOnTheList = almostFreeList;
try
{
// Clear out the muck in the byte buffer
Block current = almostFreeList;
while(current != null)
{
Block next = current.nextInList;
Block currentInChain = current;
while(currentInChain != null)
{
Block nextInChain = currentInChain.nextInChain;
_buffer.zero(currentInChain.getPosition(),
Block.BLOCK_HEADER_SIZE);
currentInChain.sequenceNumber = 0;
currentInChain.nextInChain = null;
// and, flatten this chain onto the zeroed out list.
currentInChain.nextInList = zeroedOutList;
zeroedOutList = currentInChain;
currentInChain = nextInChain;
}
current = next;
}
} catch (IOException ioex)
{
throw new StoreException(ioex);
}
// now zeroedOutList is flat and free. We can just prepend it to the free list.
lock.lock();
try
{
_metadataBlock.sequenceNumber = index;
try {
// Position plus size of two ints
_buffer.putLong(_metadataBlock.getPosition() +
METADATA_LAST_DISCARDED_LOCATION,
_metadataBlock.sequenceNumber);
} catch (IOException ex) {
throw new StoreException("Error saving last discarded: " + ex, ex);
}
if(lastOnTheList != null)
{
lastOnTheList.nextInList = _freeList;
_freeList = zeroedOutList;
}
_blocksFree.signalAll();
}
finally { lock.unlock(); }
}
public long getLastPersisted() throws StoreException
{
lock.lock();
try {
return _getLastPersisted();
}
finally { lock.unlock(); }
}
// Lock should already be held
private long _getLastPersisted() throws StoreException
{
if (_metadataBlock.sequenceNumber != 0)
{
return _metadataBlock.sequenceNumber;
}
if (_nextSequence != 1)
{
long minSeq = _getLowestUnpersisted();
long maxSeq = _getHighestUnpersisted();
if (minSeq != -1) _metadataBlock.sequenceNumber = minSeq - 1;
if (maxSeq != -1 && _nextSequence <= maxSeq)
_nextSequence = maxSeq + 1;
if (_nextSequence < _metadataBlock.sequenceNumber)
_metadataBlock.sequenceNumber = _nextSequence - 1;
}
else
{
_metadataBlock.sequenceNumber = System.currentTimeMillis()*1000000;
_nextSequence = _metadataBlock.sequenceNumber + 1;
}
try {
// Position plus size of two ints
_buffer.putLong(_metadataBlock.getPosition() +
METADATA_LAST_DISCARDED_LOCATION,
_metadataBlock.sequenceNumber);
} catch (IOException ex) {
throw new StoreException("Error saving last discarded: " + ex, ex);
}
return _metadataBlock.sequenceNumber;
}
public long getLowestUnpersisted()
{
lock.lock();
try {
return _getLowestUnpersisted();
}
finally { lock.unlock(); }
}
// Lock should already be held
private long _getLowestUnpersisted()
{
if (_usedList != null)
{
return _usedList.sequenceNumber;
}
return -1;
}
public long getHighestUnpersisted()
{
lock.lock();
try {
return _getHighestUnpersisted();
}
finally { lock.unlock(); }
}
// Lock should already be held
private long _getHighestUnpersisted()
{
if (_endOfUsedList != null)
{
return _endOfUsedList.sequenceNumber;
}
return -1;
}
public void setMessage(Message m)
{
_message = m;
}
private Block detach(long index)
{
Block keepList = null, endOfDiscardList = null, discardList = null;
lock.lock();
try
{
// advance through the list, find the first one to keep.
// we keep everything after that point, and discard everything up
// to that point.
keepList = _usedList;
while(keepList != null && keepList.sequenceNumber <= index)
{
if(discardList == null)
{
discardList = keepList;
}
endOfDiscardList = keepList;
keepList = keepList.nextInList;
}
// detach the remainder of the list, and make it the new usedList.
if(endOfDiscardList != null)
{
endOfDiscardList.nextInList = null;
}
_usedList = keepList;
if(keepList == null)
{
_endOfUsedList = null;
}
}
finally { lock.unlock(); }
return discardList;
}
/**
* Replays a single message onto a store replayer.
* @param b The first block of the message.
* @param replayer The replayer to play this message on
* @return Success returns true, false if not ready or failure.
* @throws IOException
*/
private void replayOnto(Block b, StoreReplayer replayer)
throws IOException, DisconnectedException
{
// Lock is already held
// Clear fields we use
// If message isn't set by a Client, then must not matter what type
if (_message == null)
_message = new JSONMessage(StandardCharsets.UTF_8.newEncoder(),
StandardCharsets.UTF_8.newDecoder());
_message.reset();
long bPosition = b.getPosition();
_buffer.setPosition(bPosition);
int totalLength = _buffer.getInt()- Block.CHAIN_HEADER;
// Can occur if replaying during disconnect or for queue ack messages.
if(totalLength <=0)
{
return; // this can occur when replaying during a disconnect scenario.
}
// don't care about the next in chain -- we already read that.
_buffer.getInt();
_message.setSequence(_buffer.getLong());
_buffer.getLong(); // Skip CRC until ready
while (_buffer.getInt(bPosition + Block.BLOCK_READY_POSITION) == 0) {
// An interrupted exception is ignored to recheck
try { _messageReady.await(1000, TimeUnit.MILLISECONDS); }
catch (InterruptedException e) { }
}
_buffer.setPosition(bPosition + Block.BLOCK_CRC_POSITION);
long crcVal = _buffer.getLong();
// grab the operation, lengths, flag, and ack type
_buffer.setPosition(bPosition + Block.BLOCK_HEADER_SIZE);
int operation = _buffer.getInt();
if (operation == Message.Command.Unknown) return;
_message.setCommand(operation);
int commandIdLength = _buffer.getInt();
int correlationIdLength = _buffer.getInt();
int expirationLength = _buffer.getInt();
int sowKeyLength = _buffer.getInt();
int topicLength = _buffer.getInt();
int flag = _buffer.getInt();
_message.setAckType(_buffer.getInt());
_buffer.setPosition(bPosition + Block.BLOCK_HEADER_SIZE +
Block.CHAIN_HEADER);
byte[] array = null;
int offset = 0;
if(b.nextInChain == null)
{
// everything fit into one block. replay right out of the block memory.
_argument.len = totalLength;
_buffer.getBytes(_argument);
array = _argument.array;
offset = (int)_argument.offset;
} else
{
// Discontinuous data. we'll do this the hard way.
// allocate enough space to hold it contiguously. then, copy the pieces
// in place. then call up the replayer.
if(_readBuffer == null || _readBuffer.length < totalLength)
{
_readBuffer = new byte[totalLength];
}
int position =0;
for(Block part = b; part != null; part = part.nextInChain)
{
int maxReadable = Block.BLOCK_DATA_SIZE;
if(b!=part)
{
_buffer.setPosition(part.getPosition() + Block.BLOCK_HEADER_SIZE);
}
else // read chain and extended metadata
{
maxReadable -= Block.CHAIN_HEADER;
}
int readLen = totalLength - position > maxReadable ?
maxReadable: totalLength-position;
_argument.len = readLen;
try
{
_buffer.getBytes(_argument);
} catch(Exception e)
{
throw new IOException("Corrupted message found.", e);
}
System.arraycopy(_argument.array, (int)_argument.offset, _readBuffer, position, readLen);
position += readLen;
}
if(position != totalLength)
{
throw new IOException(String.format("Corrupted message found. Expected %d bytes, read %d bytes.", totalLength, position));
}
array = _readBuffer;
offset = 0;
}
if (_crc != null)
{
_crc.reset();
try
{
_crc.update(array, offset, totalLength);
}
catch (Exception e)
{
throw new IOException("Corrupted message found.", e);
}
if (_crc.getValue() != crcVal)
{
StringBuilder sb = new StringBuilder();
for (Block part = b; part != null; part = part.nextInChain)
{
sb.append(part);
sb.append('\n');
}
String message = String
.format("Corrupted message found. Block index %d, sequence %d, topic length %d, expected CRC %d, actual CRC %d, Block list: %s",
b.index, b.sequenceNumber, topicLength,
crcVal, _crc.getValue(),
sb.toString());
throw new IOException(message);
}
}
if (commandIdLength > 0)
{
_message.setCommandId(array, offset, commandIdLength);
totalLength -= commandIdLength;
offset += commandIdLength;
}
if (correlationIdLength > 0)
{
_message.getCorrelationIdRaw().set(array, offset, correlationIdLength);
totalLength -= correlationIdLength;
offset += correlationIdLength;
}
if (expirationLength > 0)
{
_message.getExpirationRaw().set(array, offset, expirationLength);
totalLength -= expirationLength;
offset += expirationLength;
}
if (sowKeyLength > 0)
{
_message.setSowKey(array, offset, sowKeyLength);
totalLength -= sowKeyLength;
offset += sowKeyLength;
}
if (topicLength > 0)
{
_message.setTopic(array, offset, topicLength);
totalLength -= topicLength;
offset += topicLength;
}
if (operation == Message.Command.Publish ||
operation == Message.Command.DeltaPublish)
{
_message.setData(array, offset, totalLength);
replayer.execute(_message);
}
else if (operation == Message.Command.SOWDelete) {
switch (flag) {
case Store.SOWDeleteByData:
{
_message.setData(array, offset, totalLength);
replayer.execute(_message);
}
break;
case Store.SOWDeleteByFilter:
{
_message.setFilter(array, offset, totalLength);
replayer.execute(_message);
}
break;
case Store.SOWDeleteByKeys:
{
_message.setSowKeys(array, offset, totalLength);
replayer.execute(_message);
}
break;
case Store.SOWDeleteByBookmark:
{
_message.setBookmark(array, offset, totalLength);
replayer.execute(_message);
}
break;
case Store.SOWDeleteByBookmarkCancel:
{
_message.setBookmark(array, offset, totalLength).setOptions("cancel");
replayer.execute(_message);
}
break;
default:
{
String message = String.format("SOWDelete message with invalid flag found. Block index %d, sequence %d, topic length %d, expected data length %d, operation %d, flag %d",
b.index, b.sequenceNumber, topicLength,
totalLength, operation, flag);
throw new IOException(message);
}
//break;
}
} else {
String message = String.format("Message with invalid operation found. Block index %d, sequence %d, topic length %d, expected data length %d, operation %d, flag %d",
b.index, b.sequenceNumber, topicLength,
totalLength, operation, flag);
throw new IOException(message);
}
}
public void replay(StoreReplayer replayer) throws StoreException, DisconnectedException
{
lock.lock();
try {
if (_usedList == null) return;
boolean corrupted = false;
StoreException ex = null;
Block lastGood = null;
Block nextInList = null;
// for each used block
for(Block b = _usedList; b!=null; b=nextInList)
{
nextInList = b.nextInList;
if (!corrupted)
{
try
{
if (_buffer.getInt(b.getPosition() +
Block.BLOCK_READY_POSITION) == 0)
{
// Replay needs to stop here
break;
}
replayOnto(b, replayer);
lastGood = b;
}
catch(IOException ioex)
{
corrupted = true;
_endOfUsedList = lastGood == null ? _usedList:lastGood;
ex = new StoreException("Exception during replay, ignoring all following blocks", ioex);
}
}
// Can't just do an else here, need to deal with first corrupted block
if (corrupted) // We're corrupt, let's clean up bad blocks
{
Block currentInChain = b;
while(currentInChain != null)
{
Block nextInChain = currentInChain.nextInChain;
try {
_buffer.zero(currentInChain.getPosition(),
Block.BLOCK_HEADER_SIZE);
} catch(IOException e) { } // ignore
currentInChain.sequenceNumber = 0;
currentInChain.nextInChain = null;
// and, flatten this chain onto the _freeList
currentInChain.nextInList = _freeList;
_freeList = currentInChain;
currentInChain = nextInChain;
}
}
}
if (corrupted)
{
throw ex;
}
}
finally { lock.unlock(); }
}
public boolean replaySingle(StoreReplayer replayer, long index)
throws StoreException, DisconnectedException
{
lock.lock();
try
{
if (index <= _metadataBlock.sequenceNumber)
{
long lowest = _getLowestUnpersisted();
if (lowest == -1) {
lowest = _metadataBlock.sequenceNumber;
}
else {
lowest -= 1;
}
// The Client and Store are out of sync or the Store
// isn't increasing by 1 at a time. This updates
// Client with correct least value.
if (_message == null)
_message = new JSONMessage(StandardCharsets.UTF_8.newEncoder(),
StandardCharsets.UTF_8.newDecoder());
_message.reset();
_message.setSequence(lowest);
return true;
}
// find the first one that equals this index.
for(Block b = this._usedList; b != null; b = b.nextInList)
{
if(b.sequenceNumber == index)
{
while (_buffer.getInt(b.getPosition()+Block.BLOCK_READY_POSITION)==0)
{
// An interrupted exception is ignored to recheck
try {
_messageReady.await(1000, TimeUnit.MILLISECONDS);
}
catch (InterruptedException e) { }
}
_buffer.setPosition(b.getPosition());
if (_buffer.getInt() <= 0) return false;
replayOnto(b, replayer);
return true;
}
}
return false;
}
catch (IOException ioex)
{
throw new StoreException(ioex);
}
finally { lock.unlock(); }
}
public long unpersistedCount()
{
lock.lock();
try {
if (_usedList == null) return 0;
return _endOfUsedList.sequenceNumber - _usedList.sequenceNumber + 1;
}
finally { lock.unlock(); }
}
Block findOrCreate(int index, Map allBlocks)
{
if(allBlocks.containsKey(index))
{
return allBlocks.get(index);
}
else
{
Block b= new Block((int)index);
allBlocks.put(index, b);
return b;
}
}
protected void recover() throws StoreException
{
lock.lock();
try {
_usedList = _freeList = null;
// Start here and work the buffer.
HashMap allBlocks = new HashMap(); // file index -> block
TreeMap heads = new TreeMap(); // sequence number --> block
try
{
long end = _buffer.getSize();
if (end == 0) return;
long minSeq = 0;
long maxSeq = 0;
for(long offset = 0; offset < end; offset+=Block.SIZE)
{
int index = (int)(offset/Block.SIZE);
_buffer.setPosition(offset);
int size = _buffer.getInt();
int nextIndex = _buffer.getInt();
long sequence = _buffer.getLong();
_buffer.getLong(); // ignore crcVal
int ready = _buffer.getInt();
// The first block will be the metadata block
if (index == 0)
{
_metadataBlock = new Block(index);
_metadataBlock.nextInChain = null;
_metadataBlock.nextInList = null;
_metadataBlock.sequenceNumber = sequence;
_nextSequence = sequence + 1;
minSeq = _nextSequence;
continue;
}
Block b = findOrCreate(index, allBlocks);
b.nextInChain = ((size == 0 || nextIndex == index) ? null : findOrCreate(nextIndex, allBlocks));
// All empty blocks are free. That's the rule.
if(size == 0)
{
b.nextInList = _freeList;
_freeList = b;
}
else if(sequence != 0)
{ // Only head blocks have their sequence number set.
if (ready == 0) // Block wasn't completed, add to free list
{
Block currentInChain = b;
while(currentInChain != null)
{
Block nextInChain = currentInChain.nextInChain;
_buffer.zero(currentInChain.getPosition(),
Block.BLOCK_HEADER_SIZE);
currentInChain.sequenceNumber = 0;
currentInChain.nextInChain = null;
// and, flatten this chain onto the free list
currentInChain.nextInList = _freeList;
_freeList = currentInChain;
currentInChain = nextInChain;
}
}
else
{
b.sequenceNumber = sequence;
if (sequence < minSeq) minSeq = sequence;
if (sequence > maxSeq) maxSeq = sequence;
heads.put(sequence, b);
}
}
// else, Belongs to part of a "chain"
}
if (_metadataBlock.sequenceNumber < (minSeq - 1))
_metadataBlock.sequenceNumber = minSeq - 1;
if (_nextSequence <= maxSeq) _nextSequence = maxSeq + 1;
// Need to put the heads onto the list in ascending order.
for(Entry e: heads.entrySet())
{
Block b = e.getValue();
b.nextInList = null;
if(_endOfUsedList != null)
{
_endOfUsedList.nextInList = b;
_endOfUsedList = b;
}
else
{
_usedList = _endOfUsedList = b;
}
}
}
catch(IOException ioex)
{
throw new StoreException(ioex);
}
}
finally { lock.unlock(); }
}
protected void growFreeListIfEmpty() throws StoreException
{
lock.lock();
try {
_growFreeListIfEmpty();
}
finally { lock.unlock(); }
}
private void _growFreeListIfEmpty() throws StoreException
{
// Lock must already be acquired.
while (_resizing) {
// An interrupted exception is ignored to recheck the time
try { _blocksFree.await(1000, TimeUnit.MILLISECONDS); }
catch(InterruptedException ex) { }
}
if (_freeList != null) return;
try {
long oldSize = _buffer.getSize();
long newSize = oldSize + ((long)Block.SIZE * (long)_blocksPerRealloc);
_resizing = true;
try {
while (_buffer.getSize() == oldSize)
{
boolean changeSize = true;
if (_resizeHandler != null)
{
lock.unlock();
try {
changeSize = _resizeHandler.invoke(this, newSize);
}
finally {
lock.lock();
}
}
// During unlock time, something may have freed some space.
if (_freeList != null) return;
// Unless resize handler denied us, increase size
if (changeSize) _buffer.setSize(newSize);
}
} finally {
_resizing = false;
}
if (_buffer.getSize() < newSize)
throw new StoreException("Publish store could not resize, possibly due to resize handler.");
int idx = (int) (oldSize / Block.SIZE);
if (idx == 0)
{
_metadataBlock = new Block(idx);
_metadataBlock.nextInChain = null;
_metadataBlock.nextInList = null;
_metadataBlock.sequenceNumber = 0;
long metadataPosition = _metadataBlock.getPosition();
_buffer.zero(metadataPosition, Block.SIZE);
_buffer.setPosition(metadataPosition +
METADATA_VERSION_LOCATION);
// Default to lowest version where first block is last discarded
int version;
try
{
version = Client.getVersionAsInt(Client.getVersion());
}
catch (CommandException ex)
{
// An exception will occur for develop branch
version = AMPS_MIN_PUB_STORE_DISCARDED_VERSION;
}
_buffer.putInt(version);
_buffer.setPosition(metadataPosition +
METADATA_LAST_DISCARDED_LOCATION);
_buffer.putLong(0);
++idx;
}
while (idx < (int) (newSize / Block.SIZE))
{
Block b = new Block(idx);
_buffer.zero(b.getPosition(), Block.SIZE);
b.nextInChain = null;
b.nextInList = _freeList;
_freeList = b;
++idx;
}
_blocksFree.signalAll();
} catch (IOException e)
{
throw new StoreException(e);
}
}
public void flush() throws TimedOutException
{
lock.lock();
try {
if (_usedList == null) return;
long current = _endOfUsedList.sequenceNumber;
while (_usedList != null && current >= _usedList.sequenceNumber)
{
try { _blocksFree.await(); }
catch (InterruptedException e) { }
}
}
finally { lock.unlock(); }
}
public void flush(long timeout) throws TimedOutException
{
lock.lock();
try {
if (_usedList == null) return;
long current = _endOfUsedList.sequenceNumber;
long start = System.currentTimeMillis();
long end;
while (_usedList != null && current >= _usedList.sequenceNumber)
{
end = System.currentTimeMillis();
if (timeout > 0 && (end - start) >= timeout)
throw new TimedOutException("Timed out waiting to flush publish store.");
try
{
long remaining = timeout - (end - start);
// An interrupted exception is ignored to recheck
_blocksFree.await(remaining, TimeUnit.MILLISECONDS);
}
catch (InterruptedException e) { }
}
}
finally { lock.unlock(); }
}
public void setResizeHandler(PublishStoreResizeHandler handler)
{
_resizeHandler = handler;
}
/********* PRIVATE METHODS *************/
private Block get(boolean assignSequence, long sequence)
throws StoreException
{
Block b = null;
lock.lock();
try {
_growFreeListIfEmpty();
b = _freeList;
if(b != null)
{
_freeList = b.nextInList;
b.nextInList = null;
if (assignSequence)
{
// Make sure we start at a good value
if (_nextSequence <= 1) _getLastPersisted();
b.sequenceNumber = _nextSequence++;
}
else if (sequence != 0)
{
b.sequenceNumber = sequence;
}
if (assignSequence || sequence != 0)
{
if(_endOfUsedList != null)
{
_endOfUsedList.nextInList = b;
_endOfUsedList = b;
}
else
{
// _endOfUsedList is null if _usedList is null, as well.
_endOfUsedList = _usedList = b;
}
}
}
}
finally { lock.unlock(); }
return b;
}
}
