Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2010-2021 60East Technologies Inc., All Rights Reserved.
//
// This computer software is owned by 60East Technologies Inc. and is
// protected by U.S. copyright laws and other laws and by international
// treaties. This computer software is furnished by 60East Technologies
// Inc. pursuant to a written license agreement and may be used, copied,
// transmitted, and stored only in accordance with the terms of such
// license agreement and with the inclusion of the above copyright notice.
// This computer software or any other copies thereof may not be provided
// or otherwise made available to any other person.
//
// U.S. Government Restricted Rights. This computer software: (a) was
// developed at private expense and is in all respects the proprietary
// information of 60East Technologies Inc.; (b) was not developed with
// government funds; (c) is a trade secret of 60East Technologies Inc.
// for all purposes of the Freedom of Information Act; and (d) is a
// commercial item and thus, pursuant to Section 12.212 of the Federal
// Acquisition Regulations (FAR) and DFAR Supplement Section 227.7202,
// Government's use, duplication or disclosure of the computer software
// is subject to the restrictions set forth by 60East Technologies Inc..
//
////////////////////////////////////////////////////////////////////////////
package com.crankuptheamps.client;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import com.crankuptheamps.client.exception.AMPSException;
import com.crankuptheamps.client.exception.CommandException;
import com.crankuptheamps.client.fields.BookmarkField;
import com.crankuptheamps.client.fields.BookmarkRangeField;
import com.crankuptheamps.client.fields.Field;
/**
* RingBookmarkStore is an implementation of BookmarkStore that stores state on disk and in memory.
* For each subscription, RingBookmarkStore keeps the most recent bookmark B for which all
* messages prior to B have been discard()ed by the subscriber. As messages are logged and
* discarded, an in-memory array is used to track when a new bookmark can be logged.
*/
public class RingBookmarkStore implements BookmarkStore
{
// Manages the portion of the store for a single subscription.
// RingBookmarkStore uses a memory mapped file for persistent storage.
// Each subscription has four elements: one subId of size BYTES_SUBID,
// and three bookmarks of size BYTES_BOOKMARK.
// On disk, each bookmark entry begins with one byte designating the next bookmark entry
// to be written. The value '*', aka the Cursor, in this position indicates this is the oldest bookmark
// stored for the subscription, and is the next to be written.
// Writing always follows a predictable sequence: the '*' for the next entry is written,
// the bookmark id is written to the current entry, and then the cursor is erased for the current entry.
// On recovery, the entry *before* the earliest one with the '*' is the most recently
// written: in case of crash during write, there may be two entries with a '*', and the third one is the one to recover.
// In memory, each Subscription has an array of entry's. Each entry holds a bookmark and whether or not it has been discard()ed.
// Whenever the 0th entry in the array is discarded, we know it's time to write a bookmark to disk.
// We truncate the front of the array up through the last one that is inactive, and then write the next one to disk.
/**
* The Subscription object is used to represent internal bookmark state
* for the messages received and discarded on a specific subscription
* within the bookmark store.
*/
protected static class Subscription implements com.crankuptheamps.client.Subscription
{
// The buffer holding the memory-mapped file of all subscriptions
ByteBuffer _buffer;
// The offset into _buffer where our subscription begins.
int _offset;
// The disk entry to be written next.
short _currentDiskPosition = 0;
BookmarkRingBuffer _ring = new BookmarkRingBuffer();
// On-disk bytes for a bookmark entry.
static final int BYTES_BOOKMARK = BookmarkField.MAX_BOOKMARK_LENGTH*6+8;
// On-disk bytes for each entry
static final int BYTES_ENTRY = 1024;
// Number of bookmarks stored per subscription. Must be at least 3 for recovery guarantee.
static final short POSITIONS = 3;
// On-disk bytes for the subscription ID.
static final int BYTES_SUBID = BYTES_ENTRY-(BYTES_BOOKMARK * POSITIONS);
private BookmarkRangeField _range = new BookmarkRangeField();
// The Subscription lock
final Lock _lock = new ReentrantLock();
public Subscription()
{
}
public void init(ByteBuffer buffer, int offset) throws AMPSException
{
_buffer = buffer;
_offset = offset;
recover();
}
// A new bookmark has arrived for this subscription. Just remember it:
// nothing to write about until the message is discarded.
public long log(BookmarkField bookmark) throws CommandException
{
_lock.lock();
try {
if (!bookmark.isRange()) {
long seq = _ring.log(bookmark);
while (seq == 0) {
_lock.unlock();
try {
_ring.checkResize();
}
finally {
_lock.lock();
}
seq = _ring.log(bookmark);
}
return seq;
}
else {
_range.copyFrom(bookmark);
if (!_range.isValid()) {
_range.reset();
throw new CommandException("Invalid bookmark range specified");
}
long seq = 0;
// Only exclusive start does log/discard
if (_range.isStartExclusive()) {
// Log/discard in ring directly
seq = _ring.log(_range.getStart());
_ring.discard(seq);
}
// Write the range in case we disconnect with no messages
write(_range);
return seq;
}
}
finally {
_lock.unlock();
}
}
// This is never called but required by the interface
public boolean isDiscarded(BookmarkField bookmark)
{
return false;
}
// User is finished with a bookmark. Mark the entry as inactive,
// and if it's the 0th entry, forward-truncate and log the most recent.
public void discard(long bookmark)
{
_lock.lock();
try {
// If discard changes most recent, save updated recent
if(_ring.discard(bookmark))
{
// If we have a range, update that
if (_range.isValid()) {
Field recent = _ring.getLastDiscarded();
if (recent.length > 1
&& (_range.isStartInclusive()
|| !recent.equals(_range.getStart())))
{
_range.replaceStart(recent, true);
write(_range);
}
}
else {
write(_ring.getLastDiscarded());
}
}
}
finally {
_lock.unlock();
}
}
public BookmarkRangeField getRange()
{
_lock.lock();
try {
return _range.copy();
}
finally {
_lock.unlock();
}
}
public Field getMostRecent()
{
_lock.lock();
try {
return _ring.getLastDiscarded();
}
finally {
_lock.unlock();
}
}
public Field getMostRecentList(boolean useList)
{
_lock.lock();
try {
if (_range.isValid()) {
return _range;
}
else {
return _ring.getLastDiscarded();
}
}
finally {
_lock.unlock();
}
}
private void write(Field bookmark)
{
_lock.lock();
try {
// We want to write to _currentDiskPosition.
short nextDiskPosition = (short)((_currentDiskPosition + 1) % POSITIONS);
// Mark the next position with the 'cursor'
_buffer.put( _offset + BYTES_SUBID + (BYTES_BOOKMARK * nextDiskPosition), (byte)'*');
// write the current position and validate it
_buffer.position ( _offset + BYTES_SUBID + (BYTES_BOOKMARK * _currentDiskPosition) + 1 );
for(int i = 0; i < bookmark.length; i++)
{
_buffer.put(bookmark.byteAt(i));
}
for(int i = 0; i < BYTES_BOOKMARK - (bookmark.length+2); i++)
{
_buffer.put((byte)0);
}
_buffer.put( _offset + BYTES_SUBID + (BYTES_BOOKMARK * _currentDiskPosition), (byte)'+');
// advance _currentDiskPosition
_currentDiskPosition = nextDiskPosition;
}
finally {
_lock.unlock();
}
}
private void recover() throws AMPSException
{
// find the first cursor
short foundCursor = 0;
for(; foundCursor < POSITIONS; foundCursor++)
{
byte b = _buffer.get( _offset + BYTES_SUBID + (BYTES_BOOKMARK * foundCursor));
if(b == (byte)'*') break;
}
if(foundCursor == 0)
{
byte b = _buffer.get(_offset + BYTES_SUBID + (BYTES_BOOKMARK * (POSITIONS - 1)));
if(b==(byte)'*')
{
foundCursor = POSITIONS - 1;
}
}
if(foundCursor < POSITIONS)
{
// Found an existing "cursor": start the writing there.
_currentDiskPosition = foundCursor;
int mostRecentValid = _currentDiskPosition==0?POSITIONS-1:_currentDiskPosition-1;
byte[] buf = new byte[BYTES_BOOKMARK-1];
_buffer.position(_offset + BYTES_SUBID + (BYTES_BOOKMARK * mostRecentValid) + 1);
_buffer.get(buf);
int bookmarkLength = 0;
for(; bookmarkLength< buf.length && buf[bookmarkLength] != 0; bookmarkLength++);
try
{
BookmarkField f = new BookmarkField();
f.set(buf, 0, bookmarkLength);
if (f.isRange()) {
log(f);
}
else {
// discard and log to make this the
// "starting point" for the subscription.
_ring.discard(_ring.log(f));
}
}
catch (Exception e)
{
throw new AMPSException("Error while recovering.",e);
}
}
else
{
_currentDiskPosition = 0;
}
}
@Deprecated
public void setLastPersisted(long bookmark)
{
// no-op
}
public void setLastPersisted(BookmarkField bookmark)
{
// no-op
}
public long getOldestBookmarkSeq()
{
return _ring.getStartIndex();
}
public void setResizeHandler(BookmarkStoreResizeHandler handler, BookmarkStore store)
{
_ring.setResizeHandler(handler, store);
}
}
MappedByteBuffer _buffer;
static final int ENTRIES=16384;
static final int ENTRY_SIZE = Subscription.BYTES_SUBID + (Subscription.POSITIONS * Subscription.BYTES_BOOKMARK);
HashMap _map;
// Initialize to ENTRIES, so that there is no free space in the log until recover() has succeeded.
int _free = ENTRIES;
RandomAccessFile _file;
FileChannel _channel;
String _path;
BookmarkStoreResizeHandler _resizeHandler = null;
private int _serverVersion = Message.MINIMUM_SERVER_VERSION;
// The Store lock
final Lock _lock = new ReentrantLock();
Pool _pool;
public RingBookmarkStore(String path) throws AMPSException
{
this(path, 1);
}
public RingBookmarkStore(String path, int targetNumberOfSubscriptions) throws AMPSException
{
try
{
_path = path;
_file = new RandomAccessFile(_path, "rw");
_pool = new Pool(Subscription.class, targetNumberOfSubscriptions);
}
catch (IOException ioex)
{
throw new AMPSException("I/O Error initializing file " + path, ioex);
}
init();
}
/**
* Called internally by the Client to log a bookmark to the persistent log.
* @param message AMPS Message
* @return Returns bookmark to be logged.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public long log(Message message) throws AMPSException
{
BookmarkField bookmark = (BookmarkField)message.getBookmarkRaw();
Subscription sub = (RingBookmarkStore.Subscription)message.getSubscription();
if (sub == null)
{
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
long seqNo = sub.log(bookmark);
message.setBookmarkSeqNo(seqNo);
return seqNo;
}
/**
* Call this when you want to mark the message specified by the Subscription Id and bookmark sequence number as discarded,
* indicating that the application has completed processing the message. Marking a message as discarded means that the
* message will not be replayed when the subscription resumes.
* @param subId The subscription ID of the message.
* @param bookmarkSeqNo The bookmark sequence number.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public void discard(Field subId, long bookmarkSeqNo) throws AMPSException
{
find(subId).discard(bookmarkSeqNo);
}
/**
* Call this when you want to mark the provided message as discarded, indicating that the application has completed
* processing the message. Marking a message as discarded means that the message will not be replayed when the
* subscription resumes.
* @param message Message to be marked as discarded.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public void discard(Message message) throws AMPSException
{
long bookmark = message.getBookmarkSeqNo();
Subscription sub = (RingBookmarkStore.Subscription)message.getSubscription();
if (sub == null)
{
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
sub.discard(bookmark);
}
/**
* Call this when you want to return to the most recent bookmark from the log that should be used for (re-)subscriptions.
* @param subId Subscription Id
* @return Returns most recent bookmark.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public Field getMostRecent(Field subId) throws AMPSException
{
return find(subId).getMostRecentList(true).copy();
}
/**
* Call this when you want to return to the most recent bookmark from the log that should be used for (re-)subscriptions.
* @param subId Subscription Id
* @param useList Ignored by this type of store.
* @return Returns most recent bookmark.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public Field getMostRecent(Field subId, boolean useList) throws AMPSException
{
return find(subId).getMostRecentList(useList).copy();
}
/**
* Called for each arriving message to determine if the application has already processed and discarded the message.
* Generally isDiscarded is called by the Client however, if needed it can be called by the application as well.
* @param message Message used to determine if the application has already
* @return Returns 'true' if the bookmark is in the log and marked as discarded. Otherwise, returns 'false'.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public boolean isDiscarded(Message message) throws AMPSException
{
BookmarkField bookmark = (BookmarkField)message.getBookmarkRaw();
Subscription sub = (RingBookmarkStore.Subscription)message.getSubscription();
if (sub == null)
{
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
return sub.isDiscarded(bookmark);
}
private void recover() throws AMPSException
{
int currentEntry = 0;
for(; currentEntry < ENTRIES; currentEntry++)
{
// is it active?
byte firstByte = _buffer.get(currentEntry * ENTRY_SIZE);
if(firstByte != 0)
{
// read it out
byte[] id = new byte[Subscription.BYTES_SUBID - 1];
_buffer.position(currentEntry * ENTRY_SIZE);
_buffer.get(id);
int idLength = 0;
for(; idLength < id.length && id[idLength] != 0; idLength++);
Field f = new Field(id, 0, idLength);
Subscription subscription = _pool.get();
subscription.init(_buffer, currentEntry * ENTRY_SIZE);
try
{
_map.put(f, subscription);
}
catch (Exception e)
{
throw new AMPSException("Bookmark store corrupted.", e);
}
subscription.recover();
}
else break;
}
if(currentEntry == ENTRIES)
{
// todo: resize here
throw new AMPSException("Unable to allocate space in this bookmark store.");
}
_free = currentEntry;
}
/**
* Finds and returns the Subscription object for the specified
* subscription id (subId).
* @param subId The subid to find or create a Subscription for.
* @return The Subscription associated with the subId.
* @throws AMPSException If there is an error adding to the file.
*/
protected Subscription find(Field subId) throws AMPSException
{
_lock.lock();
try {
if(_map.containsKey(subId))
{
return _map.get(subId);
}
if(_free >= ENTRIES)
{
throw new AMPSException("Unable to allocate space in this bookmark store.");
}
int pos = _free++;
Subscription subscription = _pool.get();
subscription.init(_buffer, pos * ENTRY_SIZE);
subscription.setResizeHandler(_resizeHandler, this);
_map.put(subId.copy(), subscription);
_buffer.position(pos * ENTRY_SIZE);
for(int i = 0; i < subId.length; i++)
{
_buffer.put(subId.buffer[subId.position+i]);
}
return subscription;
}
finally {
_lock.unlock();
}
}
/**
* Called internally by the Client to mark the message as safely persisted by AMPS to all of its sync replication
* destinations. On this store, this method has no effect.
* @param subId The subscription ID for the message.
* @param bookmark The bookmark containing the message.
* @throws AMPSException Not thrown by this implementation.
*/
public void persisted(Field subId, BookmarkField bookmark) throws AMPSException
{
// no-op
}
/**
* Old style of setting a persisted bookmark no longer used.
* destinations. On this store, this method has no effect.
* @param subId The subscription ID for the message.
* @param bookmark The bookmark number.
* @throws AMPSException Not thrown by this implementation.
* @deprecated use {@link #persisted(Field, BookmarkField)} instead.
*/
@Deprecated
public void persisted(Field subId, long bookmark) throws AMPSException
{
// no-op
}
/**
* Call this when you want to retrieve the sequence number for the oldest bookmark in the store.
* @param subId The subscription ID for the oldest bookmark in the store.
* @return Returns the oldest bookmark sequence number in the store.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public long getOldestBookmarkSeq(Field subId) throws AMPSException
{
return find(subId).getOldestBookmarkSeq();
}
/**
* Call this when you want to set a resize handler that is invoked when the store needs to resize.
* @param handler The handler to invoke for the resize.
*/
public void setResizeHandler(BookmarkStoreResizeHandler handler)
{
_resizeHandler = handler;
Iterator it = _map.entrySet().iterator();
while (it.hasNext())
{
Map.Entry pairs = (Map.Entry)it.next();
((Subscription)pairs.getValue()).setResizeHandler(handler, this);
}
}
/**
* Call this when you want to purge the contents of this store. Removes any tracking history associated with publishers
* and received messages, and may delete or trunkate on-disk representations as well.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public void purge() throws AMPSException
{
_lock.lock();
try {
// This is the quickest way I've found, to zero out the file.
int longs = _buffer.capacity()/8;
_buffer.position(0);
for(int i =0; i < longs; i++)
{
_buffer.putLong(0);
}
_buffer.position(0);
_buffer.force();
_map = new HashMap();
_free = 0;
recover();
}
finally {
_lock.unlock();
}
}
/**
* Call this when you want to purge the contents of this store for a given Subscription Id. Removes any tracking
* history associated with publishers and received messages, and may delete or truncate on-disk representations as
* well.
* @param subId_ The identifier of the subscription to purge.
* @throws AMPSException Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public void purge(Field subId_) throws AMPSException
{
_lock.lock();
try {
if (!_map.containsKey(subId_)) return;
Subscription sub = _map.get(subId_);
int pos = sub._offset/ENTRY_SIZE;
// NULL out the subs data
byte[] buf = new byte[ENTRY_SIZE];
_buffer.position(sub._offset);
_buffer.put(buf);
// Move all subs back one to fill the hole
--_free;
Field subId = new Field();
for (int j = pos; j < _free; ++j)
{
int i = j * ENTRY_SIZE;
// Read the next entry
_buffer.position((j+1)*ENTRY_SIZE);
_buffer.get(buf);
// Write it to current location
_buffer.position(i);
_buffer.put(buf);
int idLength = 0;
for (; idLength < Subscription.BYTES_SUBID && buf[idLength] != 0;
idLength++);
subId.set(buf, 0, idLength);
_map.get(subId)._offset = i;
}
// NULL out the end and reset the end
int i = _free*ENTRY_SIZE;
_buffer.position(i);
for (; i < (_free+1)*ENTRY_SIZE; ++i)
{
_buffer.put((byte)0);
}
_map.remove(subId_);
}
finally {
_lock.unlock();
}
}
private void init() throws AMPSException
{
try
{
_channel = _file.getChannel();
_buffer = _channel.map(FileChannel.MapMode.READ_WRITE, 0, ENTRIES*ENTRY_SIZE);
_channel.close();
_file.close();
}
catch (IOException ioex)
{
throw new AMPSException("error opening store.", ioex);
}
_map = new HashMap();
_free = 0;
recover();
}
/**
* Called internally by the Client when connected to an AMPS server to indicate what version the server is.
* @param version Version number
*/
public void setServerVersion(int version)
{
_serverVersion = version;
}
/**
* Called internally by the Client to return the server version detected upon logon.
* @return The server version.
*/
public int getServerVersion()
{
return _serverVersion;
}
/**
* In order to unmap the memory used to store the state, this method closes the mapped byte
* buffer.
* @throws Exception Thrown when an operation on the store fails. The exception will contain details of the failure.
*/
public void close() throws Exception
{
// NOTE: this is an imprecise mechanism for closing a mapped byte buffer;
// The JDK does not yet provide a deterministic way to unmap memory, since it
// may be referred to by other objects. We can force a write and gc, but it
// is likely that the underlying file will still be mapped.
_buffer.force();
_buffer = null;
_map = null;
//System.gc();
}
}
