Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2010-2024 60East Technologies Inc., All Rights Reserved.
//
// This computer software is owned by 60East Technologies Inc. and is
// protected by U.S. copyright laws and other laws and by international
// treaties. This computer software is furnished by 60East Technologies
// Inc. pursuant to a written license agreement and may be used, copied,
// transmitted, and stored only in accordance with the terms of such
// license agreement and with the inclusion of the above copyright notice.
// This computer software or any other copies thereof may not be provided
// or otherwise made available to any other person.
//
// U.S. Government Restricted Rights. This computer software: (a) was
// developed at private expense and is in all respects the proprietary
// information of 60East Technologies Inc.; (b) was not developed with
// government funds; (c) is a trade secret of 60East Technologies Inc.
// for all purposes of the Freedom of Information Act; and (d) is a
// commercial item and thus, pursuant to Section 12.212 of the Federal
// Acquisition Regulations (FAR) and DFAR Supplement Section 227.7202,
// Government's use, duplication or disclosure of the computer software
// is subject to the restrictions set forth by 60East Technologies Inc..
//
////////////////////////////////////////////////////////////////////////////
package com.crankuptheamps.client;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import com.crankuptheamps.client.exception.AMPSException;
import com.crankuptheamps.client.exception.CommandException;
import com.crankuptheamps.client.fields.BookmarkField;
import com.crankuptheamps.client.fields.BookmarkRangeField;
import com.crankuptheamps.client.fields.Field;
/**
* RingBookmarkStore is an implementation of BookmarkStore that stores state on
* disk and in memory. This BookmarkStore implementation is designed for use
* when messages will be strictly discarded in the order in which they are
* received. Discarding messages out of order may result in duplicate or missed
* messages on failover. For stores that support out of order discard, see
* (links) LoggedBookmarkStore or MemoryBookmarkStore. ( For each subscription,
* RingBookmarkStore keeps the most recent bookmark B for which all messages
* prior to B have been discard()ed by the subscriber. As messages are logged
* and discarded, an in-memory array is used to track when a new bookmark can be
* logged.
*/
public class RingBookmarkStore implements BookmarkStore {
// Manages the portion of the store for a single subscription.
// RingBookmarkStore uses a memory mapped file for persistent storage.
// Each subscription has four elements: one subId of size BYTES_SUBID,
// and three bookmarks of size BYTES_BOOKMARK.
// On disk, each bookmark entry begins with one byte designating the next
// bookmark entry
// to be written. The value '*', aka the Cursor, in this position indicates this
// is the oldest bookmark
// stored for the subscription, and is the next to be written.
// Writing always follows a predictable sequence: the '*' for the next entry is
// written,
// the bookmark id is written to the current entry, and then the cursor is
// erased for the current entry.
// On recovery, the entry *before* the earliest one with the '*' is the most
// recently
// written: in case of crash during write, there may be two entries with a '*',
// and the third one is the one to recover.
// In memory, each Subscription has an array of entry's. Each entry holds a
// bookmark and whether or not it has been discard()ed.
// Whenever the 0th entry in the array is discarded, we know it's time to write
// a bookmark to disk.
// We truncate the front of the array up through the last one that is inactive,
// and then write the next one to disk.
/**
* The Subscription object is used to represent internal bookmark state
* for the messages received and discarded on a specific subscription
* within the bookmark store.
*/
protected static class Subscription implements com.crankuptheamps.client.Subscription {
// The buffer holding the memory-mapped file of all subscriptions
ByteBuffer _buffer;
// The offset into _buffer where our subscription begins.
int _offset;
// The disk entry to be written next.
short _currentDiskPosition = 0;
BookmarkRingBuffer _ring = new BookmarkRingBuffer();
// On-disk bytes for a bookmark entry.
static final int BYTES_BOOKMARK = BookmarkField.MAX_BOOKMARK_LENGTH * 6 + 8;
// On-disk bytes for each entry
static final int BYTES_ENTRY = 1024;
// Number of bookmarks stored per subscription. Must be at least 3 for recovery
// guarantee.
static final short POSITIONS = 3;
// On-disk bytes for the subscription ID.
static final int BYTES_SUBID = BYTES_ENTRY - (BYTES_BOOKMARK * POSITIONS);
private BookmarkRangeField _range = new BookmarkRangeField();
// The Subscription lock
final Lock _lock = new ReentrantLock();
public Subscription() {
}
public void init(ByteBuffer buffer, int offset) throws AMPSException {
_buffer = buffer;
_offset = offset;
recover();
}
// A new bookmark has arrived for this subscription. Just remember it:
// nothing to write about until the message is discarded.
public long log(BookmarkField bookmark) throws CommandException {
_lock.lock();
try {
if (!bookmark.isRange()) {
long seq = _ring.log(bookmark);
while (seq == 0) {
_lock.unlock();
try {
_ring.checkResize();
} finally {
_lock.lock();
}
seq = _ring.log(bookmark);
}
return seq;
} else {
_range.copyFrom(bookmark);
if (!_range.isValid()) {
_range.reset();
throw new CommandException("Invalid bookmark range specified");
}
long seq = 0;
// Only exclusive start does log/discard
if (_range.isStartExclusive()) {
// Log/discard in ring directly
seq = _ring.log(_range.getStart());
_ring.discard(seq);
}
// Write the range in case we disconnect with no messages
write(_range);
return seq;
}
} finally {
_lock.unlock();
}
}
// This is never called but required by the interface
public boolean isDiscarded(BookmarkField bookmark) {
return false;
}
// User is finished with a bookmark. Mark the entry as inactive,
// and if it's the 0th entry, forward-truncate and log the most recent.
public void discard(long bookmark) {
_lock.lock();
try {
// If discard changes most recent, save updated recent
if (_ring.discard(bookmark)) {
// If we have a range, update that
if (_range.isValid()) {
Field recent = _ring.getLastDiscarded();
if (recent.length > 1
&& (_range.isStartInclusive()
|| !recent.equals(_range.getStart()))) {
_range.replaceStart(recent, true);
write(_range);
}
} else {
write(_ring.getLastDiscarded());
}
}
} finally {
_lock.unlock();
}
}
public BookmarkRangeField getRange() {
_lock.lock();
try {
return _range.copy();
} finally {
_lock.unlock();
}
}
public Field getMostRecent() {
_lock.lock();
try {
return _ring.getLastDiscarded();
} finally {
_lock.unlock();
}
}
public Field getMostRecentList(boolean useList) {
_lock.lock();
try {
if (_range.isValid()) {
return _range;
} else {
return _ring.getLastDiscarded();
}
} finally {
_lock.unlock();
}
}
private void write(Field bookmark) {
_lock.lock();
try {
// We want to write to _currentDiskPosition.
short nextDiskPosition = (short) ((_currentDiskPosition + 1) % POSITIONS);
// Mark the next position with the 'cursor'
_buffer.put(_offset + BYTES_SUBID + (BYTES_BOOKMARK * nextDiskPosition), (byte) '*');
// write the current position and validate it
_buffer.position(_offset + BYTES_SUBID + (BYTES_BOOKMARK * _currentDiskPosition) + 1);
for (int i = 0; i < bookmark.length; i++) {
_buffer.put(bookmark.byteAt(i));
}
for (int i = 0; i < BYTES_BOOKMARK - (bookmark.length + 2); i++) {
_buffer.put((byte) 0);
}
_buffer.put(_offset + BYTES_SUBID + (BYTES_BOOKMARK * _currentDiskPosition), (byte) '+');
// advance _currentDiskPosition
_currentDiskPosition = nextDiskPosition;
} finally {
_lock.unlock();
}
}
private void recover() throws AMPSException {
// find the first cursor
short foundCursor = 0;
for (; foundCursor < POSITIONS; foundCursor++) {
byte b = _buffer.get(_offset + BYTES_SUBID + (BYTES_BOOKMARK * foundCursor));
if (b == (byte) '*')
break;
}
if (foundCursor == 0) {
byte b = _buffer.get(_offset + BYTES_SUBID + (BYTES_BOOKMARK * (POSITIONS - 1)));
if (b == (byte) '*') {
foundCursor = POSITIONS - 1;
}
}
if (foundCursor < POSITIONS) {
// Found an existing "cursor": start the writing there.
_currentDiskPosition = foundCursor;
int mostRecentValid = _currentDiskPosition == 0 ? POSITIONS - 1 : _currentDiskPosition - 1;
byte[] buf = new byte[BYTES_BOOKMARK - 1];
_buffer.position(_offset + BYTES_SUBID + (BYTES_BOOKMARK * mostRecentValid) + 1);
_buffer.get(buf);
int bookmarkLength = 0;
for (; bookmarkLength < buf.length && buf[bookmarkLength] != 0; bookmarkLength++)
;
try {
BookmarkField f = new BookmarkField();
f.set(buf, 0, bookmarkLength);
if (f.isRange()) {
log(f);
} else {
// discard and log to make this the
// "starting point" for the subscription.
_ring.discard(_ring.log(f));
}
} catch (Exception e) {
throw new AMPSException("Error while recovering.", e);
}
} else {
_currentDiskPosition = 0;
}
}
@Deprecated
public void setLastPersisted(long bookmark) {
// no-op
}
public void setLastPersisted(BookmarkField bookmark) {
// no-op
}
public long getOldestBookmarkSeq() {
return _ring.getStartIndex();
}
public void setResizeHandler(BookmarkStoreResizeHandler handler, BookmarkStore store) {
_ring.setResizeHandler(handler, store);
}
}
// A memory-mapped buffer used for in-memory data storage.
MappedByteBuffer _buffer;
static final int ENTRIES = 16384;
// This constant defines the size of each entry in the store. It is computed
// based on the subscription ID size and the size of multiple bookmarks from
// the `Subscription` class.
static final int ENTRY_SIZE = Subscription.BYTES_SUBID + (Subscription.POSITIONS * Subscription.BYTES_BOOKMARK);
// A hash map used to associate fields with subscription objects
HashMap _map;
// An integer that tracks the number of free entries in the log. It is
// initialized to `ENTRIES` to ensure that there is no free space in the log
// until the `recover()` method succeeds.
int _free = ENTRIES;
// A random access file that facilitates reading from and writing to a file on
// disk
RandomAccessFile _file;
FileChannel _channel;
// A string containing the file path where data is stored.
String _path;
// An optional handler that can be used to manage resizing of the bookmark
// store.
BookmarkStoreResizeHandler _resizeHandler = null;
// A private attribute that holds the server version. Its default value is set
// to `Message.MINIMUM_SERVER_VERSION`.
private int _serverVersion = Message.MINIMUM_SERVER_VERSION;
// The Store lock
// A lock object used to synchronize access to the `RingBookmarkStore` instance,
// ensuring thread safety.
final Lock _lock = new ReentrantLock();
// A pool used to manage subscription objects. The pool is designed to reduce
// memory overhead by allowing the reuse of subscription instances.
Pool _pool;
/**
* Initializes a `RingBookmarkStore` object with the provided file path. It
* internally calls another constructor with a default target number of
* subscriptions (`targetNumberOfSubscriptions` set to 1).
*
* @param path The path to the file.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public RingBookmarkStore(String path) throws AMPSException {
this(path, 1);
}
/**
* Initializes a `RingBookmarkStore` object with a specified file path and a
* target number of subscriptions. It creates a random access file, initializes
* the subscription pool (`_pool`), and calls the `init()` method to set up
* the object. If any I/O errors occur during initialization, an `AMPSException`
* is thrown with details of the error.
*
* @param path The path to the file.
* @param targetNumberOfSubscriptions The target number of subscriptions.
* @throws AMPSException Thrown when an operation on the store fails.
* The exception will contain details of the failure.
*/
public RingBookmarkStore(String path, int targetNumberOfSubscriptions) throws AMPSException {
try {
_path = path;
_file = new RandomAccessFile(_path, "rw");
_pool = new Pool(Subscription.class, targetNumberOfSubscriptions);
} catch (IOException ioex) {
throw new AMPSException("I/O Error initializing file " + path, ioex);
}
init();
}
/**
* This method is called internally by the AMPS Client to log a bookmark to
* the persistent log. It takes an `AMPS Message` as an argument and returns
* a bookmark to be logged. If a subscription is not associated with the
* message, it attempts to find or create one using the message's subscription
* ID. The method logs the bookmark and assigns a sequence number to it. If any
* operation on the store fails, it throws an `AMPSException` with details
* of the failure.
*
* @param message The AMPS Message.
* @return Returns bookmark to be logged.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public long log(Message message) throws AMPSException {
BookmarkField bookmark = (BookmarkField) message.getBookmarkRaw();
Subscription sub = (RingBookmarkStore.Subscription) message.getSubscription();
if (sub == null) {
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
long seqNo = sub.log(bookmark);
message.setBookmarkSeqNo(seqNo);
return seqNo;
}
/**
* Marks the message specified by the Subscription ID and bookmark sequence
* number as discarded, indicating that the application has completed processing
* the message.
* Marking a message as discarded means that the message will not be replayed
* when the subscription resumes. This method is intended for internal use by
* the AMPS Client to manage message processing. Application code typically does
* not need to call this method directly.
*
* @param subId The subscription ID of the message.
* @param bookmarkSeqNo The bookmark sequence number.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public void discard(Field subId, long bookmarkSeqNo) throws AMPSException {
find(subId).discard(bookmarkSeqNo);
}
/**
* Call this when you want to mark the provided message as discarded, indicating
* that the application has completed processing the message. Marking a message
* as discarded means that the message will not be replayed when the
* subscription resumes.
*
* @param message Message to be marked as discarded.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public void discard(Message message) throws AMPSException {
long bookmark = message.getBookmarkSeqNo();
Subscription sub = (RingBookmarkStore.Subscription) message.getSubscription();
if (sub == null) {
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
sub.discard(bookmark);
}
/**
* Call this when you want to return to the most recent bookmark from the log
* that should be used for (re-)subscriptions based on the provided subscription
* ID.
*
* @param subId Subscription Id
* @return Returns most recent bookmark.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public Field getMostRecent(Field subId) throws AMPSException {
return find(subId).getMostRecentList(true).copy();
}
/**
* Call this when you want to return to the most recent bookmark from the log
* that should be used for (re-)subscriptions based on the provided subscription
* ID.
*
* @param subId Subscription Id
* @param useList Ignored by this type of store.
* @return Returns most recent bookmark.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public Field getMostRecent(Field subId, boolean useList) throws AMPSException {
return find(subId).getMostRecentList(useList).copy();
}
/**
* Called for each arriving message to determine if the application has already
* processed and discarded the message. Generally isDiscarded is called by the
* Client however, if needed it can be called by the application as well.
*
* @param message Message used to determine if the application has already
* @return Returns 'true' if the bookmark is in the log and marked as discarded.
* Otherwise, returns 'false'.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public boolean isDiscarded(Message message) throws AMPSException {
BookmarkField bookmark = (BookmarkField) message.getBookmarkRaw();
Subscription sub = (RingBookmarkStore.Subscription) message.getSubscription();
if (sub == null) {
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
return sub.isDiscarded(bookmark);
}
// Recovers subscription information from the persistent store during
// initialization. This method iterates through the entries in the store,
// checks if an entry is active, and initializes a `Subscription` object for
// each active entry. The `Subscription` objects are added to a map for future
// reference.
private void recover() throws AMPSException {
int currentEntry = 0;
for (; currentEntry < ENTRIES; currentEntry++) {
// Check if the entry is active
byte firstByte = _buffer.get(currentEntry * ENTRY_SIZE);
if (firstByte != 0) {
// Read subscription information
byte[] id = new byte[Subscription.BYTES_SUBID - 1];
_buffer.position(currentEntry * ENTRY_SIZE);
_buffer.get(id);
int idLength = 0;
for (; idLength < id.length && id[idLength] != 0; idLength++)
;
// Create and initialize a Subscription object
Field f = new Field(id, 0, idLength);
Subscription subscription = _pool.get();
subscription.init(_buffer, currentEntry * ENTRY_SIZE);
try {
_map.put(f, subscription);
} catch (Exception e) {
throw new AMPSException("Bookmark store corrupted.", e);
}
subscription.recover();
} else
break;
}
if (currentEntry == ENTRIES) {
// Handle case where no space is available. todo: resize here.
throw new AMPSException("Unable to allocate space in this bookmark store.");
}
_free = currentEntry;
}
/**
* Finds and returns the `Subscription` object associated with the specified
* subscription ID (`subId`). If the `subId` exists in the store, it returns
* the existing `Subscription` object. If the `subId` is not found, it checks
* if there is space available to create a new `Subscription` object and adds
* it to the store.
*
* @param subId The subscription ID for which to find or create a Subscription.
* @return The Subscription associated with the specified subId.
* @throws AMPSException Thrown when there is an error in the bookmark store or
* when there is insufficient space to create a new
* Subscription.
*/
protected Subscription find(Field subId) throws AMPSException {
_lock.lock();
try {
if (_map.containsKey(subId)) {
return _map.get(subId);
}
if (_free >= ENTRIES) {
throw new AMPSException("Unable to allocate space in this bookmark store.");
}
int pos = _free++;
Subscription subscription = _pool.get();
subscription.init(_buffer, pos * ENTRY_SIZE);
subscription.setResizeHandler(_resizeHandler, this);
_map.put(subId.copy(), subscription);
_buffer.position(pos * ENTRY_SIZE);
for (int i = 0; i < subId.length; i++) {
_buffer.put(subId.buffer[subId.position + i]);
}
return subscription;
} finally {
_lock.unlock();
}
}
/**
* Called internally by the Client to mark the message as safely persisted by
* AMPS to all of its sync replication destinations. On this store, this method
* has no effect.
*
* @param subId The subscription ID for the message.
* @param bookmark The bookmark containing the message.
* @throws AMPSException Not thrown by this implementation.
*/
public void persisted(Field subId, BookmarkField bookmark) throws AMPSException {
// no-op
}
/**
* Old style of setting a persisted bookmark no longer used.
* destinations. On this store, this method has no effect.
*
* @param subId The subscription ID for the message.
* @param bookmark The bookmark number.
* @throws AMPSException Not thrown by this implementation.
* @deprecated use {@link #persisted(Field, BookmarkField)} instead.
*/
@Deprecated
public void persisted(Field subId, long bookmark) throws AMPSException {
// no-op
}
/**
* Call this when you want to retrieve the sequence number for the oldest
* bookmark in the store.
*
* @param subId The subscription ID for the oldest bookmark in the store.
* @return Returns the oldest bookmark sequence number in the store.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public long getOldestBookmarkSeq(Field subId) throws AMPSException {
return find(subId).getOldestBookmarkSeq();
}
/**
* Call this when you want to set a resize handler that is invoked when the
* store needs to resize.
*
* @param handler The handler to invoke for the resize.
*/
public void setResizeHandler(BookmarkStoreResizeHandler handler) {
_resizeHandler = handler;
Iterator it = _map.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pairs = (Map.Entry) it.next();
((Subscription) pairs.getValue()).setResizeHandler(handler, this);
}
}
/**
* Call this when you want to purge the contents of this store. Removes any
* tracking history associated with publishers and received messages, and may
* delete or truncate on-disk representations as well.
*
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public void purge() throws AMPSException {
_lock.lock();
try {
// This is the quickest way I've found, to zero out the file.
int longs = _buffer.capacity() / 8;
_buffer.position(0);
for (int i = 0; i < longs; i++) {
_buffer.putLong(0);
}
_buffer.position(0);
_buffer.force();
_map = new HashMap();
_free = 0;
recover();
} finally {
_lock.unlock();
}
}
/**
* Call this when you want to purge the contents of this store for a given
* Subscription Id. Removes any tracking history associated with publishers
* and received messages, and may delete or truncate on-disk representations as
* well.
*
* @param subId_ The identifier of the subscription to purge.
* @throws AMPSException Thrown when an operation on the store fails. The
* exception will contain details of the failure.
*/
public void purge(Field subId_) throws AMPSException {
_lock.lock();
try {
if (!_map.containsKey(subId_))
return;
Subscription sub = _map.get(subId_);
int pos = sub._offset / ENTRY_SIZE;
// NULL out the subs data
byte[] buf = new byte[ENTRY_SIZE];
_buffer.position(sub._offset);
_buffer.put(buf);
// Move all subs back one to fill the hole
--_free;
Field subId = new Field();
for (int j = pos; j < _free; ++j) {
int i = j * ENTRY_SIZE;
// Read the next entry
_buffer.position((j + 1) * ENTRY_SIZE);
_buffer.get(buf);
// Write it to current location
_buffer.position(i);
_buffer.put(buf);
int idLength = 0;
for (; idLength < Subscription.BYTES_SUBID && buf[idLength] != 0; idLength++)
;
subId.set(buf, 0, idLength);
_map.get(subId)._offset = i;
}
// NULL out the end and reset the end
int i = _free * ENTRY_SIZE;
_buffer.position(i);
for (; i < (_free + 1) * ENTRY_SIZE; ++i) {
_buffer.put((byte) 0);
}
_map.remove(subId_);
} finally {
_lock.unlock();
}
}
// Initializes the RingBookmarkStore by mapping a portion of a file into memory
// for read-write access.
private void init() throws AMPSException {
try {
_channel = _file.getChannel();
_buffer = _channel.map(FileChannel.MapMode.READ_WRITE, 0, ENTRIES * ENTRY_SIZE);
_channel.close();
_file.close();
} catch (IOException ioex) {
throw new AMPSException("error opening store.", ioex);
}
_map = new HashMap();
_free = 0;
recover();
}
/**
* Called internally by the Client when connected to an AMPS server to indicate
* what version the server is.
*
* @param version Version number
*/
public void setServerVersion(int version) {
_serverVersion = version;
}
/**
* Called internally by the Client to return the server version detected upon
* logon.
*
* @return The server version.
*/
public int getServerVersion() {
return _serverVersion;
}
/**
* In order to unmap the memory used to store the state, this method closes the
* mapped byte buffer.
*
* @throws Exception Thrown when an operation on the store fails. The exception
* will contain details of the failure.
*/
public void close() throws Exception {
// NOTE: this is an imprecise mechanism for closing a mapped byte buffer;
// The JDK does not yet provide a deterministic way to unmap memory, it
// may be referred to by other objects. We can force a write and gc, but it
// is likely that the underlying file will still be mapped.
_buffer.force();
_buffer = null;
_map = null;
// System.gc();
}
}
