com.crankuptheamps.client.LoggedBookmarkStore Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of amps-client Show documentation
Show all versions of amps-client Show documentation
AMPS Java client by 60East Technologies, Inc.
The newest version!
///////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2010-2024 60East Technologies Inc., All Rights Reserved.
//
// This computer software is owned by 60East Technologies Inc. and is
// protected by U.S. copyright laws and other laws and by international
// treaties. This computer software is furnished by 60East Technologies
// Inc. pursuant to a written license agreement and may be used, copied,
// transmitted, and stored only in accordance with the terms of such
// license agreement and with the inclusion of the above copyright notice.
// This computer software or any other copies thereof may not be provided
// or otherwise made available to any other person.
//
// U.S. Government Restricted Rights. This computer software: (a) was
// developed at private expense and is in all respects the proprietary
// information of 60East Technologies Inc.; (b) was not developed with
// government funds; (c) is a trade secret of 60East Technologies Inc.
// for all purposes of the Freedom of Information Act; and (d) is a
// commercial item and thus, pursuant to Section 12.212 of the Federal
// Acquisition Regulations (FAR) and DFAR Supplement Section 227.7202,
// Government's use, duplication or disclosure of the computer software
// is subject to the restrictions set forth by 60East Technologies Inc..
//
////////////////////////////////////////////////////////////////////////////
package com.crankuptheamps.client;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.math.BigInteger;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.lang.Long;
import java.util.*;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import com.crankuptheamps.client.fields.BookmarkField;
import com.crankuptheamps.client.fields.BookmarkRangeField;
import com.crankuptheamps.client.fields.Field;
import com.crankuptheamps.client.exception.*;
/**
* LoggedBookmarkStore implements a sequentially written log
* of incoming and discarded messages. This store tracks every
* bookmark processed in a file. An application should periodically call
* {@link #prune} to manage the size of the file by removing
* outdated entries.
* When using LoggedBookmarkStore, it is important to note that this store
* is intended to provide the ability for the application to fail and resume.
* If your application does not require this fail-and-resume functionality,
* it is recommended to use a MemoryBookmarkStore.
*
* Usage:
*
* The LoggedBookmarkStore is automatically set on the returned client
* when `createFileBacked` is called. This store is responsible for
* tracking and managing bookmarks associated with the client's state.
*/
public class LoggedBookmarkStore implements BookmarkStore {
// Each entry begins with a single byte indicating the type of entry:
// a new bookmark, or a discard of a previous one.
static final byte ENTRY_BOOKMARK = (byte) 'b';
static final byte ENTRY_DISCARD = (byte) 'd';
static final byte ENTRY_PERSISTED = (byte) 'p';
/**
* The Subscription object is used to represent internal bookmark state
* for the messages received and discarded on a specific subscription
* within the bookmark store.
*/
protected static class Subscription implements com.crankuptheamps.client.Subscription {
// This subscription's ID.
Field _sub;
// Keeps track of the last persisted bookmark
BookmarkField _lastPersisted;
// If the subscription used a range, save it here
BookmarkRangeField _range = new BookmarkRangeField();
/**
* The last-modified timestamp of the backing bookmark log file, just
* before recovery is initiated. When this is not null we intend to
* include it in the list of bookmarks returned by getMostRecentList()
* until a message is discarded on the sub or the sub state is purged.
*
* Originally we thought the recovery timestamp could be stored
* as the last-persisted for each recovered sub, but it must be
* stored separately here because setting it as a sub's
* last-persisted could cause us to lose a persisted ack that
* is earlier than the backing file's last-modified time.
*/
protected volatile String _recoveryTimestamp = null;
static final Field EPOCH_FIELD = new Field(Client.Bookmarks.EPOCH);
// A set of all of the entries recovered from the bookmark
// log file after the file's most-recent -- i.e., whatever entries
// are left in the bookmark ring buffer after recover().
HashMap _recovered = new HashMap();
// A bookmark ring buffer
BookmarkRingBuffer _ring = new BookmarkRingBuffer();
// The per-subscription memory of what we've seen from publishers
HashMap _publishers = new HashMap();
// The store that we log to when the time comes.
LoggedBookmarkStore _parent;
// The Subscription lock
final Lock _lock = new ReentrantLock();
// The encoder/decoder for the Subscription
final CharsetEncoder _encoder = StandardCharsets.UTF_8.newEncoder();
final CharsetDecoder _decoder = StandardCharsets.UTF_8.newDecoder();
public Subscription() {
// Default starting point for last persisted is EPOCH
_lastPersisted = new BookmarkField();
_lastPersisted.copyFrom(EPOCH_FIELD);
}
/**
* Reset the state of this subscription object such that it can be
* returned to the pool for reuse. This method clears all internal data
* structures and sets necessary attributes to their initial values.
* This helps manage file size and resource usage efficiently.
*/
public void reset() {
_sub.reset();
_lastPersisted.copyFrom(EPOCH_FIELD);
_recoveryTimestamp = null;
_recovered.clear();
_ring.reset();
_publishers.clear();
_range.reset();
}
/**
* Initializes the subscription with the given subscription ID and the parent
* LoggedBookmarkStore.
* This method sets up the initial state of the subscription, including
* associating it with a specific store.
*
* @param subscriptionId The ID of the subscription to initialize.
* @param parent The parent LoggedBookmarkStore to associate with this
* subscription.
*/
public void init(Field subscriptionId, LoggedBookmarkStore parent) {
_sub = subscriptionId.copy();
_ring.setSubId(_sub);
_parent = parent;
}
/**
* Get the recovery timestamp associated with this message.
*
* The recovery timestamp represents the time at which the message
* was recovered by the client during a resubscription after a
* disconnection. This timestamp provides information about when
* the message was originally published or received by the server.
*
* @return The recovery timestamp as a string in a format that
* represents the time and date when the message was
* recovered by the client.
*
*/
public String getRecoveryTimestamp() {
return _recoveryTimestamp;
}
/**
* Sets the recovery timestamp associated with this message.
* @param rts The recovery timestamp to be set.
*/
protected final void setRecoveryTimestamp(String rts) {
_recoveryTimestamp = rts;
}
/**
* This is a method the Client calls and is not for customer user.
* Logs a bookmark into the subscription. If the bookmark is a range, it
* processes the range accordingly.
*
* @param bookmark The bookmark to log.
* @return The sequence number associated with the logged bookmark or 0 if no
* bookmark was logged.
* @throws IOException If there is an I/O error during logging.
* @throws CommandException Thrown when an invalid bookmark range is specified.
*/
public long log(BookmarkField bookmark) throws IOException, CommandException {
_lock.lock();
try {
// check whether the given 'bookmark' is a single bookmark or a range
// of bookmarks.
if (!bookmark.isRange()) {
// Check to see if this is a recovered bookmark.
final Long recoveredValue = _recovered.remove(bookmark);
if (recoveredValue != null) {
return recoveredValue;
}
// Add this entry onto our list to remember in order.
if (!bookmark.isBookmarkList()) {
return _log(bookmark);
} else {
// If we're logging a list, we need to mark all items
// in the list as discarded.
long seq = 0;
for (BookmarkField bm : bookmark.parseBookmarkList()) {
isDiscarded(bm);
// if a valid sequence number is returned, it
// discards the sequence in the '_ring' and updates
// '_parent._recentChanged' to indicate a recent
// change.
seq = _log(bm);
if (seq != 0) {
_ring.discard(seq);
_parent._recentChanged = true;
}
}
if (_parent._adapter != null) {
_parent.adapterUpdate(_sub, bookmark);
}
return 0;
}
} else {
_range.copyFrom(bookmark);
if (!_range.isValid()) {
throw new CommandException("Invalid bookmark range specified");
}
long seq = 0;
if (_range.isStartExclusive()) {
// Parse the start of the range and log/discard each
BookmarkField start = _range.getStart();
if (!start.isBookmarkList()) {
isDiscarded(start);
seq = _log(start);
if (seq != 0) {
_ring.discard(seq);
_parent._recentChanged = true;
}
} else {
for (BookmarkField bm : start.parseBookmarkList()) {
isDiscarded(bm);
seq = _log(bm);
if (seq != 0) {
_ring.discard(seq);
_parent._recentChanged = true;
}
}
seq = 0;
}
}
if (_parent._adapter != null) {
_parent.adapterUpdate(_sub, _range);
}
return seq;
}
} finally {
_lock.unlock();
}
}
/**
* This is a method the Client calls and is not for customer user. Logs a
* bookmark
*
* @param bm The bookmark to log.
* @return The sequence number associated with the logged bookmark, or 0 if no
* bookmark was logged.
* @throws IOException If there is an I/O error during the log operation.
*/
private long _log(BookmarkField bm) throws IOException {
long seq = 0;
if (!bm.isTimestamp()) {
seq = _ring.log(bm);
while (seq == 0) {
_lock.unlock();
try {
_ring.checkResize();
} finally {
_lock.lock();
}
seq = _ring.log(bm);
}
} else {
setLastPersisted(bm);
}
return seq;
}
/**
* Discards an entry in the bookmark ring buffer and updates the status
* accordingly.
*
* @param index The index of the entry to discard.
* @throws IOException If there is an I/O error during the discard operation.
*/
public void discard(long index) throws IOException {
_lock.lock();
try {
BookmarkRingBuffer.Entry entry = _ring.getByIndex(index);
if (entry == null || !entry.isActive()) {
return;
}
if (!_parent._recovering) {
_parent.write(_sub, LoggedBookmarkStore.ENTRY_DISCARD,
entry.getBookmark());
}
if (_ring.discard(index)) {
_parent._recentChanged = true;
_recoveryTimestamp = null;
if (_parent._adapter != null) {
_parent.adapterUpdate(_sub,
(BookmarkField) getMostRecentList(false));
}
}
} finally {
_lock.unlock();
}
}
/**
* Check to see if this message is older than the most recent one seen,
* and if it is, then check to see if it is discarded.
*
* @param bookmark The bookmark to check for discard status.
* @return True if the message is discarded, false otherwise.
* @throws IOException If there is an I/O error during the check.
*/
public boolean isDiscarded(BookmarkField bookmark) throws IOException {
_lock.lock();
try {
if (bookmark.isRange() || bookmark.isBookmarkList())
return false;
Long recoveredIndex = _recovered.get(bookmark);
long publisher = bookmark.getPublisherId();
long sequence = bookmark.getSequenceNumber();
if(!_publishers.containsKey(publisher) ||
BookmarkField.unsignedLongLess(_publishers.get(publisher), sequence))
{
_publishers.put(publisher, sequence);
if (recoveredIndex == null) {
return false;
}
}
if (recoveredIndex != null) {
long recoveredVal = recoveredIndex;
BookmarkRingBuffer.Entry entry = _ring.getByIndex(recoveredVal);
if (entry == null && (recoveredVal < _ring.getStartIndex() || _ring.isEmpty())) {
// If the ring buffer no longer has an entry for this index
// and the index is before the ring's start index, then
// this is a stale recovered map entry whose bookmark has
// already been discarded. Don't let it get logged again.
// Remove the stale mapping and return true (it's discarded).
_recovered.remove(bookmark);
return true;
}
// Need current active state
return (entry == null) ? false : !entry.isActive();
}
return !_parent._recovering;
} finally {
_lock.unlock();
}
}
/**
* Retrieves the last persisted bookmark
*
* @return The last persisted bookmark.
*/
public Field getLastPersisted() {
_lock.lock();
try {
return _lastPersisted;
} finally {
_lock.unlock();
}
}
/**
* Retrieves the current bookmark range.
*
* @return The current bookmark range.
*/
public BookmarkRangeField getRange() {
_lock.lock();
try {
return _range;
} finally {
_lock.unlock();
}
}
/**
* Retrieves the most recent bookmark.
*
* @return The most recent bookmark.
*/
public Field getMostRecent() {
_lock.lock();
try {
return getMostRecent(false);
} finally {
_lock.unlock();
}
}
/**
* Retrieves the most recent bookmark with an option to update recovery status.
*
* @param updateRecovery_ Whether to update the recovery status before
* retrieving the most recent bookmark.
* @return The most recent bookmark.
*/
protected Field getMostRecent(boolean updateRecovery_) {
_lock.lock();
try {
// when this is called, we'll take a moment to update the list of things
// recovered, so we don't accidentally log anything we ought not to.
if (updateRecovery_ && _parent._recentChanged)
updateRecovery();
return _ring.getLastDiscarded();
} finally {
_lock.unlock();
}
}
/**
* publisherId_ and sequence are signed, but the actual id's are
* unsigned, so we have to do some work to create a valid string.
*
* @param publisherId_ The signed publisher ID to convert.
* @return The string representation of the unsigned publisher ID.
*/
private static String convertUnsignedLongToString(long publisherId_) {
final BigInteger offset = BigInteger.valueOf(Long.MAX_VALUE)
.shiftLeft(1).add(BigInteger.valueOf(2));
if (publisherId_ < 0) {
return offset.add(BigInteger.valueOf(publisherId_)).toString();
} else {
return Long.toString(publisherId_);
}
}
public Field getMostRecentList(boolean useList)
{
_lock.lock();
try {
boolean rangeIsValid = _range.isValid();
BookmarkField lastDiscarded = (BookmarkField) _ring.getLastDiscarded();
boolean useLastDiscarded = (lastDiscarded != null &&
!lastDiscarded.isNull());
// Check if we haven't discarded the first message yet
if (useLastDiscarded && lastDiscarded.equals(EPOCH_FIELD)) {
if (rangeIsValid) {
// Return the unmodified range
return _range;
}
else if (_recoveryTimestamp != null) {
BookmarkField ret = new BookmarkField();
ret.setValue(_recoveryTimestamp, _encoder);
return ret;
}
// Return EPOCH
return lastDiscarded;
}
long lastDiscardedPub = 0;
long lastDiscardedSeq = 0;
boolean useLastPersisted = (_lastPersisted != null &&
_lastPersisted.length > 1);
long lastPersistedPub = 0;
long lastPersistedSeq = 0;
if (useLastPersisted) {
lastPersistedPub = _lastPersisted.getPublisherId();
lastPersistedSeq = _lastPersisted.getSequenceNumber();
}
if (useLastDiscarded) {
if (useLastPersisted && _ring.isEmpty()
&& (!rangeIsValid || _range.getEnd().equals(_lastPersisted))) {
useLastDiscarded = false;
} else {
lastDiscardedPub = lastDiscarded.getPublisherId();
lastDiscardedSeq = lastDiscarded.getSequenceNumber();
// Only use one if they are same publisher
if (useLastPersisted && (lastDiscardedPub == lastPersistedPub)) {
useLastDiscarded = (lastDiscardedSeq < lastPersistedSeq);
useLastPersisted = !useLastDiscarded;
}
}
}
// create a StringBuilder named recentStr, which will be used to build a string
// representation of the most recent bookmark.
StringBuilder recentStr = new StringBuilder();
// create a BookmarkField named recentList that will be used to store the most
// recent bookmark.
BookmarkField recentList = new BookmarkField();
if (_recoveryTimestamp != null) {
recentStr.append(_recoveryTimestamp);
}
if (useLastDiscarded) {
if (recentStr.length() > 0)
recentStr.append(',');
recentStr.append((lastDiscarded).getValue(_decoder));
}
// If we don't have a last persisted or a last discarded OR we are
// expecting persisted acks but haven't received one yet, then we
// should try to build a list of bookmarks based on publishers we
// have seen so far, if any, or return EPOCH.
if (useList &&
((!useLastPersisted && !useLastDiscarded)
|| (_lastPersisted != null &&
_lastPersisted.equals(EPOCH_FIELD)))) {
if (_publishers.isEmpty() && !rangeIsValid) {
// Set last persisted to EPOCH and return it
if (_lastPersisted == null) {
_lastPersisted = new BookmarkField();
_lastPersisted.copyFrom(EPOCH_FIELD);
}
return _lastPersisted;
}
// If an EPOCH lastDiscarded value was added, remove it.
if (useLastDiscarded && lastDiscarded.equals(EPOCH_FIELD)) {
int len = recentStr.length();
if (len == 1) // Only contains EPOCH
recentStr.setLength(0);
else if (len > 2) // Ends with ",0" (,EPOCH)
recentStr.setLength(len - 2);
}
Iterator it = _publishers.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pairs = (Map.Entry) it.next();
long pubId = (Long) pairs.getKey();
if (pubId == 0)
continue;
if (useLastDiscarded && pubId == lastDiscardedPub)
continue;
long seq = (Long) pairs.getValue();
if (recentStr.length() > 0)
recentStr.append(',');
recentStr.append(convertUnsignedLongToString(pubId))
.append(BookmarkField.SEPARATOR_CHAR)
.append(convertUnsignedLongToString(seq))
.append(BookmarkField.SEPARATOR_CHAR);
}
recentList.setValue(recentStr.toString(), _encoder);
if (rangeIsValid) {
if (recentList.length > 1
&& (_range.isStartInclusive()
|| !recentList.equals(_range.getStart()))) {
_range.replaceStart(recentList, true);
}
return _range;
}
return recentList;
}
if (useLastPersisted) {
if (recentStr.length() > 0)
recentStr.append(',');
recentStr.append(_lastPersisted.getValue(_decoder));
}
recentList.setValue(recentStr.toString(), _encoder);
if (rangeIsValid) {
if (recentList.length > 1
&& (_range.isStartInclusive()
|| !recentList.equals(_range.getStart()))) {
_range.replaceStart(recentList, true);
}
return _range;
}
return recentList;
} finally {
_lock.unlock();
}
}
/**
* Clears the internal recovery map and updates it by scanning the ring buffer
* to identify active entries and their corresponding indices.
*/
private void updateRecovery() {
_recovered.clear();
long end = _ring.getEndIndex();
for (long index = _ring.getStartIndex(); index < end; ++index) {
BookmarkRingBuffer.Entry entry = _ring.getByIndex(index);
if (entry != null && entry._bookmark != null && !entry._bookmark.isNull()) {
_recovered.put(entry.getBookmark().copy(), index);
}
}
}
/**
* Retrieves active entries from the ring buffer and populates the provided
* ArrayList with these entries.
*
* @param entryList_ An ArrayList to store active entries.
*/
private void getActiveEntries(ArrayList entryList_) {
_ring.getRecoveryEntries(entryList_);
}
/**
* [DEPRECATED] Sets the last persisted bookmark using the old style of
* specifying the bookmark as a sequence number.
*
* @deprecated Use {@link #setLastPersisted(BookmarkField)} instead.
* @param bookmark The sequence number of the bookmark (no longer used).
* @throws IOException If an I/O error occurs.
*/
@Deprecated
public void setLastPersisted(long bookmark) throws IOException {
_lock.lock();
try {
BookmarkRingBuffer.Entry entry = _ring.getByIndex(bookmark);
if (entry == null)
return;
BookmarkField bookmarkField = entry.getBookmark();
if (bookmarkField == null || bookmarkField.isNull())
return;
long publisherId = bookmarkField.getPublisherId();
boolean lastPersistedNull = (_lastPersisted == null);
if (!lastPersistedNull &&
publisherId == _lastPersisted.getPublisherId() &&
BookmarkField.unsignedLongLessEqual(bookmarkField.getSequenceNumber(),
_lastPersisted.getSequenceNumber()))
{
return;
}
if (!lastPersistedNull)
_lastPersisted.reset();
_lastPersisted = bookmarkField.copy();
if (!_parent._recovering) {
_parent.write(_sub, ENTRY_PERSISTED, bookmarkField);
}
if (lastPersistedNull
|| publisherId == _ring.getLastDiscarded().getPublisherId()) {
_parent._recentChanged = true;
_recoveryTimestamp = null;
if (!_parent._recovering && _parent._adapter != null) {
_parent.adapterUpdate(_sub,
(BookmarkField) getMostRecentList(false));
}
}
} finally {
_lock.unlock();
}
}
/**
* Sets the last persisted bookmark using a provided BookmarkField object.
* This method updates the internal state and writes the persisted bookmark
* to the parent with appropriate handling.
*
* @param bookmark The BookmarkField object representing the last persisted
* bookmark.
* @throws IOException If an I/O error occurs.
*/
public void setLastPersisted(BookmarkField bookmark) throws IOException {
_lock.lock();
try {
if (bookmark == null || bookmark.isNull()
|| bookmark.equals(_lastPersisted)
|| bookmark.isRange())
return;
if (bookmark.isTimestamp())
{
_recoveryTimestamp = bookmark.toString();
_parent.write(_sub, ENTRY_PERSISTED, bookmark);
_parent._recentChanged = true;
;
if (_parent._adapter != null) {
_parent.adapterUpdate(_sub,
(BookmarkField) getMostRecentList(false));
}
return;
}
long publisherId = bookmark.getPublisherId();
boolean lastPersistedUnset = (_lastPersisted == null
|| _lastPersisted.equals(EPOCH_FIELD));
if (!lastPersistedUnset &&
publisherId == _lastPersisted.getPublisherId() &&
BookmarkField.unsignedLongLessEqual(bookmark.getSequenceNumber(),
_lastPersisted.getSequenceNumber()))
{
return;
}
if (_lastPersisted != null) _lastPersisted.reset();
_lastPersisted = bookmark.copy();
BookmarkField lastDiscarded = (BookmarkField)_ring.getLastDiscarded();
if (lastPersistedUnset && _ring.isEmpty()
&& lastDiscarded.equals(EPOCH_FIELD)) {
discard(_log(bookmark));
}
if (!_parent._recovering) {
_parent.write(_sub, ENTRY_PERSISTED, bookmark);
}
if (lastPersistedUnset || _ring.isEmpty()
|| publisherId == lastDiscarded.getPublisherId()) {
_recoveryTimestamp = null;
_parent._recentChanged = true;
if (!_parent._recovering && _parent._adapter != null) {
_parent.adapterUpdate(_sub,
(BookmarkField) getMostRecentList(false));
}
}
} finally {
_lock.unlock();
}
}
/**
* Retrieves the sequence number of the oldest bookmark in the subscription's
* bookmark ring buffer. This represents the starting point of the subscription.
*
* @return The sequence number of the oldest bookmark in the ring buffer.
*/
public long getOldestBookmarkSeq() {
_lock.lock();
try {
return _ring.getStartIndex();
} finally {
_lock.unlock();
}
}
/**
* Call on a Subscription object just after recovery is performed to
* convert logged entries into recovery entries and set the publishers
* cache state to the earliest sequence seen for each publisher minus
* one.
*
* NOTE: If after recovery lastDiscarded for this sub is null (i.e.
* nothing was discarded) and lastPersisted is null or EPOCH (i.e.
* no persisted ack was recorded in the log), then we can throw away
* this subscription state and return this Subscription object to the
* pool.
*
* @return Indicates whether this subscription's state was reset
* (because it's unneeded due to the lack of any discards and
* persisted acks) and this object should be returned to the
* Subscription pool.
*/
public boolean justRecovered() {
_lock.lock();
try {
BookmarkField ld = _ring.getLastDiscarded();
if ((ld == null || ld.isNull() || ld.equals(EPOCH_FIELD))
&& (_lastPersisted == null || _lastPersisted.isNull()
|| _lastPersisted.equals(EPOCH_FIELD))
&& _recoveryTimestamp == null
&& !_range.isValid())
{
// Reset this sub for reuse.
reset();
return true;
}
updateRecovery();
ArrayList active = new ArrayList();
getActiveEntries(active);
setPublishersToDiscarded(active, _publishers);
} finally {
_lock.unlock();
}
return false;
}
/**
* Update the provided publishers map with the highest sequence number
* that has been discarded for each publisher based on the given list of
* active bookmark entries.
*
* @param active A list of active bookmark entries to consider for
* updating the publishers' highest discarded sequence
* numbers.
* @param publishers A map that associates publisher IDs with their highest
* discarded sequence numbers. This map will be updated
* with the latest discarded sequence numbers for each
* publisher.
*/
public static void setPublishersToDiscarded(
List active,
Map publishers) {
if (active == null || publishers == null || publishers.isEmpty())
return;
Iterator it = active.iterator();
while (it.hasNext()) {
BookmarkRingBuffer.Entry entry = it.next();
if (!entry.isActive())
continue;
BookmarkField bf = entry.getBookmark();
long seq = bf.getSequenceNumber();
if (seq == 0)
continue;
long publisher = bf.getPublisherId();
Long pubSeq = publishers.get(publisher);
if (pubSeq != null && BookmarkField.unsignedLongLessEqual(seq, pubSeq)) {
publishers.put(publisher, seq - 1);
}
}
}
/**
* Sets the resize handler for the bookmark store. The resize handler is
* responsible for managing file size when the bookmark store needs to be
* resized.
*
* @param handler The `BookmarkStoreResizeHandler` responsible for handling file
* size management.
* @param store The `BookmarkStore` associated with the resize handler.
*/
public void setResizeHandler(BookmarkStoreResizeHandler handler, BookmarkStore store) {
_ring.setResizeHandler(handler, store);
}
/**
* Lock self's internal lock. This method is used by the `LoggedBookmarkStore`
* when gathering subscriptions to prune.
*/
void lock() {
_lock.lock();
}
/**
* Unlock self's internal lock. This method is used by the `LoggedBookmarkStore`
* when finishing a prune operation.
*/
void unlock() {
_lock.unlock();
}
}
// A map of subscriptions associated with this bookmark store.
HashMap _subs = new HashMap();
// The random access file used for storing bookmarks.
RandomAccessFile _file;
// The name of the bookmark log file.
String _fileName;
// A flag indicating whether the bookmark store is in the process of recovery.
volatile boolean _recovering = false;
// A flag indicating whether recent changes have occurred in the bookmark store.
volatile boolean _recentChanged = true;
// The version of the bookmark store.
final int VERSION = 2;
// A pool of subscription objects for reuse.
Pool _pool;
// The resize handler responsible for managing file size.
BookmarkStoreResizeHandler _resizeHandler = null;
private int _serverVersion = Client.MIN_MULTI_BOOKMARK_VERSION;
// The encoder for character set UTF-8.
final CharsetEncoder _encoder = StandardCharsets.UTF_8.newEncoder();
// The decoder for character set UTF-8.
final CharsetDecoder _decoder = StandardCharsets.UTF_8.newDecoder();
// The lock used for controlling access to the bookmark store.
final Lock _lock = new ReentrantLock();
// The lock used for controlling access to the subscription data.
final Lock _subsLock = new ReentrantLock();
RecoveryPointAdapter _adapter = null;
RecoveryPointFactory _factory = null;
/**
* This constructor is equivalent to calling `LoggedBookmarkStore(path, 1,
* false)`. Creates a new `LoggedBookmarkStore` instance with the specified
* path to the backing bookmark log file and default settings.
*
* @see #LoggedBookmarkStore(String, int, boolean)
* @param path The path to the backing bookmark log file.
* @throws IOException If there is a problem creating, reading, or writing
* the backing file.
*/
public LoggedBookmarkStore(String path) throws IOException {
this(path, 1);
}
/**
* This constructor is equivalent to calling `LoggedBookmarkStore(path,
* targetNumberOfSubscriptions, false)`. Creates a new `LoggedBookmarkStore`
* instance with the specified path to the backing bookmark log file and an
* initial capacity for subscriptions.
*
* @see #LoggedBookmarkStore(String, int, boolean)
* @param path The path to the backing bookmark log file.
* @param targetNumberOfSubscriptions The initial capacity for the number.
* @throws IOException If there is a problem creating, reading, or writing
* the backing file.
*/
public LoggedBookmarkStore(String path, int targetNumberOfSubscriptions) throws IOException {
this(path, targetNumberOfSubscriptions, false);
}
/**
* A file-backed bookmark store implementation that fully supports
* discarding messages in an order different from the order they arrived
* (i.e. out-of-order discards) and fail-over to a replicated server.
* All messages must eventually be discarded, otherwise memory usage and
* disk space used will increase in proportion to the number of messages
* received on a bookmark subscription since the first undiscarded message.
* This implementation requires that the prune() method be called
* periodically to removed unneeded discarded bookmarks from the
* backing-file, otherwise it will continue to grow without bound.
* The prune() method is thread-safe and can be called from any thread.
*
* @param path The path to the backing bookmark log file.
* @param targetNumberOfSubscriptions The initial capacity for the number
* of bookmark subscriptions you anticipate
* creating on the AMPS client instance that
* this bookmark store is registered on. This
* will grow as needed if more subscriptions
* are created than anticipated.
* @param useLastModifiedTime Indicates whether the recovery timestamp
* feature should be used. If true, the
* last-modified time of the
* backing file is included (as an AMPS
* timestamp bookmark in a
* comma-separated list of bookmarks) when
* getMostRecent() is called
* after recovering from a bookmark file.
* This feature could be
* useful if you have an infrequently run
* process that is run on a
* schedule that is longer than the AMPS
* server keeps messages in
* its transaction log. When this process is
* started and recovers
* a bookmark log full of old bookmarks that
* are no longer available
* using the MOST_RECENT bookmark indicator,
* the recovery timestamp
* will cause the bookmark subscription to
* begin at the start of the
* transaction log (e.g. EPOCH), rather than
* its tail (e.g. NOW).
*
* @throws IOException If there is a problem creating, reading, or writing
* the backing file.
*/
public LoggedBookmarkStore(String path,
int targetNumberOfSubscriptions,
boolean useLastModifiedTime) throws IOException {
File bmLogFile = new File(path);
String recoveryTimestamp = null;
if (useLastModifiedTime && bmLogFile.exists()) {
Date lastMod = new Date(bmLogFile.lastModified());
TimeZone tz = TimeZone.getTimeZone("UTC");
DateFormat df = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
df.setTimeZone(tz);
recoveryTimestamp = df.format(lastMod);
}
_pool = new Pool(Subscription.class, targetNumberOfSubscriptions);
_file = new RandomAccessFile(path, "rw");
_fileName = path;
try {
recover();
if (recoveryTimestamp != null) {
// Initialize the recovery timestamp on each recovered sub.
for (Subscription sub : _subs.values()) {
sub.setRecoveryTimestamp(recoveryTimestamp);
}
}
} catch (IOException ioex) {
try {
_file.close();
} catch (IOException ignoreTheCloseException) {
} finally {
_file = null;
}
throw ioex;
}
}
/**
* A file-backed bookmark store implementation that fully supports
* discarding messages in an order different from the order they arrived
* (i.e. out-of-order discards) and fail-over to a replicated server.
* All messages must eventually be discarded, otherwise memory usage and
* disk space used will increase in proportion to the number of messages
* received on a bookmark subscription since the first undiscarded message.
* This implementation requires that the prune() method be called
* periodically to removed unneeded discarded bookmarks from the
* backing-file, otherwise it will continue to grow without bound.
* The prune() method is thread-safe and can be called from any thread.
* The store also has a backup RecoveryPointAdapter used in case the file
* is deleted.
*
* @param path The path to the backing bookmark log file.
* @param targetNumberOfSubscriptions The initial capacity for the number
* of bookmark subscriptions you anticipate
* creating on the AMPS client instance that
* this bookmark store is registered on. This
* will grow as needed if more subscriptions
* are created than anticipated.
* @param useLastModifiedTime Indicates whether the recovery timestamp
* feature should be used. If true, the
* last-modified time of the
* backing file is included (as an AMPS
* timestamp bookmark in a
* comma-separated list of bookmarks) when
* getMostRecent() is called
* after recovering from a bookmark file.
* This feature could be
* useful if you have an infrequently run
* process that is run on a
* schedule that is longer than the AMPS
* server keeps messages in
* its transaction log. When this process is
* started and recovers
* a bookmark log full of old bookmarks that
* are no longer available
* using the MOST_RECENT bookmark indicator,
* the recovery timestamp
* will cause the bookmark subscription to
* begin at the start of the
* transaction log (e.g. EPOCH), rather than
* its tail (e.g. NOW).
* @param adapter The RecoveryPointAdapter backing up the
* store. The adapter will be sent
* FixedRecoveryPoints.
*
* @throws IOException If there is a problem creating, reading, or writing
* the backing file.
*/
public LoggedBookmarkStore(String path,
int targetNumberOfSubscriptions,
boolean useLastModifiedTime,
RecoveryPointAdapter adapter) throws IOException {
this(path, targetNumberOfSubscriptions, useLastModifiedTime, adapter, new FixedRecoveryPointFactory());
}
/**
* A file-backed bookmark store implementation that fully supports
* discarding messages in an order different from the order they arrived
* (i.e. out-of-order discards) and fail-over to a replicated server.
* All messages must eventually be discarded, otherwise memory usage and
* disk space used will increase in proportion to the number of messages
* received on a bookmark subscription since the first undiscarded message.
* This implementation requires that the prune() method be called
* periodically to removed unneeded discarded bookmarks from the
* backing-file, otherwise it will continue to grow without bound.
* The prune() method is thread-safe and can be called from any thread.
* The store also has a backup RecoveryPointAdapter used in case the file
* is deleted.
*
* @param path The path to the backing bookmark log file.
* @param targetNumberOfSubscriptions The initial capacity for the number
* of bookmark subscriptions you anticipate
* creating on the AMPS client instance that
* this bookmark store is registered on. This
* will grow as needed if more subscriptions
* are created than anticipated.
* @param useLastModifiedTime Indicates whether the recovery timestamp
* feature should be used. If true, the
* last-modified time of the
* backing file is included (as an AMPS
* timestamp bookmark in a
* comma-separated list of bookmarks) when
* getMostRecent() is called
* after recovering from a bookmark file.
* This feature could be
* useful if you have an infrequently run
* process that is run on a
* schedule that is longer than the AMPS
* server keeps messages in
* its transaction log. When this process is
* started and recovers
* a bookmark log full of old bookmarks that
* are no longer available
* using the MOST_RECENT bookmark indicator,
* the recovery timestamp
* will cause the bookmark subscription to
* begin at the start of the
* transaction log (e.g. EPOCH), rather than
* its tail (e.g. NOW).
* @param adapter The RecoveryPointAdapter backing up the
* store.
* @param factory The RecoveryPointFactory used to create
* RecoveryPoints that are sent to the
* adapter.
*
* @throws IOException If there is a problem creating, reading, or writing
* the backing file.
*/
public LoggedBookmarkStore(String path,
int targetNumberOfSubscriptions,
boolean useLastModifiedTime,
RecoveryPointAdapter adapter,
RecoveryPointFactory factory) throws IOException {
String recoveryTimestamp = null;
File bmLogFile = new File(path);
if (useLastModifiedTime && bmLogFile.exists()) {
Date lastMod = new Date(bmLogFile.lastModified());
TimeZone tz = TimeZone.getTimeZone("UTC");
DateFormat df = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
df.setTimeZone(tz);
recoveryTimestamp = df.format(lastMod);
}
_pool = new Pool(Subscription.class, targetNumberOfSubscriptions);
_file = new RandomAccessFile(path, "rw");
_fileName = path;
try {
_recovering = true;
Message m = new JSONMessage(_encoder, _decoder);
for (RecoveryPoint rp : adapter) {
if (rp == null)
break;
Field subId = rp.getSubId();
m.reset();
m.setSubId(subId.buffer, subId.position, subId.length);
BookmarkField bookmark = rp.getBookmark();
if (bookmark.isRange()) {
m.setBookmark(bookmark.buffer, bookmark.position,
bookmark.length);
try {
log(m);
} catch (AMPSException e) {
} // Always a valid range
} else {
try {
for (BookmarkField bm : bookmark.parseBookmarkList()) {
if (!bm.isTimestamp()) {
m.setBookmark(bm.buffer, bm.position,
bm.length);
isDiscarded(m);
if (log(m) > 0)
discard(m);
} else {
find(subId).setRecoveryTimestamp(bm.toString());
}
}
} catch (AMPSException e) {
} // Always valid
}
}
} finally {
_recovering = false;
}
try {
recover();
if (recoveryTimestamp != null) {
// Initialize the recovery timestamp on each recovered sub.
for (Subscription sub : _subs.values()) {
sub.setRecoveryTimestamp(recoveryTimestamp);
}
}
} catch (IOException ioex) {
try {
_file.close();
} catch (IOException ignoreTheCloseException) {
} finally {
_file = null;
}
throw ioex;
}
// Set after recovery complete to avoid sending any updates
// during recovery
_adapter = adapter;
_factory = factory;
}
/**
* Remove outdated entries in the bookmark store.
* This function creates a temporary file, copies current entries to
* that file, and then replaces the current file with the temporary
* file.
*
* @throws IOException Thrown when an operation on the file fails.
* @throws StoreException Thrown when any other operation fails with details on
* the failure.
*/
public void prune() throws IOException, StoreException {
String name = _fileName + ".tmp";
prune(name);
}
/**
* Remove outdated entries in the bookmark store.
* This function creates a temporary file, copies current entries to
* that file, and then replaces the current file with the temporary
* file.
*
* @param tmpFileName_ The name of the temporary file.
* @throws IOException Thrown when an operation on the file fails.
* @throws StoreException Thrown when any other operation fails with details on
* the failure.
*/
public void prune(String tmpFileName_) throws IOException, StoreException {
_subsLock.lock();
try {
List> subs = _lockSubs();
try {
prune(tmpFileName_, subs);
} finally {
_unlockSubs(subs);
}
} finally {
_subsLock.unlock();
}
}
/**
* Remove outdated entries in the bookmark store.
* This function is used internally and processes a list of subscriptions
* whose locks are already held, so that the locking order of subscription
* then store can be preserved.
*
* @param tmpFileName_ The name of the temporary file.
* @param subs_ The list of Subscriptions to prune.
*/
private void prune(String tmpFileName_,
List> subs_)
throws IOException, StoreException {
_lock.lock();
try {
if (_file == null) {
throw new StoreException("Store not open.");
}
if (!_recentChanged) {
return;
}
// Create a new temporary file for storing pruned data.
RandomAccessFile file = new RandomAccessFile(tmpFileName_, "rw");
try {
// This line forces any updates to the file channel to be written to the storage
// device but not necessarily to the disk.
file.getChannel().force(false);
file.writeInt(VERSION);
file.writeByte((byte) '\n');
StringBuilder bookmarkBuilder = new StringBuilder(64);
// Create a Field object, which is used to store bookmark data.
Field bookmark = new Field();
// Enters a loop over a list of subscriptions (subs_).
for (Map.Entry sub : subs_) {
// Retrieves the subscription ID from the entry.
Field subId = sub.getKey();
assert (subId != null);
// Gets the subscription object associated with the entry.
Subscription subscription = sub.getValue();
// Retrieves the most recent bookmark associated with the subscription and
// makes a copy of it.
BookmarkField recent = (BookmarkField) subscription.getMostRecent().copy();
// If the recent bookmark is null, it copies data from `EPOCH_FIELD` to ensure
// it has a valid value.
if (recent.isNull()) {
recent.copyFrom(subscription.EPOCH_FIELD);
}
long recentPub = recent.getPublisherId();
// Check for the Subscription's range
if (subscription.getRange().isValid()) {
// Update and return the range
Field range = subscription.getMostRecentList(true);
// A range doesn't need to be discarded so only need log
writeBookmarkToFile(file, subId, range, ENTRY_BOOKMARK);
// Ignore recent after this because it's a range
recentPub = 0;
recent.reset();
}
HashMap publishers = new HashMap(subscription._publishers);
ArrayList recovered = new ArrayList();
subscription.getActiveEntries(recovered);
Subscription.setPublishersToDiscarded(recovered, publishers);
// First write the highest discard for each publisher other
// than most recent.
Iterator> pubIter = publishers.entrySet().iterator();
while (pubIter.hasNext()) {
Map.Entry publisher = pubIter.next();
long pubId = publisher.getKey();
long pubSeq = publisher.getValue();
if (pubId == 0 || pubSeq == 0 || pubId == recentPub)
continue;
bookmarkBuilder.setLength(0);
bookmarkBuilder.append(Subscription.convertUnsignedLongToString(pubId))
.append(BookmarkField.SEPARATOR_CHAR)
.append(Subscription.convertUnsignedLongToString(pubSeq))
.append(BookmarkField.SEPARATOR_CHAR);
bookmark.set(bookmarkBuilder.toString().getBytes(StandardCharsets.UTF_8), 0,
bookmarkBuilder.length());
writeBookmarkToFile(file, subId, bookmark, ENTRY_BOOKMARK);
writeBookmarkToFile(file, subId, bookmark, ENTRY_DISCARD);
}
// Now write the most recent
if (recent.length > 1) {
writeBookmarkToFile(file, subId, recent, ENTRY_BOOKMARK);
writeBookmarkToFile(file, subId, recent, ENTRY_DISCARD);
}
// If there is a last persisted bookmark for the subscription, it is written to
// the temporary file.
if (subscription._lastPersisted != null
&& subscription._lastPersisted.length > 1) {
writeBookmarkToFile(file, subId,
subscription._lastPersisted,
ENTRY_PERSISTED);
}
// It iterates over the recovered entries, writing bookmarks and discard entries
// for each.
for (BookmarkRingBuffer.Entry entry : recovered) {
BookmarkField entryBookmark = entry.getBookmark();
if (entryBookmark == null
|| entryBookmark.isNull()) {
continue;
}
writeBookmarkToFile(file, subId, entryBookmark,
ENTRY_BOOKMARK);
if (!entry.isActive())
writeBookmarkToFile(file, subId,
entryBookmark,
ENTRY_DISCARD);
}
}
}
// If we end up in catch because of an IOException, need to clean tmp file
catch (IOException e) {
file.close();
File tmp = new File(tmpFileName_);
// We can ignore a failure of delete here, prune has failed
tmp.delete();
throw new StoreException("Failed attempting to prune file " + _fileName + " to " + tmpFileName_, e);
}
// Close the files, delete the original, move the pruned file
file.close();
_file.close();
_file = null;
int retries = 0;
File origTmp = new File(_fileName);
while (retries++ < 3) {
if (!origTmp.delete()) {
if (retries >= 3)
throw new StoreException(
"Failed to delete original file " + _fileName
+ " after completing prune to " + tmpFileName_);
} else {
break;
}
}
// Let file system catch up
while (origTmp.exists()) {
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
// Ignore
}
}
retries = 0;
while (retries++ < 3) {
File tmp = new File(tmpFileName_);
if (!tmp.renameTo(origTmp)) {
if (retries >= 3)
throw new StoreException(
"Failed to rename pruned file " + tmpFileName_
+ " to original file name: " + _fileName);
try {
Thread.sleep(50);
} catch (InterruptedException ex) {
}
} else {
break;
}
}
_file = new RandomAccessFile(_fileName, "rw");
if (_file.length() > 0)
_file.seek(_file.length());
_recentChanged = false;
} finally {
_lock.unlock();
}
}
/**
* Writes a bookmark entry to a RandomAccessFile.
*
* @param file_ The RandomAccessFile to write to.
* @param subId_ The Field containing subscription ID.
* @param bookmark_ The Field containing the bookmark data.
* @param entry_ The byte representing the type of entry.
* @throws IOException If there is an error while writing to the file.
*/
private void writeBookmarkToFile(RandomAccessFile file_, Field subId_, Field bookmark_, byte entry_)
throws IOException {
file_.writeInt(subId_.length);
file_.write(subId_.buffer, subId_.position, subId_.length);
file_.writeByte(entry_);
file_.writeInt(bookmark_.length);
file_.write(bookmark_.buffer, bookmark_.position, bookmark_.length);
file_.writeByte((byte) '\n');
}
/**
* Writes a bookmark entry with Field data to the bookmark store.
*
* @param sub The Field containing subscription ID.
* @param entry The byte representing the type of entry.
* @param data The Field containing the bookmark data.
* @throws IOException If there is an error while writing to the bookmark store
* file.
*/
void write(Field sub, byte entry, Field data) throws IOException {
_lock.lock();
try {
if (!_recovering) {
writeBookmarkToFile(_file, sub, data, entry);
}
} finally {
_lock.unlock();
}
}
/**
* Writes a bookmark entry with long data to the bookmark store.
*
* @param sub The Field containing subscription ID.
* @param entry The byte representing the type of entry.
* @param data The long data to be written as a bookmark.
* @throws IOException If there is an error while writing to the bookmark store
* file.
*/
void write(Field sub, byte entry, long data) throws IOException {
_lock.lock();
try {
if (!_recovering) {
_file.writeInt(sub.length);
_file.write(sub.buffer, sub.position, sub.length);
_file.writeByte(entry);
_file.writeLong(data);
_file.writeByte((byte) '\n');
}
} finally {
_lock.unlock();
}
}
/**
* This method is used internally by this {@link BookmarkStore}
* implementation to recover its state from the bookmark store file.
* This allows it to know on restart what messages have been discarded by
* a subscriber and what the most recent message a subscriber should receive
* from a publisher for a bookmark subscription.
*
* IMPORTANT NOTE: When making changes to this method, please run the
* manual test case in
* gfs/sixty/amps-client-java/wip/manual_tests/LargeUnprunedBookmarkStore
* to ensure proper function in certain hard to automate cases such
* as ensuring reasonable memory footprint.
*
* @throws IOException If an unrecoverable problem is detected while
* processing the bookmark log file.
*/
private final void recover() throws IOException {
// Mark that the recovery process is active.
_recovering = true;
// If the file is smaller than the version marker size (4 bytes), initialize it.
if (_file.length() < 4) {
try {
_file.writeInt(VERSION);
_file.writeByte((byte) '\n');
} finally {
_recovering = false;
}
return;
}
// Read the version marker.
int version = 0;
try {
version = _file.readInt();
} catch (IOException ex) {
_recovering = false;
throw ex;
}
// In prior versions, length for subid and bookmark was limited to 255
// Now, it can be up to max int. Check that this isn't an old file.
boolean readInts = version == VERSION;
// If it uses integer lengths, read the newline character.
if (readInts) {
try {
_file.readByte();
} catch (IOException ex) {
_recovering = false;
throw ex;
}
// If the file only contains the version marker and newline character, no
// recovery is needed since no data is present.
if (_file.length() == 5) {
_recovering = false;
return;
}
}
else {
// Prior versions also didn't write the version to the file.
_file.seek(0);
}
// We have something in the file, so we need to clear anything that
// the recovery point adapter had.
try {
_purge();
}
catch (AMPSException ex) { }
// This is used to track bookmark entries for faster discard
HashMap> ids = new HashMap>();
// Variables for reading in subId and bookmark
int maxSubLen = 255;
byte[] sub = new byte[maxSubLen];
int maxBookmarkLen = 255;
byte[] bookmark = new byte[maxBookmarkLen];
int subLen = 0;
int bmLen = 0;
Field subId = new Field();
BookmarkField bookmarkField = new BookmarkField();
Long zero = Long.valueOf(0);
// Create temporary buffered stream for bookmark file. We use a new file
// channel so that closing the stream later won't close _file.
Path path = FileSystems.getDefault().getPath(_fileName);
FileChannel channel = FileChannel.open(path, StandardOpenOption.READ);
long lastGoodPosition = _file.getFilePointer();
channel.position(lastGoodPosition);
DataInputStream in = null;
try {
// Create a DataInputStream for reading from the buffered stream.
in = new DataInputStream(
new BufferedInputStream(Channels.newInputStream(channel),
8192));
long position = lastGoodPosition;
long line = 0;
// Iterate through the bookmark entries in the file.
while (lastGoodPosition < _file.length()) {
long entryStartPos = position;
// Read subId length, then subId
if (readInts) {
subLen = in.readInt();
position += 4;
} else {
subLen = in.readUnsignedByte();
position++;
}
// Check for invalid length or truncated entry.
if (position < 0 || position + subLen > _file.length()) {
// truncated entry, throw an exception.
_file.seek(entryStartPos);
throw new IOException("Invalid subid length " + subLen + " starting at position " + entryStartPos
+ " line " + line + " in file " + _fileName + " of size " + _file.length());
}
// Resize the sub byte array if needed.
if (subLen > maxSubLen) {
do {
maxSubLen *= 2;
} while (subLen > maxSubLen);
sub = new byte[maxSubLen];
}
// Read the subscription ID bytes.
in.readFully(sub, 0, subLen);
subId.set(sub, 0, subLen);
position += subLen;
Subscription subscription = find(subId);
HashMap subscriptionMap = ids.get(subId);
if (subscriptionMap == null)
{
subscriptionMap = new HashMap();
ids.put(subId.copy(), subscriptionMap);
}
// Read the entry type.
int entryType = in.readUnsignedByte();
position++;
// Read bookmark length and bookmark
if (readInts) {
bmLen = in.readInt();
position += 4;
} else {
bmLen = in.readUnsignedByte();
position++;
}
// Check for invalid length or truncated entry.
if (bmLen < 0 || position + bmLen > _file.length()) {
// truncated entry, throw an exception.
_file.seek(entryStartPos);
throw new IOException("Invalid bookmark len " + bmLen + " starting at position " + entryStartPos
+ " line " + line + " in file " + _fileName + " of size " + _file.length());
}
// Resize the bookmark byte array if needed.
if (bmLen > maxBookmarkLen) {
do {
maxBookmarkLen *= 2;
} while (bmLen > maxBookmarkLen);
bookmark = new byte[maxBookmarkLen];
}
// Read the bookmark bytes.
in.readFully(bookmark, 0, bmLen);
position += bmLen;
// Read trailing newline
if (in.readUnsignedByte() != (byte)'\n') {
// bad read
_file.seek(entryStartPos);
throw new IOException("Invalid record didn't end with newline starting at position " + entryStartPos
+ " line " + line + " in file " + _fileName + " of size " + _file.length());
}
position++;
bookmarkField.set(bookmark, 0, bmLen);
// Process the entry based on its type.
switch (entryType) {
case ENTRY_BOOKMARK:
if (bookmarkField.isRange()) {
// Handle range entries.
try {
// Log does all we need
subscription.log(bookmarkField);
} catch (CommandException e) {
} // Range always valid, so ignore exceptions.
} else {
// Handle non-range bookmark entries.
String bmStr = bookmarkField.getValue(_decoder);
if (subscriptionMap.get(bmStr) != null) {
// Bookmark already encountered, clear subscription map.
subscription.getMostRecent(true);
subscriptionMap.clear();
}
if (!subscription.isDiscarded(bookmarkField)) {
// Bookmark is not discarded, log it.
try {
long addedIdx = subscription.log(bookmarkField);
subscriptionMap.put(bmStr, addedIdx);
} catch (CommandException e) {
} // No range, ignore exceptions.
} else {
// Bookmark is discarded, mark it as zero in the subscription map.
subscriptionMap.put(bmStr, zero);
}
}
break;
case ENTRY_DISCARD:
// Handle discard entries.
String bkmStr = bookmarkField.getValue(_decoder);
Long subscriptionMapEntry = subscriptionMap.get(bkmStr);
if (subscriptionMapEntry != null) {
// Remove the entry from the subscription map.
subscriptionMap.remove(bkmStr);
if (subscriptionMapEntry > 0) {
// Discard if the entry has a valid index.
subscription.discard(subscriptionMapEntry);
}
}
break;
case ENTRY_PERSISTED:
// Handle persisted entries.
subscription.setLastPersisted(bookmarkField);
break;
default:
// Corrupt file found, throw an exception.
throw new IOException("Corrupt file found.");
}
lastGoodPosition = position;
++line;
}
// Ensure the file pointer is set to the last valid position.
if (_file.getFilePointer() != lastGoodPosition) {
_file.seek(lastGoodPosition);
}
// Close the input stream.
in.close();
in = null;
} catch (IOException ex) {
// Handle IO exceptions during recovery.
if (in == null) {
// Failed to create DataInputStream, recovery is impossible.
_recovering = false;
throw ex;
}
boolean onLastTruncatedLine = true;
try {
// Try to determine if we are on the last line of the file,
// which may have been corrupted due to truncation from an
// abrupt exit.
int len = 0;
byte[] buffer = new byte[255];
while ((len = in.read(buffer, 0, buffer.length)) != -1) {
// Scan for a newline char from the current position.
for (int i = 0; i < len; i++) {
// If we found a newline, then we're not on the
// last line of the file that has been truncated.
if ((byte) '\n' == buffer[i]) {
onLastTruncatedLine = false;
break;
}
}
}
// Now close the stream.
in.close();
in = null;
} catch (IOException ex2) {
// Add the suppressed exception and rethrow the original exception.
ex.addSuppressed(ex2);
_recovering = false;
// Something seems to be seriously wrong, so throw.
throw ex;
}
if (lastGoodPosition > 0 && onLastTruncatedLine) {
// We only want to seek to the last good position if
// we know we're on the last line of the file and it
// has been truncated (due to an abrupt shutdown).
try {
_file.seek(lastGoodPosition);
} catch (IOException ex2) {
// Add the suppressed exception and rethrow the original exception.
ex.addSuppressed(ex2);
_recovering = false;
throw ex;
}
} else {
// The corruption seems to be somewhere earlier in the
// file, so throw the exception.
_recovering = false;
throw ex;
}
} finally {
try {
if (in != null)
in.close();
} catch (IOException ex) {
// Ignore any exceptions while closing the stream.
}
// Because this is only called either in constructor or in
// setServerVersion already under the _subsLock, there is
// no reason to lock _subsLock here.
// Cleanup and dispose of subscriptions that were just recovered.
Iterator> it = _subs.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pairs = it.next();
Subscription theSub = pairs.getValue();
boolean disposeSub = theSub.justRecovered();
if (disposeSub) {
it.remove();
_pool.returnToPool(theSub);
}
}
_recovering = false;
}
// If this was an older file version, rewrite it
if (!readInts) {
try {
prune();
} catch (StoreException ex) {
throw new IOException("Failed to rewrite older file version, see inner exception", ex);
}
}
}
/**
* This is a method the Client calls and is not for customer user. Logs a
* message's bookmark to the bookmark store and updates the message's
* bookmark sequence number.
*
* @param message The message to log, containing the bookmark to be stored.
* @return The index at which the bookmark is stored in the subscription.
* @throws AMPSException If an error occurs while logging to the bookmark store.
*/
public long log(Message message) throws AMPSException {
BookmarkField bookmark = (BookmarkField) message.getBookmarkRaw();
// Check if the bookmark is the epoch field, no logging is needed.
if (bookmark.equals(Subscription.EPOCH_FIELD))
return 0;
Subscription sub = (LoggedBookmarkStore.Subscription) message.getSubscription();
Field subId = message.getSubIdRaw();
// If subId is null or empty, attempt to retrieve it from the message's
// subIdsRaw.
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
_lock.lock();
try {
// Check if the store is open for writing, and not in recovery mode.
if (!_recovering && _file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
long index = 0;
try {
// If the subscription is not provided, attempt to find it using subId.
if (sub == null) {
sub = find(subId);
message.setSubscription(sub);
}
// Log the bookmark and retrieve the index where it's stored.
index = sub.log(bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error logging to bookmark store", ioex);
}
// Update the message's bookmark sequence number.
message.setBookmarkSeqNo(index);
_lock.lock();
try {
// Log the arrival of this bookmark.
write(subId, LoggedBookmarkStore.ENTRY_BOOKMARK, bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error logging to bookmark store", ioex);
} finally {
_lock.unlock();
}
return index;
}
/**
* Discards a bookmark entry associated with a specific subscription and
* bookmark sequence number.
*
* @param subId The identifier of the subscription.
* @param bookmarkSeqNo The sequence number of the bookmark entry to discard.
* @throws AMPSException If an error occurs while discarding from the bookmark
* store.
*/
public void discard(Field subId, long bookmarkSeqNo) throws AMPSException {
_lock.lock();
try {
// Check if the store is open for writing, and not in recovery mode.
if (!_recovering && _file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
try {
// Find the subscription associated with subId and discard the bookmark entry.
find(subId).discard(bookmarkSeqNo);
} catch (IOException ioex) {
throw new AMPSException("Error discarding from bookmark store", ioex);
}
}
/**
* Discards a bookmark entry based on the information provided in the message.
*
* @param message The message containing information about the bookmark to
* discard.
* @throws AMPSException If an error occurs while discarding from the bookmark
* store.
*/
public void discard(Message message) throws AMPSException {
_lock.lock();
try {
// Check if the store is open for writing.
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
BookmarkField bookmarkField = message.getBookmarkRaw();
// Check if the bookmark is the epoch field or has specific characteristics,
// no logging is needed.
if (bookmarkField.equals(Subscription.EPOCH_FIELD)
|| bookmarkField.isTimestamp() || bookmarkField.isRange()
|| bookmarkField.isBookmarkList()) {
return;
}
long bookmark = message.getBookmarkSeqNo();
Subscription sub = (LoggedBookmarkStore.Subscription) message.getSubscription();
// If the subscription is not provided, attempt to find it using subId from the
// message.
if (sub == null) {
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
try {
// Discard the bookmark entry.
sub.discard(bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error discarding from bookmark store", ioex);
}
}
/**
* Retrieves the most recent bookmark associated with a subscription identified
* by the given subId.
*
* @param subId The identifier of the subscription for which to retrieve the
* most recent bookmark.
* @return The most recent bookmark for the specified subscription.
* @throws AMPSException If an error occurs while retrieving the most recent
* bookmark or if the store is not open.
*/
public Field getMostRecent(Field subId) throws AMPSException {
// Delegate to the overloaded method with 'useList' set to true.
return getMostRecent(subId, true);
}
/**
* Retrieves the most recent bookmark associated with a subscription identified
* by the given subId.
*
* @param subId The identifier of the subscription for which to retrieve the
* most recent bookmark.
* @param useList A flag indicating whether to retrieve the most recent bookmark
* list or a single bookmark.
* @return The most recent bookmark for the specified subscription.
* @throws AMPSException If an error occurs while retrieving the most recent
* bookmark or if the store is not open.
*/
public Field getMostRecent(Field subId, boolean useList) throws AMPSException {
_lock.lock();
try {
// Check if the store is open for reading.
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
// Retrieve and return the most recent bookmark (list or single) for the
// specified subscription.
return find(subId).getMostRecentList(useList).copy();
}
/**
* This method is called internally by the client to determine whether a
* message's bookmark received from the amps server has already been discarded
* OR already been delivered to the subscriber during this run.
*
* @param message The message containing the bookmark to check for discarding.
* @return True if the bookmark has been discarded; false otherwise.
* @throws AMPSException If an error occurs while checking if the bookmark is
* discarded or if the store is not open.
*/
public boolean isDiscarded(Message message) throws AMPSException {
_lock.lock();
try {
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
BookmarkField bookmark = (BookmarkField) message.getBookmarkRaw();
if (bookmark.equals(Subscription.EPOCH_FIELD))
return true;
if (bookmark.isTimestamp() || bookmark.isBookmarkList())
return false;
Subscription sub = (LoggedBookmarkStore.Subscription) message.getSubscription();
// If the subscription is not provided, attempt to find it using subId from the
// message.
if (sub == null) {
Field subId = message.getSubIdRaw();
if (subId == null || subId.isNull())
subId = message.getSubIdsRaw();
sub = find(subId);
message.setSubscription(sub);
}
try {
// Check if the bookmark is discarded within the subscription.
return sub.isDiscarded(bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error checking is discarded in bookmark store", ioex);
}
}
/**
* Deprecated method for setting a persisted bookmark. Use
* {@link #persisted(Field, BookmarkField)} instead.
*
* @param subId The identifier of the subscription for which to set the
* persisted bookmark.
* @param bookmark The persisted bookmark value to set.
* @throws AMPSException If an error occurs while setting the persisted bookmark
* or if the store is not open.
* @deprecated Use {@link #persisted(Field, BookmarkField)} instead.
*/
@Deprecated
public void persisted(Field subId, long bookmark) throws AMPSException {
_lock.lock();
try {
// Check if the store is open for writing.
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
try {
// Set the last persisted bookmark for the specified subscription.
find(subId).setLastPersisted(bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error logging persisted to bookmark store", ioex);
}
}
/**
* This method is called by the client and it is used to process persisted
* acknowledgements that track a safe recovery point in the txlog. Sets a
* persisted bookmark for a subscription.
*
* @param subId The identifier of the subscription for which to set the
* persisted bookmark.
* @param bookmark The persisted bookmark value to set.
* @throws AMPSException If an error occurs while setting the persisted bookmark
* or if the store is not open.
*/
public void persisted(Field subId, BookmarkField bookmark) throws AMPSException {
_lock.lock();
try {
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
// Check if the provided bookmark is the epoch field or a range, and return if
// so.
if (bookmark.equals(Subscription.EPOCH_FIELD) || bookmark.isRange()) {
return;
}
try {
// Set the last persisted bookmark for the specified subscription.
find(subId).setLastPersisted(bookmark);
} catch (IOException ioex) {
throw new AMPSException("Error logging persisted to bookmark store", ioex);
}
}
/**
* Retrieves the oldest bookmark sequence number associated with a subscription.
*
* @param subId The identifier of the subscription for which to retrieve the
* oldest bookmark sequence number.
* @return The oldest bookmark sequence number for the specified subscription.
* @throws AMPSException If an error occurs while retrieving the oldest bookmark
* sequence number or if the store is not open.
*/
public long getOldestBookmarkSeq(Field subId) throws AMPSException {
_lock.lock();
try {
if (_file == null) {
throw new StoreException("Store not open.");
}
} finally {
_lock.unlock();
}
// Retrieve and return the oldest bookmark sequence number for the specified
// subscription.
return find(subId).getOldestBookmarkSeq();
}
/**
* Sets a handler for bookmark store resize events.
*
* @param handler The handler to set for resize events.
*/
public void setResizeHandler(BookmarkStoreResizeHandler handler) {
// The _resizeHandler is only touched under _subsLock
_subsLock.lock();
try {
// holds the handler responsible for managing resize events in the bookmark
// store.
_resizeHandler = handler;
Iterator it = _subs.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pairs = (Map.Entry) it.next();
// Calls a setResizeHandler method on the Subscription object, passing two
// arguments: the handler and this. The Subscription object is responsible
// for handling resize events and is being configured with the provided handler.
((Subscription) pairs.getValue()).setResizeHandler(handler, this);
}
} finally {
_subsLock.unlock();
}
}
/**
* Finds and returns the Subscription object for the specified
* subscription id (subId).
*
* @param subId The subId to find or create a Subscription for.
* @return The Subscription associated with the subId.
*/
protected Subscription find(Field subId) {
_subsLock.lock();
try {
Subscription s = _subs.get(subId);
if (s == null) {
s = _pool.get();
s.init(subId, this);
s.setResizeHandler(_resizeHandler, this);
_subs.put(subId.copy(), s);
}
return s;
} finally {
_subsLock.unlock();
}
}
/**
* Remove all entries in the bookmark store, completely
* clearing all record of messages received and discarded.
* This should NOT be called on store that has active
* subscriptions or as there is a chance of creating an
* inconsistency between the file and the current state.
*/
public void purge() throws AMPSException
{
if (_file == null)
{
throw new StoreException("Store not open.");
}
_purge();
_lock.lock();
_recentChanged = true;
try
{
// delete the file on disk.
try
{
_file.setLength(0);
_file.writeInt(VERSION);
_file.writeByte((byte) '\n');
} catch (IOException ioex) {
throw new StoreException("Error truncating file", ioex);
}
if (_adapter != null) {
try {
_adapter.purge();
}
catch (AMPSException e) {
throw e;
}
catch (Exception e) {
throw new StoreException("Exception in RecoveryPointAdapter.purge()", e);
}
}
}
finally {
_lock.unlock();
}
}
/**
* Removes all entries in the bookmark store and clears all records of messages
* received and discarded.
* This method is used to purge the entire bookmark store, effectively resetting
* it to an empty state.
* It acquires necessary locks to ensure safe access to shared resources.
*
* @throws AMPSException If an error occurs during the purging process.
*/
public void _purge() throws AMPSException {
_subsLock.lock();
_lock.lock();
try {
for (Subscription sub: _subs.values())
{
sub.reset();
_pool.returnToPool(sub);
}
// Clear the _subs map to remove all subscription entries.
_subs.clear();
} finally {
_lock.unlock();
_subsLock.unlock();
}
}
/**
* Removes all entries in the bookmark store associated with a specific
* subscription ID (subId_).
* This method is used to clear messages received and discarded for a particular
* subscription.
* It also acquires necessary locks to ensure safe access to shared resources.
*
* @param subId_ The subscription ID for which to remove entries from the store.
* @throws AMPSException If an error occurs during the purging process.
*/
public void purge(Field subId_) throws AMPSException {
if (_file == null) {
throw new StoreException("Store not open.");
}
// Delegate the purging operation to the private _purge method.
_purge(subId_);
if (_adapter != null) {
try {
_adapter.purge(subId_);
} catch (AMPSException e) {
throw e;
} catch (Exception e) {
throw new StoreException("Exception in RecoveryPointAdapter.purge(" + subId_.toString() + ")", e);
}
}
}
/**
* Removes all entries in the bookmark store associated with a specific
* subscription ID (subId_).
* This method is used to clear messages received and discarded for a particular
* subscription.
* It also acquires necessary locks to ensure safe access to shared resources.
*
* @param subId_ The subscription ID for which to remove entries from the store.
* @throws AMPSException If an error occurs during the purging process.
*/
public void _purge(Field subId_) throws AMPSException {
_subsLock.lock();
// initializes an empty string variable name to store the name of the temporary
// bookmark log file.
String name = "";
try {
// Need to acquire all the Subscription locks here before the
// store lock, because prune(name) will acquire all of them below.
// If we didn't acquire these and some other Subscription is active
// (not the one we're purging) we could get a deadlock if that
// active sub is executing a discard() or persisted() call.
List> subs = _lockSubs();
try {
_lock.lock();
try {
Subscription sub = _subs.remove(subId_);
if (sub == null)
return;
sub.reset();
_pool.returnToPool(sub);
// A temporary bookmark log file name is generated.
name = _fileName + ".tmp";
// prune(name) is called to remove any entries associated with
// this subscription from the bookmark store file
prune(name);
} finally {
_lock.unlock();
}
} finally {
_unlockSubs(subs);
}
} catch (IOException e) {
throw new StoreException("Underlying IOException while pruning. "
+ "temp bookmark log file = " + name, e);
} finally {
_subsLock.unlock();
}
}
/**
* Change the RecoveryPointFactory used by this store for its adapter.
*
* @param factory_ The new RecoveryPointFactory
* @throws AMPSException If one of factory or adapter is null.
*/
public void setRecoveryPointFactory(RecoveryPointFactory factory_) throws AMPSException {
if (factory_ == null || _adapter == null) {
throw new CommandException("Factory and Adapter must not be null.");
}
_factory = factory_;
}
/**
* Closes the bookmark store. If it is already closed, a StoreException is
* thrown.
*
* @throws StoreException If there is an error closing the file backing in the
* store, or the store already closed.
*/
public void close() throws AMPSException {
_lock.lock();
try {
StoreException ex = null;
if (_adapter != null) {
try {
_adapter.close();
} catch (Exception e) {
ex = new StoreException("Error closing adapter", e);
}
}
if (_file != null) {
try {
_file.close();
} catch (IOException ioex) {
if (ex == null) {
throw new StoreException("Error closing file", ioex);
} else {
throw new StoreException("Error closing adapter and file " + ioex.toString(), ex);
}
}
} else {
throw new StoreException("Store not open.");
}
if (ex != null)
throw ex;
} finally {
_adapter = null;
_factory = null;
_file = null;
_lock.unlock();
}
}
/**
* Used to change the version of the AMPS server that this bookmark store's
* client has connected to.
*
* @param version An AMPS server version integer of the form 03080000 for
* version 3.8.0.0.
*/
public void setServerVersion(int version) {
_serverVersion = version;
}
/**
* Called by Client when connected to an AMPS server in order to retrieve
* the version number of the server. Returns retrieved version number.
*
* @return the server version, represented as an integer
*/
public int getServerVersion() {
return _serverVersion;
}
/**
* Used internally to update the RecoveryPointAdapter if there is one.
* This method is used for internal bookkeeping and communication with the
* RecoveryPointAdapter, which is responsible for handling recovery points
* associated with the bookmark store.
*
* @param subId The subId to update.
* @param bookmark The latest bookmark.
* @throws IOException If there is an exception from the adapter.
*/
protected void adapterUpdate(Field subId, BookmarkField bookmark) throws IOException {
if (_adapter != null) {
try {
_adapter.update(_factory.createRecoveryPoint(subId, bookmark));
} catch (Exception e) {
throw new IOException("Exception in LoggedBookmarkStore updating the RecoveryPointAdapter", e);
}
}
}
/**
* Locks and returns the current list of Subscriptions.
* Used internally by prune() to gather and lock the list of subscriptions
* that we need to prune. We hold self's lock here to get the subscription
* list and then release it to lock up all of the subscriptions.
*/
private List> _lockSubs() {
// The _subsLock MUST already be held
// No need to hold self's _lock here.
List> subs = new Vector>(_subs.size());
for (Map.Entry entry : _subs.entrySet()) {
subs.add(entry);
}
// Lock the subscriptions without self's lock held.
for (Map.Entry entry : subs) {
entry.getValue().lock();
}
return subs;
}
/**
* Unlocks the provided list of Subscriptions.
* Used internally by prune() to gather and lock the list of subscriptions
* that we need to prune. We hold self's lock here to get the subscription
* list and then release it to lock up all of the subscriptions.
*/
private void _unlockSubs(List> subs_) {
// The _subsLock MUST already be held
// No need to hold self's _lock here.
// We are just unlocking the individual Subscriptions that were
// held for the duration of prune().
for (Map.Entry entry : subs_) {
entry.getValue().unlock();
}
}
}