com.tangosol.internal.net.topic.impl.paged.PagedTopicCaches Maven / Gradle / Ivy
Show all versions of coherence Show documentation
/*
* Copyright (c) 2000, 2022, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
*/
package com.tangosol.internal.net.topic.impl.paged;
import com.oracle.coherence.common.base.Exceptions;
import com.oracle.coherence.common.base.Logger;
import com.oracle.coherence.common.collections.ConcurrentHashMap;
import com.tangosol.internal.net.NamedCacheDeactivationListener;
import com.tangosol.internal.net.topic.impl.paged.agent.EnsureSubscriptionProcessor;
import com.tangosol.internal.net.topic.impl.paged.agent.EvictSubscriber;
import com.tangosol.internal.net.topic.impl.paged.filter.UnreadTopicContentFilter;
import com.tangosol.internal.net.topic.impl.paged.model.NotificationKey;
import com.tangosol.internal.net.topic.impl.paged.model.Page;
import com.tangosol.internal.net.topic.impl.paged.model.ContentKey;
import com.tangosol.internal.net.topic.impl.paged.model.PagedPosition;
import com.tangosol.internal.net.topic.impl.paged.model.SubscriberGroupId;
import com.tangosol.internal.net.topic.impl.paged.model.SubscriberId;
import com.tangosol.internal.net.topic.impl.paged.model.SubscriberInfo;
import com.tangosol.internal.net.topic.impl.paged.model.Subscription;
import com.tangosol.internal.net.topic.impl.paged.model.Usage;
import com.tangosol.io.ClassLoaderAware;
import com.tangosol.io.Serializer;
import com.tangosol.net.CacheService;
import com.tangosol.net.DistributedCacheService;
import com.tangosol.net.Member;
import com.tangosol.net.MemberEvent;
import com.tangosol.net.MemberListener;
import com.tangosol.net.NamedCache;
import com.tangosol.net.PartitionedService;
import com.tangosol.net.cache.TypeAssertion;
import com.tangosol.net.partition.PartitionSet;
import com.tangosol.net.topic.NamedTopic;
import com.tangosol.net.topic.Position;
import com.tangosol.net.topic.Subscriber;
import com.tangosol.util.AbstractMapListener;
import com.tangosol.util.Aggregators;
import com.tangosol.util.Base;
import com.tangosol.util.Binary;
import com.tangosol.util.Filter;
import com.tangosol.util.Filters;
import com.tangosol.util.HashHelper;
import com.tangosol.util.InvocableMap;
import com.tangosol.util.InvocableMapHelper;
import com.tangosol.util.MapEvent;
import com.tangosol.util.NullImplementation;
import com.tangosol.util.ValueExtractor;
import com.tangosol.util.aggregator.Count;
import com.tangosol.util.aggregator.GroupAggregator;
import com.tangosol.util.extractor.EntryExtractor;
import com.tangosol.util.extractor.ReflectionExtractor;
import com.tangosol.util.filter.AlwaysFilter;
import com.tangosol.util.filter.PartitionedFilter;
import java.io.PrintStream;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static com.tangosol.net.cache.TypeAssertion.withTypes;
/**
* This class encapsulates operations on the set of {@link NamedCache}s
* that are used to hold the underlying data for a topic.
*
* @author jk 2015.06.19
* @since Coherence 14.1.1
*/
@SuppressWarnings("rawtypes")
public class PagedTopicCaches
implements ClassLoaderAware, AutoCloseable
{
// ----- constructors ---------------------------------------------------
/**
* Create a {@link PagedTopicCaches}.
*
* @param sName the name of the topic
* @param cacheService the {@link CacheService} owning the underlying caches
* @param dependencies the {@link PagedTopicDependencies}
*/
public PagedTopicCaches(String sName, CacheService cacheService, PagedTopicDependencies dependencies)
{
this(sName, cacheService, dependencies, null);
}
/**
* Create a {@link PagedTopicCaches}.
*
* @param sName the name of the topic
* @param cacheService the {@link CacheService} owning the underlying caches
* @param dependencies the {@link PagedTopicDependencies}
* @param functionCache the function to invoke to obtain each underlying cache
*/
PagedTopicCaches(String sName, CacheService cacheService,
PagedTopicDependencies dependencies,
BiFunction functionCache)
{
if (sName == null || sName.isEmpty())
{
throw new IllegalArgumentException("The name argument cannot be null or empty String");
}
if (cacheService == null)
{
throw new IllegalArgumentException("The cacheService argument cannot be null");
}
if (functionCache == null)
{
functionCache = cacheService::ensureCache;
}
f_sTopicName = sName;
f_cacheService = cacheService;
f_sCacheServiceName = cacheService.getInfo().getServiceName();
f_cPartition = ((DistributedCacheService) cacheService).getPartitionCount();
f_dependencies = dependencies == null ? new DefaultPagedTopicDependencies(f_cPartition) : dependencies;
f_functionCache = functionCache;
initializeCaches();
m_state = State.Active;
}
// ----- TopicCaches methods --------------------------------------
/**
* Return the serializer.
*
* @return the serializer
*/
public Serializer getSerializer()
{
return f_cacheService.getSerializer();
}
@Override
public void close()
{
release();
}
/**
* Destroy the PagedTopicCaches.
*/
public void release()
{
releaseOrDestroy(/* destroy */ false);
}
/**
* Destroy the PagedTopicCaches.
*/
public void destroy()
{
releaseOrDestroy(/* destroy */ true);
}
/**
* Determines {@code true} if the caches are active.
*
* @return true if the caches are active; false otherwise
*/
public boolean isActive()
{
State state = m_state;
return state == State.Active || state == State.Disconnected;
}
/**
* Returns {@code true} if the caches are destroyed,
* specifically the page cache for the topic.
*
* @return true if the caches are destroyed; false otherwise
*/
public boolean isDestroyed()
{
return Pages.isDestroyed();
}
/**
* Returns whether or not the caches are released,
* specifically the page cache for the topic.
*
* @return true if the caches are released; false otherwise
*/
public boolean isReleased()
{
return Pages.isReleased();
}
public synchronized void addListener(Listener listener)
{
ensureListeners();
m_mapListener.put(listener, Boolean.TRUE);
}
public synchronized void removeListener(Listener listener)
{
m_mapListener.remove(listener);
}
public void ensureConnected()
{
if (m_state == State.Disconnected)
{
synchronized (this)
{
if (m_state == State.Disconnected)
{
m_state = State.Active;
f_setCaches.forEach(NamedCache::size);
Set setListener = m_mapListener.keySet();
for (Listener listener : setListener)
{
try
{
listener.onConnect();
}
catch (Throwable t)
{
Logger.err(t);
}
}
}
}
}
}
// ----- ClassLoaderAware methods ---------------------------------------
@Override
public ClassLoader getContextClassLoader()
{
return f_cacheService.getContextClassLoader();
}
@Override
public void setContextClassLoader(ClassLoader classLoader)
{
throw new UnsupportedOperationException();
}
// ----- accessor methods -----------------------------------------------
/**
* Return the topic name.
*
* @return the topic name
*/
public String getTopicName()
{
return f_sTopicName;
}
/**
* Get the start page for this topic upon creation.
*
* @return the start page
*/
public int getBasePage()
{
return Math.abs(f_sTopicName.hashCode() % f_cPartition);
}
/**
* Return the partition count for this topic.
*
* @return the partition count for this topic
*/
public int getPartitionCount()
{
return f_cPartition;
}
/**
* Return the channel count for this topic.
*
* @return the channel count for this topic
*/
public int getChannelCount()
{
return getDependencies().getChannelCount();
}
/**
* Return the set of NotificationKeys covering all partitions for the given notifier
*
* @param nNotifier the notifier id
*
* @return the NotificationKeys
*/
public Set getPartitionNotifierSet(int nNotifier)
{
Set setKey = new HashSet<>();
for (int i = 0; i < f_cPartition; ++i)
{
setKey.add(new NotificationKey(i, nNotifier));
}
return setKey;
}
/**
* Return the unit of order for a topic partition.
*
* @param nPartition the partition
*
* @return the unit of order
*/
public int getUnitOfOrder(int nPartition)
{
return f_sTopicName.hashCode() + nPartition;
}
/**
* Return the associated CacheService.
*
* @return the cache service
*/
public CacheService getCacheService()
{
return f_cacheService;
}
/**
* Return the {@link PagedTopicDependencies}.
*
* @return the {@link PagedTopicDependencies}
*/
public PagedTopicDependencies getDependencies()
{
return f_dependencies;
}
/**
* Returns the {@link Usage.Key} for the {@link Usage} entry that tracks the topic's tail for
* a given channel.
*
* We don't just use partition zero as that would concentrate extra load on a single partitions
* when there are many channels.
*
* @param nChannel the channel number
*
* @return the {@link Usage.Key} for the {@link Usage} entry that tracks the topic's head and tail
*/
public Usage.Key getUsageSyncKey(int nChannel)
{
int nPart = Math.abs((HashHelper.hash(f_sTopicName.hashCode(), nChannel) % f_cPartition));
return new Usage.Key(nPart, nChannel);
}
/**
* Returns {@code true} if a subscriber in the specified group has committed the specified {@link Position}
* in the specified channel.
*
* @param groupId the subscriber group identifier
* @param nChannel the channel identifier
* @param position the {@link Position} to check
*
* @return {@code true} if a subscriber in the specified group has committed the specified {@link Position}
* in the specified channel; or {@code false} if the position is not committed or the subscriber
* group does not exist
*/
public boolean isCommitted(SubscriberGroupId groupId, int nChannel, Position position)
{
if (position instanceof PagedPosition && nChannel >= 0 && nChannel < getChannelCount())
{
Map map = getLastCommitted(groupId);
Position posCommitted = map.get(nChannel);
return posCommitted != null && posCommitted.compareTo(position) >= 0;
}
return false;
}
/**
* Returns a {@link Map} of channel numbers to the latest {@link Position} committed
* for that channel. If a channel has had zero commits it will be missing
* from the map's keySet.
*
* @param subscriberGroupId the {@link SubscriberGroupId identifier} of the subscriber
* group to obtain the commits for
*
* @return {@code true} if the specified {@link Position} has been committed in the
* specified channel
*/
public Map getLastCommitted(SubscriberGroupId subscriberGroupId)
{
ValueExtractor extractorChannel = new ReflectionExtractor<>("getChannelId", new Object[0], EntryExtractor.KEY);
ValueExtractor extractorGroup = new ReflectionExtractor<>("getGroupId", new Object[0], EntryExtractor.KEY);
Filter filter = Filters.equal(extractorGroup, subscriberGroupId);
Filter filterPosition = Filters.not(Filters.equal(PagedPosition::getPage, Page.NULL_PAGE));
InvocableMap.EntryAggregator aggregatorPosn
= Aggregators.comparableMax(Subscription::getCommittedPosition);
// Aggregate the subscription commits and remove any null values from the returned map
return Subscriptions.aggregate(filter, GroupAggregator.createInstance(extractorChannel, aggregatorPosn, filterPosition))
.entrySet()
.stream()
.filter(e -> e.getKey() != Page.EMPTY && e.getValue() != null)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
/**
* Returns a {@link Map} of the {@link Position} of the head for each channel for a subscriber group.
*
* @param subscriberGroupId the {@link SubscriberGroupId identifier} of the subscriber
* group to obtain the heads for
*
* @return a {@link Map} of the {@link Position} of the head for each channel for a subscriber group
*/
public Map getHeads(SubscriberGroupId subscriberGroupId, long nSubscriberId)
{
ValueExtractor extractorChannel = new ReflectionExtractor<>("getChannelId", new Object[0], EntryExtractor.KEY);
ValueExtractor extractorGroup = new ReflectionExtractor<>("getGroupId", new Object[0], EntryExtractor.KEY);
Filter filter = Filters.equal(extractorGroup, subscriberGroupId);
Filter filterPosition = Filters.not(Filters.equal(PagedPosition::getPage, Page.NULL_PAGE));
InvocableMap.EntryAggregator aggregatorPosn
= Aggregators.comparableMin(new Subscription.HeadExtractor(nSubscriberId));
// Aggregate the subscription commits and remove any null values from the returned map
return Subscriptions.aggregate(filter, GroupAggregator.createInstance(extractorChannel, aggregatorPosn, filterPosition))
.entrySet()
.stream()
.filter(e -> e.getKey() != Page.EMPTY && e.getValue() != null)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
/**
* Returns a {@link Map} of the {@link Position} of the head for each channel.
*
* @return a {@link Map} of the {@link Position} of the head for each channel
*/
public Map getHeads()
{
ValueExtractor extractorChannel
= new ReflectionExtractor<>("getChannelId", new Object[0], EntryExtractor.KEY);
InvocableMap.EntryAggregator aggregatorTail
= Aggregators.comparableMin(Page.HeadExtractor.INSTANCE);
return Pages.aggregate(GroupAggregator.createInstance(extractorChannel, aggregatorTail));
}
/**
* Returns a {@link Map} of the {@link Position} of the tail for each channel.
*
* @return a {@link Map} of the {@link Position} of the tail for each channel
*/
public Map getTails()
{
ValueExtractor extractorChannel
= new ReflectionExtractor<>("getChannelId", new Object[0], EntryExtractor.KEY);
InvocableMap.EntryAggregator aggregatorTail
= Aggregators.comparableMax(Page.TailExtractor.INSTANCE);
return Pages.aggregate(GroupAggregator.createInstance(extractorChannel, aggregatorTail));
}
/**
* Returns the {@link NamedTopic.ElementCalculator} to use to calculate message sizes.
*
* @return the {@link NamedTopic.ElementCalculator} to use to calculate message sizes
*/
public NamedTopic.ElementCalculator getElementCalculator()
{
return getDependencies().getElementCalculator();
}
/**
* Returns the identifiers for all the subscribers belonging to a subscriber group.
*
* There is no guarantee that all of the subscribers are actually still active. If a subscriber
* process exits without closing the subscriber, the identifier remains in the cache until it
* is timed-out.
*
* @param sSubscriberGroup the subscriber group name to get subscribers for
*
* @return the identifiers for all the subscribers belonging to a subscriber group
*/
public Set getSubscribers(String sSubscriberGroup)
{
ValueExtractor extractor
= ValueExtractor.of(SubscriberInfo.Key::getGroupId).fromKey();
return Subscribers.keySet(Filters.equal(extractor, SubscriberGroupId.withName(sSubscriberGroup)));
}
/**
* Return the set of {@link Subscriber.Name named} subscriber group(s) and statically configured subscriber-group(s).
*
* @return the set of named subscriber groups
*/
public Set getSubscriberGroups()
{
return getSubscriberGroupsIds(false)
.stream()
.map(SubscriberGroupId::getGroupName)
.collect(Collectors.toSet());
}
/**
* Return the set of subscriber group(s) for the topic, optionally including pseudo-groups
* created for anonymous subscribers.
*
* @param fAnonymous {@code true} to include anonymous subscriber groups
*
* @return the set of named subscriber groups
*/
public Set getSubscriberGroupsIds(boolean fAnonymous)
{
int cParts = ((PartitionedService) Subscriptions.getCacheService()).getPartitionCount();
PartitionSet setPart = new PartitionSet(cParts);
setPart.add(Base.getRandom().nextInt(cParts));
Set setSubs = Subscriptions.keySet(new PartitionedFilter<>(AlwaysFilter.INSTANCE(), setPart));
Stream stream = setSubs.stream()
.map(Subscription.Key::getGroupId);
if (!fAnonymous)
{
stream = stream.filter((key) -> !key.isAnonymous());
}
return stream.collect(Collectors.toSet());
}
/**
* Returns an immutable map of subscriber id to channels allocated to that subscriber.
*
* @param sGroup the subscriber group to obtain allocations for
*
* @return an immutable map of subscriber id to channels allocated to that subscriber
*/
public Map> getChannelAllocations(String sGroup)
{
Subscription.Key key = new Subscription.Key(0, 0, SubscriberGroupId.withName(sGroup));
Subscription subscription = Subscriptions.get(key);
if (subscription != null)
{
return Collections.unmodifiableMap(subscription.getAllocationMap());
}
return Collections.emptyMap();
}
/**
* Print the channel allocations for the specified subscriber group.
*
* @param sGroup the name of the subscriber group
* @param out the {@link PrintStream} to print to
*/
public void printChannelAllocations(String sGroup, PrintStream out)
{
Map mapMember = f_cacheService.getCluster()
.getMemberSet()
.stream()
.collect(Collectors.toMap(Member::getId, Member::toString));
out.println("Subscriber channel allocations for topic \"" + f_sTopicName + "\" subscriber group \"" + sGroup + "\":");
for (Map.Entry> entry : getChannelAllocations(sGroup).entrySet())
{
long nId = entry.getKey();
int nMember = PagedTopicSubscriber.memberIdFromId(nId);
out.println("SubscriberId=" + nId + " channels=" + entry.getValue() + " " + mapMember.get(nMember));
}
}
/**
* Disconnect a specific subscriber from a topic.
*
* Disconnecting a subscriber will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param sGroup the name of the subscriber group
* @param nId the subscriber id
*
* @return {@code true} if the subscriber was disconnected
*/
public boolean disconnectSubscriber(String sGroup, long nId)
{
return Subscribers.invoke(new SubscriberInfo.Key(SubscriberGroupId.withName(sGroup), nId), EvictSubscriber.INSTANCE);
}
/**
* Disconnect a specific subscriber from a topic.
*
* Disconnecting a subscriber will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param key the {@link SubscriberInfo.Key key} of the subscriber to disconnect
*
* @return {@code true} if the subscriber was disconnected
*/
public boolean disconnectSubscriber(SubscriberInfo.Key key)
{
return Subscribers.invoke(key, EvictSubscriber.INSTANCE);
}
/**
* Disconnect all group subscribers from a topic.
*
* Disconnecting subscribers will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param sGroup the name of the subscriber group
* @param nMember the cluster member to disconnect subscribers for
*
* @return the identifiers of the disconnected subscribers
*/
public long[] disconnectAllSubscribers(String sGroup, int nMember)
{
return disconnectAllSubscribers(SubscriberGroupId.withName(sGroup), nMember);
}
/**
* Disconnect all group subscribers from a topic.
*
* Disconnecting subscribers will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param id the {@link SubscriberGroupId id} of the subscriber group
* @param nMember the cluster member to disconnect subscribers for
*
* @return the identifiers of the disconnected subscribers
*/
public long[] disconnectAllSubscribers(SubscriberGroupId id, int nMember)
{
Filter filter = Filters.equal(SubscriberInfo.GroupIdExtractor.INSTANCE, id)
.and(Filters.equal(SubscriberInfo.MemberIdExtractor.INSTANCE, nMember));
Map map = Subscribers.invokeAll(filter, EvictSubscriber.INSTANCE);
return map.keySet().stream().mapToLong(SubscriberInfo.Key::getSubscriberId).toArray();
}
/**
* Disconnect all subscribers in a group.
*
* Disconnecting subscribers will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param sGroup the name of the subscriber group
*
* @return the identifiers of the disconnected subscribers
*/
public long[] disconnectAllSubscribers(String sGroup)
{
return disconnectAllSubscribers(SubscriberGroupId.withName(sGroup));
}
/**
* Disconnect all subscribers in a group.
*
* Disconnecting subscribers will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @param id the {@link SubscriberGroupId id} of the subscriber group
*
* @return the identifiers of the disconnected subscribers
*/
public long[] disconnectAllSubscribers(SubscriberGroupId id)
{
Filter> filter = Filters.equal(SubscriberInfo.GroupIdExtractor.INSTANCE, id);
Map map = Subscribers.invokeAll(filter, EvictSubscriber.INSTANCE);
return map.keySet().stream().mapToLong(SubscriberInfo.Key::getSubscriberId).toArray();
}
/**
* Disconnect all group subscribers from all groups.
*
* Disconnecting subscribers will cause channels to be reallocated and
* positions to be rolled back to the last commit for the channels
* owned by the disconnected subscriber.
*
* @return the identifiers of the disconnected subscribers
*/
public long[] disconnectAllSubscribers()
{
long[] alId = Subscribers.keySet()
.stream()
.mapToLong(SubscriberInfo.Key::getSubscriberId)
.toArray();
for (SubscriberGroupId id : getSubscriberGroupsIds(false))
{
PagedTopicSubscriber.notifyClosed(Subscriptions, id, SubscriberId.NullSubscriber);
}
Subscribers.clear();
return alId;
}
/**
* Ensure the specified subscriber group exists.
*
* @param sName the name of the group
* @param filter the filter to use to filter messages received by the group
* @param fnConverter the converter function to convert the messages received by the group
*/
protected void ensureSubscriberGroup(String sName, Filter filter, Function fnConverter)
{
SubscriberGroupId subscriberGroupId = SubscriberGroupId.withName(sName);
initializeSubscription(subscriberGroupId, SubscriberId.NullSubscriber, filter, fnConverter, false, true, false);
}
/**
* Initialise a subscription.
*
* @param subscriberGroupId the subscriber group identifier
* @param subscriberId the subscriber identifier
* @param filter the filter to use to filter messages received by the subscription
* @param fnConverter the converter function to convert the messages received by the subscription
* @param fReconnect {@code true} if this is a reconnection
* @param fCreateGroupOnly {@code true} if this is to only create a subscriber group
* @param fDisconnected {@code true} if this is an existing, disconnected subscription
*
* @return the pages that are the heads of the channels
*/
protected long[] initializeSubscription(SubscriberGroupId subscriberGroupId,
SubscriberId subscriberId,
Filter filter,
Function fnConverter,
boolean fReconnect,
boolean fCreateGroupOnly,
boolean fDisconnected)
{
try
{
int cChannel = getChannelCount();
String sName = subscriberGroupId.getGroupName();
Set setSubKeys = new HashSet<>(f_cPartition);
for (int i = 0; i < f_cPartition; ++i)
{
// Note: we ensure against channel 0 in each partition, and it will in turn initialize all channels
setSubKeys.add(new Subscription.Key(i, /*nChannel*/ 0, subscriberGroupId));
}
// outside any lock discover if pages are already pinned. Note that since we don't
// hold a lock, this is only useful if the group was already fully initialized (under lock) earlier.
// Otherwise, there is no guarantee that there isn't gaps in our pinned pages.
// check results to verify if initialization has already completed
EnsureSubscriptionProcessor processor = new EnsureSubscriptionProcessor(EnsureSubscriptionProcessor.PHASE_INQUIRE,
null, filter, fnConverter, subscriberId, fReconnect, fCreateGroupOnly);
Collection results;
if (sName == null)
{
results = null;
}
else
{
CompletableFuture> future
= InvocableMapHelper.invokeAllAsync(Subscriptions,
setSubKeys,
key -> getUnitOfOrder(key.getPartitionId()), processor);
try
{
long cMillis = getDependencies().getSubscriberTimeoutMillis();
results = future.get(cMillis, TimeUnit.MILLISECONDS).values();
}
catch (TimeoutException e)
{
try
{
future.cancel(true);
}
catch (Throwable ignored)
{
// ignored
}
throw Exceptions.ensureRuntimeException(e);
}
}
Collection colPages = EnsureSubscriptionProcessor.Result.assertPages(results);
long[] alHead = new long[cChannel];
if (colPages == null || colPages.contains(null) || fDisconnected)
{
alHead = initialiseSubscriptionPages(subscriberGroupId, subscriberId, filter, fnConverter, fReconnect, fCreateGroupOnly, setSubKeys);
}
else
{
// all partitions were already initialized, min is our head
for (int nChannel = 0; nChannel < cChannel; ++nChannel)
{
final int finChan = nChannel;
alHead[nChannel] = colPages.stream().mapToLong((alResult) -> alResult[finChan])
.min().orElse(Page.NULL_PAGE);
}
}
return alHead;
}
catch (InterruptedException | ExecutionException e)
{
throw Exceptions.ensureRuntimeException(e);
}
}
/**
* Initialise the head pages for the subscriber.
*
* @param subscriberGroupId the subscriber group identifier
* @param subscriberId the subscriber identifier
* @param filter the filter to use to filter messages received by the subscription
* @param fnConverter the converter function to convert the messages received by the subscription
* @param fReconnect {@code true} if this is a reconnection
* @param fCreateGroupOnly {@code true} if this is to only create a subscriber group
* @param setSubKeys the set of {@link Subscription.Key keys} to use to initialise the subscription pages
*
* @return an array of head pages
*
* @throws InterruptedException if any asynchronous operations are interrupted
* @throws ExecutionException if any asynchronous operations fail
*/
@SuppressWarnings("OptionalGetWithoutIsPresent")
protected long[] initialiseSubscriptionPages(SubscriberGroupId subscriberGroupId,
SubscriberId subscriberId,
Filter filter,
Function fnConverter,
boolean fReconnect,
boolean fCreateGroupOnly,
Set setSubKeys)
throws InterruptedException, ExecutionException
{
String sGroupName = subscriberGroupId.getGroupName();
// The subscription doesn't exist in at least some partitions, create it under lock. A lock is used only
// to protect against concurrent create/destroy/create resulting an gaps in the pinned pages. Specifically
// it would be safe for multiple subscribers to concurrently "create" the subscription, it is only unsafe
// if there is also a concurrent destroy as this could result in gaps in the pinned pages.
if (sGroupName != null)
{
Subscriptions.lock(subscriberGroupId, -1);
}
try
{
EnsureSubscriptionProcessor processor
= new EnsureSubscriptionProcessor(EnsureSubscriptionProcessor.PHASE_PIN, null,
filter, fnConverter, subscriberId, fReconnect, fCreateGroupOnly);
CompletableFuture> future
= InvocableMapHelper.invokeAllAsync(Subscriptions,
setSubKeys,
key -> getUnitOfOrder(key.getPartitionId()),
processor);
Collection results;
try
{
long cMillis = getDependencies().getSubscriberTimeoutMillis();
results = future.get(cMillis, TimeUnit.MILLISECONDS).values();
}
catch (TimeoutException e)
{
try
{
future.cancel(true);
}
catch (Throwable ignored)
{
// ignored
}
throw Exceptions.ensureRuntimeException(e);
}
Collection colPages = EnsureSubscriptionProcessor.Result.assertPages(results);
PagedTopicDependencies dependencies = getDependencies();
int cChannel = getChannelCount();
long lPageBase = getBasePage();
long[] alHead = new long[cChannel];
// mapPages now reflects pinned pages
for (int nChannel = 0; nChannel < cChannel; ++nChannel)
{
final int finChan = nChannel;
if (fReconnect || dependencies.isRetainConsumed())
{
// select lowest page in each channel as our channel heads
alHead[nChannel] = colPages.stream()
.mapToLong((alPage) -> Math.max(alPage[finChan], lPageBase))
.min()
.getAsLong();
}
else
{
// select highest page in each channel as our channel heads
alHead[nChannel] = colPages.stream()
.mapToLong((alPage) -> Math.max(alPage[finChan], lPageBase))
.max()
.getAsLong();
}
}
// finish the initialization by having subscription in all partitions advance to our selected heads
processor = new EnsureSubscriptionProcessor(EnsureSubscriptionProcessor.PHASE_ADVANCE, alHead, filter, fnConverter, subscriberId, fReconnect, fCreateGroupOnly);
InvocableMapHelper.invokeAllAsync(Subscriptions, setSubKeys,
key -> getUnitOfOrder(key.getPartitionId()), processor).join();
return alHead;
}
finally
{
if (sGroupName != null)
{
Subscriptions.unlock(subscriberGroupId);
}
}
}
/**
* Return an immutable set of all the caches for the topic.
*
* @return an immutable set of all the caches for the topic
*/
public Set getCaches()
{
return Collections.unmodifiableSet(f_setCaches);
}
/**
* Returns the number of remaining messages to be read from the topic for the specific {@link SubscriberGroupId}.
*
* This method is a sum of the remaining messages for each channel from the last committed message (exclusive)
* to the current tail.This result returned by this method is somewhat transient in situations where there are
* active Subscribers with in-flight commit requests, so the count may change just after the method returns.
* Message expiry may also affect the returned value, if messages expire after the count is returned.
*
* @param id the {@link SubscriberGroupId}
*
* @return the number of remaining messages for the {@link SubscriberGroupId}
*/
@SuppressWarnings("unchecked")
public int getRemainingMessages(SubscriberGroupId id, int... anChannel)
{
if (Subscriptions.containsKey(new Subscription.Key(0, 0, id)))
{
Map mapHeads = getLastCommitted(id);
Map mapTails = getTails();
for (int i = 0; i < getChannelCount(); i++)
{
mapHeads.putIfAbsent(i, new PagedPosition(-1L, -1));
}
if (anChannel.length > 0)
{
List listChannel = IntStream.of(anChannel).boxed().collect(Collectors.toList());
mapHeads.keySet().retainAll(listChannel);
mapTails.keySet().retainAll(listChannel);
}
InvocableMap.EntryAggregator counter = new Count();
Binary bin = (Binary) Data.aggregate(new UnreadTopicContentFilter(mapHeads, mapTails), counter);
return ((Number) f_cacheService.getBackingMapManager().getContext()
.getValueFromInternalConverter().convert(bin)).intValue();
}
// subscriber group does not exist, return the total number of messages
return Data.size();
}
// ----- object methods -------------------------------------------------
@Override
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (o == null || getClass() != o.getClass())
{
return false;
}
PagedTopicCaches that = (PagedTopicCaches) o;
return f_sTopicName.equals(that.f_sTopicName) && f_cacheService.equals(that.f_cacheService);
}
@Override
public int hashCode()
{
return HashHelper.hash(f_sTopicName, 31);
}
@Override
public String toString()
{
return "TopicCaches(name='" + f_sTopicName
+ ", service=" + f_sCacheServiceName
+ ", state=" + m_state
+ ")";
}
// ----- helper methods -------------------------------------------------
@SuppressWarnings("unchecked")
private synchronized void initializeCaches()
{
f_cacheService.start();
ClassLoader loader = f_cacheService.getContextClassLoader();
Pages = f_functionCache.apply(Names.PAGES.cacheNameForTopicName(f_sTopicName), loader);
Subscribers = f_functionCache.apply(Names.SUBSCRIBERS.cacheNameForTopicName(f_sTopicName), loader);
Notifications = f_functionCache.apply(Names.NOTIFICATIONS.cacheNameForTopicName(f_sTopicName), loader);
Usages = f_functionCache.apply(Names.USAGE.cacheNameForTopicName(f_sTopicName), loader);
Subscriptions = f_functionCache.apply(Names.SUBSCRIPTIONS.cacheNameForTopicName(f_sTopicName), loader);
Data = f_functionCache.apply(Names.CONTENT.cacheNameForTopicName(f_sTopicName), NullImplementation.getClassLoader());
Set setCaches = f_setCaches = new HashSet<>();
setCaches.add(Pages);
setCaches.add(Data);
setCaches.add(Subscriptions);
setCaches.add(Subscribers);
setCaches.add(Notifications);
setCaches.add(Usages);
ensureListeners();
}
@SuppressWarnings("unchecked")
private synchronized void ensureListeners()
{
DeactivationListener listener = m_deactivationListener;
Pages.addMapListener(listener);
f_cacheService.addMemberListener(listener);
}
@SuppressWarnings("unchecked")
private synchronized void removeListeners()
{
DeactivationListener listener = m_deactivationListener;
if (Pages.isActive())
{
Pages.removeMapListener(listener);
}
f_cacheService.removeMemberListener(listener);
}
/**
* Release or destroy the PagedTopicCaches.
*
* @param fDestroy true to destroy, false to release
*/
private void releaseOrDestroy(boolean fDestroy)
{
if (!isActive())
{
return;
}
synchronized (this)
{
if (isActive())
{
m_state = fDestroy ? State.Destroyed : State.Released;
Set setListener = m_mapListener.keySet();
for (Listener listener : setListener)
{
try
{
if (fDestroy)
{
listener.onDestroy();
}
else
{
listener.onRelease();
}
}
catch (Throwable t)
{
Logger.err(t);
}
}
removeListeners();
if (f_setCaches != null)
{
Consumer function = fDestroy ? this::destroyCache : this::releaseCache;
synchronized (this)
{
if (f_setCaches != null)
{
f_setCaches.forEach(function);
f_setCaches = null;
}
}
}
}
}
}
private void releaseCache(NamedCache cache)
{
if (cache.isActive() && !cache.isReleased())
{
cache.release();
}
}
private void destroyCache(NamedCache cache)
{
if (cache.isActive() && !cache.isDestroyed())
{
cache.destroy();
}
}
void disconnected()
{
if (m_state == State.Active)
{
synchronized (this)
{
if (m_state == State.Active)
{
m_state = State.Disconnected;
Set setListener = m_mapListener.keySet();
for (Listener listener : setListener)
{
try
{
listener.onDisconnect();
}
catch (Throwable t)
{
Logger.err(t);
}
}
}
}
}
}
// ----- inner class: Names ---------------------------------------------
/**
* An pseudo enumeration representing the different caches used
* by topic and topic implementations.
*
* @author jk 2015.06.08
* @since Coherence 14.1.1
*/
public static class Names
{
// ----- constructors ---------------------------------------------------
/**
* Create a TopicCacheNames with the given cache name prefix.
*
* @param sName the name of this TopicCacheNames.
* @param sPrefix the prefix to use to obtain the cache name from the topic name
* @param classKey the type of the cache keys
* @param classValue the type of the cache values
*/
private Names(String sName, String sPrefix, Class classKey, Class classValue, Storage storage)
{
f_sName = sName;
f_sPrefix = sPrefix;
f_classKey = classKey;
f_classValue = classValue;
f_typeAssertion = withTypes(f_classKey, f_classValue);
f_storage = storage;
s_setValues.add(this);
}
// ----- TopicCacheNames methods ----------------------------------------
/**
* Return the cache name from the specified topic name.
*
* @param sTopicName the topic name
*
* @return the cache name
*/
public String cacheNameForTopicName(String sTopicName)
{
return f_sPrefix + sTopicName;
}
/**
* Return the {@link Names} that matches the specified cache name.
*
* @param sCacheName the cache name;
*
* @return the {@link Names} that matches the specified cache name
*/
public static Names fromCacheName(String sCacheName)
{
for (Names pagedTopicCacheNames : values())
{
if (sCacheName.startsWith(pagedTopicCacheNames.f_sPrefix))
{
return pagedTopicCacheNames;
}
}
throw new IllegalArgumentException("Cache name " + sCacheName + " is not a valid TopicCacheName");
}
/**
* For a given cache name return the topic name.
*
* @param sCacheName the cache name
*
* @return the topic name
*/
public static String getTopicName(String sCacheName)
{
for (Names pagedTopicCacheNames : values())
{
if (sCacheName.startsWith(pagedTopicCacheNames.f_sPrefix))
{
return sCacheName.substring(pagedTopicCacheNames.f_sPrefix.length());
}
}
return sCacheName;
}
/**
* Obtain the set of all possible {@link Names}.
*
* @return the set of all possible {@link Names}
*/
public static Set values()
{
return Collections.unmodifiableSet(s_setValues);
}
// ----- accessor methods -----------------------------------------------
/**
* Obtain the {@link TypeAssertion} to use when obtaining a type safe
* version of the cache this {@link Names} represents.
*
* @return the {@link TypeAssertion} to use when obtaining a type safe
* version of the cache this {@link Names} represents
*/
public TypeAssertion getTypeAssertion()
{
return f_typeAssertion;
}
/**
* Obtain the prefix used to get the cache name from the
* topic name.
*
* @return the prefix used to get the cache name from the
* topic name
*/
public String getPrefix()
{
return f_sPrefix;
}
/**
* Obtain the Class of the cache keys.
*
* @return the Class of the cache keys
*/
public Class getKeyClass()
{
return f_classKey;
}
/**
* Obtain the Class of the cache values.
*
* @return the Class of the cache values
*/
public Class getValueClass()
{
return f_classValue;
}
/**
* Obtain the storage location for the cache of this type.
*
* @return the storage location for the cache of this type
*/
public Storage getStorage()
{
return f_storage;
}
/**
* Return true if corresponding cache should be considered an internal submapping.
*
* @return true if corresponding sub cache mapping should be considered internal.
*/
public boolean isInternal()
{
return Storage.MetaData.equals(getStorage());
}
// ----- object methods -------------------------------------------------
@Override
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (o == null || getClass() != o.getClass())
{
return false;
}
Names names = (Names) o;
return f_sName.equals(names.f_sName);
}
@Override
public int hashCode()
{
return f_sName.hashCode();
}
@Override
public String toString()
{
return f_sName;
}
// ----- constants ------------------------------------------------------
/**
* The {@link Set} of all legal {@link Names}.
*/
private static final Set s_setValues = new HashSet<>();
/**
* The name prefix for all meta-data {@link Names} used to implement a {@link NamedTopic}.
*/
public static final String METACACHE_PREFIX="$meta$topic";
/**
* The cache that holds the topic content.
*
* Use of no prefix rather than METACACHE_PREFIX since it is being used to filter out internal topic meta caches.
*/
public static final Names CONTENT =
new Names<>("content", "$topic$", ContentKey.class, Object.class, Names.Storage.Data);
/**
* The cache that holds the topic pages.
*/
public static final Names PAGES =
new Names<>("pages", METACACHE_PREFIX +"$pages$", Page.Key.class,
Page.class, Names.Storage.MetaData);
/**
* The cache that holds the topic subscriber partitions.
*/
public static final Names SUBSCRIPTIONS =
new Names<>("subscriptions", METACACHE_PREFIX + "$subscriptions$",
Subscription.Key.class, Subscription.class,
Names.Storage.MetaData);
/**
* The cache that holds subscriber information.
*/
public static final Names SUBSCRIBERS =
new Names<>("subscribers", METACACHE_PREFIX + "$subscribers$",
SubscriberInfo.Key.class, SubscriberInfo.class,
Names.Storage.MetaData);
/**
* The cache used for notifying publishers and subscribers of full/empty events
*/
public static final Names USAGE =
new Names<>("usage", METACACHE_PREFIX + "$usage$",
Usage.Key.class, Usage.class, Names.Storage.Data);
// ----- inner class: StorageType ---------------------------------------
public enum Storage {Data, MetaData}
// ----- data members ---------------------------------------------------
/**
* The name of this {@link Names}.
*/
private final String f_sName;
/**
* The prefix to add to the topic name to obtain the cache name.
*/
private final String f_sPrefix;
/**
* The key class of the cache.
*/
private final Class f_classKey;
/**
* The value class of the cache.
*/
private final Class f_classValue;
/**
* The {@link TypeAssertion} to use when obtaining the cache.
*/
private final TypeAssertion f_typeAssertion;
/**
* The storage location for the cache data.
*/
private final Storage f_storage;
}
// ----- inner class: DeactivationListener ------------------------------
/**
* A listener to detect the topic caches being deactivated due to release, destroy,
* service shutdown or all storage members departing.
*/
class DeactivationListener
extends AbstractMapListener
implements NamedCacheDeactivationListener, MemberListener
{
@Override
@SuppressWarnings("rawtypes")
public void entryDeleted(MapEvent evt)
{
// destroy/disconnect event
NamedCache cache = (NamedCache) evt.getMap();
boolean fReleased = cache.isReleased();
boolean fDestroyed = cache.isDestroyed();
if (fReleased || fDestroyed)
{
String sReason = fReleased ? "release" : "destroy";
Logger.fine("Detected " + sReason + " of topic " + f_sTopicName);
PagedTopicCaches.this.releaseOrDestroy(fDestroyed);
}
}
@Override
public void memberLeft(MemberEvent evt)
{
DistributedCacheService service = (DistributedCacheService) evt.getService();
if (evt.isLocal())
{
Logger.fine("Detected local member disconnect in service " + PagedTopicCaches.this);
disconnected();
}
else if (service.getOwnershipEnabledMembers().isEmpty())
{
Logger.fine("Detected loss of all storage members in service " + PagedTopicCaches.this);
disconnected();
}
}
@Override
public void memberJoined(MemberEvent evt)
{
}
@Override
public void memberLeaving(MemberEvent evt)
{
}
@Override
public boolean equals(Object oThat)
{
return oThat instanceof DeactivationListener && hashCode() == oThat.hashCode();
}
@Override
public int hashCode()
{
return System.identityHashCode(this);
}
}
// ----- inner interface: Listener --------------------------------------
/**
* A listener that can be registered with a {@link PagedTopicCaches} instance
* to be notified of disconnection events.
*/
public interface Listener
{
/**
* The caches have been disconnected.
*/
void onDisconnect();
/**
* The caches have been connected.
*/
void onConnect();
/**
* The caches have been destroyed.
*/
void onDestroy();
/**
* The caches have been released.
*/
void onRelease();
}
// ----- inner enum: State ----------------------------------------------
public enum State
{
Active,
Released,
Destroyed,
Disconnected,
Closed
}
// ----- data members ---------------------------------------------------
/**
* The topic name.
*/
protected final String f_sTopicName;
/**
* The cache service.
*/
protected final CacheService f_cacheService;
/**
* The topic dependencies.
*/
protected final PagedTopicDependencies f_dependencies;
/**
* The cache service name (mainly used in logging but calls to serv,ce.getInfo().getServiceName()
* can trigger a service restart - which is not always desirable).
*/
protected final String f_sCacheServiceName;
/**
* The current state of this {@link PagedTopicCaches}.
*/
private volatile State m_state;
/**
* The number of partitions in the cache service.
*/
protected final int f_cPartition;
/**
* The caches which back the topic.
*/
protected volatile Set f_setCaches;
/**
* The cache that holds the topic pages.
*/
public NamedCache Pages;
/**
* The cache that holds the topic elements.
*/
public NamedCache Data;
/**
* The cache that holds the topic subscriber partitions.
*/
public NamedCache Subscriptions;
/**
* The cache that holds the topic subscriber information.
*/
public NamedCache Subscribers;
/**
* The cache that is used to notify blocked publishers and subscribers that they topic is no longer full/empty.
*/
public NamedCache Notifications;
/**
* The cache that holds the highest used topic pages for a cache partition.
*/
public NamedCache Usages;
private final BiFunction f_functionCache;
/**
* The deactivation listener used to detect topic destroy and service restarts.
*/
private final DeactivationListener m_deactivationListener = new DeactivationListener();
/**
* The {@link Listener} instances to be notified of connection and disconnection events.
*/
private final Map m_mapListener = new ConcurrentHashMap<>();
}