com.gemstone.gemfire.internal.cache.tier.sockets.ClientHealthMonitor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of gemfire-core Show documentation
Show all versions of gemfire-core Show documentation
SnappyData store based off Pivotal GemFireXD
The newest version!
/*
* Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.gemstone.gemfire.internal.cache.tier.sockets;
import com.gemstone.gemfire.CancelException;
import com.gemstone.gemfire.SystemFailure;
import com.gemstone.gemfire.cache.Cache;
import com.gemstone.gemfire.i18n.LogWriterI18n;
import com.gemstone.gemfire.internal.LogWriterImpl;
import com.gemstone.gemfire.internal.cache.CacheClientStatus;
import com.gemstone.gemfire.internal.cache.IncomingGatewayStatus;
import com.gemstone.gemfire.internal.cache.tier.Acceptor;
import com.gemstone.gemfire.internal.concurrent.AIArray;
import com.gemstone.gemfire.internal.concurrent.CFactory;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.shared.Version;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Class ClientHealthMonitor is a server-side singleton that
* monitors the health of clients by looking at their heartbeats. If too much
* time elapses between heartbeats, the monitor determines that the client is
* dead and interrupts its threads.
*
* @author Barry Oglesby
*
* @since 4.2.3
*/
public class ClientHealthMonitor
{
/**
* The map of known clients
*/
protected volatile Map _clientHeartbeats = Collections.EMPTY_MAP;
/**
* An object used to lock the map of known clients
*/
final protected Object _clientHeartbeatsLock = new Object();
/**
* The map of known client threads
*/
final protected Map _clientThreads;
/**
* An object used to lock the map of client threads
*/
final private Object _clientThreadsLock = new Object();
/**
* THe GemFire Cache
*/
final protected Cache _cache;
/**
* The GemFire LogWriterI18n
*/
final protected LogWriterI18n _logger;
/**
* A thread that validates client connections
*/
final private ClientHealthMonitorThread _clientMonitor;
/**
* The singleton CacheClientNotifier instance
*/
static ClientHealthMonitor _instance;
/**
* Reference count in the event that multiple bridge servers are using the health monitor
*/
private static int refCount = 0;
/**
* The interval between client monitor iterations
*/
final protected static long CLIENT_MONITOR_INTERVAL = 1000;
final private CacheClientNotifierStats stats;
/**
* Used to track the number of handshakes in a VM primary use, license
* enforcement.
*
* note, these were moved from static fields in ServerConnection
* so that they will be cleaned up when the client health monitor is shutdown.
*/
private final HashMap cleanupTable = new HashMap();
private final HashMap cleanupProxyIdTable = new HashMap();
/**
* Gives, version-wise, the number of clients connected to the cache servers
* in this cache, which are capable of processing recieved deltas.
*
* NOTE: It does not necessarily give the actual number of clients per version
* connected to the cache servers in this cache.
*
* @see CacheClientNotifier#addClientProxy(CacheClientProxy)
*/
AIArray numOfClientsPerVersion = CFactory.createAIArray(Version.NUM_OF_VERSIONS);
/**
* Factory method to construct or return the singleton
* ClientHealthMonitor instance.
*
* @param cache
* The GemFire Cache
* @param maximumTimeBetweenPings
* The maximum time allowed between pings before determining the
* client has died and interrupting its sockets.
* @return The singleton ClientHealthMonitor instance
*/
public static ClientHealthMonitor getInstance(Cache cache,
int maximumTimeBetweenPings, CacheClientNotifierStats stats)
{
createInstance(cache, maximumTimeBetweenPings, stats);
return _instance;
}
/**
* Factory method to return the singleton ClientHealthMonitor
* instance.
*
* @return the singleton ClientHealthMonitor instance
*/
public static ClientHealthMonitor getInstance()
{
return _instance;
}
/**
* Shuts down the singleton ClientHealthMonitor instance.
*/
public synchronized static void shutdownInstance()
{
refCount--;
if (_instance == null)
return;
if (refCount > 0)
return;
_instance.shutdown();
boolean interrupted = false; // Don't clear, let join fail if already interrupted
try{
if (_instance._clientMonitor != null) {
_instance._clientMonitor.join();
}
}
catch(InterruptedException e) {
interrupted = true;
if(_instance._logger.fineEnabled()) {
_instance._logger
.fine(":Interrupted joining with the ClientHealthMonitor Thread",e);
}
}
finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
_instance = null;
refCount = 0;
}
/**
* Registers a new client to be monitored.
*
* @param proxyID
* The id of the client to be registered
*/
public void registerClient(ClientProxyMembershipID proxyID)
{
boolean registerClient = false;
synchronized (_clientHeartbeatsLock) {
Map oldClientHeartbeats = this._clientHeartbeats;
if (!oldClientHeartbeats.containsKey(proxyID)) {
Map newClientHeartbeats = new HashMap(oldClientHeartbeats);
newClientHeartbeats.put(proxyID, Long.valueOf(System.currentTimeMillis()));
this._clientHeartbeats = newClientHeartbeats;
registerClient = true;
}
}
if (registerClient) {
if (this.stats != null) {
this.stats.incClientRegisterRequests();
}
if (this._logger.infoEnabled()) {
this._logger.info(
LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITOR_REGISTERING_CLIENT_WITH_MEMBER_ID_0,
proxyID);
}
}
}
/**
* Takes care of unregistering from the _clientHeatBeats map.
* @param proxyID
* The id of the client to be unregistered
*/
private void unregisterClient(ClientProxyMembershipID proxyID) {
boolean unregisterClient = false;
synchronized (_clientHeartbeatsLock) {
Map oldClientHeartbeats = this._clientHeartbeats;
if (oldClientHeartbeats.containsKey(proxyID)) {
unregisterClient = true;
Map newClientHeartbeats = new HashMap(oldClientHeartbeats);
newClientHeartbeats.remove(proxyID);
this._clientHeartbeats = newClientHeartbeats;
}
}
if (unregisterClient) {
if (this._logger.infoEnabled()) {
this._logger.info(
LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITOR_UNREGISTERING_CLIENT_WITH_MEMBER_ID_0,
proxyID);
}
if (this.stats != null) {
this.stats.incClientUnRegisterRequests();
}
expireTXStates(proxyID);
}
}
/**
* Unregisters a client to be monitored.
*
* @param proxyID
* The id of the client to be unregistered
* @param acceptor
* non-null if the call is from a ServerConnection (as
* opposed to a CacheClientProxy).
* @param clientDisconnectedCleanly
* Whether the client disconnected cleanly or crashed
*/
public void unregisterClient(ClientProxyMembershipID proxyID,
AcceptorImpl acceptor,
boolean clientDisconnectedCleanly)
{
unregisterClient(proxyID);
// Unregister any CacheClientProxy instances associated with this member id
// if this method was invoked from a ServerConnection and the client did
// not disconnect cleanly.
if (acceptor != null) {
CacheClientNotifier ccn = acceptor.getCacheClientNotifier();
if (ccn != null) {
try {
ccn.unregisterClient(proxyID, clientDisconnectedCleanly);
}
catch (CancelException ignore) {
}
}
}
}
/**
* expire the transaction states for the given client. This uses the
* transactionTimeToLive setting that is inherited from the CacheServer.
* If that setting is non-positive we expire the states immediately
* @param proxyID
*/
private void expireTXStates(ClientProxyMembershipID proxyID) {
/* TODO: merge: trunk merge code for client TX
final TXManagerImpl txMgr = (TXManagerImpl)this._cache.getCacheTransactionManager();
final Set txids = txMgr.getTransactionsForClient(
(InternalDistributedMember)proxyID.getDistributedMember());
CacheClientNotifier notifier = CacheClientNotifier.getInstance();
if (notifier == null || this._cache.isClosed()) {
return; // notifier is null when shutting down
}
long timeout = notifier.getTransactionTimeToLive() * 1000;
if (txids.size() > 0) {
if (this._logger.fineEnabled()) {
this._logger.fine("expiring " + txids.size() + " transaction contexts for " + proxyID + " timeout=" + timeout/1000);
}
if (timeout <= 0) {
txMgr.removeTransactions(txids, true);
} else {
SystemTimerTask task = new SystemTimerTask() {
@Override
public LogWriterI18n getLoggerI18n() { return _logger; }
@Override
public void run2() {
txMgr.removeTransactions(txids, true);
}
};
((GemFireCacheImpl)this._cache).getCCPTimer().schedule(task, timeout);
}
}
*/
}
public void removeAllConnectionsAndUnregisterClient(
ClientProxyMembershipID proxyID)
{
// Remove all connections
cleanupClientThreads(proxyID, false);
unregisterClient(proxyID);
}
/**
* Adds a ServerConnection to the client's processing threads
*
* @param proxyID
* The membership id of the client to be updated
* @param connection
* The thread processing client requests
*/
public void addConnection(ClientProxyMembershipID proxyID,
ServerConnection connection)
{
//this._logger.info("ClientHealthMonitor: Adding " + connection + " to
// client with member id " + proxyID);
synchronized (_clientThreadsLock) {
Set serverConnections = (Set)this._clientThreads.get(proxyID);
if (serverConnections == null) {
serverConnections = new HashSet();
this._clientThreads.put(proxyID, serverConnections);
}
serverConnections.add(connection);
//this._logger.info("ClientHealthMonitor: The client with member id " +
// proxyID + " contains " + serverConnections.size() + " threads");
}
}
/**
* Removes a ServerConnection from the client's processing
* threads
*
* @param proxyID
* The id of the client to be updated
* @param connection
* The thread processing client requests
*/
public void removeConnection(ClientProxyMembershipID proxyID,
ServerConnection connection)
{
//this._logger.info("ClientHealthMonitor: Removing " + connection + " from
// client with member id " + proxyID);
synchronized (_clientThreadsLock) {
Set serverConnections = (Set)this._clientThreads.get(proxyID);
if (serverConnections != null) { // fix for bug 35343
serverConnections.remove(connection);
//this._logger.info("ClientHealthMonitor: The client with member id " +
// proxyID + " contains " + serverConnections.size() + " threads");
if (serverConnections.isEmpty()) {
//this._logger.info("ClientHealthMonitor: The client with member id "
// + proxyID + " is being removed since it contains 0 threads");
this._clientThreads.remove(proxyID);
}
}
}
}
/**
* Processes a received ping for a client.
*
* @param proxyID
* The id of the client from which the ping was received
*/
public void receivedPing(ClientProxyMembershipID proxyID)
{
if (this._clientMonitor == null) {
return;
}
if (this._logger.finerEnabled()) {
this._logger
.finer("ClientHealthMonitor: Received ping from client with member id "
+ proxyID);
}
synchronized (_clientHeartbeatsLock) {
if (!this._clientHeartbeats.containsKey(proxyID)) {
registerClient(proxyID);
}
else {
this._clientHeartbeats.put(proxyID,
Long.valueOf(System.currentTimeMillis()));
}
}
}
// /**
// * Returns modifiable map (changes do not effect this class) of memberId
// * to connection count.
// */
// public Map getConnectedClients() {
// Map map = new HashMap(); // KEY=memberId, VALUE=connectionCount (Integer)
// synchronized (_clientThreadsLock) {
// Iterator connectedClients = this._clientThreads.entrySet().iterator();
// while (connectedClients.hasNext()) {
// Map.Entry entry = (Map.Entry) connectedClients.next();
// String memberId = (String) entry.getKey();// memberId includes FQDN
// Set connections = (Set) entry.getValue();
// int socketPort = 0;
// InetAddress socketAddress = null;
// ///*
// Iterator serverConnections = connections.iterator();
// // Get data from one.
// while (serverConnections.hasNext()) {
// ServerConnection sc = (ServerConnection) serverConnections.next();
// socketPort = sc.getSocketPort();
// socketAddress = sc.getSocketAddress();
// break;
// }
// //*/
// int connectionCount = connections.size();
// String clientString = null;
// if (socketAddress == null) {
// clientString = "client member id=" + memberId;
// } else {
// clientString = "host name=" + socketAddress.toString() + " host ip=" +
// socketAddress.getHostAddress() + " client port=" + socketPort + " client
// member id=" + memberId;
// }
// map.put(memberId, new Object[] {clientString, new
// Integer(connectionCount)});
// /* Note: all client addresses are same...
// Iterator serverThreads = ((Set) entry.getValue()).iterator();
// while (serverThreads.hasNext()) {
// ServerConnection connection = (ServerConnection) serverThreads.next();
// InetAddress clientAddress = connection.getClientAddress();
// getLogger().severe("getConnectedClients: memberId=" + memberId +
// " clientAddress=" + clientAddress + " FQDN=" +
// clientAddress.getCanonicalHostName());
// }*/
// }
// }
// return map;
// }
/**
* Returns modifiable map (changes do not effect this class) of client
* membershipID to connection count. This is different from the map contained
* in this class as here the key is client membershipID & not the the proxyID.
* It is to be noted that a given client can have multiple proxies.
*
* @param filterProxies
* Set identifying the Connection proxies which should be fetched.
* These ConnectionProxies may be from same client member or
* different. If it is null this would mean to fetch the Connections
* of all the ConnectionProxy objects.
*
*/
public Map getConnectedClients(Set filterProxies)
{
Map map = new HashMap(); // KEY=proxyID, VALUE=connectionCount (Integer)
synchronized (_clientThreadsLock) {
Iterator connectedClients = this._clientThreads.entrySet().iterator();
while (connectedClients.hasNext()) {
Map.Entry entry = (Map.Entry)connectedClients.next();
ClientProxyMembershipID proxyID = (ClientProxyMembershipID)entry
.getKey();// proxyID includes FQDN
if (filterProxies == null || filterProxies.contains(proxyID)) {
String membershipID = null;
Set connections = (Set)entry.getValue();
int socketPort = 0;
InetAddress socketAddress = null;
///*
Iterator serverConnections = connections.iterator();
// Get data from one.
while (serverConnections.hasNext()) {
ServerConnection sc = (ServerConnection)serverConnections.next();
socketPort = sc.getSocketPort();
socketAddress = sc.getSocketAddress();
membershipID = sc.getMembershipID();
break;
}
//*/
int connectionCount = connections.size();
String clientString = null;
if (socketAddress == null) {
clientString = "client member id=" + membershipID;
}
else {
clientString = "host name=" + socketAddress.toString()
+ " host ip=" + socketAddress.getHostAddress()
+ " client port=" + socketPort + " client member id="
+ membershipID;
}
Object[] data = null;
data = (Object[])map.get(membershipID);
if (data == null) {
map.put(membershipID, new Object[] { clientString,
Integer.valueOf(connectionCount) });
}
else {
data[1] = Integer.valueOf(((Integer)data[1]).intValue()
+ connectionCount);
}
/*
* Note: all client addresses are same... Iterator serverThreads =
* ((Set) entry.getValue()).iterator(); while
* (serverThreads.hasNext()) { ServerConnection connection =
* (ServerConnection) serverThreads.next(); InetAddress clientAddress =
* connection.getClientAddress();
* getLogger().severe("getConnectedClients: proxyID=" + proxyID + "
* clientAddress=" + clientAddress + " FQDN=" +
* clientAddress.getCanonicalHostName()); }
*/
}
}
}
return map;
}
/**
* This method returns the CacheClientStatus for all the clients that are
* connected to this server. This method returns all clients irrespective of
* whether subscription is enabled or not.
*
* @return Map of ClientProxyMembershipID against CacheClientStatus objects.
*/
public Map getStatusForAllClients() {
Map result = new HashMap();
synchronized (_clientThreadsLock) {
Iterator connectedClients = this._clientThreads.entrySet().iterator();
while (connectedClients.hasNext()) {
Map.Entry entry = (Map.Entry)connectedClients.next();
ClientProxyMembershipID proxyID = (ClientProxyMembershipID)entry.getKey();
CacheClientStatus cci = new CacheClientStatus(proxyID);
Set connections = (Set)this._clientThreads.get(proxyID);
if (connections != null) {
String memberId = null;
Iterator connectionsIterator = connections.iterator();
while (connectionsIterator.hasNext()) {
ServerConnection sc = (ServerConnection)connectionsIterator.next();
byte communicationMode = sc.getCommunicationMode();
/* Check for all modes that could be used for Client-Server communication*/
if (communicationMode == Acceptor.CLIENT_TO_SERVER
|| communicationMode == Acceptor.PRIMARY_SERVER_TO_CLIENT
|| communicationMode == Acceptor.SECONDARY_SERVER_TO_CLIENT
|| communicationMode == Acceptor.CLIENT_TO_SERVER_FOR_QUEUE) {
memberId = sc.getMembershipID(); // each ServerConnection has the same member id
cci.setMemberId(memberId);
cci.setNumberOfConnections(connections.size());
result.put(proxyID, cci);
break;
}
}
}
}
}
return result;
}
public void fillInClientInfo(Map allClients)
{
// The allClients parameter includes only actual clients (not remote
// gateways). This monitor will include remote gateway connections,
// so weed those out.
synchronized (_clientThreadsLock) {
Iterator allClientsIterator = allClients.entrySet().iterator();
while (allClientsIterator.hasNext()) {
Map.Entry entry = (Map.Entry)allClientsIterator.next();
ClientProxyMembershipID proxyID = (ClientProxyMembershipID)entry
.getKey();// proxyID includes FQDN
CacheClientStatus cci = (CacheClientStatus)entry.getValue();
Set connections = (Set)this._clientThreads.get(proxyID);
if (connections != null) {
String memberId = null;
cci.setNumberOfConnections(connections.size());
List socketPorts = new ArrayList();
List socketAddresses = new ArrayList();
Iterator connectionsIterator = connections.iterator();
while (connectionsIterator.hasNext()) {
ServerConnection sc = (ServerConnection)connectionsIterator.next();
socketPorts.add(Integer.valueOf(sc.getSocketPort()));
socketAddresses.add(sc.getSocketAddress());
memberId = sc.getMembershipID(); // each ServerConnection has the
// same member id
}
cci.setMemberId(memberId);
cci.setSocketPorts(socketPorts);
cci.setSocketAddresses(socketAddresses);
}
}
}
}
public Map getConnectedIncomingGateways()
{
Map connectedIncomingGateways = new HashMap();
synchronized (_clientThreadsLock) {
Iterator connectedClients = this._clientThreads.entrySet().iterator();
while (connectedClients.hasNext()) {
Map.Entry entry = (Map.Entry)connectedClients.next();
ClientProxyMembershipID proxyID = (ClientProxyMembershipID)entry
.getKey();
Set connections = (Set)entry.getValue();
Iterator connectionsIterator = connections.iterator();
while (connectionsIterator.hasNext()) {
ServerConnection sc = (ServerConnection)connectionsIterator.next();
if (sc.getCommunicationMode() == Acceptor.GATEWAY_TO_GATEWAY) {
IncomingGatewayStatus status = new IncomingGatewayStatus(proxyID
.getDSMembership(), sc.getSocketAddress(), sc.getSocketPort());
connectedIncomingGateways.put(proxyID.getDSMembership(), status);
}
}
}
}
return connectedIncomingGateways;
}
protected boolean cleanupClientThreads(ClientProxyMembershipID proxyID, boolean timedOut) {
boolean result = false;
Set serverConnections = null;
synchronized (this._clientThreadsLock) {
serverConnections = (Set) this._clientThreads.remove(proxyID);
// It is ok to modify the set after releasing the sync
// because it has been removed from the map while holding
// the sync.
} // end sync here to fix bug 37576 and 36740
{
if (serverConnections != null) { // fix for bug 35343
result = true;
//getLogger().warning("Terminating " + serverConnections.size() + "
// connections");
for (Iterator it = serverConnections.iterator(); it.hasNext();) {
ServerConnection serverConnection = (ServerConnection)it.next();
//getLogger().warning("Terminating " + serverConnection);
serverConnection.handleTermination(timedOut);
}
}
}
return result;
}
protected boolean isAnyThreadProcessingMessage(ClientProxyMembershipID proxyID) {
boolean processingMessage = false;
synchronized (this._clientThreadsLock) {
Set serverConnections = (Set) this._clientThreads.get(proxyID);
if (serverConnections != null) {
for (Iterator it = serverConnections.iterator(); it.hasNext();) {
ServerConnection serverConnection = (ServerConnection) it.next();
if (serverConnection.isProcessingMessage()) {
processingMessage = true;
break;
}
}
}
}
return processingMessage;
}
protected void validateThreads(ClientProxyMembershipID proxyID) {
Set serverConnections = null;
synchronized (this._clientThreadsLock) {
serverConnections = (Set) this._clientThreads.get(proxyID);
if (serverConnections != null) {
serverConnections = new HashSet(serverConnections);
}
}
// release sync and operation on copy to fix bug 37675
if (serverConnections != null) {
for (Iterator it = serverConnections.iterator(); it.hasNext();) {
ServerConnection serverConnection = (ServerConnection) it.next();
if (serverConnection.hasBeenTimedOutOnClient()) {
if (getLogger().warningEnabled()) {
getLogger().warning(
LocalizedStrings.ClientHealtMonitor_0_IS_BEING_TERMINATED_BECAUSE_ITS_CLIENT_TIMEOUT_OF_1_HAS_EXPIRED,
new Object[] {serverConnection, Integer.valueOf(serverConnection.getClientReadTimeout())});
}
try {
serverConnection.handleTermination(true);
// Not all the code in a ServerConnection correctly
// handles interrupt. In particular it is possible to be doing
// p2p distribution and to have sent a message to one peer but
// to never send it to another due to interrupt.
//serverConnection.interruptOwner();
}
finally {
// Just to be sure we clean it up.
// This call probably isn't needed.
removeConnection(proxyID, serverConnection);
}
}
}
}
}
/**
* Returns the GemFire LogWriterI18n.
*
* @return the GemFire LogWriterI18n
*/
protected LogWriterI18n getLogger()
{
return this._logger;
}
/**
* Returns the map of known clients.
*
* @return the map of known clients
*/
public Map getClientHeartbeats()
{
return this._clientHeartbeats;
}
/**
* Shuts down the singleton CacheClientNotifier instance.
*/
protected synchronized void shutdown() {
// Stop the client monitor
if (this._clientMonitor != null) {
this._clientMonitor.stopMonitoring();
}
}
/**
* Creates the singleton CacheClientNotifier instance.
*
* @param cache
* The GemFire Cache
* @param maximumTimeBetweenPings
* The maximum time allowed between pings before determining the
* client has died and interrupting its sockets.
*/
protected static synchronized void createInstance(Cache cache,
int maximumTimeBetweenPings, CacheClientNotifierStats stats) {
refCount++;
if (_instance != null) {
return;
}
_instance = new ClientHealthMonitor(cache, maximumTimeBetweenPings, stats);
}
/**
*
* Constructor.
*
* @param cache
* The GemFire Cache
* @param maximumTimeBetweenPings
* The maximum time allowed between pings before determining the
* client has died and interrupting its sockets.
*/
private ClientHealthMonitor(Cache cache, int maximumTimeBetweenPings,
CacheClientNotifierStats stats) {
// Set the Cache
this._cache = cache;
// Set the LogWriterI18n
this._logger = cache.getLoggerI18n();
// Initialize the client threads map
this._clientThreads = new HashMap();
if (maximumTimeBetweenPings > 0) {
if (this._logger.fineEnabled())
this._logger.fine(this + ": Initializing client health monitor thread");
this._clientMonitor = new ClientHealthMonitorThread(maximumTimeBetweenPings);
this._clientMonitor.start();
} else {
if (this._logger.configEnabled()) {
this._logger.config(LocalizedStrings.ClientHealthMonitor_CLIENT_HEALTH_MONITOR_THREAD_DISABLED_DUE_TO_MAXIMUMTIMEBETWEENPINGS_SETTING__0, maximumTimeBetweenPings);
}
this._clientMonitor = null;
}
this.stats = stats;
}
/**
* Returns a brief description of this ClientHealthMonitor
*
* @since 5.1
*/
@Override
public String toString()
{
return "ClientHealthMonitor@"
+ Integer.toHexString(System.identityHashCode(this));
}
public Map getCleanupProxyIdTable() {
return cleanupProxyIdTable;
}
public Map getCleanupTable() {
return cleanupTable;
}
public int getNumberOfClientsAtVersion(Version version) {
return numOfClientsPerVersion.get(version.ordinal());
}
public int getNumberOfClientsAtOrAboveVersion(Version version) {
int number = 0;
for (int i = version.ordinal(); i < numOfClientsPerVersion.length(); i++) {
number += numOfClientsPerVersion.get(i);
}
return number;
}
public boolean hasDeltaClients() {
return getNumberOfClientsAtOrAboveVersion(Version.GFE_61) > 0;
}
/**
* Class ClientHealthMonitorThread is a Thread
* that verifies all clients are still alive.
*/
class ClientHealthMonitorThread extends Thread
{
/**
* The maximum time allowed between pings before determining the client has
* died and interrupting its sockets.
*/
final protected int _maximumTimeBetweenPings;
/**
* Whether the monitor is stopped
*/
protected volatile boolean _isStopped = false;
/**
* Constructor.
*
* @param maximumTimeBetweenPings
* The maximum time allowed between pings before determining the
* client has died and interrupting its sockets
*/
protected ClientHealthMonitorThread(int maximumTimeBetweenPings) {
super(LogWriterImpl.createThreadGroup("ClientHealthMonitor Thread Group",
getLogger()), "ClientHealthMonitor Thread");
setDaemon(true);
// Set the client connection timeout
this._maximumTimeBetweenPings = maximumTimeBetweenPings;
if (getLogger().configEnabled()) {
getLogger().config(
LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITORTHREAD_MAXIMUM_ALLOWED_TIME_BETWEEN_PINGS_0,
this._maximumTimeBetweenPings);
}
if (maximumTimeBetweenPings == 0) {
if (getLogger().fineEnabled()) {
getLogger().fine("zero ping interval detected",
new Exception(LocalizedStrings.ClientHealthMonitor_STACK_TRACE_0.toLocalizedString()));
}
}
}
/**
* Notifies the monitor to stop monitoring.
*/
protected synchronized void stopMonitoring()
{
if (_logger.fineEnabled()) {
_logger.fine(ClientHealthMonitor.this + ": Stopping monitoring");
}
this._isStopped = true;
this.interrupt();
if (_logger.fineEnabled()) {
_logger.fine(ClientHealthMonitor.this + ": Stopped dispatching");
}
}
/**
* Returns whether the dispatcher is stopped
*
* @return whether the dispatcher is stopped
*/
protected boolean isStopped()
{
return this._isStopped;
}
/**
* Runs the monitor by iterating the map of clients and testing the latest
* ping time received against the current time.
*/
@Override
public void run()
{
if (_logger.fineEnabled()) {
_logger.fine(ClientHealthMonitor.this
+ ": Beginning to monitor clients");
}
while (!this._isStopped) {
SystemFailure.checkFailure();
try {
Thread.sleep(CLIENT_MONITOR_INTERVAL);
if (getLogger().finerEnabled()) {
getLogger().finer(
"Monitoring " + getClientHeartbeats().size() + " client(s)");
}
//getLogger().warning("Monitoring " + getClientHeartbeats().size() +
// " client(s).");
// Get the current time
long currentTime = System.currentTimeMillis();
if (getLogger().finerEnabled()) {
getLogger().finer(
ClientHealthMonitor.this.toString() + " starting sweep at " + currentTime);
}
// Iterate through the clients and verify that they are all still
// alive
for (Iterator i = getClientHeartbeats().entrySet().iterator(); i
.hasNext();) {
Map.Entry entry = (Map.Entry)i.next();
ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();
// Validate all ServerConnection threads. If a thread has been
// processing a message for more than the socket timeout time,
// close it it since the client will have timed out and resent.
validateThreads(proxyID);
Long latestHeartbeatValue = (Long)entry.getValue();
// Compare the current value with the current time if it is not null
// If it is null, that means that the client was just registered
// and has not done a heartbeat yet.
if (latestHeartbeatValue != null) {
long latestHeartbeat = latestHeartbeatValue.longValue();
if (getLogger().finerEnabled()) {
getLogger()
.finer(
(currentTime - latestHeartbeat)
+ " ms have elapsed since the latest heartbeat for client with member id "
+ proxyID);
}
if ( (currentTime-latestHeartbeat) > this._maximumTimeBetweenPings) {
// This client has been idle for too long. Determine whether
// any of its ServerConnection threads are currently processing
// a message. If so, let it go. If not, disconnect it.
if (isAnyThreadProcessingMessage(proxyID)) {
if (getLogger().fineEnabled()) {
getLogger().fine("Monitoring client with member id " + entry.getKey() + ". It has been " + (currentTime - latestHeartbeat) + " ms since the latest heartbeat. This client would have been terminated but at least one of its threads is processing a message.");
}
} else {
if (cleanupClientThreads(proxyID, true)) {
if (getLogger().warningEnabled()) {
getLogger().warning(LocalizedStrings.ClientHealthMonitor_MONITORING_CLIENT_WITH_MEMBER_ID_0_IT_HAD_BEEN_1_MS_SINCE_THE_LATEST_HEARTBEAT_MAX_INTERVAL_IS_2_TERMINATED_CLIENT, new Object[] {entry.getKey(), currentTime - latestHeartbeat, this._maximumTimeBetweenPings});
}
}
}
} else {
if (getLogger().finerEnabled()) {
getLogger()
.finer(
"Monitoring client with member id "
+ entry.getKey()
+ ". It has been "
+ (currentTime - latestHeartbeat)
+ " ms since the latest heartbeat. This client is healthy.");
}
//getLogger().warning("Monitoring client with member id " +
// entry.getKey() + ". It has been " + (currentTime -
// latestHeartbeat) + " ms since the latest heartbeat. This
// client is healthy.");
}
}
}
}
catch (InterruptedException e) {
// no need to reset the bit; we're exiting
if (this._isStopped) {
break;
}
if (_logger.warningEnabled()) {
_logger.warning(LocalizedStrings.ClientHealthMonitor_UNEXPECTED_INTERRUPT_EXITING, e);
}
break;
}
catch (Exception e) {
// An exception occurred while monitoring the clients. If the monitor
// is not stopped, log it and continue processing.
if (!this._isStopped) {
_logger.severe(
LocalizedStrings.ClientHealthMonitor_0_AN_UNEXPECTED_EXCEPTION_OCCURRED,
ClientHealthMonitor.this, e);
}
}
} // while
}
} // ClientHealthMonitorThread
}