All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.alachisoft.ncache.client.internal.communication.Broker Maven / Gradle / Ivy

There is a newer version: 5.3.0
Show newest version
package com.alachisoft.ncache.client.internal.communication;

import Alachisoft.NCache.Caching.EventId;
import Alachisoft.NCache.Common.BitSet;
import Alachisoft.NCache.Common.Communication.Secure.SslConfiguration;
import Alachisoft.NCache.Common.DataStructures.NewHashmap;
import Alachisoft.NCache.Common.Enum.RequestStatus;
import Alachisoft.NCache.Common.Extensibility.Client.RPC.PartitioningStrategy;
import Alachisoft.NCache.Common.Logger.ILogger;
import Alachisoft.NCache.Common.Logger.JLogger;
import Alachisoft.NCache.Common.Logger.LoggerNames;
import Alachisoft.NCache.Common.Net.Address;
import Alachisoft.NCache.Common.Threading.*;
import Alachisoft.NCache.Common.Util.ReaderWriterLock;
import com.alachisoft.ncache.licensing.LicenseManager;
import Alachisoft.NCache.Management.Statistics.StatisticsCounter;

import com.alachisoft.ncache.client.*;
import com.alachisoft.ncache.client.internal.caching.*;
import com.alachisoft.ncache.client.internal.command.*;
import com.alachisoft.ncache.client.internal.util.ClientConfiguration;
import com.alachisoft.ncache.client.internal.util.Logs;
import com.alachisoft.ncache.common.protobuf.*;
import Alachisoft.NCache.Common.ErrorHandling.ErrorCodes;
import Alachisoft.NCache.Common.ErrorHandling.ErrorMessages;
import com.alachisoft.ncache.runtime.events.EventDataFilter;
import com.alachisoft.ncache.runtime.exceptions.*;
import com.alachisoft.ncache.runtime.exceptions.SecurityException;
import com.alachisoft.ncache.runtime.exceptions.runtime.CacheRuntimeException;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;

import static Alachisoft.NCache.Common.EncryptionUtil.Encrypt;
import static com.alachisoft.ncache.licensing.LicenseManager.getNCacheBuildType;

public class Broker implements ServerLostListener {
    public static final int ForcedViewId = -5;
    private static java.util.Map _requestDic = new java.util.HashMap();
    public boolean _perfStatsEnabled = false;
    public ClientConfiguration _clientConfig;
    public StatisticsCounter _perfStatsColl;
    public Latch _hashMapStatus = new Latch(HashMapStatus.UNINITIALIZE);
    private byte[] _value;
    private AtomicLong _requestId = new AtomicLong(-1);

    // use max request Id = 9,999,999,999 long sequence value; if request id excede the max value we reset the request sequence from 0
    // This maz limit of requestid is set ; java and dotnet directly not serailze the short and long and int directly there must have work arround for this apporach
    // Java data types have not unsigned values and its ending (little) may differ from java
    // that why we move request id serialization with help of string conversion ;
    // As long have max value is 9,223,372,036,854,775,807 and if string conversion required then 20 bytes are use for request id that not practical apporach
    // so we limit the of request id till 9,999,999,999. that minizise the hit on memory during network call.
    private final Long MAXREQUESTID = new Long("9999999999");
    private long _allowedRequests = 200000;
    private long _shutdownTimeout = 180;
    private double _retryConnectionDelayInMinutes = 1; // total mintues;
    private int _connectionRetries = 5;
    private int _connectionTimeout = 5000;
    private int _connectionMutexTimeout = -1;
    private int _port;
    private int _asyncProccesorThreadCount = 1;
    private boolean _connectionRectified;
    private boolean _retryConnection = true;
    private boolean _isLocalAddress = false;
    private boolean _connectingFirstTime = true;
    private boolean _balanceNode;
    private boolean _notifyAsync = true;
    private String _cacheId;
    private String _cacheConfigId;
    private String _monitoringSessionId;
    private String _licenceCode;
    private ConnectionKeepAlive _connectionPinger;
    private java.util.ArrayList missingEvents = new java.util.ArrayList();
    private HashMap _requestTable = null;
    private ReaderWriterLock _lock = new ReaderWriterLock();
    private LocalDateTime _retryConnectionStartTime =LocalDateTime.now();
    private RemoteCache _cache = null;
    //this processor will be used to reconnect with connection who are disconnected.
    private AsyncProcessor _processor = null;
    //this is for bulk events
    private AsyncProcessor _eventProcessor = null;
    private ResponseIntegrator _responseIntegrator = new ResponseIntegrator();
    private RequestModerator requestModerator = new RequestModerator();
    private Address _serverIP;
    private AddressUtil _addressUtil;
    private Object _requestsLock = new Object();
    private LicenseManager.LicenseType _type;
    private ClientInfo _clientInfo;
    private Object _hashmapUpdateMutex = new Object();
    private java.util.HashMap _shutdownServers = new java.util.HashMap();
    private PersistenceManager _persistenceManager = null;
    private ThrottlingManager _throttleManager = new ThrottlingManager(100);
    private java.util.Map _reconnectTasks = new java.util.HashMap();
    private Logs privateLogger = new Logs();
    private ILogger _ncacheLogger;
    private boolean privateIsPersistenceEnabled;
    private int privatePersistInterval;
    private PartitioningStrategy privatePartitioningStrategy;
    private ClientLicenseType clientLicenseType = ClientLicenseType.values()[0];
    private InetAddress nodeAddress;
    private int newServerPort;
    private boolean importHashmap = false;
    private Connection connection;
    private boolean privateIsDisposing;
    private ConnectionPool connectionPool = null;
    private SocketManagerHandler privateSocketManagerHandler;
    private int retryInterval = 1;
    private int operationTimeout = 90000;//default 90 sec.
    private boolean _pipeliningEnabled;
    private int _pipeliningBatchInterval;

    public Broker(RemoteCache cache, CacheConnectionOptions cacheConnectionOptions, StatisticsCounter statsCol, ClientInfo clientInfo) {
        this(cache, true, statsCol, cacheConnectionOptions);
        _clientInfo = clientInfo;
    }

    private Broker(RemoteCache cache, boolean importHashMap, StatisticsCounter perfStatsColl, CacheConnectionOptions cacheConnectionOptions) {
        _clientConfig = new ClientConfiguration(cache.getName(), cacheConnectionOptions);
        _cache = cache;
        _licenceCode = "LicenceCode";

        _balanceNode = _clientConfig.getBalanceNodes();
        importHashmap = _clientConfig.getImportHashmap();

        operationTimeout = _clientConfig.getClientRequestTimeout();
        _connectionTimeout = _clientConfig.getConnectionTimeout();
        _connectionRetries = 3;
        retryInterval = 3;
        _perfStatsColl = perfStatsColl;

        this._requestTable = new HashMap();
        this.setPool(new ConnectionPool());
    }
    public String getMonitoringSessionId()
    {
        return _monitoringSessionId;
    }

    public String getCacheConfigID(){
        return _cacheConfigId;
    }

    public ClientConfiguration getClientConfig() {
        return _clientConfig;
    }

    public Latch getHashMapStatus() {
        return _hashMapStatus;
    }

    public int getPort() {
        return _port;
    }


    public HashMap getRequestTable() {
        return _requestTable;
    }

    public ReaderWriterLock getlock() {
        return _lock;
    }

    public ResponseIntegrator getResponseIntegrator() {
        return _responseIntegrator;
    }

    Address getServerIP() {
        return _serverIP;
    }

    public Object getHashmapUpdateMutex() {
        return _hashmapUpdateMutex;
    }

    public HashMap getShutdownServers() {
        return _shutdownServers;
    }

    public LicenseManager.LicenseType getLicenseType() {
        return _type;
    }

    public RemoteCache getCache() {
        return _cache;
    }

    public final Logs getLogger() {
        return privateLogger;
    }

    public final void setLogger(Logs value) {
        privateLogger = value;
    }
    public final void setNCacheLog(ILogger logger)
    {
        _ncacheLogger=logger;
    }
    public final ILogger getNCacheLog()
    {
        return _ncacheLogger;
    }

    public final int getRetryInterval() {
        return retryInterval;
    }

    public final boolean getIsPersistenceEnabled() {
        return privateIsPersistenceEnabled;
    }

    private void setIsPersistenceEnabled(boolean value) {
        privateIsPersistenceEnabled = value;
    }

    public final int getPersistInterval() {
        return privatePersistInterval;
    }

    private void setPersistInterval(int value) {
        privatePersistInterval = value;
    }

    private long getRequestId() {
        long requestId = _requestId.incrementAndGet();
        if(requestId > MAXREQUESTID)
        {
            _requestId.set(0);
             requestId = _requestId.incrementAndGet();
        }
        return requestId;
    }

    public final long getClientLastViewId() {
        return this.getPool().getLastViewId();
    }

    public final PartitioningStrategy getPartitioningStrategy() {
        return privatePartitioningStrategy;
    }

    public final void setPartitioningStrategy(PartitioningStrategy value) {
        privatePartitioningStrategy = value;
    }

    public final ClientLicenseType getClientLicenseType() {
        return clientLicenseType;
    }

    public final void setClientLicenseType(ClientLicenseType value) {
        clientLicenseType = value;
    }

    public final InetAddress getNodeIP() {
        return nodeAddress;
    }

    public final void setNodeIP(InetAddress value) {
        nodeAddress = value;
    }

    public final int getNewServerPort() {
        return newServerPort;
    }

    public final void setNewServerPort(int value) {
        newServerPort = value;
    }

    public final boolean getIsConnected() {
        return getConnection() != null && getConnection().getIsConnected();
    }

    public final boolean getPoolHasAllServers() {
        return _clientConfig.getServerCount() == getPool().getServers().size();
    }

    public final boolean getPoolFullyConnected() {
        synchronized (_hashmapUpdateMutex) {
            boolean poolFullyConnected = getPool().getFullyConnnected();

            if (!poolFullyConnected) {
                if (getPool().getConnections() != null) {
                    try {
                        //as pool is fully disconnected,let's start reconnection task
                        for (Object item : getPool().getConnections().values()) {
                            Connection connection = (Connection) item;
                            if (connection != null && !connection.getIsConnected()) {
                                ReconectInBackground(connection.getServerAddress(), connection);
                            }
                        }
                    } catch (Exception e) {
                        //enumeration exception can occur
                    }
                }
            }

            if (_shutdownServers.size() > 1) {
                return false;
            }
            return poolFullyConnected;
        }
    }

    public final boolean poolFullyDisConnected() {
        synchronized (_hashmapUpdateMutex) {
            if (getPool().getFullyDisConnnected()) {
                if (getPool().getConnections() != null) {
                    try {
                        //as pool is fully disconnected,let's start reconnection task
                        for (Object item : getPool().getConnections().values()) {
                            Connection connection = (Connection) item;
                            if (connection != null) {
                                ReconectInBackground(connection.getServerAddress(), connection);
                            }
                        }
                    } catch (RuntimeException e) {
                        //enumeration exception can occur
                    }
                }
                return true;
            }
        }

        return false;
    }

    public final boolean getImportHashmap() {
        return importHashmap;
    }

    public final void setImportHashmap(boolean value) {
        importHashmap = value;
    }

    public final java.util.ArrayList getClientServerList() {
        return _clientConfig.getServerList();
    }

    public final byte[] getValue() {
        return _value;
    }

    public final void setValue(byte[] value) {
        if (value != null) {
            _value = new byte[value.length];
            System.arraycopy(value, 0, _value, 0, value.length);
        }
    }

    final Connection getConnection() {
        return connection;
    }

    public final void setConnection(Connection value) {
        connection = value;
    }


    public final int getOperationTimeout() {
        return operationTimeout;
    }

    public final void setOperationTimeout(int value) {
        operationTimeout = value;
    }

    public final boolean getIsDisposing() {
        return privateIsDisposing;
    }

    public final void setIsDisposing(boolean value) {
        privateIsDisposing = value;
    }

    public final ConnectionPool getPool() {
        return connectionPool;
    }

    public final void setPool(ConnectionPool value) {
        connectionPool = value;
    }

    public final SocketManagerHandler getSocketManagerHandler() {
        return privateSocketManagerHandler;
    }

    public final void setSocketManagerHandler(SocketManagerHandler value) {
        privateSocketManagerHandler = value;
    }

    public final ServerInfo GetInitialServer() {
        ServerInfo serverInfo = new ServerInfo();
        List list = _clientConfig.getCacheConnectionOptions().getServerList();
        if (list.size() > 0) {
            serverInfo = list.get(0);
        }
        return serverInfo;

    }

    public final void StartServices(String cacheId, String server, int port) throws Exception {
        try {

            setSocketManagerHandler(new SocketManagerHandler(this));
            getSocketManagerHandler().StartSocketManager(true);

            this._cacheId = cacheId;

            try {
                String AsynEventNotification = System.getProperty("NCacheClient.AsynchronousEventNotification");
                if (AsynEventNotification != null && !AsynEventNotification.isEmpty())
                    _notifyAsync = Boolean.parseBoolean(AsynEventNotification);
            } catch (Exception ex) {
                throw new OperationFailedException(ErrorCodes.Common.INVALID_VALUE_ASYNC_EVENT_NOTIF, ErrorMessages.getErrorMessage(ErrorCodes.Common.INVALID_VALUE_ASYNC_EVENT_NOTIF));
            }

            if (!_notifyAsync) {
                try {
                    String asyncProccesorThreadCountString = System.getProperty("NCacheClient.NumberofEventProccesingThreads");
                    if (asyncProccesorThreadCountString != null && asyncProccesorThreadCountString.isEmpty())
                        _asyncProccesorThreadCount = Integer.getInteger(asyncProccesorThreadCountString);

                } catch (Exception ex) {
                    throw new OperationFailedException(ErrorCodes.Common.NUMBER_OF_EVENTS_PROCESSING_THREADS, ErrorMessages.getErrorMessage(ErrorCodes.Common.NUMBER_OF_EVENTS_PROCESSING_THREADS));
                }

                if (_asyncProccesorThreadCount <= 0) {
                    _asyncProccesorThreadCount = 1;
                }

                if (_asyncProccesorThreadCount > 5) {
                    _asyncProccesorThreadCount = 5;
                }

                _eventProcessor = new AsyncProcessor(_asyncProccesorThreadCount);
                _eventProcessor.Start();
            }

            //Check the type of license the application
            try {
                _clientConfig.loadConfiguration();
            } catch (ConfigurationException e) {

            }

            boolean enable_logs = false;
            boolean detailed_logs = false;

            if (System.getProperty("enableNCWebLogs") != null) {
                enable_logs = Boolean.parseBoolean(System.getProperty("enableNCWebLogs"));
            } else {
                enable_logs = _clientConfig.getEnableClientLogs();
            }

            if (System.getProperty("enableDetailedNCWebLogs") != null) {
                detailed_logs = Boolean.parseBoolean(System.getProperty("enableDetailedNCWebLogs"));
            } else {
                detailed_logs = _clientConfig.getEnableDetailedClientLogs();
            }

            InitializeLogs(enable_logs, detailed_logs);

            if (System.getProperty("enablePerfStats") != null) {
                _perfStatsEnabled = Boolean.parseBoolean(System.getProperty("enablePerfStat"));
            }
            try {
                LicenseVerification licenseVerification = new LicenseVerification();
                _type = licenseVerification.verifyLicense();

                if (_type == LicenseManager.LicenseType.ActivePerProcessor)
                    clientLicenseType = ClientLicenseType.CommunityPaidClient;
                else
                    clientLicenseType = ClientLicenseType.CommunityFreeClient;
                if (_type == LicenseManager.LicenseType.InEvaluation)
                    _licenceCode = "InEvaluation";
            } catch (LicensingException e) {
                this.setClientLicenseType(ClientLicenseType.CommunityFreeClient);


                // We are not throwing exception intentionally so that client can continue to work as a free client in case of invalid eval or license.
            }
                 // Not needed here as the server activated with client-server licensing will throw licensing exception in case a free clien tries connect.

            int conTimeout = _connectionRetries * (_connectionTimeout + getRetryInterval());
            if (conTimeout > 0) {
                _connectionMutexTimeout = conTimeout;
            }
            if (getOperationTimeout() < 60000) //minimum timeout is 60 seconds.
            {
                setOperationTimeout(60000);
            }

            setConnection(new Connection(this, getLogger(), _perfStatsColl, _responseIntegrator, _clientConfig.getBindIP(), this._cacheId));

            ServerInfo remoteServer = new ServerInfo(server, port);

            if (this.getImportHashmap()) {
                this._processor = new AsyncProcessor();
            }

            if (remoteServer.getIP() != null) {
                remoteServer.setIsUserProvidedInternal(true);

                _clientConfig.addServer(remoteServer);

                try {
                    ConnectRemoteServer(getConnection(), remoteServer, true);
                } catch (SecurityException se) {
                    if (getLogger().getIsErrorLogsEnabled()) {
                        getLogger().getNCacheLog().Error("Broker.StartServices", se.toString());
                    }
                } catch (InternalCommandException ex) {
                    if (getLogger().getIsErrorLogsEnabled()) {
                        getLogger().getNCacheLog().Error("Broker.StartServices", ex.toString());
                    }
                }
            }
            if (!getIsConnected()) {
                try {
                    TryNextServer();
                } catch (SecurityException ex) {
                    if (getLogger().getIsErrorLogsEnabled()) {
                        getLogger().getNCacheLog().Error("Broker.StartServices", ex.toString());
                    }
                    throw ex;
                }
            }

            if (_pipeliningEnabled) {
                try {
                    Extensions.setTimeout(_pipeliningBatchInterval);
                    getSocketManagerHandler().StartPipelining();
                } catch (Exception ex) {
                    if (getLogger().getIsErrorLogsEnabled()) {
                        getLogger().getNCacheLog().Error("Broker.StartPipelining", "Problem occured while starting forceful pipelining. " + ex.toString());
                    }
                }
            }
        } catch (Exception e2) {
            getSocketManagerHandler().StopWriter();
            throw e2;
        }
    }

    private void InitializeLogs(boolean enablelogs, boolean detailedlogs) {
        if (enablelogs) {
            Logs localLogger = new Logs();
            localLogger.setIsErrorLogsEnabled(enablelogs);
            if (localLogger.getIsErrorLogsEnabled()) {
                localLogger.setIsDetailedLogsEnabled(detailedlogs);
            }

            long pid = ProcessHandle.current().pid();
            localLogger.setNCacheLog(new JLogger());
            try {

                localLogger.getNCacheLog().Initialize(LoggerNames.ClientLogs, _cacheId);


                if (detailedlogs) {
                    localLogger.getNCacheLog().SetLevel("ALL");
                } else {
                    localLogger.getNCacheLog().SetLevel("INFO");
                }

                localLogger.getNCacheLog().Info("Broker.InitializeLogs", "PID :" + pid + " ClientID : " + _cache.getClientID());
                setLogger(localLogger);
                setNCacheLog(localLogger.getNCacheLog());
            } catch (Exception e) {
                System.out.println(e);
            }
        } else {
            if (getLogger().getNCacheLog() != null) {
                getLogger().getNCacheLog().Flush();
                getLogger().getNCacheLog().SetLevel("OFF");
            }
        }
    }

    private void disconnection(Address address) {
        if (getConnection() != null) {
            getConnection().Disconnect();
            getConnection().dispose();
            try {
                ResetBroker(Address.Parse(getConnection().getIpAddress()));
            } catch (UnknownHostException e) {
            }
        }
        try {
            _lock.AcquireWriterLock();
            if (getPool() != null && getPool().getConnections() != null) {
                for (Object item : getPool().getConnections().entrySet()) {
                    Map.Entry entry = (Map.Entry) item;
                    Address ip = entry.getKey();
                    Connection connection = entry.getValue();
                    connection.Disconnect();
                    connection.dispose();
                    ResetBroker(ip);
                }

            }
        } finally {
            _lock.ReleaseWriterLock();
        }

    }

    int NextWaitInterval(tangible.RefObject totalTimeToWait, int timeSlice) {
        if (totalTimeToWait.argvalue == 0) {
            timeSlice = 0;
        } else if (timeSlice > totalTimeToWait.argvalue) {
            timeSlice = totalTimeToWait.argvalue;
            totalTimeToWait.argvalue = 0;
        } else {
            totalTimeToWait.argvalue -= timeSlice;
        }
        return timeSlice;
    }

    private void SecureConnectionIfEnabled(Connection connection, boolean enabledOnServer) {
        if (!SslConfiguration.getSslConnectionEnabled() && !enabledOnServer) {
            return;
        }

        if (SslConfiguration.getSslConnectionEnabled() && enabledOnServer) {

            // connection.Secure(SslConfiguration.getCertificateName(), SslConfiguration.getRequireClientCertificate(), SslConfiguration.getSslProtocols(), SslConfiguration.getEncryptionPolicy());
            return;
        }

        throw new RuntimeException(String.format("Mistmatch between Client-Server connection security detected. The %1$s node must have Secured-Connection (SSL/TLS)" + "enabled in order to communicate with a %2$s node with Secured-Connection.", enabledOnServer ? "Client" : "Server", enabledOnServer ? "Server" : "Client"));
    }

    public final void InitializeSecondarySocket(Connection connection, InetAddress address, int port) throws Exception {
        connection.ConnectSecondarySocket(address, port);
        InitSecondarySocketCommand command = new InitSecondarySocketCommand(_cache.getClientID());
        DoSendCommand(connection, command, false, false);
        CommandResponse res = connection.RecieveCommandResponse(true);
        if (res != null) {
            res.parseResponse();
        }
    }

    public final void ProcessResponse(CommandResponse response, Address remoteServerAddress) throws IOException, OperationFailedException {
        Command command = null;
        Request request = null;

        synchronized (_requestTable) {
            if (getLogger().getIsDetailedLogsEnabled()) {
                getLogger().getNCacheLog().Info("Broker.ProcessResponse", "Response recieved for request ID:"+response.getRequestId());
            }
            request = _requestTable.get(response.getRequestId());
//            if(request!=null && getLogger().getIsErrorLogsEnabled())
//            {
//                getLogger().getNCacheLog().Error("Broker.ProcessResponse", "Request found in requesttable for request ID:"+request.getRequestId());
//
//            }
            if(request==null)

            if (!response.isNeedsDeserialization() && request != null) {
                //The async Add/Insert/Remove complete events need to verify the command type to raise events specific to a commmand

                HashMap commands = request.getCommands();
                if (commands.size() > 0) {
                    if (commands.containsKey(remoteServerAddress)) {
                        command = request.getCommands().get(remoteServerAddress);
                    } else {
                        //incase if request is sent to some other server for server down or dedicated call scenerio
                        for (Command cmd : request.getCommands().values()) {
                            if (cmd.getFinalDestinationAddress().equals(remoteServerAddress)) {
                                command = cmd;
                                break;
                            }
                        }
                    }
                }
            }
        }


        response.setCacheId(_cacheId);
        if (command != null) {
            response.setCommandId(command.getCommandID());
        }

        switch (response.getType()) {
            case INIT:
            case ADD:
            case REMOVE:
            case GET:
            case INSERT:
            case CLEAR:
            case COUNT:
            case REGISTER_NOTIF:
            case GET_OPTIMAL_SERVER:
            case GET_ENUMERATOR:
            case ADD_BULK:
            case INSERT_BULK:
            case GET_BULK:
            case BULK_GET_CACHEITEM:
            case REMOVE_BULK:
            case CONTAINS:
            case CONTAINS_BULK:
            case GET_CACHE_ITEM:
            case RAISE_CUSTOM_EVENT:
            case SEARCH:
            case SEARCH_ENTRIES:
            case REGISTER_KEY_NOTIF:
            case REGISTER_BULK_KEY_NOTIF:
            case UNREGISTER_KEY_NOTIF:
            case UNREGISTER_BULK_KEY_NOTIF:
            case GET_TYPEINFO_MAP:
            case GET_HASHMAP:
            case GET_COMPACT_TYPES:
            case UNLOCK:
            case LOCK:
            case ISLOCKED:
            case GET_TAG:
            case GET_LOGGING_INFO:
            case DISPOSE:
            case REMOVE_TAG:
            case GET_KEYS_TAG:
            case DELETE:
            case DELETE_BULK:
            case EXCEPTION:
            case GET_GROUP_NEXT_CHUNK:
            case GET_NEXT_CHUNK:
            case ADD_ATTRIBUTE:
            case SYNC_EVENTS:
            case DELETE_QUERY:
            case REMOVE_QUERY:
            case GET_SERVER_MAPPING:
            case INQUIRY_REQUEST_RESPONSE:
            case EXECUTE_READER:
            case DISPOSE_READER:
            case GET_READER_CHUNK:
            case EXPIRATION_RESPONSE:
            case POLL:
            case REGISTER_POLL_NOTIF:
            case GET_CONNECTED_CLIENTS:
            case TOUCH:
            case GET_TOPIC:
            case REMOVE_TOPIC:
            case SUBSCRIBE_TOPIC:
            case UNSUBSCRIBE_TOPIC:
            case GET_MESSAGE:
            case MESSAGE_PUBLISH:
            case MESSAGE_ACKNOWLEDGEMENT:
            case PING:
            case MESSAGE_COUNT:
            case GET_SERIALIZATION_FORMAT:
            case GETMODULESTATE:
            case SETMODULESTATE:
            case MODULE:
                ProcessRawResponse(request, response, remoteServerAddress);
                break;
            case SURROGATE:
                response.setSurrogate(true);
                ProcessRawResponse(request, response, remoteServerAddress);
                break;
            default:
                ProcessInternalResponse(response, remoteServerAddress, command);
                break;

        }
    }

    private void ProcessRawResponse(Request request, CommandResponse response, Address remoteServerAddress) {
        if (request == null) {
            return;
        }
        synchronized (request) {
            request.addResponse(remoteServerAddress, response);
            request.setCacheId(_cacheId);
            Monitor.pulse(request);
        }
    }

    //For Internal Responses and Notifications Callback
    private void ProcessInternalResponse(CommandResponse response, Address remoteServerAddress, Command command) throws IOException, OperationFailedException {
        Address clusterAddress = null;
        Address serverAddress = null;

        //Deserializing response incase of InternalCommands and CallBacks
        response.deserializeResponse();
        switch (response.getType()) {
            case BULK_EVENT:
                if (response.getEventList().size() > 0) {

                    BulkEventStructure bulkEventStructure = new BulkEventStructure(remoteServerAddress, response.getEventList(), this, _perfStatsColl, _eventProcessor, _notifyAsync);
                    ThreadPool.getInstance().executeTask(bulkEventStructure);
                    break;
                }
                break;

            case NODE_LEFT_EVENT:
                ServerInfo serverLeft = new ServerInfo();

                if (response.getServerPort() > 0) {
                    //Get Mapped Server will return the same "IP and port" incase of non-existance of the map
                    serverLeft = _clientConfig.getMappedServer(response.getIp(), response.getServerPort());
                    _clientConfig.removeServer(serverLeft);
                }

                clusterAddress = new Address(serverLeft.getIP(), serverLeft.getPort());
                serverAddress = new Address(serverLeft.getIP(), serverLeft.getPort());
                _cache.getCacheClusterEventsListener().OnMemberLeft(clusterAddress, serverAddress);
                if (getImportHashmap()) {
                    _cache.InvalidateReaders(serverAddress.getIpAddress().toString());
                }
                break;

            case CACHE_STOPPED_EVENT:
                break;

            case NODE_JOINED_EVENT:
                StartBalancingClients startBalancingClients = new StartBalancingClients();
                startBalancingClients.setResponse(response);
                startBalancingClients.setParent(this);
                ThreadPool.getInstance().executeTask(startBalancingClients);
                break;

            case HASHMAP_CHANGED_EVENT:
                UpdateHashmapAsync updateHashmapAsync = new UpdateHashmapAsync(this);
                updateHashmapAsync.setData(response.getValue());
                ThreadPool.getInstance().executeTask(updateHashmapAsync);
                break;

            case CONFIG_MODIFIED_EVENT:
                _cache.setCompressionEnabled(response.getHotConfig().is_compressionEnabled());
                _cache.setCompressionThresholdSize(response.getHotConfig().get_compressionThreshold());
                break;

            case COMPACT_TYPE_REGISTER_EVENT:

                UpdateCompactTypes updateCompactTypes = new UpdateCompactTypes();
                updateCompactTypes.setData(response.getValue());
                updateCompactTypes.setParent(this);
                ThreadPool.getInstance().executeTask(updateCompactTypes);
                break;

            case LOGGING_INFO_MODIFIED_EVENT:
                this.InitializeLogs(response.isEnableErrorLogs(), response.isEnableDetailedLogs());
                break;

            case DS_UPDATE_CALLBACK:
                if (_cache != null && _cache.getAsyncEventListener() != null)
                {
                    _cache.getAsyncEventListener().onDataSourceUpdated(response.getCallbackId(), response.getResultMap(), response.getDSOperationCode(), true);
                }
                break;
            case BLOCK_ACTIVITY:

                ShutDownServerInfo ssinfo = new ShutDownServerInfo();
                BlockActivityEventResponseProtocol.BlockActivityEventResponse blockActivityEventResponse = response.getProtobufResponse().getBlockActivityEvent();
                ssinfo.setUniqueBlockingId(blockActivityEventResponse.getUniqueKey());
                ssinfo.setBlockServerAddress(new Address(blockActivityEventResponse.getServerIP(), blockActivityEventResponse.getPort()));
                ssinfo.setBlockInterval(blockActivityEventResponse.getTimeoutInterval());

                ssinfo.setStartBlockingTime(new Date());

                if (!_shutdownServers.containsKey(ssinfo.getBlockServerAddress())) {
                    _shutdownServers.put(ssinfo.getBlockServerAddress(), ssinfo);

                    long maxTimeout = 0;

                    for (ShutDownServerInfo sInfo : _shutdownServers.values()) {
                        if (maxTimeout == 0) {
                            maxTimeout = sInfo.getBlockInterval();
                        }
                        if (maxTimeout < sInfo.getBlockInterval()) {
                            maxTimeout = sInfo.getBlockInterval();
                        }
                    }

                    double additionaltime = maxTimeout * 0.05f;
                    maxTimeout = (maxTimeout + (int) additionaltime) * 1000;
                    _shutdownTimeout = maxTimeout;
                    try {
                        Iterator iter = null;
                        synchronized (_requestTable) {
                            for (Map.Entry entry : _requestTable.entrySet()) {
                                Request req = entry.getValue();
                                synchronized (req) {
                                    if (!req.getIsRequestTimeoutReset()) {
                                        req.setRequestTimeout(req.getRequestTimeout() + maxTimeout);
                                        req.setIsRequestTimeoutReset(true);
                                    }
                                }
                            }
                        }
                    } catch (Exception ex) {
                        getLogger().getNCacheLog().Error("Broker.ProcessResponse", ex.toString());
                    }
                } else {
                    ShutDownServerInfo oldInfo = _shutdownServers.get(ssinfo.getBlockServerAddress());
                    if (!oldInfo.getUniqueBlockingId().equals(ssinfo.getUniqueBlockingId())) {

                        long startTime = oldInfo.getStartBlockingTime().getTime() - System.currentTimeMillis();
                        int timeout = (int) (oldInfo.getBlockInterval() * 1000) - (int) (System.currentTimeMillis() - startTime);
                        if (timeout <= 0) {
                            _shutdownServers.put(oldInfo.getBlockServerAddress(), ssinfo);
                        }
                    }
                }

                break;

            case UNBLOCK_ACTIVITY:
                UnBlockActivityEventResponseProtocol.UnBlockActivityEventResponse unblockActivityEventResponse = response.getProtobufResponse().getUnblockActivityEvent();
                Address blockServer = null;
                blockServer = new Address(unblockActivityEventResponse.getServerIP(), unblockActivityEventResponse.getPort());
                if (_shutdownServers.containsKey(blockServer)) {
                    ShutDownServerInfo ssInfo = _shutdownServers.get(blockServer);
                    if (ssInfo != null) {
                        if (ssInfo.getUniqueBlockingId().equals(unblockActivityEventResponse.getUniqueKey())) {
                            Connection shutdownCon = null;
                            if (getPool().Contains(blockServer)) {
                                shutdownCon = getPool().getItem(blockServer);
                                getPool().Remove(blockServer);
                            }
                            if (getConnection().getServerAddress().equals(blockServer)) {
                                Connection con = getPool().GetAnyConnection(); //TryPool or GetAnyConnection here?
                                if (con != null) {
                                    setConnection(con);
                                }
                            }

                            if (getConnection() != null) {
                                ReconectInBackground(getConnection().getServerAddress(), getConnection());
                            }

                            synchronized (ssInfo.getWaitForBlockedActivity()) {
                                _shutdownServers.remove(blockServer);
                                Monitor.pulse(ssInfo.getWaitForBlockedActivity());
                            }

                            if (shutdownCon != null) {
                                shutdownCon.Disconnect();
                            }

                        }
                    }
                }

                break;

            case OPERATIONCHANGEDEVNET:
                Address address = null;
                try {
                    address = Address.Parse(response.getProtobufResponse().getOperationModeChangeEventResponse().getServerIP());
                } catch (UnknownHostException e) {
                    throw new OperationFailedException(e);
                }
                disconnection(address);
                break;
        }
    }

    public final void GetHashmap() throws LicensingException, CommandException, OperationFailedException, InternalCommandException, SecurityException, ConfigurationException, ActivityBlockedException {
        GetHashmap(null);
    }

    public final boolean getKeysDistributionMap(String[] keys, CacheItem[] items, java.util.HashMap> keysDistributionMap) {
        boolean result = false;
        boolean itemsAvailable = items != null;

        if (getImportHashmap()) {
            java.util.HashMap keysDistributionList = new java.util.HashMap();
            java.util.HashMap keysAndItems = null;
            String key = "";
            CacheItem item = null;

            for (int i = 0; i < keys.length; i++) {
                key = keys[i];
                if (itemsAvailable) {
                    item = items[i];
                }

                Address address;
                _hashMapStatus.WaitForAny(HashMapStatus.INITIALIZE);
                synchronized (_hashmapUpdateMutex) {
                    address = getPool().GetIp(key);
                }

                if (keysDistributionList.containsKey(address)) {
                    keysAndItems = keysDistributionList.get(address);
                    keysAndItems.put(key, item);
                } else {
                    keysAndItems = new java.util.HashMap<>();
                    keysAndItems.put(key, item);
                    keysDistributionList.put(address, keysAndItems);
                }
            }

            java.util.Map.Entry tmp;
            Address serverAddress;

            for (java.util.Map.Entry pair : keysDistributionList.entrySet()) {
                int index = 0;

                serverAddress = pair.getKey();
                keysAndItems = pair.getValue();

                String[] distributedKeys = new String[keysAndItems.size()];
                CacheItem[] distributedItems = null;
                if (itemsAvailable) {
                    distributedItems = new CacheItem[keysAndItems.size()];
                }


                for (Object entryObject : keysAndItems.entrySet()) {
                    Map.Entry entry = (Map.Entry) entryObject;
                    distributedKeys[index] = (entry.getKey() instanceof String) ? (String) entry.getKey() : null;
                    if (itemsAvailable) {
                        distributedItems[index] = (entry.getValue() instanceof CacheItem) ? (CacheItem) entry.getValue() : null;
                    }
                    index++;
                }

                CacheItem[] finalDistributedItems = distributedItems;
                tmp = new AbstractMap.SimpleEntry<>(distributedKeys,distributedItems);
                Connection conn = getPool().getItem(serverAddress); //for balanced connection
                Address adrs = serverAddress;

                if (conn != null && !conn.getIsConnected()) {
                    Address loadbaanced = GetLoadBalancedAddress();
                    if (loadbaanced != null) {
                        adrs = loadbaanced;
                    }
                }

                if (keysDistributionMap.containsKey(adrs)) {
                    java.util.Map.Entry newtmp = keysDistributionMap.get(adrs);

                    tmp = MergeDistributioonMap(tmp, newtmp, itemsAvailable);
                    keysDistributionMap.put(adrs, tmp);
                    result = true;
                } else {
                    keysDistributionMap.put(adrs, tmp);
                }
            }
        }

        return result;
    }

    public final java.util.Map.Entry MergeDistributioonMap(java.util.Map.Entry map1, java.util.Map.Entry map2, boolean itemsAvailable) {

        String[] distributedKeys = new String[map1.getKey().length + map2.getKey().length];
        CacheItem[] distributedItems = null;
        if (itemsAvailable) {
            distributedItems = new CacheItem[map1.getValue().length + map2.getValue().length];
        }

        int size = map1.getKey().length;

        for (int i = 0; i < map1.getKey().length; i++) {
            distributedKeys[i] = map1.getKey()[i];
            if (itemsAvailable) {
                distributedItems[i] = map1.getValue()[i];
            }
        }

        for (int i = 0; i < map2.getKey().length; i++) {
            distributedKeys[size + i] = map2.getKey()[i];
            if (itemsAvailable) {
                distributedItems[size + i] = map2.getValue()[i];
            }
        }

        return new AbstractMap.SimpleEntry<>(distributedKeys, distributedItems);
    }

    public final Request createRequest(Command command) throws OperationFailedException {
        Request request = null;

        switch (command.getCommandType()) {
            case GET_GROUP:
            case GET_TAG:
            case SEARCH:
            case SEARCH_CQ:
            case GET_KEYS_TAG:
            case GETGROUP_NEXT_CHUNK:
            case GET_NEXT_CHUNK:
            case REMOVE_BY_TAG:
            case REMOVE_GROUP:
            case DELETEQUERY:
            case TASK_ENUMERATOR:
            case EXECUTE_READER:
            case EXECUTE_READER_CQ:
            case DISPOSE_READER:
            case GET_READER_CHUNK:
            case POLL:
            case GETMESSAGE:
            case REGISTER_NOTIF:

                if (getImportHashmap()) {
                    if (poolFullyDisConnected()) {
                        throw new OperationFailedException(ErrorCodes.Common.NO_SERVER_AVAILABLE, ErrorMessages.getErrorMessage(ErrorCodes.Common.NO_SERVER_AVAILABLE));
                    }

                    if (!getPoolFullyConnected()) {
                        request = createDedicatedRequest(command);
                    } else {
                        request = new Request(true, getOperationTimeout());
                        synchronized (_hashmapUpdateMutex) {
                            command.setClientLastViewId(this.getClientLastViewId());
                            for (Object item : getPool().getServers()) {
                                request.addCommand((Address) item, command);
                            }
                        }
                    }
                } else {
                    request = new Request(false, getOperationTimeout());
                    request.addCommand(getConnection().getServerAddress(), command);
                }
                request.setIsAsync(command.isAsync);
                request.setIsAyncCallbackSpecified(command.asyncCallbackSpecified);
                break;

            default:
                request = new Request(false, getOperationTimeout());
                request.setIsAsync(command.isAsync);
                request.setIsAyncCallbackSpecified(command.asyncCallbackSpecified);
                Address ipAddress = null;
                try {
                    ipAddress = GetConnectionIP(command);
                } catch (CacheException e) {
                    throw new OperationFailedException(ErrorCodes.Common.NO_SERVER_AVAILABLE, ErrorMessages.getErrorMessage(ErrorCodes.Common.NO_SERVER_AVAILABLE));
                }
                if (ipAddress == null) {
                    ipAddress = getConnection().getServerAddress();
                }
                request.addCommand(ipAddress, command);
                break;
        }

        return request;
    }

    public final Request CreateRequestOnServer(String nodeAddress, Command command) throws OperationFailedException {
        Request request = null;
        request = new Request(true, this.getOperationTimeout());
        Connection conn = null;
        synchronized (_hashmapUpdateMutex) {
            conn = getPool().GetConnection(nodeAddress);
        }
        if (conn != null) {
            request.addCommand(conn.getServerAddress(), command);
        } else {
            return createDedicatedRequest(command);
        }

        return request;
    }

    public final Request createDedicatedRequest(Command command) throws OperationFailedException {
        Request request = null;
        command.resetCommand();
        if (getImportHashmap()) {
            request = new Request(true, getOperationTimeout());
            command.setClientLastViewId(ForcedViewId);

            Address server = GetLoadBalancedAddress();
            if (server != null) {
                request.addCommand(server, command);
            } else {
                throw new OperationFailedException(ErrorCodes.Common.NO_SERVER_AVAILABLE, ErrorMessages.getErrorMessage(ErrorCodes.Common.NO_SERVER_AVAILABLE));
            }
        } else {
            request = new Request(false, getOperationTimeout());
            command.setClientLastViewId(ForcedViewId);
            request.addCommand(getConnection().getServerAddress(), command);
        }
        request.setIsAsync(command.isAsync);
        request.setIsAyncCallbackSpecified(command.asyncCallbackSpecified);

        return request;
    }

    public final Address GetLoadBalancedAddress() {
        if (getPool() != null) {
            return getPool().GetNextAddress();
        }
        return null;
    }

    private void RetrySendCommand(Command command) throws OperationFailedException {
        //Try to resend command to any other available server
        int tries = 1;
        boolean retry = true;
        int _numberOfRetries = 5;
        while (retry) {
            retry = tries < _numberOfRetries;
            try {
                if (getLogger().getIsErrorLogsEnabled()) {
                    getLogger().getNCacheLog().CriticalInfo("RetrySendCommand", "Command: " + command.getCommandType() + " retried. RequestId: " + command.getParent().getRequestId() + ", Sent as dedicated. command.ClientLastViewId == ForcedViewId => " + (command.getClientLastViewId() == ForcedViewId));
                }
                command.setIsRetry(true);
                SendCommand(command, true);
                retry = false;
            } catch (ConnectionException e) {
                if (getLogger().getIsErrorLogsEnabled()) {
                    getLogger().getNCacheLog().Error("Broker.RetrySendCommand", "RequestID :" + command.getParent().getRequestId() + " " + command.getCommandName() + " can not sent to server " + (new Address(e.getaddress(), e.getport())).toString() + e.toString());
                }
                if (retry) {
                    tries++;
                    if (_numberOfRetries != 0) {
                        try {
                            Thread.sleep((long) (5000));
                        } catch (InterruptedException ex) {
                        }
                    }
                } else {
                    throw new OperationFailedException(ErrorCodes.Common.NO_SERVER_AVAILABLE, ErrorMessages.getErrorMessage(ErrorCodes.Common.NO_SERVER_AVAILABLE));
                }
            } catch (ActivityBlockedException e) {
                if (getLogger().getIsErrorLogsEnabled()) {
                    getLogger().getNCacheLog().Error("Broker.RetrySendCommand", "RequestID :" + command.getParent().getRequestId() + " " + command.getCommandName() + " can not sent to server " + e.getBlockedServerIp() + e.toString());
                }
                if (retry) {
                    tries++;
                    if (1 != 0) {
                        try {
                            Thread.sleep((long) (1000));
                        } catch (InterruptedException ex) {
                        }
                    }
                } else {
                    throw new OperationFailedException(ErrorCodes.Common.NO_SERVER_AVAILABLE, ErrorMessages.getErrorMessage(ErrorCodes.Common.NO_SERVER_AVAILABLE));
                }
            } catch (Exception e) {
                if (getLogger().getIsErrorLogsEnabled()) {
                    getLogger().getNCacheLog().Error("Broker.RetrySendCommand", "RequestID :" + command.getParent().getRequestId() + " " + command.getCommandName() + " can not sent to server. " + e.toString());
                }
                throw new OperationFailedException( e);
            }
        }
    }

    private boolean RetrySendBulkCommand(Command command) throws OperationFailedException {
        Command resendCommand = null;
        boolean isDedicatedResend = false;

        //Only key bulk write commands, when parallel
        if (command.getCommandRequestType() == RequestType.KeyBulkWrite && !command.getParent().getIsDedicatedRequest()) {
            //key based unsafe parallel commands-- command can be sent to any other- server wil re-route according to keys
            resendCommand = command;
            command.getParent().RemoveResponse(command.getFinalDestinationAddress(), command.getCommandID());
            command.resetCommand();
        } else {
            //if request is dedicated or command is safe, a dedicated request can be executed on any other server
            //command is parallel and unsafe
            //if command is non-key bulk write e.g. delete query etc, create a dedicated request and execute and merge responses with already present responses
            resendCommand = Command.getDedicatedCommand(command.getParent().getCommands().values(), _serverIP.toString());
            isDedicatedResend = true;
            resendCommand.getParent().ClearResponses();
        }

        RetrySendCommand(resendCommand);
        return isDedicatedResend;
    }

    private void RetryBulkRequest(Iterable> faliedCommands) throws OperationFailedException, InternalCommandException {
        for (java.util.Map.Entry pair : faliedCommands) {
            Address address = pair.getKey();
            Command command = pair.getValue();

            if (command.getSendError() == null) {
                continue;
            }

            ErrorType errorType = command.getSendError().getType();
            Exception exception = command.getSendError().getException();
            switch (errorType) {
                case ConnectionException:
                case ActivityBlocked:
                    if (command.getCommandRequestType() == RequestType.ChunkRead) {
                        throw new OperationFailedException("Enumeration has been modified");
                    }
                    if (!command.getIsInternalCommand()) {
                        boolean dedicatedResend = RetrySendBulkCommand(command);
                        //if resent as dedicated command, no need to execute remaining commands
                        if (dedicatedResend) {
                            break;
                        }
                    } else {
                        throw new InternalCommandException(exception.getMessage(), (CacheException) exception);
                    }
                    break;
                case Exception:
                    if (getLogger().getIsErrorLogsEnabled()) {
                        getLogger().getNCacheLog().Error("Broker.RetryBulkRequest", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + address + exception.toString());
                    }
                    throw new OperationFailedException(exception);
            }
        }
    }

    private void RetryNonBulkRequest(Iterable> faliedCommands) throws OperationFailedException {
        for (java.util.Map.Entry pair : faliedCommands) {
            Command command = pair.getValue();

            if (command.getSendError() == null) {
                continue;
            }

            ErrorType errorType = command.getSendError().getType();
            Exception exception = command.getSendError().getException();
            switch (errorType) {
                case ConnectionException:
                case ActivityBlocked:
                    //should check for request type or not?
                    if (!command.getIsInternalCommand()) {
                        RetrySendCommand(command);
                    } else {
                        throw new OperationFailedException(exception);
                    }
                    break;
                case Exception:
                    throw new OperationFailedException(exception);
            }
        }
    }

    public final void executeRequest(Request request, Connection connection, boolean checkConnected, boolean waitForResponse) throws LicensingException, OperationFailedException, InternalCommandException, ActivityBlockedException, CommandException, SecurityException, ConfigurationException {
        Throttle();

        if (waitForResponse && _shutdownServers.size() > 0) {
            if (request.getRequestTimeout() == getOperationTimeout()) {
                request.setRequestTimeout(request.getRequestTimeout() + _shutdownTimeout);
            }
        }

        //Feast your eyes
        //for the code is dark, and full of errors
        try {
            //1. Add request to request table
            if (waitForResponse) {
//                if (getLogger().getIsErrorLogsEnabled()) {
//                    getLogger().getNCacheLog().Error("Broker.executeRequest","Request added to request Table with request id" +request.getRequestId());
//                }
                AddRequestToRequestTable(request);
            }

            //2. send each command. This method not only takes care of the specific connection for
            //sending command, it also intializes the response for the connection.
            //This case will have only one command, since IsBulk returns true for more than one commands
            //In case of send failure for only one command, command can be resend to any other available connection
            if (!request.getIsBulk()) {
                for (Command command : request.getCommands().values()) {
                    boolean nullConnection = connection == null;
                    try {
                        if (nullConnection) {
                            SendCommand(command, false);
                        } else {
                            SendCommand(connection, command, checkConnected, false);
                        }
                    } catch (ConnectionException | ActivityBlockedException ex) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.executeRequest", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + ex.toString());
                        }
                        //should check for request type or not?
                        if (!command.getIsInternalCommand()) {
                            RetrySendCommand(command);
                        } else {
                            throw new InternalCommandException(ex.getMessage(), ex);
                        }
                    }
                }
            } else {
                for (java.util.Map.Entry pair : request.getCommands().entrySet()) {
                    Address ip = pair.getKey();
                    Command command = pair.getValue();
                    boolean optimizeConnection = (command.getCommandType() != CommandType.GET_NEXT_CHUNK)
                            || (command.getCommandType() == CommandType.GET_NEXT_CHUNK
                            && ((command.getIntendedRecipient() != null && !command.getIntendedRecipient().isEmpty()) || !getImportHashmap()));

                    connection = VerifyServerConnectivity(ip, optimizeConnection);

                    if (connection == null && command.getCommandType() == CommandType.GET_NEXT_CHUNK) {
                        throw new OperationFailedException(ErrorCodes.Common.ENUMERATION_MODIFIED, ErrorMessages.getErrorMessage(ErrorCodes.Common.ENUMERATION_MODIFIED));
                    }
                    try {
                        if (command.getSupportsSurrogation()) {
                            command = GetSurrogateCommand(request, command, connection, ip);
                        }
                        SendCommand(connection, command, true, false);
                    } catch (ActivityBlockedException ex) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.executeRequest", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + ex.toString());
                        }
                        if (command.getCommandRequestType() == RequestType.ChunkRead) {
                            throw new OperationFailedException(ErrorCodes.Common.ENUMERATION_MODIFIED, ErrorMessages.getErrorMessage(ErrorCodes.Common.ENUMERATION_MODIFIED));
                        }
                        if (!command.getIsInternalCommand()) {
                            boolean dedicatedResend = RetrySendBulkCommand(command);
                            //if resent as dedicated command, no need to execute remaining commands
                            if (dedicatedResend) {
                                break;
                            }
                        } else {
                            throw new InternalCommandException(ex.getMessage(), ex);
                        }
                    } catch (ConnectionException ex) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.executeRequest", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + ex.toString());
                        }
                        if (command.getCommandRequestType() == RequestType.ChunkRead) {
                            throw new OperationFailedException(ErrorCodes.Common.ENUMERATION_MODIFIED, ErrorMessages.getErrorMessage(ErrorCodes.Common.ENUMERATION_MODIFIED));
                        }
                        if (!command.getIsInternalCommand()) {
                            if (command.getSupportsSurrogation()) {

                                    connection = VerifyServerConnectivity(ip, true);

                                command = GetSurrogateCommand(request, command, connection, ip);
                                SendCommand(connection, command, true, true);
                            } else {
                                boolean dedicatedResend = RetrySendBulkCommand(command);
                                //if resent as dedicated command, no need to execute remaining commands
                                if (dedicatedResend) {
                                    break;
                                }
                            }
                        } else {
                            throw new InternalCommandException(ex.getMessage(), ex);
                        }
                    } catch (Exception e) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.executeRequest", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getAddress() + e.toString());
                        }
                        throw new OperationFailedException(e);
                    }
                }
            }

            if (waitForResponse) {
                boolean reacquiredLock = true;
                int timeout = (int) request.getRequestTimeout();
                long startTime = (Calendar.getInstance().getTime().getTime() - 621355968000000000l) / 10000l;
                boolean recheckCommandStatus = false;

                try {
                    while (timeout > 0) {
                        recheckCommandStatus = false;
                        if (request.getIsAsync()) {
                            break;
                        }

                        java.util.ArrayList> failedSendCommands = request.GetSendFailureCommands();
                        if (failedSendCommands != null) {
                            request.RemoveResponse(failedSendCommands);

                            if (request.getIsBulk()) {
                                RetryBulkRequest(failedSendCommands);
                            } else {
                                RetryNonBulkRequest(failedSendCommands);
                            }
                        }

                        if (request.getIsCompleteResponseReceived()) {
                            if (request.getHasFailedResponses()) {
                                continue;
                            }

                            if (request.getFailedCommands().size() > 0) {
                                if (getLogger().getIsErrorLogsEnabled()) {
                                    getLogger().getNCacheLog().Error("executeRequest", "failed to receive response(s) from " + request.getFailedCommands().size() + " command(s) for requestId: " + request.getRequestId());
                                }
                                Command[] failedCommands = request.getFailedCommands().toArray(new Command[0]);
                                request.getFailedCommands().clear();
                                //All commands will be of same type as of first command e.g. IsSafe,CommandRequestType etc

                                if (failedCommands[0].getIsInternalCommand()) {
                                    throw new InternalCommandException("Could not receive response for internal command. ");
                                }
                                //There will be at-max one command for dedicated requests
                                if (failedCommands[0].getIsSafe()) {
                                    // Retrying safe command, consider all parallel commands as failed
                                    RetrySafeCommand(request, request.getCommands().values());
                                } else {
                                    recheckCommandStatus = RetryUnsafeCommand(request, failedCommands, startTime, failedCommands[0].getCommandRequestType() == RequestType.NonKeyBulkWrite);
                                }

                            } else {
                                break;
                            }
                        }
                        synchronized (request) {

                            if (recheckCommandStatus) {
                                continue;
                            }
                            timeout = (int) request.getRequestTimeout() - (int) ((Calendar.getInstance().getTime().getTime() - 621355968000000000l) / 10000l - startTime);
                            if (!request.getIsCompleteResponseReceived() && timeout > 0) // fix for non-negative value exception
                            {
                                try {
//                                    if (getLogger().getIsErrorLogsEnabled()) {
//                                        getLogger().getNCacheLog().Error("Broker.executeRequest","waiting for response for request id "+request.getRequestId());
//                                    }
                                    reacquiredLock = Monitor.wait(request, timeout);
                                } catch (InterruptedException e) {
                                    throw new OperationFailedException(e);
                                }
                            }

                            if (!reacquiredLock) {
                                if (request.getIsRequestTimeoutReset()) {
                                    timeout = (int) request.getRequestTimeout() - (int) ((Calendar.getInstance().getTime().getTime() - 621355968000000000l) / 10000l - startTime);
                                    if (timeout > 0) {
                                        reacquiredLock = true;
                                        try {
                                            reacquiredLock = Monitor.wait(request, timeout);
                                        } catch (InterruptedException e) {
                                            throw new OperationFailedException(e);
                                        }
                                    }
                                }
                            }

                            if (!reacquiredLock && !request.getIsCompleteResponseReceived()) {
                                if (_perfStatsColl != null) {
                                    _perfStatsColl.DecrementRequestQueueSizeStats();
                                }

                                if (getLogger().getIsErrorLogsEnabled()) {
                                    getLogger().getNCacheLog().Error("Broker.SendCommand", request.getTimeoutMessage());
                                    getLogger().getNCacheLog().Error("Broker.executeRequest->timeout", (Long.valueOf(request.getRequestId())).toString());
                                }
                                throw new OperationFailedException(request.getTimeoutMessage());
                            }
                        }
                    }
                }  finally {
                    synchronized (_requestTable) {
                        if (!request.getIsAsync()) {
                            _requestTable.remove(request.getRequestId());
                        }
                    }
                    if (_perfStatsColl != null) {
                        _perfStatsColl.DecrementRequestQueueSizeStats();
                    }

                }
            }

        } finally {
            // Async requests are removed from table when Async completion response is recieved
            if (waitForResponse && !request.getIsAsync()) {
                RemoveRequestFromRequestTable(request);
            }
            requestModerator.unRegisterRequest(request.getRequestId());
        }

    }

    private Command GetSurrogateCommand(Request request, Command command, Connection connection, Address actualTargetNode) {
        SurrogateCommand surrogateCommand = null;
        if (command instanceof SurrogateCommand) {
            //unregister old surrogate command
            surrogateCommand = (SurrogateCommand) ((command instanceof SurrogateCommand) ? command : null);
            request.RemoveSurrogateCommand(connection.getServerAddress());
            command = surrogateCommand.getWrappedCommand();
        }

        if (command.getSupportsSurrogation() && connection != null && !connection.getServerAddress().equals(actualTargetNode)) {
            surrogateCommand = new SurrogateCommand(command, actualTargetNode, connection.getRequestInquiryEnabled());
            request.AddSurrogateCommand(surrogateCommand, connection.getServerAddress());
            command = surrogateCommand;
        }

        return command;
    }

    private void RetrySafeCommand(Request request, Collection failedCommands) throws OperationFailedException {
        request.ClearResponses();
        //Create dedicated command
        Command dedicatedCommand = Command.getDedicatedCommand(failedCommands, _serverIP.toString());
        //Set retry flag true
        dedicatedCommand.setIsRetry(true);
        RetrySendCommand(dedicatedCommand);
    }

    private boolean RetryUnsafeCommand(Request request, Command[] failedCommands, long executionStartTime, boolean sendDedicatedRequest) throws OperationFailedException, InternalCommandException, LicensingException, CommandException, ActivityBlockedException, SecurityException, ConfigurationException {
        boolean checkRequestStatus = false;
        boolean retrySend = false;
        boolean retryDedicated = false;

        if (failedCommands == null || failedCommands.length == 0) {
            return false;
        }

        for (Command command : failedCommands) {
            retrySend = false;
            request.RemoveResponse(command.getFinalDestinationAddress(), command.getCommandID());

            //Inquiry command
            int timeout = (int) request.getRequestTimeout() - (int) ((Calendar.getInstance().getTime().getTime() - 621355968000000000l) / 10000l - executionStartTime);
            if (getLogger().getIsDetailedLogsEnabled()) {
                getLogger().getNCacheLog().CriticalInfo("executeRequest", "Inquiring for command " + command.getCommandType() + " of RequestId: " + command.getParent().getRequestId());
            }

            CommandResponse inquiryResponse = Inquire(command, timeout);
            //If inquiry fails, re-execute command, else add command resposne(returned by inquiry)
            if (inquiryResponse.getInquiryResponse() != null) {
                if (getLogger().getIsDetailedLogsEnabled()) {
                    getLogger().getNCacheLog().CriticalInfo("executeRequest", "Inquiry Result for command " + command.getCommandType() + ": " + inquiryResponse.getInquiryResponse().getStatus());
                }

                switch (inquiryResponse.getInquiryResponse().getStatus()) {
                    case RequestStatus.RECEIVED_AND_EXECUTED:

                        ByteArrayOutputStream stream = new ByteArrayOutputStream();

                        for (com.google.protobuf.ByteString bytesString : inquiryResponse.getInquiryResponse().getValueList()) {
                            stream.write(bytesString.toByteArray(), 0, bytesString.toByteArray().length);
                        }
                        CommandResponse cmdResponse = new CommandResponse(false, command.getFinalDestinationAddress());
                        cmdResponse.setRawResult(stream.toByteArray());
                        request.InitializeResponse(command.getFinalDestinationAddress(), command);
                        request.addResponse(command.getFinalDestinationAddress(), cmdResponse);
                        break;
                    case Alachisoft.NCache.Common.Enum.RequestStatus.NOT_RECEIVED:
                    case Alachisoft.NCache.Common.Enum.RequestStatus.RECEIVED_WITH_ERROR:
                    case Alachisoft.NCache.Common.Enum.RequestStatus.NODE_DOWN:
                        retrySend = true;
                        break;
                    case Alachisoft.NCache.Common.Enum.RequestStatus.RECEIVED_AND_INEXECUTION:
                        //If command is in execution on server, add again to failed commands, will be inquired again in next cycle
                        request.getFailedCommands().add(command);
                        request.InitializeFailedResponse(command.getFinalDestinationAddress(), command);
                        checkRequestStatus = true;
                        break;
                }
            } else {
                retrySend = true;
            }

            if (retrySend) {
                if (sendDedicatedRequest) {
                    retryDedicated |= retrySend;
                } else {
                    RetrySendCommand(command);
                }
            }
        }

        if (retryDedicated) {
            Command dedicatedCommmand = null;
            if (sendDedicatedRequest) {
                dedicatedCommmand = Command.getDedicatedCommand(Arrays.asList(failedCommands), _serverIP.toString());
                RetrySendCommand(dedicatedCommmand);
            }
        }
        return checkRequestStatus;
    }

    public final CommandResponse Inquire(Command command, long timeout) throws OperationFailedException, LicensingException, CommandException, InternalCommandException, ActivityBlockedException, SecurityException, ConfigurationException {
        Request inquiryRequest = new Request(false, timeout);
        InquiryRequestCommand inquiryCommand = new InquiryRequestCommand(command.getRequestId(), command.getCommandID(), command.getFinalDestinationAddress().getIpAddress().toString());
        Address ipAddress= null;
        try {
            ipAddress = GetConnectionIP(command);
        } catch (CacheException e) {
            throw new OperationFailedException(e);
        }
        inquiryRequest.addCommand(ipAddress, inquiryCommand);
        executeRequestForInternalCommand(inquiryRequest);
        return inquiryRequest.getResponse();
    }

    public final void executeRequestForInternalCommand(Request request) throws OperationFailedException, LicensingException, CommandException, InternalCommandException, SecurityException, ActivityBlockedException, ConfigurationException {
       executeRequest(request, null, true, true);
    }


    public final void executeRequest(Request request) throws OperationFailedException, LicensingException, CommandException {
        try {
            executeRequest(request, null, true, true);
        } catch (SecurityException e) {
            throw new OperationFailedException(e);
        }
         catch (InternalCommandException | ActivityBlockedException | ConfigurationException e) {
        }
    }

    public final Address GetConnectionIP(Command command) throws SecurityException, ConfigurationException, LicensingException, CommandException, GeneralFailureException, OperationFailedException, OperationNotSupportedException, StreamException, StreamAlreadyLockedException, StreamNotFoundException, AggregateException, ActivityBlockedException {
        Connection connection = getConnection();
        if (command.getKey() != null && !command.getKey().equals("") && this.getImportHashmap()) {
            Address ip; //= String.Empty;
            synchronized (_hashmapUpdateMutex) {
                ip = this.getPool().GetIp(command.getKey());
            }
            if (ip != null) {
                connection = GetConnection(ip, true);
                if (connection == null || (connection != null && !connection.getIsConnected())) {
                    connection = TryPool();
                }
            }
            //done everything to get a working connection.
        }

        return connection.getServerAddress();
    }

    public final void SendCommand(Command command, boolean waitUntilSend) throws SecurityException, ConfigurationException, LicensingException, ActivityBlockedException, OperationFailedException, CommandException {
        Connection connection = getConnection();
        if (connection != null) {
            connection.setIntendedRecipientIPAddress("");
        }

        if (command.getKey() != null && !command.getKey().equals("") && this.getImportHashmap()) {
            Address ip;

            synchronized (_hashmapUpdateMutex) {
                ip = this.getPool().GetIp(command.getKey());
            }

            if (ip != null) {
                connection = GetConnection(ip, true);
                if (connection == null || (connection != null && !connection.getIsConnected())) {
                    connection = TryPool();
                }
            }
            //done everything to get a working connection.
        } else if (this.getImportHashmap() && !connection.getIsConnected()) {
            connection = TryPool();

            if (connection != null && connection.getServerAddress() != null) {
                setConnection(connection);
            }
        }

        //If not POR or Partitioned and connection is not connected, try to get another connection before sending command
        if (!this.getImportHashmap() && connection != null && !connection.getIsConnected()) {

            TryNextServer();

            connection = getConnection();
        }

        SendCommand(connection, command, true, waitUntilSend);
    }

    public final void SendCommand(Connection connection, Command command, boolean checkConnected, boolean waitUntilSend) throws OperationFailedException, ConnectionException,ActivityBlockedException {
        Address ip = connection.getServerAddress();
        if (checkConnected) {
            connection.getStatusLatch().waitForAnyUpdated((byte) (ConnectionStatus.Connected | ConnectionStatus.Disconnected | ConnectionStatus.LoadBalance));
        }
        try {
            DoSendCommand(connection, command, checkConnected, waitUntilSend);
            if (getLogger().getIsDetailedLogsEnabled()) {
                getLogger().getNCacheLog().Debug("Broker.SendCommand", "RequestID : " + command.getRequestId() + " " + command.getCommandName() + " sent to server " + connection.getIpAddress());
            }
        } catch (ConnectionException connectionException) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.SendCommand", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + connectionException.toString());
            }

            throw connectionException;
        } catch (OperationFailedException e) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.SendCommand", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + e.toString());
            }

            throw e;
        } catch (ActivityBlockedException ex) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.SendCommand", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + ex.toString());
            }
            throw ex;
        } catch (Exception e) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.SendCommand", "RequestID :" + command.getRequestId() + " " + command.getCommandName() + " can not sent to server " + connection.getIpAddress() + " " + e.toString());
            }
            throw new OperationFailedException(e);
        }
    }

    public final void dispose() {
        dispose(false);
    }

    public final void dispose(boolean disposingGracefully) {
        setIsDisposing(true);

        if (_connectionPinger != null) {
            _connectionPinger.Stop();
        }

        ConcurrentHashMap connections = this.getPool().CloneConnectionTable();

        for (Object item : connections.entrySet()) {
            Map.Entry entry = (Map.Entry) item;
            Connection connection = (entry.getValue() instanceof Connection) ? (Connection) entry.getValue() : null;
            if (connection != null) {
                if (connection.getIsConnected() && disposingGracefully) {
                    DisposeCommand command = new DisposeCommand();

                    try {
                        Request request = this.createRequest(command);

                        this.executeRequest(request, connection, true, false);
                    } catch (CacheException e) {
                    }
                }

                connection.Disconnect();
                this.getPool().Remove(connection.getServerAddress());
            }


            _perfStatsColl.dispose();

            connection.dispose();
        }

        if (this._processor != null) {
            this._processor.Stop();
        }
        if (_eventProcessor != null) {
            _eventProcessor.Stop();
        }



        try {
            if(requestModerator !=null)
                requestModerator.close();

            if (getLogger().getNCacheLog() != null) {
                getLogger().getNCacheLog().Flush();
                getLogger().getNCacheLog().close();
            }
            getSocketManagerHandler().close();
        } catch (Exception exception) {
        }
    }


    public final Connection VerifyServerConnectivity(Address serverIP, boolean optimizeConnection) throws LicensingException, ConfigurationException, SecurityException, CommandException, OperationFailedException, ActivityBlockedException {
        Connection con = null;
        if (optimizeConnection) {
            con = GetConnection(serverIP, false);
            if (!con.getIsConnected()) {
                con = TryPool();
            }
        } else {
            con = GetConnection(serverIP, true);
        }
        return con;
    }

    public final void DoSendCommand(Connection connection, Command command, boolean checkConnected, boolean waitUntilSend) throws Exception {
        if (_shutdownServers.size() > 0) {
            boolean reacquiredLock = true;

            if (command.getCommandRequestType() != RequestType.InternalCommand) {
                ShutDownServerInfo ssInfo = _shutdownServers.get(connection.getServerAddress());
                if (ssInfo != null) {
                    synchronized (ssInfo.getWaitForBlockedActivity()) {
                        if (_shutdownServers.containsKey(connection.getServerAddress())) {
                            long startTime = (ssInfo.getStartBlockingTime().getTime() - 621355968000000000l) / 100001;
                            int timeout = (int) (ssInfo.getBlockInterval() * 1000) - (int) ((Calendar.getInstance().getTime().getTime() - 621355968000000000l) / 10000l - startTime);
                            if (timeout > 0) {
                                try {
                                    reacquiredLock = Monitor.wait(ssInfo.getWaitForBlockedActivity(), timeout);
                                } catch (InterruptedException e) {
                                }
                                throw new ActivityBlockedException(ErrorCodes.Common.REQUEST_TIMEOUT, ErrorMessages.getErrorMessage(ErrorCodes.Common.REQUEST_TIMEOUT), ssInfo.getBlockServerAddress());
                            }
                        }
                    }
                }
            }
        }

        if (command.getCommandType() == CommandType.INQUIRY_REQUEST) {
            if (!connection.getRequestInquiryEnabled()) {
                throw new OperationFailedException(ErrorCodes.Common.OPERATION_INTERRUPTED, ErrorMessages.getErrorMessage(ErrorCodes.Common.OPERATION_INTERRUPTED));
            }
        }


        InitializeResponse(connection, command);
        command.setCacheId(_cacheId);

        try {
            long acknowledgement = -1;
            if (connection.getRequestInquiryEnabled() && !command.getIsSafe()) {
                acknowledgement = requestModerator.registerRequest(connection.getIpAddress(), command.getRequestId());
                command.setAcknowledgmentId(acknowledgement);
            }

            //Following code is to be refactored, it's written poorly for the reason, to keep the newly introduced
            //secure-communication path as isolated it can be from the old communication path.
            if (!connection.getOptimized()) {
                if (connection.getIsSecured()) {
                    connection.AssureSendSecure(command.toByte(acknowledgement, connection.getRequestInquiryEnabled()), connection.getPrimarySecureStream(), checkConnected);
                } else {

                    connection.AssureSendDirect(command.toByte(acknowledgement, connection.getRequestInquiryEnabled()), connection.getPrimaryClientSocket(), checkConnected);
                }
            } else if (command instanceof InitSecondarySocketCommand) {
                if (connection.getIsSecured()) {
                    connection.AssureSendSecure(command.toByte(acknowledgement, connection.getRequestInquiryEnabled()), connection.getSecondarySecureStream(), checkConnected);
                } else {
                    connection.AssureSendDirect(command.toByte(acknowledgement, connection.getRequestInquiryEnabled()), connection.getSecondaryClientSocket(), checkConnected);
                }
            } else {
                if (checkConnected) {
                    command.setPulseOnSend(waitUntilSend);
                    command.setSentOverWire(false);
                    command.setSendError(null);
                    connection.TryEnqueue(command, checkConnected);

                    if (waitUntilSend) {
                        synchronized (command) {
                            if (!command.getSentOverWire()) {
                                Monitor.wait(command);
                            }

                            if (command.getSendError() != null) {
                                throw command.getSendError().getException();
                            }
                        }
                    }
                } else {
                    connection.SendCommand(command.toByte(acknowledgement, connection.getRequestInquiryEnabled()), checkConnected);
                    connection.Flush();
                }
            }
        } catch (ConnectionException e) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.DoSendCommand", e.toString());
            }
            RemoveResponse(connection, command);
            throw e;
        } catch (Exception e) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.DoSendCommand", e.toString());
            }
            RemoveResponse(connection, command);
            throw e;
        }
    }

    private void AddRequestToRequestTable(Request request) {
        if (!(request.getIsAsync() && !request.getIsAyncCallbackSpecified())) {
            request.setRequestId(this.getRequestId());
            synchronized (_requestTable) {
                _requestTable.put(request.getRequestId(), request);
            }
            if (_perfStatsColl != null) {
                _perfStatsColl.incrementRequestQueueSizeStats();
            }

        }
    }

    public final Request GetRequest(long requestId) {
        Request request = null;
        synchronized (_requestTable) {
            if (_requestTable.containsKey(requestId)) {
                request = _requestTable.get(requestId);
            }
        }

        return request;
    }

    private void RemoveRequestFromRequestTable(Request request) {
        boolean requestExistedInTable = false;
        synchronized (_requestTable) {
            requestExistedInTable = _requestTable.containsKey(request.getRequestId());
            if (requestExistedInTable) {
                _requestTable.remove(request.getRequestId());
            }
        }
        if (_perfStatsColl != null && requestExistedInTable) {
            _perfStatsColl.DecrementRequestQueueSizeStats();
        }

    }

    private void InitializeResponse(Connection connection, Command command) {
        command.getParent().InitializeResponse(connection.getServerAddress(), command);
    }

    private void RemoveResponse(Connection connection, Command command) {
        command.getParent().RemoveResponse(connection.getServerAddress(), command.getCommandID());
    }

    public final void ServerLost(Address ip, boolean forcedDisconnected)  {
            if (getLogger().getIsDetailedLogsEnabled()) {
            getLogger().getNCacheLog().Info("ServerLost", "Server lost " + ip + "; forcedDisconnected = " + forcedDisconnected);
        }
        try {
            if (this.getImportHashmap()) {
                if (!forcedDisconnected) {
                    try {
                        ReregisterEvents(ip);
                        Connection connection = this.getPool().getItem(ip);
                        if (connection != null) {
                            ReconectInBackground(connection.getServerAddress(), connection);
                        }
                    } catch (Exception exc) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.ServerLost", exc.getMessage());
                        }
                    }
                }
            } else {
                this.getPool().Remove(ip);
                if (_shutdownServers.containsKey(ip)) {
                    ShutDownServerInfo ssInfo = _shutdownServers.get(ip);

                    synchronized (ssInfo.getWaitForBlockedActivity()) {

                        Monitor.pulse(ssInfo.getWaitForBlockedActivity());
                        _shutdownServers.remove(ip);
                    }
                }
            }

            ResetBroker(ip);
            this._clientConfig.removeServer(new ServerInfo(ip.getIpAddress(), ip.getPort()));

            if (forcedDisconnected || getConnection().getStatusLatch().IsAnyBitsSet(ConnectionStatus.Connecting)) {
                return;
            }
            try {
                if (!this.getConnection().getIsConnected() && !this.getImportHashmap()) {
                    try {
                        TryNextServer();
                    } catch (Exception ex) {
                        if (getLogger().getIsErrorLogsEnabled()) {
                            getLogger().getNCacheLog().Error("Broker.ServerLost", ex.toString());
                        }
                    }
                }
            } finally {
                // raise this event if client once disconnected and then was not able to connect again.
                if (!getConnection().getIsConnected() && !TryPool().getIsConnected()) {
                    _cache.getEventsListener().OnCacheStopped(_cacheId, _notifyAsync);
                } else if (getConnection().getIsConnected()) {
                    _cache.getEventsListener().OnReregisterTopic();
                }
            }
        } catch (Exception e) {
            if (getLogger().getIsErrorLogsEnabled()) {
                getLogger().getNCacheLog().Error("Broker.ServerLost", e.toString());
            }
        }
    }


    private void ReregisterEvents(Address ip) throws Exception {
        ReregisterEvents(getPool().getItem(ip));
    }

    private void ReregisterEvents(Connection connection) throws SecurityException, ConfigurationException, OperationFailedException, LicensingException, ActivityBlockedException, CommandException {
        if (connection != null && connection.getNotifRegistered()) {
            connection.setNotifRegistered(false);

            Connection selected = getConnection();

            if (!selected.getIsConnected()) {
                selected = TryPool();
            }
            if (selected != null && selected.getIsConnected()) {
                _cache.reRegisterGeneralNotification(selected);
                selected.setNotifRegistered(true);
            }
        }
    }

    void ResetBroker(Address ip) {
        try {
            java.util.Iterator iter = null;

            synchronized (_requestTable) {
                iter = _requestTable.values().iterator();

                while (iter.hasNext()) {
                    Request request = (Request) iter.next();
                    if (request.ExpectingResponseFrom(ip)) {
                        synchronized (request) {
                            request.Reset(ip);
//                            request.notify();
                            Monitor.pulse(request);
                        }
                    }
                }
            }
        } catch (Exception ex) {
            getLogger().getNCacheLog().Error("Broker.ResetBroker", ex.toString());
        }
    }

    void TryNextServer() throws SecurityException, LicensingException, ConfigurationException, OperationFailedException, CommandException, ActivityBlockedException {
        boolean connected = false;
        ServerInfo startingServer = null;
        java.util.ArrayList deniedServers = null;
        java.util.Hashtable invalidServers = new java.util.Hashtable();
        int retries = this._connectionRetries;


        try {
            _lock.AcquireWriterLock();
        } catch (Throwable e) {
            return; //lock could not be granted before the timeout expires.
        }

        try {
            CheckRetryConnectionDelay(); //[KS: Checking if retry connection Interval is over or not]
            if (!_retryConnection) {
                return;
            }

            while (retries-- > 0 && !getIsDisposing()) {
                try {
                    if (!this.getConnection().getIsConnected()) {
                        this.getConnection().getStatusLatch().SetStatusBit(ConnectionStatus.Connecting, (byte) (ConnectionStatus.Connected | ConnectionStatus.Disconnected));

                        if (_clientConfig == null) {
                            _clientConfig = new ClientConfiguration(_cacheId);
                        }

                        int nretries = 3;
                        while (true) {
                            try {
                                _clientConfig.loadConfiguration();
                                break;
                            } catch (ConfigurationException ie) {
                                if (--nretries == 0) {
                                    throw ie;
                                }
                                try {
                                    Thread.sleep(500);
                                } catch (InterruptedException e) {
                                }
                            }
                        }

                        if (_clientConfig.getServerCount() > 0) {
                            if (!_clientConfig.getBalanceNodes()) {
                                _clientConfig.setCurrentServerIndex(0);
                            }
                            ServerInfo nextServer = _clientConfig.getNextServer();
                            startingServer = nextServer;
                            boolean triedWithParamPort = true;
                            while (!connected) {
                                if (nextServer == null) {
                                    break;
                                }
                                if (nextServer.getIP() != null) {
                                    for (int i = 0; i < nextServer.getPortRangeInternal(); i++) {
                                        try {
                                            if (!connected) {
                                                Exception exception = null;
                                                tangible.RefObject tempRef_exception = new tangible.RefObject(exception);
                                                connected = ConnectRemoteServer(this.getConnection(), nextServer.getIP(), nextServer.getPort() + i, this._balanceNode, this.getImportHashmap(), true, tempRef_exception, true);
                                                exception = tempRef_exception.argvalue;

                                            }
                                            if (connected) {
                                                break;
                                            }
                                        } catch (SecurityException e2) {
                                            if (deniedServers == null) {
                                                deniedServers = new java.util.ArrayList();
                                            }
                                            if (!deniedServers.contains(nextServer.getName())) {
                                                deniedServers.add(nextServer.getName());
                                            }
                                        }
                                        //if free clients try to connect with licensed servers
                                        catch (LicensingException ex) {
                                            if (!invalidServers.contains(nextServer.getName())) {
                                                invalidServers.put(nextServer.getName(), ex);
                                            }
                                        }catch (InternalCommandException internalException)
                                        { }
                                    }
                                }

                                if (!connected) {
                                    if (triedWithParamPort && _clientConfig.isDifferentParamPort(nextServer.getPort())) {
                                        triedWithParamPort = false;
                                        nextServer.setPort(_clientConfig.getConfigServerPort());
                                    } else {
                                        triedWithParamPort = true;
                                        nextServer = _clientConfig.getNextServer();
                                        if (startingServer.equals(nextServer)) {
                                            break;
                                        }
                                    }
                                }
                            }
                            //muds:
                            //if the connection is established, exit the outer loop.
                            //otherwise sleep for the sleep interval and retry.
                            if (connected) {
                                break;
                            }
                            Thread.sleep(getRetryInterval());
                            continue;
                        } else {
                            throw new ConfigurationException(ErrorCodes.CacheInit.SERVER_INFO_NOT_FOUND, ErrorMessages.getErrorMessage(ErrorCodes.CacheInit.SERVER_INFO_NOT_FOUND));
                        }
                    } else {
                        connected = true;
                    }

                } catch (InterruptedException e) {

                }
            }
        } finally {
            //set the connection status
            byte setStatus = connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected;
            byte unsetStatus = (byte) ((!connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected) | ConnectionStatus.Connecting);
            getConnection().getStatusLatch().SetStatusBit(setStatus, unsetStatus);

            _retryConnection = connected;

            //release the lock
            _lock.ReleaseWriterLock();

            //muds:
            //if security exception was thrown from some server(s), then we
            //must pass it to the client application.
            if (!connected) {
                if (deniedServers != null && deniedServers.size() > 0) {
                    boolean first = true;
                    StringBuilder sb = new StringBuilder("You do not have permissions to perform the operation on : ");
                    java.util.Iterator ie = deniedServers.iterator();
                    while (ie.hasNext()) {
                        if (!first) {
                            sb.append(" ,");
                            first = false;
                        }
                        Object next= ie.next();
                        sb.append("\'" + ((next instanceof String) ? next : null) + "\'");
                    }

                    throw new SecurityException(sb.toString());

                }
                //if free clients try to connect with licensed servers
                if (invalidServers != null && invalidServers.size() > 0) {
                    boolean first = true;
                    StringBuilder sb = new StringBuilder();
                    Iterator ie = invalidServers.entrySet().iterator();
                    while (ie.hasNext()) {
                        if (!first) {
                            sb.append(" ,");
                            first = false;
                        }
                        Map.Entry entry = (Map.Entry) ie.next();
                        sb.append(entry.getKey());
                        sb.append(' ');
                        sb.append(entry.getValue());
                    }

                    throw new LicensingException(sb.toString());

                }
            }
        }
    }
    /**
     Get server address according to key distribution map

     @param key Hash of this value decides the server address to be used
     @param serverAddress Server address to which request must be sent
     @return true if call is a dedicated call
     */
    public final boolean GetServerAddressWithCallStatus(String key, tangible.RefObject
serverAddress) { _hashMapStatus.WaitForAny(HashMapStatus.INITIALIZE); Address hashMapAddress; synchronized (_hashmapUpdateMutex) { // get address to which the key belongs hashMapAddress = connectionPool.GetIp(key); } // get connection of acquired server address Connection conn = connectionPool.getItem(hashMapAddress); // check if connection is not found if (conn == null || !conn.getIsConnected()) { // if connection not found, get next available address Address nextAvailableAddress = GetLoadBalancedAddress(); if (nextAvailableAddress != null) { // set next available address as the one to be used serverAddress.argvalue = nextAvailableAddress; // return as a dedicated call return true; } } // return address from hashmap as it is now validated serverAddress.argvalue = hashMapAddress; // return as a normal call return false; } private Connection ReconnectServer(Connection connection, ServerInfo nextServer) throws SecurityException, LicensingException, CommandException, OperationFailedException, ConfigurationException, ActivityBlockedException { boolean connected = false; java.util.ArrayList deniedServers = null; java.util.ArrayList invalidServers = null; int retries = this._connectionRetries; if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().Debug("ReconnectServer", "Trying to reconnect to :" + connection.getIpAddress()); } try { _lock.AcquireWriterLock(); } catch (Exception e) { if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().Debug("reconnectServer:\tFailed to acquire lock"); } return connection; //lock could not be granted before the timeout expires. } try { CheckRetryConnectionDelay(); //[KS: Checking if retry connection Interval is over or not] if (!_retryConnection) { return connection; } while (retries-- > 0 && !getIsDisposing()) { if (!connection.getIsConnected()) { connection.getStatusLatch().SetStatusBit(ConnectionStatus.Connecting, (byte) (ConnectionStatus.Connected | ConnectionStatus.Disconnected)); if (nextServer == null) { break; } if (nextServer.getIP() != null) { for (int i = 0; i < nextServer.getPortRangeInternal(); i++) { try { if (!connected) { Exception exception = null; tangible.RefObject tempRef_exception = new tangible.RefObject(exception); connected = ConnectRemoteServer(connection, nextServer.getIP(), nextServer.getPort() + i, this._balanceNode, this.getImportHashmap(), true, tempRef_exception, true); exception = tempRef_exception.argvalue; } if (connected) { if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().CriticalInfo("ReconnectServer", "Reconnected with: " + connection.getIpAddress()); } break; } } catch (SecurityException e2) { if (deniedServers == null) { deniedServers = new java.util.ArrayList(); } if (!deniedServers.contains(nextServer.getName())) { deniedServers.add(nextServer.getName()); } } catch (LicensingException e3) { if (invalidServers == null) { invalidServers = new java.util.ArrayList(); } if (!invalidServers.contains(nextServer.getName())) { invalidServers.add(nextServer.getName()); } } catch (InternalCommandException internalException) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("ReconnectServer", "Reconnected with: " + connection.getIpAddress()); } } } } try { Thread.sleep(getRetryInterval()); } catch (InterruptedException e) { } continue; } else { connected = true; break; } } } finally { byte setStatus = connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected; byte unsetStatus = (byte) ((!connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected) | ConnectionStatus.Connecting); connection.getStatusLatch().SetStatusBit(setStatus, unsetStatus); _retryConnection = connected; //[KS : Connection is up again so we can retry ] _lock.ReleaseWriterLock(); //muds: //if security exception was thrown from some server(s), then we //must pass it to the client application. if (!connected) { if (deniedServers != null && deniedServers.size() > 0) { boolean first = true; StringBuilder sb = new StringBuilder("You do not have permissions to perform the operation on : "); java.util.Iterator ie = deniedServers.iterator(); while (ie.hasNext()) { if (!first) { sb.append(" ,"); first = false; } sb.append("\'" + ((ie.next() instanceof String) ? ie.next() : null) + "\'"); } throw new SecurityException(sb.toString()); } //if free clients try to connect with licensed servers if (invalidServers != null && invalidServers.size() > 0) { boolean first = true; StringBuilder sb = new StringBuilder("You do not have valid license to communicate with : "); java.util.Iterator ie = invalidServers.iterator(); while (ie.hasNext()) { if (!first) { sb.append(" ,"); first = false; } sb.append("\'" + ((ie.next() instanceof String) ? ie.next() : null) + "\'"); } throw new LicensingException(sb.toString()); } } } return connection; } private Connection GetConnection(Address ip, boolean strictMatch) throws SecurityException, OperationFailedException, LicensingException, ConfigurationException, ActivityBlockedException, CommandException { Connection connection = this.getPool().getItem(ip); if (connection != null) { if (!connection.getIsConnected()) { if (connection.getNotifRegistered()) { ReregisterEvents(connection); } if (!connection.getIsReconnecting()) { ReconectInBackground(connection.getServerAddress(), connection); } } else { return connection; } } if (this.getImportHashmap() && !strictMatch) { connection = TryPool(); } if (connection != null) { connection.setIntendedRecipientIPAddress(""); } return connection; } private boolean NeedsNotifRegistration() { try { _lock.AcquireReaderLock(); ConcurrentHashMap connections = this.getPool().getConnections(); synchronized (connections) { for (Map.Entry entry : connections.entrySet()) { if (entry.getValue().getIsConnected()) { return false; } } return true; } } finally { _lock.ReleaseReaderLock(); } } public final boolean TryConnecting(Connection connection, tangible.RefObject exception, int connectionWaitInterval, boolean importHashmap, boolean changeStatus) throws CommandException, OperationFailedException, ConfigurationException, ActivityBlockedException { boolean connected = false; try { this._lock.AcquireWriterLock(); connected = (connection != null && connection.getIsConnected()); if (!connected) { try { CheckRetryConnectionDelay(); //[KS: Checking if retry connection Interval is over or not] if (!_retryConnection) { if (getLogger() != null && getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().CriticalInfo("Broker.TryConnecting", "Can not continue reconnect due to reconnection delay"); } return false; } connection.getStatusLatch().SetStatusBit(ConnectionStatus.Connecting, (byte) (ConnectionStatus.Connected | ConnectionStatus.Disconnected)); if (connectionWaitInterval > 0) { try { Thread.sleep(connectionWaitInterval); } catch (InterruptedException e) { } } connected = ConnectRemoteServer(connection, connection.getAddress(), this._port, false, importHashmap, NeedsNotifRegistration(), exception, changeStatus); if (connected) { if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().Info("Broker.TryConnecting", "Connection established with " + connection.getIpAddress()); } } } catch (LicensingException e) { changeStatus = true; if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.TryConnecting", "Client does not have a valid license to communicate with the cache server : " + connection.getIpAddress()); } } catch (SecurityException e2) { changeStatus = true; if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.TryConnecting", "You do not have permissions to perform the operation on : " + connection.getIpAddress()); } }catch (InternalCommandException e2) { changeStatus = true; if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.TryConnecting", e2.toString()); } } catch (Exception e4) { changeStatus = true; throw e4; } finally { byte setStatus = connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected; byte unsetStatus = (byte) ((!connected ? ConnectionStatus.Connected : ConnectionStatus.Disconnected) | ConnectionStatus.Connecting); if (connected || changeStatus) { connection.getStatusLatch().SetStatusBit(setStatus, unsetStatus); } _retryConnection = connected; } } } finally { this._lock.ReleaseWriterLock(); } return connected; } public final Connection TryPool() throws ConfigurationException, SecurityException, LicensingException, OperationFailedException, CommandException, ActivityBlockedException { Connection connection = null; if (_clientConfig == null) { _clientConfig = new ClientConfiguration(_cacheId); } int retries = 3; while (true && !getIsDisposing()) { try { _clientConfig.loadConfiguration(); break; } catch (ConfigurationException ie) { if (--retries == 0) throw ie; try { Thread.sleep(500); } catch (InterruptedException e) { } } } if (!_clientConfig.getBalanceNodes()) { _clientConfig.setCurrentServerIndex(0); } int index = 0; ServerInfo nextServer = _clientConfig.getServerAt(index); ServerInfo startingServer = nextServer; while (true && !getIsDisposing()) { if (_clientConfig.getServerCount() > 0 && index < _clientConfig.getServerCount()) { connection = this.getPool().getItem(new Address(nextServer.getName(), nextServer.getPort())); if (connection != null && connection.getIsConnected()) { break; } else if (connection != null && !connection.getIsReconnecting()) { ReconectInBackground(connection.getServerAddress(), connection); } else { index++; if (index >= _clientConfig.getServerCount()) { index = 0; } nextServer = _clientConfig.getServerAt(index); if (startingServer.equals(nextServer)) { break; } } } else { if (index >= _clientConfig.getServerCount()) { nextServer = startingServer; } break; } } if (connection == null || !connection.getIsConnected()) { index = 0; nextServer = _clientConfig.getServerAt(index); while (true && !getIsDisposing()) { if (_clientConfig.getServerCount() > 0 && index < _clientConfig.getServerCount()) { boolean found = false; Address address = new Address(nextServer.getName(), nextServer.getPort()); if (this.getPool().Contains(address)) { connection = this.getPool().getItem(address); found = true; } else { connection = new Connection(this, this.getLogger(), _perfStatsColl, _responseIntegrator, _clientConfig.getBindIP(), this._cacheId); } if (!connection.getIsConnected()) { if (!connection.getIsReconnecting()) { connection = ReconnectServer(connection, nextServer); //in case we could not acquire lock and no call to connect is made if (connection.getServerAddress() == null) { connection.setServerAddress(address); } if (connection.getIsConnected()) { if (!this.getPool().Contains(address)) { this.getPool().setItem(address, connection); } break; } } index++; if (index >= _clientConfig.getServerCount()) { index = 0; } nextServer = _clientConfig.getServerAt(index); if (startingServer.equals(nextServer)) { break; } } else { break; } } else { if (index >= _clientConfig.getServerCount()) { nextServer = startingServer; } break; } } } return connection; } private void CheckRetryConnectionDelay() { LocalDateTime currentTime = LocalDateTime.now(); Duration duration = Duration.between(currentTime, _retryConnectionStartTime); long diff = Math.abs(duration.toMinutes()); if (diff >= _retryConnectionDelayInMinutes) { _retryConnectionStartTime = LocalDateTime.now(); _retryConnection = true; } } private void Throttle() throws LicensingException { if (LicenseManager.isDeveloper()) { if (_addressUtil != null && !_isLocalAddress) { { _throttleManager.throttle(1); //throttling rate 1 CheckPendingRequests(_cacheId.toLowerCase()); } } } } public boolean ConnectRemoteServer(Connection connection, ServerInfo server, boolean registerNotifs) throws Exception { boolean connected = false; Exception exception = null; if (server != null && server.getIP() != null) { tangible.RefObject tempRef_exception = new tangible.RefObject(exception); connected = ConnectRemoteServer(connection, server.getIP(), server.getPort(), this._balanceNode, this.getImportHashmap(), registerNotifs, tempRef_exception, true); exception = tempRef_exception.argvalue; } return connected; } public boolean ConnectRemoteServer(Connection connection, InetAddress addr, int port, boolean balanceNodes, boolean importHashmap, boolean registerNotifs, tangible.RefObject exception, boolean changeStatus) throws SecurityException, LicensingException, InternalCommandException, CommandException, ConfigurationException, OperationFailedException, ActivityBlockedException { boolean connected = connection.connect(addr, port); _responseIntegrator.RemoveServer(new Address(addr, port)); if (connected) { try { java.util.HashMap runningServers; if (_clientConfig.getIPMappingConfigured()) { GetServerMapping(connection, true); } // connect with cache host connected = ConnectWithCacheHost(connection, addr, port); if (balanceNodes) { CommandResponse response = IsOptimalServer(connection, addr, port); ServerInfo rm = _clientConfig.getMappedServer(response.getIp(), response.getServerPort()); if (response != null && (!addr.toString().equals(rm.getName()) || port != rm.getPort())) { connected = TryConnectTo(connection, rm); } } if (connected) { if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().Info("Broker.ConnectRemoteServer", "[Local : (" + (connection.getPrimaryClientSocket() != null ? (connection.getPrimaryClientSocket().getInetAddress() != null ? connection.getPrimaryClientSocket().getInetAddress().toString() : "localendpointNULL") : "primaryclientsocketNULL") + ") Server : (" + addr.toString() + ":" + port + ")] connected successfully"); } InitializeCache(connection, addr, port, balanceNodes); StartThrottlingOnRemoteServer(connection); if (connection.getSupportDualSocket()) { InitializeSecondarySocket(connection, addr, port); } runningServers = GetRunningServers(connection, addr, port); connection.Init(); if (runningServers != null) { int outPort = 0; for (String str : runningServers.keySet()) { ServerInfo rServer = new ServerInfo(); outPort = runningServers.get(str); rServer.setName(str); Address address = new Address(str, outPort); rServer.setIP(address.getIpAddress()); rServer.setPort(address.getPort()); rServer.setPriorityInternal(Short.MAX_VALUE); _clientConfig.addServer(rServer); } } if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().CriticalInfo("Broker.ConnectRemoteServer", "[Local : (" + connection.getPrimaryClientSocket().getInetAddress().toString() + ") Server : (" + addr.toString() + ":" + port + ")] initialized cache successfully"); } } } catch (SecurityException sec_ex) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", sec_ex.toString()); } connection.Disconnect(); connected = false; throw sec_ex; } catch (LicensingException lic) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", lic.toString()); } connection.Disconnect(); connected = false; throw lic; } catch (Exception e) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", e.toString()); } connection.Disconnect(changeStatus); connected = false; exception.argvalue = e; } } else { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "Could not connect to server (" + addr.toString() + ":" + port + ")"); } } if (connected) { if (_processor != null) { _processor.Start(); } connection.getStatusLatch().SetStatusBit(ConnectionStatus.Connected, (byte) (ConnectionStatus.Disconnected | ConnectionStatus.Connecting)); _serverIP = connection.getServerAddress(); _port = connection.getPort(); getPool().Add(connection.getServerAddress(), connection); if (importHashmap) { GetHashmap(connection); } if (registerNotifs) { _cache.reRegisterGeneralNotification(connection); connection.setNotifRegistered(true); } if (getImportHashmap()) { _cache.registerHashmapChangedEvent(connection); } _cache.getTypeInfoMap(connection); _cache.getExpirationFromServer(connection); _cache.getSerializationFormat(connection); if (getLogger() != null && getLogger().getIsErrorLogsEnabled() && _cache != null) getLogger().getNCacheLog().CriticalInfo("Broker.ConnectRemoteServer", "Compression Enbaled : { " +_cache.getCompressionEnabled() + "} ; " + "Compression Threshold : { " + _cache.getCompressionThreshold()+ "} ; Encryption Enabled : { " + _cache.getEncryptionEnabled()+ "} ; " + "Pipelining Enabled : { " + _pipeliningEnabled + "} ; Pipelining Batch Interval : { " + _pipeliningEnabled+ "}"); //Not to re-register if (importHashmap) { _cache.getCompactTypesFromServer(connection); } _connectingFirstTime = false; //Last moment fix for _connection becomes null or it's server address get's null if (getConnection() == null || getConnection().getServerAddress() == null) { //DONT REMOVE IT; used for dump analysis _connectionRectified = true; if (getLogger() != null && getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "_connection anamoly detected connection ? " + (getConnection() == null)); } setConnection(connection); } } return connected; } public final void GetHashmap(Connection connection) throws OperationFailedException, CommandException, InternalCommandException, ActivityBlockedException, SecurityException, ConfigurationException, LicensingException { GetHashmapCommand command = new GetHashmapCommand(); Request request = new Request(false, getOperationTimeout()); Address ipAddress = connection == null ? getConnection().getServerAddress() : connection.getServerAddress(); request.addCommand(ipAddress, command); if (connection != null) { try { executeRequest(request, connection, true, true); } catch (CacheException e) { throw new OperationFailedException(e); } } else { executeRequestForInternalCommand(request); } CommandResponse res = request.getResponse(); try { res.parseResponse(); } catch (CacheException e) { throw new OperationFailedException(e); } NewHashmap hashmap = res.getNewHashmap(); if (hashmap == null) { if (this._processor != null) { this._processor.Stop(); } this.setImportHashmap(false); return; } UpdateHashmapAsync updateHashmapAsync = new UpdateHashmapAsync(this); updateHashmapAsync.sethashmap(hashmap); connectionPool.setBucketSize(res.getBucketSize()); ThreadPool.getInstance().executeTask(updateHashmapAsync); } private void StartThrottlingOnRemoteServer(Connection connection) throws LicensingException { if (LicenseManager.isDeveloper()) { if (connection != null && connection.getAddress() != null) { if (_addressUtil == null) { _addressUtil = new AddressUtil(); _isLocalAddress = _addressUtil.verifyLocalAddress(connection.getAddress()); if (!_isLocalAddress) { _throttleManager.start(); CheckPendingRequests(_cacheId.toLowerCase()); } } } } } private void CheckPendingRequests(String cacheId) throws LicensingException { long val = 0; synchronized (_requestsLock) { if (_requestDic.containsKey(cacheId.toLowerCase())) { val = _requestDic.get(cacheId.toLowerCase()); if (val >= 0) { if (val >= _allowedRequests) { throw new LicensingException("Clients running under DEV license cannot send more than 200,000 requests to remote cache."); } val += 1; _requestDic.put(cacheId.toLowerCase(), val); } } else { _requestDic.put(cacheId.toLowerCase(), (long) 1); } } } public final void InitializeCache(Connection connection, InetAddress address, int port, boolean balanceNodes) throws OperationFailedException, InternalCommandException, AggregateException, GeneralFailureException, LicensingException, SecurityException, com.alachisoft.ncache.runtime.exceptions.SecurityException, ConfigurationException, StreamException, StreamAlreadyLockedException, OperationNotSupportedException, StreamNotFoundException, IOException, InterruptedException, CommandException { //server boolean isLicenseclient = false; InitCommand initCommand = new InitCommand(_cache.getClientID(), _cacheId, null, null, _licenceCode, LicenseManager.getLicenseInfo(), isLicenseclient, connection.getClientLocalIP(), connection.getAddress(), _clientInfo, getClientLicenseType(), SslConfiguration.getSslConnectionEnabled(), getOperationTimeout()); Request request = new Request(false, getOperationTimeout()); request.addCommand(connection.getServerAddress(), initCommand); try { executeRequest(request, connection, false, false); } catch (CacheException e) { throw new OperationFailedException(e); } connection.setOptimized(true); CommandResponse res = connection.RecieveCommandResponse(false); SecureConnectionIfEnabled(connection, res.isSecureConnectionEnabled()); switch (res.getCacheType()) { case "partitioned-server": case "partitioned-replicas-server": this._balanceNode = false; setPartitioningStrategy(new DistributedPartitioningStrategy(this)); break; case "local-cache": this._balanceNode = false; this.setImportHashmap(false); setPartitioningStrategy(new LocalCachePartitioningStrategy(connection)); case "mirror-server": this._balanceNode = false; this.setImportHashmap(false); case "replicated-server": this.setImportHashmap(false); } connection.setRequestInquiryEnabled(res.isRequestLoggingEnabled()); if (res != null) { res.parseResponse(); } this._pipeliningEnabled = res.getIspipeliningEnabled(); this._pipeliningBatchInterval = res.getPipeliningBatchInterval(); getSocketManagerHandler().setEnablePipelining(_pipeliningEnabled); this.setIsPersistenceEnabled(res.getIsPersistenceEnabled()); this.setPersistInterval(res.getPersistInterval()); if (getIsPersistenceEnabled() && _persistenceManager == null) { _persistenceManager = new PersistenceManager(getPersistInterval() + 10); } else if (!getIsPersistenceEnabled() && _persistenceManager != null) { _persistenceManager.dispose(); _persistenceManager = null; } else if (getIsPersistenceEnabled() && _persistenceManager != null) { _persistenceManager.StartEventDuplicationCheck(); } if (res.getProtobufResponse().getInitCache().getIsShutDownProcessEnabled()) { for (ShutDownServerInfoProtocol.ShutDownServerInfo sInfo : res.getProtobufResponse().getInitCache().getShutDownServerInfoList()) { Address blockedServer = new Address(sInfo.getServerIP(), sInfo.getPort()); if (!_shutdownServers.containsKey(blockedServer)) { ShutDownServerInfo shutdownServer = new ShutDownServerInfo(); shutdownServer.setBlockInterval(sInfo.getTimeoutInterval()); shutdownServer.setBlockServerAddress(blockedServer); shutdownServer.setUniqueBlockingId(sInfo.getUniqueKey()); _shutdownServers.put(blockedServer, shutdownServer); } } } _monitoringSessionId= res.getProtobufResponse().getInitCache().getMonitoringSessionId(); _cacheConfigId = res.getProtobufResponse().getInitCache().getCacheConfigId(); } private java.util.HashMap GetRunningServers(Connection conn, InetAddress coonectedServerAddress, int port) throws IOException, OperationFailedException, InternalCommandException, AggregateException, GeneralFailureException, LicensingException, com.alachisoft.ncache.runtime.exceptions.SecurityException, ConfigurationException, StreamException, StreamAlreadyLockedException, OperationNotSupportedException, StreamNotFoundException, InterruptedException, CommandException, ActivityBlockedException { GetRunningServersCommand command = new GetRunningServersCommand(_cacheId, null, null); Request request = new Request(false, getOperationTimeout()); request.addCommand(conn.getServerAddress(), command); executeRequest(request, conn, false, false); CommandResponse runningServers = conn.RecieveCommandResponse(false); if (runningServers != null) { runningServers.parseResponse(); return runningServers.getRunningServers(); } return null; } public void GetServerMapping(Connection connection, boolean initialRequest) throws OperationFailedException { GetServerMappingCommand command = new GetServerMappingCommand(); CommandResponse commandResponse = null; Request request = createRequest(command); try { if (initialRequest) { executeRequest(request, connection, false, false); commandResponse = connection.RecieveCommandResponse(false); } else { executeRequest(request); commandResponse = request.getResponse(); } if (commandResponse != null) { commandResponse.parseResponse(); } _clientConfig.addMappedServers(commandResponse.getServerMappingList()); } catch (Exception exp) { if (getLogger().getIsDetailedLogsEnabled()) { getLogger().getNCacheLog().Debug(exp.getMessage()); } } } private int GetCachePort(Connection connection) throws OperationFailedException, InternalCommandException, AggregateException, GeneralFailureException, LicensingException, com.alachisoft.ncache.runtime.exceptions.SecurityException, ConfigurationException, StreamException, StreamAlreadyLockedException, OperationNotSupportedException, StreamNotFoundException, IOException, InterruptedException, CommandException, ActivityBlockedException { GetCacheHostPortCommand command = new GetCacheHostPortCommand(_cacheId, null, null); Request request = new Request(false, getOperationTimeout()); request.addCommand(connection.getServerAddress(), command); executeRequest(request, connection, false, false); CommandResponse response = connection.RecieveCommandResponse(false); if (response != null) { response.parseResponse(); } return response.getCacheManagementPort(); } private CommandResponse IsOptimalServer(Connection connection, InetAddress connectedServerAddress, int port) throws IOException, OperationFailedException, AggregateException, GeneralFailureException, InternalCommandException, LicensingException, com.alachisoft.ncache.runtime.exceptions.SecurityException, ConfigurationException, StreamException, StreamAlreadyLockedException, OperationNotSupportedException, StreamNotFoundException, InterruptedException, CommandException, ActivityBlockedException { GetOptimalServer command = new GetOptimalServer(_cacheId, null, null); Request request = new Request(false, getOperationTimeout()); request.addCommand(connection.getServerAddress(), command); executeRequest(request, connection, false, false); CommandResponse balanceNodeRes = connection.RecieveCommandResponse(false); if (balanceNodeRes != null) { balanceNodeRes.parseResponse(); } return balanceNodeRes; } private boolean TryConnectTo(Connection connection, ServerInfo remoteServer) { int mainPort = connection.getPort(); InetAddress mainIp = connection.getAddress(); boolean mainDisconnected = false, connected = true; try { connection.Disconnect(); mainDisconnected = true; if (connected = connection.connect(remoteServer.getIP(), remoteServer.getPort())) { connected = ConnectWithCacheHost(connection, remoteServer.getIP(), remoteServer.getPort()); } else { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "Unable to connect to [" + remoteServer.getIP() + ":" + remoteServer.getPort() + "], restoring existing connection with [" + mainIp + ":" + mainPort + "]"); } try { connection.Disconnect(); } catch (java.lang.Exception e) { } if (connected = connection.connect(mainIp, mainPort)) { connected = ConnectWithCacheHost(connection, mainIp, mainPort); } else { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "Unable to restoring connection to [" + mainIp + ":" + mainPort + "]"); } } } } catch (RuntimeException | IOException e) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "Error connecting. Error: " + e.toString()); } if (mainDisconnected) { try { connection.Disconnect(); } catch (java.lang.Exception e2) { } if (connected = connection.connect(mainIp, mainPort)) { try { connected = ConnectWithCacheHost(connection, mainIp, mainPort); } catch (Exception ex) { } } else { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectRemoteServer", "Unable to restoring connection to [" + mainIp + ":" + mainPort + "]"); } } } } catch (Exception e) { } return connected; } private boolean ConnectWithCacheHost(Connection connection, InetAddress address, int port) throws IOException, InternalCommandException, GeneralFailureException, AggregateException, OperationFailedException, LicensingException, com.alachisoft.ncache.runtime.exceptions.SecurityException, StreamNotFoundException, StreamException, StreamAlreadyLockedException, ConfigurationException, OperationNotSupportedException, InterruptedException, CommandException, ActivityBlockedException { boolean connectedWithHost = true; // getting management port to transfer connection int cachePort = GetCachePort(connection); //.Net core does not support socket transfer from one process to other. In this case, client needs to directly connect to the cache host. //However for .net framework, service will transfer connection to the cache host. if (cachePort != port) { connection.Disconnect(false); connectedWithHost = connection.SwitchTo(this, this.getLogger(), _perfStatsColl, _responseIntegrator, _clientConfig.getBindIP(), this._cacheId, address, cachePort); if (!connectedWithHost) { if (getLogger().getIsErrorLogsEnabled()) { getLogger().getNCacheLog().Error("Broker.ConnectWithCacheHost", "Failed to connect with cache host directly on port :" + cachePort); } } } return connectedWithHost; } public final boolean GetMessageDistribution(java.util.Map> topicWiseMessageIds, java.util.HashMap>> messageDistributionMap) { boolean result = false; for (java.util.Map.Entry> pair : topicWiseMessageIds.entrySet()) { String topicName = pair.getKey(); java.util.List messageList = pair.getValue(); for (String messageId : messageList) { Address address; synchronized (_hashmapUpdateMutex) { address = getPool().GetIp(messageId); } Connection conn = getPool().getItem(address); //for balanced connection Address adrs = address; if (conn != null && !conn.getIsConnected()) { Address loadbaanced = GetLoadBalancedAddress(); if (loadbaanced != null) { adrs = loadbaanced; } } java.util.HashMap> topicDic; if (messageDistributionMap.containsKey(adrs)) { topicDic = messageDistributionMap.get(adrs); PopulateList(topicDic, topicName, messageId); result = true; } else { topicDic = new java.util.HashMap>(); PopulateList(topicDic, topicName, messageId); messageDistributionMap.put(adrs, topicDic); } } } return result; } private void PopulateList(java.util.HashMap> topicDic, String topicName, String messageId) { if (topicDic.containsKey(topicName)) { java.util.List list = topicDic.get(topicName); list.add(messageId); } else { java.util.List list = new java.util.ArrayList(); list.add(messageId); topicDic.put(topicName, list); } } private void ReconectInBackground(Address address, Connection connection) { synchronized (_reconnectTasks) { if (!_reconnectTasks.containsKey(address)) { if (!connection.getIsReconnecting()) { ReconnectTask task = new ReconnectTask(this, connection); _reconnectTasks.put(address, task); this._processor.Enqueue(task); } } } } public final void UnregisterReconnectTask(Address address) { synchronized (_reconnectTasks) { _reconnectTasks.remove(address); } } @Override public void OnServerLost(Address ip, boolean foredDisocnnect) { ServerLost(ip, foredDisocnnect); } public boolean poolHasAllServers() { return _clientConfig.getServerCount() == connectionPool.getServers().size(); } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy