com.gemstone.gemfire.internal.cache.GatewayImpl Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of gemfire-core Show documentation
SnappyData store based off Pivotal GemFireXD
The newest version!
/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */

package com.gemstone.gemfire.internal.cache;

import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;


import com.gemstone.gemfire.CancelCriterion;
import com.gemstone.gemfire.CancelException;
import com.gemstone.gemfire.SystemFailure;
import com.gemstone.gemfire.cache.Cache;
import com.gemstone.gemfire.cache.CacheException;
import com.gemstone.gemfire.cache.CacheListener;
import com.gemstone.gemfire.cache.EntryEvent;
import com.gemstone.gemfire.cache.GatewayConfigurationException;
import com.gemstone.gemfire.cache.GatewayException;
import com.gemstone.gemfire.cache.RegionDestroyedException;
import com.gemstone.gemfire.cache.SerializedCacheValue;
import com.gemstone.gemfire.cache.client.PoolManager;
import com.gemstone.gemfire.cache.client.internal.EndpointManagerImpl;
import com.gemstone.gemfire.cache.client.internal.PoolImpl;
import com.gemstone.gemfire.cache.util.CacheListenerAdapter;
import com.gemstone.gemfire.cache.util.Gateway;
import com.gemstone.gemfire.cache.util.GatewayEventListener;
import com.gemstone.gemfire.cache.util.GatewayQueueAttributes;
import com.gemstone.gemfire.distributed.DistributedSystem;
import com.gemstone.gemfire.distributed.GatewayCancelledException;
import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
import com.gemstone.gemfire.i18n.LogWriterI18n;
import com.gemstone.gemfire.internal.Assert;
import com.gemstone.gemfire.internal.LogWriterImpl;
import com.gemstone.gemfire.internal.SocketCreator;
import com.gemstone.gemfire.internal.cache.EntryEventImpl.SerializedCacheValueImpl;
import com.gemstone.gemfire.internal.cache.tier.sockets.CacheServerHelper;
import com.gemstone.gemfire.internal.concurrent.AI;
import com.gemstone.gemfire.internal.concurrent.CFactory;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.offheap.StoredObject;
import com.gemstone.gemfire.internal.offheap.annotations.Unretained;
import com.gemstone.gemfire.security.GemFireSecurityException;
import static com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier.ENTRY_EVENT_NEW_VALUE;


/**
 * Represents a {@link Gateway}.
 *
 * @author Barry Oglesby
 * @since 4.2
 */
public class GatewayImpl extends AbstractGateway
{

  /**
   * Timeout tokens in the unprocessedEvents map after this many milliseconds.
   */
  static protected final int TOKEN_TIMEOUT
    = Integer.getInteger("Gateway.TOKEN_TIMEOUT", 15000).intValue();

  /**
   * Size of the oplog file used for the persistent queue in bytes
   */
  static public final int QUEUE_OPLOG_SIZE
    = Integer.getInteger("gemfire.GatewayQueueOpLogSize", 1024*1024*100).intValue();


  /**
   * Time, in seconds, that we allow before a gateway is considered
   * dead and should be aborted
   */
  static private final long GATEWAY_TIMEOUT
  = Integer.getInteger("gemfire.GATEWAY_TIMEOUT", 30).intValue();

  /**
   * The list of endpoints (host and port) to which this Gateway
   * is connected. Use pattern of replace instead of modify, which means:
   *
   *    * List newEndpoints = new ArrayList(this._endpoints);
   * newEndpoints.add(new EndpointImpl(id, host, port));
   * this._endpoints = Collections.unmodifiableList(newEndpoints);
   * 
   */
  private volatile List _endpoints; // always replace never modify

  /**
   * The list of GatewayEventListeners to which this
   * Gateway invokes the callback. Use pattern of replace
   * instead of modify, which means:   *   List newListeners = new ArrayList(this._listeners);
   *   newListeners.add(listener);
   *   this._listeners = Collections.unmodifiableList(newListeners);
   * 
   */
  private volatile List _listeners; // always replace never modify

  /**
   * The GatewayQueueAttributes for this Gateway.
   */
  private volatile GatewayQueueAttributes _queueAttributes;

  /**
   * Whether this is a primary Gateway
   */
  private volatile boolean _primary;

  /**
   * An object used by a secondary Gateway to wait - notify on
   * when becoming primary
   */
  private final Object _primaryLock = new Object();

  protected class Stopper extends CancelCriterion {

    /* (non-Javadoc)
     * @see com.gemstone.gemfire.CancelCriterion#cancelInProgress()
     */
    @Override
    public String cancelInProgress() {
      String reason = GatewayImpl.this._hub.getCancelCriterion().cancelInProgress();
      if (reason != null) {
        return reason;
      }
      if (GatewayImpl.this._isRunning) {
        return null;
      }
      return "Gateway has been stopped";
    }

    /* (non-Javadoc)
     * @see com.gemstone.gemfire.CancelCriterion#generateCancelledException(java.lang.Throwable)
     */
    @Override
    public RuntimeException generateCancelledException(Throwable e) {
      String reason = cancelInProgress();
      if (reason == null) {
        return null;
      }
      RuntimeException result = GatewayImpl.this._hub.getCancelCriterion().generateCancelledException(e);
      if (result != null) {
        return result;
      }
      return new GatewayCancelledException("Gateway has been stopped"); // TODO pick better exception
    }

  }
  protected final Stopper stopper = new Stopper();

  public CancelCriterion getCancelCriterion() {
    return stopper;
  }

  /**
   * The pool used by this Gateway.
   */
  protected volatile PoolImpl _proxy;

  /**
   * The buffer size in bytes of the socket connection between this code>Gateway
   *  and its receiving Gateway
   */
  private int _socketBufferSize;

  /**
   * The amount of time in milliseconds that a socket read between this
   * Gateway and its receiving Gateway will block.
   */
  private final int _socketReadTimeout;

  /**
   * The GatewayEventProcessor used by this Gateway
   * to do the following:
   * put events on its queue (if primary) or map (if secondary)
   * read batches of events off the queue (if primary)
   * 
manage the map as events are added to the queue (if secondary)
   * 
dispatch events to its receiver
   * 
   */
  private volatile GatewayEventProcessor _eventProcessor;

  /**
   * Whether this Gateway is used by a GatewayParallelImpl
   */
  private final boolean _usedInParallel;
  /**
   * The number of times to peek on shutdown before giving up and shutting down.
   */
  protected static final int MAXIMUM_SHUTDOWN_PEEKS = Integer.getInteger(
      "Gateway.MAXIMUM_SHUTDOWN_PEEKS", 20).intValue();

  /**
   * The queue size threshold used to warn the user. If the queue reaches this
   * size, log a warning.
   */
  protected static final int QUEUE_SIZE_THRESHOLD = Integer.getInteger(
      "Gateway.QUEUE_SIZE_THRESHOLD", 5000).intValue();


  /**
   * Unique ID for pool names
   */
  private static final AI ID_COUNTER = CFactory.createAI();

  ////////////////////// Constructors //////////////////////

  /**
   * Constructor. Creates a new GatewayImpl with the default
   * configuration.
   */
  GatewayImpl(GatewayHubImpl hub, String id) {
    this(hub, id, false, null);
  }

  /**
   * Constructor. Creates a new GatewayImpl for use in parallel.
   */
  GatewayImpl(GatewayHubImpl hub, String id, boolean usedInParallel,
      GatewayStats allStatistics) {
    super(hub, id, id, allStatistics);
    this._usedInParallel = usedInParallel;
    this._endpoints = Collections.EMPTY_LIST;
    this._listeners = Collections.EMPTY_LIST;
    this._isRunning = false;
    this._queueAttributes = new GatewayQueueAttributes();    
    this._primary = false;
    this._socketBufferSize = DEFAULT_SOCKET_BUFFER_SIZE;
    this._socketReadTimeout = DEFAULT_SOCKET_READ_TIMEOUT;
  }

  ///////////////////// Instance Methods /////////////////////

  public void addEndpoint(String id, String host, int port)
      throws GatewayException
  {
    synchronized (this.controlLock) {
      checkRunning();
      // If an endpoint with the id, host and port is already defined, throw an
      // exception
      if (alreadyDefinesEndpoint(id, host, port)) {
        throw new GatewayException(LocalizedStrings.GatewayImpl_GATEWAY_0_ALREADY_DEFINES_AN_ENDPOINT_EITHER_WITH_ID_1_OR_HOST_2_AND_PORT_3.toLocalizedString(
                new Object[] {this._id, id, host, Integer.valueOf(port)}));
      }

      // If another gateway defines this same endpoint and its not used in a
      // parallel gateway, throw an exception. If it is used in a parallel
      // gateway, it is ok to define the same endpoint in multiple gateways.
      String[] otherGateway = new String[1];
      if (!this._usedInParallel && otherGatewayDefinesEndpoint(host, port, otherGateway)) {
        throw new GatewayException(LocalizedStrings.GatewayImpl_GATEWAY_0_CANNOT_DEFINE_ENDPOINT_HOST_1_AND_PORT_2_BECAUSE_IT_IS_ALREADY_DEFINED_BY_GATEWAY_3.toLocalizedString(
              new Object[] {this._id, host, Integer.valueOf(port), otherGateway[0]}));
      }

      // If the gateway is attempting to add an endpoint to its own hub, throw
      // an exception
      if (isConnectingToOwnHub(host, port)) {
        throw new GatewayException(LocalizedStrings.GatewayImpl_GATEWAY_0_CANNOT_DEFINE_AN_ENDPOINT_TO_ITS_OWN_HUB_HOST_1_AND_PORT_2
            .toLocalizedString(new Object[] {this._id, host, Integer.valueOf(port)}));
      }

      // If listeners are already defined, throw an exception
      if (hasListeners()) {
        throw new GatewayException(LocalizedStrings.GatewayImpl_GATEWAY_0_CANNOT_DEFINE_AN_ENDPOINT_BECAUSE_AT_LEAST_ONE_LISTENER_IS_ALREADY_DEFINED_BOTH_LISTENERS_AND_ENDPOINTS_CANNOT_BE_DEFINED_FOR_THE_SAME_GATEWAY
            .toLocalizedString(this._id));
      }

      List newEndpoints = new ArrayList(this._endpoints);
      newEndpoints.add(new EndpointImpl(id, host, port));
      this._endpoints = Collections.unmodifiableList(newEndpoints);
    }
  }

  public List getEndpoints()
  {
    return this._endpoints;
  }

  public boolean hasEndpoints() {
    return getEndpoints().size() > 0;
  }

  public void addListener(GatewayEventListener listener) throws GatewayException {
    synchronized (this.controlLock) {
      checkRunning();
      // If endpoints are already defined, throw an exception
      if (hasEndpoints()) {
        throw new GatewayException(LocalizedStrings.GatewayImpl_GATEWAY_0_CANNOT_DEFINE_A_LISTENER_BECAUSE_AT_LEAST_ONE_ENDPOINT_IS_ALREADY_DEFINED_BOTH_LISTENERS_AND_ENDPOINTS_CANNOT_BE_DEFINED_FOR_THE_SAME_GATEWAY.toLocalizedString(this._id));
      }

      List newListeners = new ArrayList(this._listeners);
      newListeners.add(listener);
      this._listeners = Collections.unmodifiableList(newListeners);
    }
  }

  public List getListeners() {
    return this._listeners;
  }

  public boolean hasListeners() {
    return getListeners().size() > 0;
  }

  public void setSocketBufferSize(int socketBufferSize)
  {
    synchronized (this.controlLock) {
      checkRunning();
      this._socketBufferSize = socketBufferSize;
    }
  }

  public int getSocketBufferSize()
  {
    synchronized (this.controlLock) {
      return this._socketBufferSize;
    }
  }

  public void setSocketReadTimeout(int socketReadTimeout)
  {
    synchronized (this.controlLock) {
      checkRunning();
      getLogger().warning(LocalizedStrings.GatewayImpl_GATEWAY_SOCKET_READ_TIMEOUT_DISABLED);
      // do nothing on purpose...
      // setSocketReadTimeout is now optional and this impl ignores it
      // setSocketReadTimeout was causing too many problems because customers
      //    kept using too small of a value
    }
  }

  public int getSocketReadTimeout()
  {
    synchronized (this.controlLock) {
      return this._socketReadTimeout;
    }
  }

  /**
   * Set whether this is a primary Gateway
   */
  @Override
  protected void setPrimary(boolean primary)
  {
    this._primary = primary;
  }

  /**
   * Returns whether this is a primary Gateway
   *
   * @return whether this is a primary Gateway
   */
  public boolean getPrimary()
  {
    return this._primary;
  }

  public void setQueueAttributes(GatewayQueueAttributes queueAttributes)
  {
    synchronized (this.controlLock) {
      checkRunning();
      // See bug 44558. The gateway.stop method has been changed to close the
      // region. If the previous queue attributes enabled persistence and the
      // new ones don't, destroy the disk files. Don't use the existing queue
      // attributes since they may have been modified in place. If the event
      // processor is null, the gateway has not been started, so there isn't any
      // way to know the previous state. This method will throw an
      // IllegalStateException if the previous state is incompatible with the
      // new state.
      if (this._eventProcessor != null
          && ((SingleWriteSingleReadRegionQueue) this._eventProcessor._eventQueue)
              .isPersistent() && (!queueAttributes.getEnablePersistence())) {
        this._eventProcessor.destroyQueuePersistenceFiles();
      }
      this._queueAttributes = queueAttributes;
    }
  }

  public GatewayQueueAttributes getQueueAttributes()
  {
    synchronized (this.controlLock) {
      return this._queueAttributes;
    }
  }

  public GatewayQueueAttributes getQueueAttributesNoSync()
  {
    return this._queueAttributes;
  }
  
  /**
   * This method has been added for test only purpose. 
   * This can be used by DUnit or Hydra tests to get the RegionQueue enclosed by this GatewayImpl.
   *  
   * @return    RegionQueue     RegionQueue enclosed by this gateway
   */
  public RegionQueue getRegionQueueTestOnly() {
    return this._eventProcessor._eventQueue;
  }

  public int getConcurrencyLevel() {
    return DEFAULT_CONCURRENCY_LEVEL;
  }
  
  /**
   * Returns whether another Gateway already defines an
   * Endpoint with the same host and port.
   *
   * @param host
   *          The host of the endpoint
   * @param port
   *          The port that the endpoint is listening on
   * @param otherGateway
   *          The other Gateway defining this
   *          Endpoint
   * @return whether another Gateway already defines an
   *         Endpoint with the same host and port
   * @guarded.By {@link #controlLock}
   */
  private boolean otherGatewayDefinesEndpoint(String host, int port,
      String[] otherGateway)
  {
    boolean otherGatewayDefined = false;
    // Iterate through all gateways and compare their endpoints to the input
    // host and port.
    for (Iterator i = getGatewayHub().getGateways().iterator(); i.hasNext();) {
      Gateway gateway = (Gateway)i.next();
      // Do not compare the current gateway
      if (!getId().equals(gateway.getId())) {
        for (Iterator j = gateway.getEndpoints().iterator(); j.hasNext();) {
          Endpoint endpoint = (Endpoint)j.next();
          // If the endpoint's host and port are equal to the input host and
          // port, answer true; else continue.
          if (endpoint.getHost().equals(host) && endpoint.getPort() == port) {
            otherGatewayDefined = true;
            otherGateway[0] = gateway.getId();
            break;
          }
        }
      }
    }
    return otherGatewayDefined;
  }

  /**
   * Returns whether an Endpoint with id is already defined by
   * this Gateway.
   *
   * @param id
   *          The id to verify
   * @param host
   *          The host of the endpoint
   * @param port
   *          The port that the endpoint is listening on
   * @return whether an Endpoint with id is already defined by
   *         this Gateway
   */
  private boolean alreadyDefinesEndpoint(String id, String host, int port)
  {
    boolean alreadyDefined = false;
    for (Iterator i = this._endpoints.iterator(); i.hasNext();) {
      Endpoint endpoint = (Endpoint)i.next();
      // If the ids are equal or the host and port are equal, then the
      // requested endpoint is already defined.
      if (endpoint.getId().equals(id)
          || (endpoint.getHost().equals(host) && endpoint.getPort() == port)) {
        alreadyDefined = true;
        break;
      }
    }
    return alreadyDefined;
  }

  /**
   * Returns whether this Gateway is attempting to add an
   * Endpoint to its own GatewayHub.
   *
   * @param host
   *          The host of the endpoint
   * @param port
   *          The port that the endpoint is listening on
   * @return whether this Gateway is attempting to add an
   *         Endpoint to its GatewayHub
   */
  private boolean isConnectingToOwnHub(String host, int port)
  {
    // These tests work where the host is specified as:
    // IP address string (e.g. host=10.80.10.80)
    // Short host name (e.g. host=bishop)
    // Fully-qualified host name (e.g. bishop.gemstone.com)
    // The localhost (e.g. localhost)
    // The loopback address (e.g. 127.0.0.1)

    // First compare the port with the hub's port. If they are the same,
    // then compare the host with the local host.
    boolean isConnectingToOwnHub = false;
    if (port == this._hub.getPort()) {
      // The ports are equal. Now, compare the hosts. Do a best guess
      // determination whether the input host is the same as the local
      // host.
      try {
        String localHostName = SocketCreator.getLocalHost()
            .getCanonicalHostName();
        String requestedHostName = InetAddress.getByName(host)
            .getCanonicalHostName();
        if (localHostName.equals(requestedHostName)
            || requestedHostName.startsWith("localhost")) {
          isConnectingToOwnHub = true;
        }
      }
      catch (UnknownHostException e) {
      }
    }
    return isConnectingToOwnHub;
  }

  /**
   * Returns the GemFire cache
   *
   * @return the GemFire cache
   */
  protected Cache getCache()
  {
    return this._cache;
  }

  /**
   * Returns the pool
   *
   * @return the pool
   */
  protected PoolImpl getProxy()
  {
    synchronized (this.controlLock) {
      return this._proxy;
    }
  }

  public void start() throws IOException
  {
    start(null);
  }
  
  protected void start(GatewayParallelImpl gpi) throws IOException {  
    synchronized (this.controlLock) {
      if (this._isRunning) {
        return;
      }

      // If the current stats are closed due to previously stopping this gateway,
      // recreate them using the GatewayParallelImpl's stats if necessary
      if (getStatistics().isClosed()) {
        setStatistics(new GatewayStats(this._cache.getDistributedSystem(),
            getGatewayHubId(), getId(), gpi == null ? null
                : gpi.getStatistics()));
      }

      if(hasEndpoints()) {
        // Create ConnectionProxy to the Endpoints
        Properties properties = new Properties();
        StringBuffer buffer = new StringBuffer();
        for (Iterator i = getEndpoints().iterator(); i.hasNext();) {
          Endpoint endpoint = (Endpoint)i.next();
          buffer.append(endpoint.getId()).append('=').append(endpoint.getHost())
          .append(':').append(endpoint.getPort());
          if (i.hasNext()) {
            buffer.append(',');
          }
        }
        properties.setProperty("endpoints", buffer.toString());
        properties.setProperty("socketBufferSize", String
            .valueOf(this._socketBufferSize));
        // we don't want any connections until we call acquireConnection
        // this fixes bug 39430
        properties.setProperty("connectionsPerServer", "0");
        properties.setProperty("readTimeout", String
            .valueOf(this._socketReadTimeout));

        this._proxy = createPool(properties);
        EndpointManagerImpl emi = (EndpointManagerImpl) this._proxy.getEndpointManager();
        emi.setGatewayStats(getStatistics());
      }

      // Initialize the event processor - done after the proxy is created
      try {
        /*
         * isRunning must be false when we initialize the queue, otherwise we can encounter distributed deadlock with P2P reader threads
         * seeing isRunning=true and then trying to get the controlLock, meanwhile the controlLock is held while we are doing stateFlush
         * and waiting for replies that need the p2p reader thread to unstick.
         * 
         * See bug #41921
         */
        GatewayEventProcessor ev = initializeEventProcessor();
        /*
         * We have to set isRunning=true before we actually start the GEP thread because of bug #39772, where the thread would exit early
         */
        setRunning(true);
        ev.start();
        
        ((GemFireCacheImpl) getCache()).getPdxRegistry().gatewayStarted(this);
      }
      catch (GemFireSecurityException e) {
        setRunning(false);
        throw e;
      }
      catch (GatewayConfigurationException e) {
        setRunning(false);
        throw e;
      }

      this.getLogger().info(LocalizedStrings.SerialGatewaySenderImpl_STARTED__0, this);
    }
  }
  
  public void pause()
  {
    synchronized (this.controlLock) {
      if (!this._isRunning) {
        return;
      }
      this._eventProcessor.pauseDispatching();
      getLogger().info(LocalizedStrings.GatewayImpl_PAUSED__0, this);
    }
  }

  public void resume()
  {
    synchronized (this.controlLock) {
      if (!this._isRunning) {
        return;
      }
      this._eventProcessor.resumeDispatching();
      getLogger().info(LocalizedStrings.GatewayImpl_RESUMED__0, this);
    }
  }

  public boolean isPaused() {
    boolean isPaused = false;
    GatewayEventProcessor ev = this._eventProcessor;
    if (ev != null) {
      isPaused = ev.getIsPaused();
    }
    return isPaused;
  }

  public PoolImpl createPool(Properties props) {
    String name = "GatewayPool-" + getGatewayPoolId();
    PoolFactoryImpl pf = (PoolFactoryImpl)PoolManager.createFactory();
    // @todo grid: switch this to use PoolImpl instead of BridgePoolImpl
    try {
      pf.init(props, false, true);
    } catch (IllegalArgumentException e) {
      if(e.getMessage().contains("Couldn't find any Endpoint")){
        throw e;
      }else {
        this._logger.warning(LocalizedStrings.GatewayImpl_UnknownHost, e);  
      }
    }
    PoolImpl result = (PoolImpl)pf.create(name);
    return result;
  }


  private static int getGatewayPoolId() {
    return ID_COUNTER.incrementAndGet();
  }

  /**
   * Ensure that the ConnectionProxyImpl class gets loaded
   *
   * @see SystemFailure#loadEmergencyClasses()
   */
  public static void loadEmergencyClasses() {
    PoolImpl.loadEmergencyClasses();
  }

  /**
   * Close this instance's proxy.
   *
   * @see SystemFailure#emergencyClose()
   */
  @Override
  public void emergencyClose() {
    this._isRunning = false; // try!
    {
      PoolImpl bpi = this._proxy;
      if (bpi != null) {
        bpi.emergencyClose();
      }
    }
  }
  
  /**
   * For bug 39255
   */
  private void stompProxyDead() {
    Runnable stomper = new Runnable() {
      public void run() {
        PoolImpl bpi = GatewayImpl.this._proxy;
        if (bpi != null) {
          try {
            bpi.destroy();
          }
          catch (Exception e) {/* ignore */
          }
        }
      }
    };
    ThreadGroup tg = LogWriterImpl.createThreadGroup(
        "Proxy Stomper Group", GatewayImpl.this.getLogger());
    Thread t = new Thread(tg, stomper, "Gateway Proxy Stomper");
    t.setDaemon(true);
    t.start();
    try {
      t.join(GATEWAY_TIMEOUT * 1000);
      return;
    }
    catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }

    this.getLogger().warning( LocalizedStrings.
        GatewayImpl_GATEWAY_0_IS_NOT_CLOSING_CLEANLY_FORCING_CANCELLATION,
        this);
    // OK, either we've timed out or been interrupted.  Time  for
    // violence.
    t.interrupt(); // give up
    this._proxy.emergencyClose(); // VIOLENCE!
  }

  public void stop()
  {
    synchronized (this.controlLock) {
      if (!this._isRunning) {
        return;
      }
      setRunning(false);
      
      setFailoverComplete(false);

      // Stop the dispatcher
      GatewayEventProcessor ev = this._eventProcessor;
      if (ev != null) {
        try {
          ev.resumeDispatching();
          ev.stopProcessing();
        }
        catch (Exception e) {/* ignore */
        }
      }

      // Stop the proxy (after the dispatcher, so the socket is still
      // alive until after the dispatcher has stopped)
      stompProxyDead();

      // Close the listeners
      for (Iterator i = this._listeners.iterator(); i.hasNext();) {
        ((GatewayEventListener) i.next()).close();
      }

      // Close the statistics
      if (this._statistics != null) {
        this._statistics.close();
      }

      this.getLogger().info(LocalizedStrings.GatewayImpl_STOPPED__0, this);
    }
  }

  public int getQueueSize()
  {
    synchronized (this.controlLock) {
      GatewayEventProcessor ev = this._eventProcessor;
      if (ev == null) {
        return 0;
      }
      return ev._eventQueue.size();
    }
  }

  @Override
  public String toString()
  {
    StringBuffer sb = new StringBuffer();
    sb.append(this._primary ? "Primary" : "Secondary");
    sb.append(" Gateway to ");
    sb.append(this._id);
    if (hasEndpoints()) {
      if (!isConnectedNoSync()) {
        sb.append(" not");
      }
      sb.append(" connected to ");
      sb.append(this._endpoints); // replaced and not modified in place... safe
    } else if (hasListeners()) {
      sb.append(" with listeners ");
      sb.append(this._listeners); // replaced and not modified in place... safe
    }
    return sb.toString();
  }

  public boolean isConnected()
  {
    synchronized (this.controlLock) {
      return isConnectedNoSync();
    }
  }

  private boolean isConnectedNoSync()
  {
    // want a nosync flavor so toString does not cause deadlocks
    if (!this._isRunning) {
      return false;
    }
    GatewayEventProcessor ev = this._eventProcessor;
    if (ev == null) {
      return false;
    }
    if (!ev.isAlive()) {
      return false;
    }
    if (!this._isRunning) {
      return false;
    }
    PoolImpl bpi = this._proxy; // volatile fetch
    if (bpi == null) {
      return false;
    }
    return !bpi.isDestroyed();
  }

  /**
   * Sets this Gateway to be running or stopped
   *
   * @param running
   *          Whether this Gateway is running or stopped
   */
  private void setRunning(boolean running)
  {
    synchronized (this.controlLock) {
      this._isRunning = running;
    }
  }

  /**
   * Distributes the event and operation to this Gateway's
   * EventDispatcher
   *
   * @param operation
   *          The operation of the event (e.g. AFTER_CREATE, AFTER_UPDATE, etc.)
   * @param event
   *          The event to distribute
   */
  @Override
  public void distribute(EnumListenerEvent operation, EntryEventImpl event)
  {
    if (!isRunning()) {
      return;
    }
    
    {
      // Serialization must be done outside of the control lock because PDX
      // might perform messaging that can deadlock if we're holding the controlLock.
      @Unretained(ENTRY_EVENT_NEW_VALUE)
      Object newValue = event.getRawNewValue();
      if (CachedDeserializableFactory.preferObject()
          || newValue instanceof CachedDeserializable) {
        // What if the value contained in the CachedDeserializable has been deserialized?
        // TODO: clarify why we do not need to check for that in this case.
        // Note that if the value is stored off-heap then newValue is a CachedDeserializable and is in serialized form.
      } else if (newValue instanceof byte[]) {
        // just bytes so nothing needed
      } else {
        // So if the event's new value is in deserialized form serialize it here
        // and record it in the event.
        try {
          byte[] value = CacheServerHelper.serialize(newValue);
          // TODO this looks wrong. We shouldn't get calling setNewValue with a CachedDeserializable.
          // Instead we should be calling setSerializedNewValue with the byte[] value.
          // Also what if getRawNewValue returns a CachedDeserializable?
          event.setNewValue(CachedDeserializableFactory.create(value));
        } catch (IOException e) {
          this.getLogger().severe(
              LocalizedStrings.GatewayImpl_0_AN_EXCEPTION_OCCURRED_WHILE_QUEUEING_1_TO_PERFORM_OPERATION_2_FOR_3,
              new Object[] {this, getId(), operation, event}, e);
        }
      }
    }
    
    synchronized (this.controlLock) {
      // If this gateway is not running, return
      if (!isRunning()) {
        return;
      }

      if (this.getLogger().fineEnabled()) {
        // We can't deserialize here for logging purposes so don't
        // call getNewValue.
        //event.getNewValue(); // to deserialize the value if necessary
        this.getLogger().fine(this + ": About to queue operation " + operation + " for "+ getId() + ": "
            + event);
      }
      try {
        GatewayEventProcessor ev = this._eventProcessor;
        if (ev == null) {
          stopper.checkCancelInProgress(null);
          ((InternalDistributedSystem)this._cache.getDistributedSystem())
            .getCancelCriterion().checkCancelInProgress(null);
          // event processor will be null if there was an authorization problem
          // connecting to the other site (bug #40681)
          if (ev == null) {
            throw new GatewayCancelledException("Event processor thread is gone");
          }
        }
        if (ev != null) {
          ev.enqueueEvent(operation, event);
        }
      }
      catch (CancelException e) {
        throw e;
      }
      catch (Exception e) {
        this.getLogger().severe(
            LocalizedStrings.GatewayImpl_0_AN_EXCEPTION_OCCURRED_WHILE_QUEUEING_1_TO_PERFORM_OPERATION_2_FOR_3,
            new Object[] {this, getId(), operation, event}, e);
      }
    }
  }

  /**
   * Initializes this Gateway's GatewayEventProcessor
   */
  private GatewayEventProcessor initializeEventProcessor() {
    GatewayEventProcessor ev = new GatewayEventProcessor(this);
    this._eventProcessor = ev;

    if (this.getLogger().fineEnabled()) {
      this.getLogger().fine(this + ": Created event processor " + ev);
    }

    return ev;
    
  }

  /**
   * Wait to be told to become the primary Gateway. This method
   * is invoked by the Gateway'sEventDispatcher
   * to wait until it is primary before processing the queue.
   */
  protected void waitToBecomePrimary() throws InterruptedException
  {
    if (getPrimary()) {
      return;
    }
    synchronized (this._primaryLock) {
      while (!getPrimary()) {
        this.getLogger().info(LocalizedStrings.GatewayImpl_0__WAITING_TO_BECOME_PRIMARY_GATEWAY, this);
        this._primaryLock.wait();
      }
    } // synchronized
  }

  /**
   * @guarded.By failoverCompletedLock
   */
  private boolean failoverCompleted = false;

  private final Object failoverCompletedLock = new Object();

  protected void waitForFailoverCompletion()
  {
    synchronized (this.failoverCompletedLock) {
      if (this.failoverCompleted) {
        return;
      }
      this.getLogger().info(LocalizedStrings.GatewayImpl_0__WAITING_FOR_FAILOVER_COMPLETION, this);
      try {
        while (!this.failoverCompleted) {
          this.failoverCompletedLock.wait();
        }
      }
      catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
        this._cache.getCancelCriterion().checkCancelInProgress(ex);
        this.getLogger().info(LocalizedStrings.GatewayImpl_0_DID_NOT_WAIT_FOR_FAILOVER_COMPLETION_DUE_TO_INTERRUPTION, this);
      }
    }
  }

  protected void completeFailover()
  {
    synchronized (this.failoverCompletedLock) {
      setFailoverComplete(true);
      this.failoverCompletedLock.notifyAll();
    }
  }

  private void setFailoverComplete(boolean failoverCompleted) {
    this.failoverCompleted = failoverCompleted;
  }

  /**
   * Become the primary Gateway. This method is invoked by the
   * GatewayHub when it becomes primary.
   *
   * @see GatewayHubImpl#launchLockObtainingThread
   */
  @Override
  protected void becomePrimary()
  {
    this.getLogger().info(LocalizedStrings.GatewayImpl_0__BECOMING_PRIMARY_GATEWAY, this);
    synchronized (this._primaryLock) {
      setPrimary(true);
      this._primaryLock.notify();
    }
  }

  /**
   * Class EndpointImpl represents a Gateway's
   * endpoint
   *
   * @author Barry Oglesby
   *
   * @since 4.2
   */
  static protected class EndpointImpl implements Endpoint
  {

    /**
     * The id of the Endpoint
     */
    protected String _myid;

    /**
     * The host of the Endpoint
     */
    protected String _host;

    /**
     * The port of the Endpoint
     */
    protected int _port;

    /**
     * Constructor.
     *
     * @param id
     *          The id of the Endpoint
     * @param host
     *          The host of the Endpoint
     * @param port
     *          The port of the Endpoint
     */
    protected EndpointImpl(String id, String host, int port) {
      this._myid = id;
      this._host = host;
      this._port = port;
    }

    public String getId()
    {
      return this._myid;
    }

    public String getHost()
    {
      return this._host;
    }

    public int getPort()
    {
      return this._port;
    }

    @Override
    public String toString()
    {
      StringBuffer buffer = new StringBuffer();
      buffer.append(this._myid).append("=").append(this._host).append(":")
          .append(this._port);
      return buffer.toString();
    }
  }

  /**
   * Class GatewayEventProcessor is this Gateway's
   * event processor thread. It processor events to its corresponding
   * Gateway
   *
   * @author Barry Oglesby
   *
   * @since 4.2
   */
  @SuppressWarnings("synthetic-access")
  protected class GatewayEventProcessor extends Thread  {

    /**
     * The GatewayImpl for which this
     * GatewayEventDispatcher processes messages.
     */
    protected final GatewayImpl _gateway;

    /**
     * The RegionQueue used to queue messages by this
     * GatewayEventDispatcher.
     */
    protected final RegionQueue _eventQueue;

    /**
     * The GatewayEventDispatcher used by this
     * GatewayEventProcessor to dispatch events.
     */
    protected final GatewayEventDispatcher _eventDispatcher;

    /**
     * The conflator faciliates message conflation
     */
    //protected BridgeEventConflator _eventConflator;
    
    private final Object unprocessedEventsLock = new Object();
    /**
     * A Map of events that have not been processed by the
     * primary yet. This map is created and used by a secondary
     * Gateway to keep track of events that have been received by
     * the secondary but not yet processed by the primary. Once an event has
     * been processed by the primary, it is removed from this map. This map will
     * only be used in the event that this Gateway becomes the
     * primary. Any events contained in this map will need to be sent to other
     * Gateways.
     * Note: unprocessedEventsLock MUST be synchronized before using this map.
     */
    private Map unprocessedEvents;
    /**
     * A Map of tokens (i.e. longs) of entries that we have
     * heard of from the primary but not yet the secondary.
     * This map is created and used by a secondary
     * Gateway to keep track.
     * Note: unprocessedEventsLock MUST be synchronized before using this map.
     *       This is not a cut and paste error. sync unprocessedEventsLock
     *       when using unprocessedTokens.
     */
    private Map unprocessedTokens;

    /**
     * A boolean verifying whether a warning has already been issued if the
     * event queue has reached a certain threshold.
     */
    protected boolean _eventQueueSizeWarning = false;

    /**
     * An int id used to identify each batch.
     */
    protected int _batchId = 0;

    /**
     * A boolean verifying whether this GatewayEventDispatcher is
     * running.
     */
    protected volatile boolean _isStopped = false;

    /**
     * A boolean verifying whether this GatewayEventDispatcher is
     * paused.
     */
    protected volatile boolean _isPaused = false;

    /**
     * A lock object used to control pausing this dispatcher
     */
    protected final Object _pausedLock = new Object();

    /**
     * Constructor.
     *
     * @param gateway
     *          The Gateway on behalf of whom this dispatcher
     *          dispatches events.
     */
    protected GatewayEventProcessor(GatewayImpl gateway) {
      super(LogWriterImpl.createThreadGroup("Gateway Event Processor from "
          + gateway.getGatewayHubId() + " to " + gateway.getId(), gateway
          .getLogger()), "Gateway Event Processor from "
          + gateway.getGatewayHubId() + " to " + gateway.getId());

      this._gateway = gateway;
      this.unprocessedEvents = new LinkedHashMap();
      this.unprocessedTokens = new LinkedHashMap();

      // Create the event conflator
      //this._eventConflator = new BridgeEventConflator(this._logger);

      this._eventQueue = this.initializeMessageQueue();

      this._eventDispatcher = this.initializeEventDispatcher();
      setDaemon(true);
    }

    /**
     * Stops the dispatcher from dispatching events . The dispatcher will stay
     * alive until its queue is empty or a number of peeks is reached.
     *
     * @see GatewayImpl#MAXIMUM_SHUTDOWN_PEEKS
     */
    protected void stopProcessing()
    {
      if (this.isAlive()) {


      if (getLogger().fineEnabled()) {
        getLogger().fine(this + ":Notifying the dispatcher to terminate");
      }

      // If this is the primary, stay alive until the queue is empty
      // or a number of peeks is reached.
      if (getGateway().getPrimary()) {
        int numberOfPeeks = 0;
        try {
          try {
            while (this._eventQueue.peek() != null
                && numberOfPeeks != MAXIMUM_SHUTDOWN_PEEKS) {
              numberOfPeeks++;
              try {
                Thread.sleep(100);
              }
              catch (InterruptedException e) {/* ignore */
                // TODO this looks like a bug
              }
            }
          }
          catch (InterruptedException e) {
            /*
             * ignore, this will never be thrown by
             * SingleWriteSingleReadRegionQueue
             */
            // TODO if this won't be thrown, assert something here.
          }
        }
        catch (CacheException e) {/* ignore */
        }
      }
      setIsStopped(true);
      
      if (this.isAlive()) {
        this.interrupt();
        if (getLogger().fineEnabled()) {
          getLogger().fine(this
              + ":Joining with the dispatcher thread upto limit of 5 seconds");
        }
        try {
          this.join(5000); // wait for our thread to stop
          if (this.isAlive()) {
            getLogger().warning(
                LocalizedStrings.GatewayImpl_0_DISPATCHER_STILL_ALIVE_EVEN_AFTER_JOIN_OF_5_SECONDS,
                this);
            // if the server machine crashed or there was a nic failure, we need
            // to terminate the socket connection now to avoid a hang when closing
            // the connections later
            if (_eventDispatcher instanceof GatewayEventRemoteDispatcher) {
              GatewayEventRemoteDispatcher g = (GatewayEventRemoteDispatcher)_eventDispatcher;
              g.destroyConnection();
            }
          }
        }
        catch (InterruptedException ex) {
          Thread.currentThread().interrupt();
          if (getLogger().warningEnabled()) {
            getLogger().warning(
              LocalizedStrings.GatewayImpl_0_INTERRUPTEDEXCEPTION_IN_JOINING_WITH_DISPATCHER_THREAD,
              this);
          }
        }

      }
      }
      
      // Close the processor's queue. This preserves the disk files if persistent.
      closeProcessor();
      
      if (getLogger().fineEnabled()) {
        getLogger().fine("Stopped dispatching: " + this);
      }
    }

    protected void pauseDispatching()
    {
      if (this._isPaused) {
        return;
      }
      this._isPaused = true;
    }

    protected void resumeDispatching()
    {
      if (!this._isPaused) {
        return;
      }
      this._isPaused = false;

      // Notify thread to resume
      if (getLogger().fineEnabled()) {
        getLogger().fine(this + ": Resuming processing");
      }
      synchronized (this._pausedLock) {
        this._pausedLock.notifyAll();
      }
    }

    protected void closeProcessor()
    {
      if (getLogger().fineEnabled()) {
        getLogger().fine("Closing dispatcher");
      }
      try {
        try {
          if (this._eventQueue.peek() != null) {
            getLogger()
                .warning(LocalizedStrings.GatewayImpl_DESTROYING_GATEWAYEVENTDISPATCHER_WITH_ACTIVELY_QUEUED_DATA);
          }
        }
        catch (InterruptedException e) {
          /*
           * ignore, this will never be thrown by
           * SingleWriteSingleReadRegionQueue
           */
          // TODO if this won't be thrown, assert it.
        }
      }
      catch (CacheException ignore) {
        // just checking in case we should log a warning
      }
      finally {
        this._eventQueue.close();
        if (getLogger().fineEnabled()) {
          getLogger().fine("Closed dispatcher");
        }
      }
    }

    private void destroyQueuePersistenceFiles() {
      //Destroy a files of a persistent region
      ((SingleWriteSingleReadRegionQueue)this._eventQueue).destroyPersistentFiles(getGateway().getCache());
    }
    
    /**
     * Returns whether this dispatcher is stopped (not dispatching any more
     * events)
     *
     * @return whether this dispatcher is stopped (not dispatching any more
     *         events)
     */
    protected boolean isStopped()
    {
      return this._isStopped;
    }

    protected void setIsStopped(boolean isStopped) {
      this._isStopped = isStopped;
    }

    protected boolean getIsStopped() {
      return this._isStopped;
    }

    protected boolean getIsPaused() {
      return this._isPaused;
    }

    /**
     * Increment the batch id. This method is not synchronized because this
     * dispatcher is the caller
     */
    protected void incrementBatchId()
    {
      // If _batchId + 1 == maximum, then roll over
      if (this._batchId + 1 == Integer.MAX_VALUE) {
        this._batchId = -1;
      }
      this._batchId++;
    }

    /**
     * Reset the batch id. This method is not synchronized because this
     * dispatcher is the caller
     */
    protected void resetBatchId()
    {
      this._batchId = 0;
    }

    /**
     * Returns the current batch id to be used to identify the next batch.
     *
     * @return the current batch id to be used to identify the next batch
     */
    protected int getBatchId()
    {
      return this._batchId;
    }

    protected void eventQueueRemove(int numberOfEventsToRemove) throws CacheException {
      this._eventQueue.remove(numberOfEventsToRemove);
    }

    protected Object eventQueueTake() throws CacheException, InterruptedException {
      return this._eventQueue.take();
    }

    protected int eventQueueSize() {
      return this._eventQueue.size();
    }

    protected GatewayImpl getGateway() {
      return this._gateway;
    }

    /**
     * Returns this processor's LogWriterImpl.
     * @return this processor's LogWriterImpl
     */
    public LogWriterI18n getLogger() {
      return getGateway()._logger;
    }

    /**
     * Initializes this processor's message queue. This queue is a
     * RegionQueue
     *
     * @see RegionQueue
     */
    protected RegionQueue initializeMessageQueue()
    {
      RegionQueue queue = null;

      // Create the region name
      StringBuilder regionNameBuffer = new StringBuilder();
      regionNameBuffer
        .append(getGateway().getGatewayHubId())
        .append('_')
        .append(getGateway().getId())
        .append("_EVENT_QUEUE");
      String regionName = regionNameBuffer.toString();

      // Define the cache listener if this is a secondary
      CacheListener listener = null;
      if (!getGateway().getPrimary()) {
        listener = new SecondaryGatewayListener(this);
        initializeListenerExecutor();
      }

      // Create the region queue
      queue = new SingleWriteSingleReadRegionQueue(getGateway().getCache(), regionName, getGateway().getQueueAttributes(), listener, getGateway().getStatistics());
      // this is deadcode to fix bug 37575
//       if (!getGateway().getPrimary()) {
//         // We need to make sure and account for entries obtained from gii.
//         // This is currently done with a simple iteration which might also
//         // iterator over entries that invoked a listener but that is ok since
//         // the hub can not yet call distribute on us because our constructor
//         // has not yet returned
//         Region qRegion = queue.getRegion();
//         Collection regionValues = qRegion.values();
//         Iterator i = regionValues.iterator();
//         ArrayList al = new ArrayList(regionValues.size());
//         while (i.hasNext()) {
//           Object o = i.next();
//           // Need instanceof check because headkey/tailkey are in here too
//           if (o instanceof GatewayEventImpl) {
//             al.add(o);
//           }
//         }
//         if (al.size() > 0) {
//           handlePrimaryEvents(al);
//         }
//       }

      if (getLogger().fineEnabled()) {
        getLogger().fine("Created queue: " + queue);
      }
      return queue;
    }

    protected GatewayEventDispatcher initializeEventDispatcher() {
      GatewayEventDispatcher dispatcher = null;
      if (getGateway().hasEndpoints()) {
        dispatcher = new GatewayEventRemoteDispatcher(this);
      } else {
        dispatcher = new GatewayEventCallbackDispatcher(this);
      }
      return dispatcher;
    }

    private boolean stopped() {
      if (this._isStopped) {
        return true;
      }
      if (GatewayImpl.this.stopper.cancelInProgress() != null) {
        return true;
      }
      return false;
    }

    /**
     * @return false on failure
     */
    private boolean waitForPrimary() {

      try {
        getGateway().waitToBecomePrimary();
      }
      catch (InterruptedException e) {
        // No need to set the interrupt bit, we're exiting the thread.
        if (!stopped()) {
          getLogger().severe(
              LocalizedStrings.GatewayImpl_AN_INTERRUPTEDEXCEPTION_OCCURRED_THE_THREAD_WILL_EXIT,
              e);
        }
        shutdownListenerExecutor();
        return false;
      }
      try {
        shutdownListenerExecutor();

        // Don't allow conserve-sockets = false so that ordering is preserved.
        DistributedSystem.setThreadsSocketPolicy(true);

        // Once notification has occurred, handle failover
        if (!stopped()) {
          handleFailover();
        }
      }
      catch (RegionDestroyedException e) {
        // This happens during handleFailover
        // because the region on _eventQueue can be closed.
        if (!stopped()) {
          getLogger().fine("Terminating due to " + e);
        }
        return false;
      }
      catch (CancelException e) {
        if (!stopped()) {
          getLogger().fine("Terminating due to " + e);
        }
        return false;
      }
      finally {
        // This is a good thing even on termination, because it ends waits
        // on the part of other threads.
        completeFailover();
      }
      return true;
    }

    private void processQueue() {
      GatewayQueueAttributes gqa = getGateway().getQueueAttributes();
      final int batchSize = gqa.getBatchSize();
      final int batchTimeInterval = gqa.getBatchTimeInterval();
      final int alertThreshold = gqa.getAlertThreshold();
      final GatewayStats statistics = getGateway().getStatistics();
      List events = null;
      for (;;) {
        if (stopped()) {
          break;
        }

        try {
          // Check if paused. If so, wait for resumption
          if (this._isPaused) {
            waitForResumption();
          }

          // Peek a batch
          if (getLogger().fineEnabled()) {
            getLogger().fine("Attempting to peek a batch of " + batchSize
                + " events");
          }
          for (;;) {
            // check before sleeping
            if (stopped()) {
              break;
            }

            // Check if paused. If so, wait for resumption
            if (this._isPaused) {
              waitForResumption();
            }

            // sleep a little bit, look for events
            boolean interrupted = Thread.interrupted();
            try {
              events = this._eventQueue.peek(batchSize, batchTimeInterval);
            }
            catch (InterruptedException e) {
              interrupted = true;
              GatewayImpl.this.getCancelCriterion().checkCancelInProgress(e);
              continue; // keep trying
            }
            finally {
              if (interrupted) {
                Thread.currentThread().interrupt();
              }
            }

            if (events.isEmpty()) {
              continue; // nothing to do!
            }
            // before dispatching the events, initialize the keys if required
            Iterator it = events.iterator();
            while (it.hasNext()) {
              Object o = it.next();
              if (o != null && o instanceof GatewayEventImpl) {
                ((GatewayEventImpl)o).initializeKey();
              }
            }
            logBatchFine("During normal processing, dispatching the following ",
                events);
            boolean success = this._eventDispatcher.dispatchBatch(events, true);
            if (getLogger().fineEnabled()) {
              getLogger().fine("During normal processing, "
                  + (success ? "" : "un") + "successfully dispatched " +
                      events.size() + " events (batch #" + getBatchId() + ")" + " queue size=" + eventQueueSize());
            }

            // check again, don't do post-processing if we're stopped.
            if (stopped()) {
              break;
            }

            // If the batch is successfully processed, remove it from the queue.
            if (success) {
              // TODO - what to do if an exception occurs during remove()?
              eventQueueRemove(events.size());

              /*
               * FAILOVER TESTING CODE System.out.println(getName() + ": Removed " +
               * events.size() + " events from queue");
               */
              int queueSize = eventQueueSize();
              statistics.setQueueSize(queueSize);

              // Log an alert for each event if necessary
              if (alertThreshold > GatewayQueueAttributes.DEFAULT_ALERT_THRESHOLD) {
                it = events.iterator();
                long currentTime = System.currentTimeMillis();
                while (it.hasNext()) {
                  Object o = it.next();
                  if (o != null && o instanceof GatewayEventImpl) {
                    GatewayEventImpl ge = (GatewayEventImpl) o;
                    if (ge.getCreationTime()+alertThreshold < currentTime) {
                      getLogger().warning(
                        LocalizedStrings.GatewayImpl_EVENT_QUEUE_ALERT_OPERATION_0_REGION_1_KEY_2_VALUE_3_TIME_4,
                        new Object[] {
                          ge.getOperation(), ge.getRegionName(),
                          ge.getKey(), ge.getDeserializedValueForLogging(),
                          ge.getCallbackArgument(),
                          currentTime-ge.getCreationTime()});
                      statistics.incEventsExceedingAlertThreshold();
                    }
                  }
                }
              }

              if (this._eventQueueSizeWarning
                  && queueSize <= QUEUE_SIZE_THRESHOLD) {
               getLogger().info(
                 LocalizedStrings.GatewayImpl_THE_EVENT_QUEUE_SIZE_HAS_DROPPED_BELOW_THE_THRESHOLD_0,
                 QUEUE_SIZE_THRESHOLD);
                this._eventQueueSizeWarning = false;
              }
              incrementBatchId();
            } // successful batch
            else { // The batch was unsuccessful.
              statistics.incBatchesRedistributed();

              // Set posDup flag on each event in the batch
              it = events.iterator();
              while (it.hasNext() && !this._isStopped) {
                Object o = it.next();
                if (o != null && o instanceof GatewayEventImpl) {
                  GatewayEventImpl ge = (GatewayEventImpl) o;
                  ge.setPossibleDuplicate(true);
                }
              }
            } // unsuccessful batch
            if (getLogger().fineEnabled()) {
              getLogger().fine("Finished processing events (batch #"
                  + getBatchId() + ")");
            }
          } // for
        }
        catch (RegionDestroyedException e) {
          if (stopped()) {
            return; // don't print message
          }
          getLogger().info(LocalizedStrings.
              GatewayImpl_TERMINATED_DUE_TO_REGIONDESTROYEDEXCEPTION);
          setIsStopped(true);
        }
        catch (CancelException e) {
//          getLogger().info(LocalizedStrings.DEBUG, "caught cancel exception", e);
          setIsStopped(true);
        }
        catch (Throwable e) {
          Error err;
          if (e instanceof Error && SystemFailure.isJVMFailureError(
              err = (Error)e)) {
            SystemFailure.initiateFailure(err);
            // If this ever returns, rethrow the error. We're poisoned
            // now, so don't let this thread continue.
            throw err;
          }
          // Whenever you catch Error or Throwable, you must also
          // check for fatal JVM error (see above).  However, there is
          // _still_ a possibility that you are dealing with a cascading
          // error condition, so you also need to check to see if the JVM
          // is still usable:
          SystemFailure.checkFailure();

          // Well, OK.  Some strange nonfatal thing.
          if (stopped()) {
            return; // don't complain, just exit.
          }

          // We'll log it but continue on with the next batch.
          getLogger().severe(
              LocalizedStrings.GatewayImpl_AN_EXCEPTION_OCCURRED_THE_DISPATCHER_WILL_CONTINUE, e);
        }
      } // for
    }

    /**
     * Runs the dispatcher by taking a batch of messages from the queue and
     * sending them to the corresponding Gateway
     */

    @Override
    public void run()
    {
      try {
        // If this is not a primary gateway, wait for notification
        if (!getGateway().getPrimary()) {
          if (!waitForPrimary()) {
            return;
          }
        }
        else {
          // we are the primary so mark failover as being completed
          completeFailover();
        }

        // Begin to process the message queue after becoming primary
        if (getLogger().fineEnabled()) {
          getLogger().fine("Beginning to process the message queue");
        }

        /*
         * FAILOVER TESTING CODE this._logger.warning("Beginning to process the
         * message queue");
         */

        if (!getGateway().getPrimary()) {
          getLogger().warning(LocalizedStrings.
              GatewayImpl_ABOUT_TO_PROCESS_THE_MESSAGE_QUEUE_BUT_NOT_THE_PRIMARY);
        }

        // Sleep for a bit. The random is so that if several of these are
        // started at once, they will stagger a bit.
        try {
          Thread.sleep(new Random().nextInt(1000));
        }
        catch (InterruptedException e) {
          // no need to set the interrupt bit or throw an exception; just exit.
          return;
        }

        processQueue();
      }
      catch (CancelException e) {
        if (!this._isStopped) {
          getLogger().info( LocalizedStrings.
            GatewayImpl_A_CANCELLATION_OCCURRED_STOPPING_THE_DISPATCHER);
          setIsStopped(true);
        }
      }
      catch (Throwable e) {
        Error err;
        if (e instanceof Error && SystemFailure.isJVMFailureError(
            err = (Error)e)) {
          SystemFailure.initiateFailure(err);
          // If this ever returns, rethrow the error. We're poisoned
          // now, so don't let this thread continue.
          throw err;
        }
        // Whenever you catch Error or Throwable, you must also
        // check for fatal JVM error (see above).  However, there is
        // _still_ a possibility that you are dealing with a cascading
        // error condition, so you also need to check to see if the JVM
        // is still usable:
        SystemFailure.checkFailure();
        getLogger().severe( LocalizedStrings.
            GatewayImpl_MESSAGE_DISPATCH_FAILED_DUE_TO_UNEXPECTED_EXCEPTION, e);
      }

      //TODO:ASIF:CHECK IF CODE IS NEEDED FROM CS43_ha BRANCH ,IF WE NEED TO
      // TAKE THE
      //CODE OF TAKING UP RESIDUAL DATA
    }

    // dispatchBatch moved to GatewayEventRemoteDispatcher

    // dispatchBatch(List events) moved to GatewayEventRemoteDispatcher

    /**
     * Handle failover. This method is called when a secondary
     * Gateway becomes a primary Gateway.
     *
     * Once this secondary becomes the primary, it must:
     * 
     * Remove the queue's CacheListener
     * 
Process the map of unprocessed events (those it has seen but the
     * previous primary had not yet processed before it crashed). These will
     * include both queued and unqueued events. Remove from the queue any events
     * that were already sent
     * 
Clear the unprocessed events map
     * 
     */
    protected void handleFailover()
    {
      /* We must hold this lock while we're processing these
       * maps to prevent us from handling a secondary event while failover
       * occurs. See enqueueEvent
       */
      synchronized (this.unprocessedEventsLock) {
        // Remove the queue's CacheListener
        this._eventQueue.removeCacheListener();
        this.unprocessedTokens = null;

        // Process the map of unprocessed events
        getLogger().info(LocalizedStrings.GatewayImpl_GATEWAY_FAILOVER_INITIATED_PROCESSING_0_UNPROCESSED_EVENTS, this.unprocessedEvents.size());
        GatewayStats statistics = getGateway().getStatistics();
        statistics.setQueueSize(eventQueueSize()); // to capture an initial size
        if (!this.unprocessedEvents.isEmpty()) {
          // do a reap for bug 37603
          reapOld(statistics, true); // to get rid of timed out events
          // now iterate over the region queue to figure out what unprocessed
          // events are already in the queue
          {
            Iterator it = this._eventQueue.getRegion().values().iterator();
            while (it.hasNext() && !stopped()) {
              Object o = it.next();
              if (o != null && o instanceof GatewayEventImpl) {
                GatewayEventImpl ge = (GatewayEventImpl)o;
                if (this.unprocessedEvents.remove(ge.getEventId()) != null) {
                  if (this.unprocessedEvents.isEmpty()) {
                    break;
                  }
                }
              }
            }
          }
          // now for every unprocessed event add it to the end of the queue
          {
            getLogger().info(LocalizedStrings.GatewayImpl_GATEWAY_FAILOVER_INITIATED_ADDING_0_UNPROCESSED_EVENTS, this.unprocessedEvents.size());
            Iterator it = this.unprocessedEvents.values().iterator();
            while (it.hasNext() && !stopped()) {
              EventWrapper ew = (EventWrapper)it.next();
              GatewayEventImpl gatewayEvent = ew.event;
              try {
                // Initialize each gateway event. This initializes the key,
                // value
                // and callback arg based on the EntryEvent.
                gatewayEvent.initialize();
              }
              catch (IOException e) {
                getLogger().warning(
                  LocalizedStrings.GatewayImpl_EVENT_FAILED_TO_BE_INITIALIZED_0,
                  gatewayEvent, e);
              }
              // Verify that they GatewayEventCallbackArgument is initialized.
              // If not,
              // initialize it. It won't be initialized if a client to this
              // GatewayHub
              // VM was the creator of this event. This Gateway will be the
              // first one
              // to process it. If will be initialized if this event was sent to
              // this
              // Gateway from another GatewayHub (eityher directly or
              // indirectly).
              GatewayEventCallbackArgument geca = gatewayEvent
                .getGatewayCallbackArgument();
              if (geca.getOriginatingGatewayHubId() == null) {
                geca
                    .setOriginatingGatewayHubId(getGateway().getGatewayHubId());
                geca.initializeRecipientGateways(getGateway().getGatewayHub()
                    .getGatewayIds());
              }
              try {
                if (getLogger().fineEnabled()) {
                  getLogger().fine(getGateway() + ": Queueing unprocessed event: " + gatewayEvent);
                }
                queuePrimaryEvent(gatewayEvent);
              }
              catch (IOException ex) {
                if (!stopped()) {
                  getLogger().warning(
                    LocalizedStrings.GatewayImpl_EVENT_DROPPED_DURING_FAILOVER_0,
                    gatewayEvent, ex);
                }
              }
              catch (CacheException ex) {
                if (!stopped()) {
                  getLogger().warning(
                    LocalizedStrings.GatewayImpl_EVENT_DROPPED_DURING_FAILOVER_0,
                    gatewayEvent, ex);
                }
              }
            }
          }
          // Clear the unprocessed events map
          statistics.clearUnprocessedMaps();
        }

        // Iterate the entire queue and mark all events as possible
        // duplicate
        if (getLogger().infoEnabled()) {
          getLogger().info(LocalizedStrings.GatewayImpl_0__MARKING__1__EVENTS_AS_POSSIBLE_DUPLICATES, new Object[] {getGateway(), Integer.valueOf(this._eventQueue.size())});
        }
        Iterator it = this._eventQueue.getRegion().values().iterator();
        while (it.hasNext() && !stopped()) {
          Object o = it.next();
          if (o != null && o instanceof GatewayEventImpl) {
            GatewayEventImpl ge = (GatewayEventImpl)o;
            ge.setPossibleDuplicate(true);
          }
        }

        this.unprocessedEvents = null;

        getLogger().info(LocalizedStrings.GatewayImpl_GATEWAY_FAILOVER_COMPLETED);
      } // synchronized
    }

    /**
     * Add the input object to the event queue
     */
    protected void enqueueEvent(EnumListenerEvent operation, EntryEvent event)
        throws IOException, CacheException
    {
      // There is a case where the event is serialized for processing. The
      // region is not
      // serialized along with the event since it is a transient field. I
      // created an
      // intermediate object (GatewayEventImpl) to avoid this since the region
      // name is
      // used in the sendBatch method, and it can't be null. See EntryEventImpl
      // for details.
      GatewayEventImpl gatewayEvent = null;

      boolean isPrimary = getGateway().getPrimary();
      if(!isPrimary) {
        //Fix for #40615. We need to check if we've now become the primary
        //while holding the unprocessedEventsLock. This prevents us from failing
        //over while we're processing an event as a secondaryEvent.
        synchronized (unprocessedEventsLock) {
          // Test whether this gateway is the primary.
          if (getGateway().getPrimary()) {
            isPrimary = true;
          }
          else {
            // If it is not, create an uninitialized GatewayEventImpl and
            // put it into the map of unprocessed events.
            gatewayEvent = new GatewayEventImpl(operation, event, false);
            handleSecondaryEvent(gatewayEvent);
          }
        }
      }
      if(isPrimary) {
        waitForFailoverCompletion();
        // If it is, create and enqueue an initialized GatewayEventImpl
        gatewayEvent = new GatewayEventImpl(operation, event);
        queuePrimaryEvent(gatewayEvent);
      }
    }

    private void queuePrimaryEvent(GatewayEventImpl gatewayEvent)
        throws IOException, CacheException
    {
      // Queue the event
      GatewayStats statistics = getGateway().getStatistics();
      if (getLogger().fineEnabled()) {
        getLogger().fine(getGateway() + ": Queueing event ("
            + (statistics.getEventsQueued() + 1) + "): " + gatewayEvent);
      }
      long start = statistics.startTime();
      try {
        if(shouldEnqueue(gatewayEvent)) {
          this._eventQueue.put(gatewayEvent);
        }
      }
      catch (InterruptedException e) {
        //Asif Not expected from SingleWriteSingleReadRegionQueue as it does not throw
        //InterruptedException. But since both HARegionQueue and SingleReadSingleWriteRegionQueue
        //extend RegionQueue , it has to handle InterruptedException
        Thread.currentThread().interrupt();
        GatewayImpl.this.getCancelCriterion().checkCancelInProgress(e);
      }

      statistics.endPut(start);
      if (getLogger().fineEnabled()) {
        getLogger().fine(getGateway() + ": Queued event ("
            + (statistics.getEventsQueued()) + "): " + gatewayEvent);
      }
      //this._logger.warning(getGateway() + ": Queued event (" +
      // (statistics.getEventsQueued()) + "): " + gatewayEvent + " queue size: "
      // + this._eventQueue.size());
      /*
       * FAILOVER TESTING CODE System.out.println(getName() + ": Queued event (" +
       * (statistics.getEventsQueued()) + "): " + gatewayEvent.getId());
       */
      int queueSize = eventQueueSize();
      statistics.setQueueSize(queueSize);
      if (!this._eventQueueSizeWarning && queueSize >= QUEUE_SIZE_THRESHOLD) {
        getLogger().warning(
          LocalizedStrings.GatewayImpl_0_THE_EVENT_QUEUE_SIZE_HAS_REACHED_THE_THRESHOLD_1,
          new Object[] {getGateway(), Integer.valueOf(QUEUE_SIZE_THRESHOLD)});
        this._eventQueueSizeWarning = true;
      }
    }

    /**
     * Test to see if we should enqueue a gateway event.
     */
    private boolean shouldEnqueue(GatewayEventImpl gatewayEvent) {
      LocalRegion region = gatewayEvent.getRegion();
      //GatewayEventListeners should not see events on Meta Regions
      //This fixes 43978
      return hasEndpoints() || !region.isUsedForMetaRegion();
    }

    /**
     * Update an unprocessed event in the unprocessed events map. This method is
     * called by a secondary Gateway to store a gateway event
     * until it is processed by a primary Gateway. The
     * complexity of this method is the fact that the event could be processed
     * first by either the primary or secondary Gateway.
     *
     * If the primary processes the event first, the map will already contain an
     * entry for the event (through
     * {@link com.gemstone.gemfire.internal.cache.GatewayImpl.SecondaryGatewayListener#afterDestroy}).
     * When the secondary processes the event, it will remove it from the map.
     *
     * If the secondary processes the event first, it will add it to the map.
     * When the primary processes the event (through
     * {@link com.gemstone.gemfire.internal.cache.GatewayImpl.SecondaryGatewayListener#afterDestroy}),
     * it will then be removed from the map.
     *
     * @param gatewayEvent
     *          The event being processed
     */
    protected void handleSecondaryEvent(GatewayEventImpl gatewayEvent)
    {
      basicHandleSecondaryEvent(gatewayEvent);
    }

    /**
     * Update an unprocessed event in the unprocessed events map. This method is
     * called by a primary Gateway (through
     * {@link com.gemstone.gemfire.internal.cache.GatewayImpl.SecondaryGatewayListener#afterCreate})
     * to notify the secondary Gateway that an event has been
     * added to the queue. 
     * Once an event has been added to the queue, the
     * secondary no longer needs to keep track of it in the unprocessed events
     * map. The complexity of this method is the fact that the event could be
     * processed first by either the primary or secondary Gateway.
     *
     * If the primary processes the event first, the map will not contain an
     * entry for the event. It will be added to the map in this case so that
     * when the secondary processes it, it will know that the primary has
     * already processed it, and it can be safely removed.
     *
     * If the secondary processes the event first, the map will already contain
     * an entry for the event.
     * In this case, the event can be removed from the map.
     *
     * @param gatewayEvent
     *          The event being processed
     */
    protected void handlePrimaryEvent(final GatewayEventImpl gatewayEvent)
    {
      Executor my_executor = this._executor;
      if (my_executor == null) {
        // should mean we are now primary
        return;
      }
      try {
        my_executor.execute(new Runnable() {
            public void run() {
              basicHandlePrimaryEvent(gatewayEvent);
            }
          });
      } catch (RejectedExecutionException ex) {
        throw ex;
      }
    }

    /**
     * Called when the primary gets rid of an event from the queue
     * This method added to fix bug 37603
     */
    protected void handlePrimaryDestroy(final GatewayEventImpl gatewayEvent)
    {
      Executor my_executor = this._executor;
      if (my_executor == null) {
        // should mean we are now primary
        return;
      }
      try {
        my_executor.execute(new Runnable() {
            public void run() {
              basicHandlePrimaryDestroy(gatewayEvent);
            }
          });
      } catch (RejectedExecutionException ex) {
        throw ex;
      }
    }

//     protected void handlePrimaryEvents(final ArrayList eventList)
//     {
//       Executor my_executor = this._executor;
//       if (my_executor == null) {
//         // should mean we are now primary
//         return;
//       }
//       try {
//         my_executor.execute(new Runnable() {
//           public void run()
//           {
//             final int eventCount = eventList.size();
//             final Long to = Long.valueOf(System.currentTimeMillis() + TOKEN_TIMEOUT);
//             final Map my_unprocessedEvents = unprocessedEvents;
//             if (my_unprocessedEvents == null)
//               return;
//             synchronized (my_unprocessedEvents) {
//               if (_unprocessedEvents == null)
//                 return;
//               // now we can safely use the _unprocessedEvents field
//               for (int i = 0; i < eventCount; i++) {
//                 GatewayEventImpl gatewayEvent = (GatewayEventImpl)eventList
//                     .get(i);
//                 handleEvent(gatewayEvent, true, to);
//               }
//             }
//           }
//           });
//       } catch (RejectedExecutionException ex) {
//         throw ex;
//       }
//     }

    /**
     * Just remove the event from the unprocessed events map if it is present.
     * This method added to fix bug 37603
     */
    protected void basicHandlePrimaryDestroy(final GatewayEventImpl gatewayEvent) {
      if (getPrimary()) {
        // no need to do anything if we have become the primary
        return;
      }
      GatewayStats statistics = getGateway().getStatistics();
      // Get the event from the map
      synchronized (unprocessedEventsLock) {
        if (this.unprocessedEvents == null)
          return;
        // now we can safely use the unprocessedEvents field
        Object v = this.unprocessedEvents.remove(gatewayEvent.getEventId());
        if (v != null) {
          statistics.incUnprocessedEventsRemovedByPrimary();
        }
      }
    }

    protected void basicHandlePrimaryEvent(final GatewayEventImpl gatewayEvent) {
      if (getPrimary()) {
        // no need to do anything if we have become the primary
        return;
      }
      GatewayStats statistics = getGateway().getStatistics();
      // Get the event from the map
      synchronized (unprocessedEventsLock) {
        if (this.unprocessedEvents == null)
          return;
        // now we can safely use the unprocessedEvents field
        Object v = this.unprocessedEvents.remove(gatewayEvent.getEventId());

        if (v == null) {
          // first time for the event
          if (getLogger().finerEnabled()) {
            try {
              gatewayEvent.initialize();
            }
            catch (Exception e) {
            }
            if (getLogger().finerEnabled()) {
              getLogger().finer(GatewayImpl.this + ": fromPrimary "
                  + " event " + gatewayEvent.getEventId() + ":"
                  + gatewayEvent.getKey() + "->"
                  + deserialize(gatewayEvent.getValue())
                  + " added to unprocessed token map.");
            }
          }
          {
            Object mapValue = Long.valueOf(System.currentTimeMillis() + TOKEN_TIMEOUT);
            Object oldv = this.unprocessedTokens.put(gatewayEvent.getEventId(), mapValue);
            if (oldv == null) {
              statistics.incUnprocessedTokensAddedByPrimary();
            } else {
              // its ok for oldv to be non-null
              // this shouldn't happen anymore @todo add an assertion here
            }
          }
        }
        else {
          // already added by secondary (i.e. hub)
          // the secondary
          // gateway has already seen this event, and it can be safely
          // removed (it already was above)
          if (getLogger().finerEnabled()) {
            try {
              gatewayEvent.initialize();
            }
            catch (Exception e) {
            }
            getLogger().finer(GatewayImpl.this
                               + ": Primary create/update event "
                               + gatewayEvent.getEventId() + ":" + gatewayEvent.getKey()
                               + "->" + deserialize(gatewayEvent.getValue())
                               + " removed from unprocessed events map");
          }
          statistics.incUnprocessedEventsRemovedByPrimary();
        }
        reapOld(statistics, false);
      }
    }

    private void basicHandleSecondaryEvent(final GatewayEventImpl gatewayEvent) {
      GatewayStats statistics = getGateway().getStatistics();
      // Get the event from the map
      
        Assert.assertHoldsLock(unprocessedEventsLock, true);
        Assert.assertTrue(this.unprocessedEvents != null);
        // @todo add an assertion that !getPrimary()
        // now we can safely use the unprocessedEvents field
        Object v = this.unprocessedTokens.remove(gatewayEvent.getEventId());

        if (v == null) {
          // first time for the event
          if (getLogger().finerEnabled()) {
            try {
              gatewayEvent.initialize();
            }
            catch (Exception e) {
            }
            getLogger().finer(GatewayImpl.this + ": fromSecondary "
                + " event " + gatewayEvent.getEventId() + ":"
                + gatewayEvent.getKey() + "->"
                + deserialize(gatewayEvent.getValue())
                + " added to unprocessed events map.");
          }
          {
            Object mapValue = new EventWrapper(gatewayEvent);
            Object oldv = this.unprocessedEvents.put(gatewayEvent.getEventId(), mapValue);
            if (oldv == null) {
              statistics.incUnprocessedEventsAddedBySecondary();
            } else {
              // put old one back in
              this.unprocessedEvents.put(gatewayEvent.getEventId(), oldv);
              // already added by secondary (i.e. hub)
              if (getLogger().warningEnabled()) {
                try {
                  gatewayEvent.initialize();
                }
                catch (IOException e) {
                  getLogger().warning(
                    LocalizedStrings. GatewayImpl_EVENT_FAILED_TO_BE_INITIALIZED_0,
                    gatewayEvent, e);
                }
                getLogger().warning(
                  LocalizedStrings.GatewayImpl_0_THE_UNPROCESSED_EVENTS_MAP_ALREADY_CONTAINED_AN_EVENT_FROM_THE_HUB_1_SO_IGNORING_NEW_EVENT_2,
                  new Object[] {GatewayImpl.this, GatewayImpl.this.getGatewayHubId(), gatewayEvent});
              }
            }
          }
        }
        else {
          // token already added by primary already removed
          if (getLogger().finerEnabled()) {
            try {
              gatewayEvent.initialize();
            }
            catch (Exception e) {
            }
            getLogger().finer(GatewayImpl.this
                               + ": Secondary created event " + gatewayEvent.getEventId()
                               + ":" + gatewayEvent.getKey() + "->"
                               + deserialize(gatewayEvent.getValue())
                               + " removed from unprocessed token map");
          }
          statistics.incUnprocessedTokensRemovedBySecondary();
        }
        reapOld(statistics, false);
    }
    /**
     * When the Number of unchecked events exceeds this threshold and the number
     * of tokens in the map exceeds this threshold then a check will be done for
     * old tokens.
     */
    static private final int REAP_THRESHOLD = 1000;

    /*
     * How many events have happened without a reap check being done?
     */
    private int uncheckedCount = 0;

    /**
     * Call to check if a cleanup of tokens needs to be done
     */
    private void reapOld(final GatewayStats statistics, boolean forceEventReap)
    {
      synchronized (this.unprocessedEventsLock) {
        if (uncheckedCount > REAP_THRESHOLD) { // only check every X events
          uncheckedCount = 0;
          long now = System.currentTimeMillis();
          if (!forceEventReap && this.unprocessedTokens.size() > REAP_THRESHOLD) {
            Iterator it = this.unprocessedTokens.entrySet().iterator();
            int count = 0;
            while (it.hasNext()) {
              Map.Entry me = (Map.Entry)it.next();
              long meValue = ((Long)me.getValue()).longValue();
              if (meValue <= now) {
                // @todo log fine level message here
                // it has expired so remove it
                it.remove();
                count++;
              }
              else {
                // all done try again
                break;
              }
            }
            if (count > 0) {
              statistics.incUnprocessedTokensRemovedByTimeout(count);
            }
          }
          if (forceEventReap || this.unprocessedEvents.size() > REAP_THRESHOLD)  {
            Iterator it = this.unprocessedEvents.entrySet().iterator();
            int count = 0;
            while (it.hasNext()) {
              Map.Entry me = (Map.Entry)it.next();
              EventWrapper ew = (EventWrapper)me.getValue();
              if (ew.timeout <= now) {
                // @todo log fine level message here
                // it has expired so remove it
                it.remove();
                count++;
              }
              else {
                // all done try again
                break;
              }
            }
            if (count > 0) {
              statistics.incUnprocessedEventsRemovedByTimeout(count);
            }
          }
        }
        else {
          uncheckedCount++;
        }
      }
    }

    protected void waitForResumption() throws InterruptedException {
      synchronized (this._pausedLock) {
        if (!this._isPaused) {
          return;
        }
        if (getLogger().fineEnabled()) {
          getLogger().fine(this + ": Pausing processing");
        }
        while (this._isPaused) {
          this._pausedLock.wait();
        }
      }
    }

    /**
     * Logs a batch of events to the LogWriterI18n.
     *
     * @param events
     *          The batch of events to log
     **/
    protected void logBatchFine(String message, List events)
    {
      if (getLogger().fineEnabled()) {
        StringBuilder buffer = new StringBuilder();
        buffer.append(message);
        buffer.append(events.size()).append(" events");
        buffer.append(" (batch #" + getBatchId());
        buffer.append("):\n");
        for (Iterator i = events.iterator(); i.hasNext();) {
          GatewayEventImpl ge = (GatewayEventImpl)i.next();
          buffer.append("\tEvent ").append(ge.getEventId()).append(":");
          buffer.append(ge.getKey()).append("->");
          buffer.append(ge.getDeserializedValueForLogging());
          buffer.append("\n");
        }
        getLogger().fine(buffer.toString());
      }
    }

    protected Object deserialize(byte[] serializedBytes)
    {
      Object deserializedObject = serializedBytes;
      // This is a debugging method so ignore all exceptions like
      // ClassNotFoundException
      try {
        deserializedObject = EntryEventImpl.deserialize(serializedBytes);
      }
      catch (Exception e) {
      }
      return deserializedObject;
    }

    @Override
    public String toString()
    {
      StringBuffer buffer = new StringBuffer();
      GatewayQueueAttributes gqa = getGateway().getQueueAttributesNoSync();
      buffer
        .append("GatewayEventProcessor[")
        .append("gatewayId=")
        .append(getGateway().getId())
        .append(";gatewayHubId=")
        .append(getGateway().getGatewayHubId());
      if (gqa.getDiskStoreName()!=null) {
        buffer.append(";diskStoreName=")
        .append(gqa.getDiskStoreName());
      } else {
        buffer.append(";overflowDirectory=")
        .append(gqa.getOverflowDirectory());
      }
      buffer
        .append(";batchSize=")
        .append(gqa.getBatchSize())
        .append(";batchTimeInterval=")
        .append(gqa.getBatchTimeInterval())
        .append(";batchConflation=")
        .append(gqa.getBatchConflation())
        .append(";enablePersistence=")
        .append(gqa.getEnablePersistence())
        .append("]");
      return buffer.toString();
    }

    private ExecutorService _executor;

    /**
     * Initialize the Executor that handles listener events. Only used by
     * non-primary gateways
     */
    private void initializeListenerExecutor()
    {
      // Create the ThreadGroups
      final ThreadGroup loggerGroup = LogWriterImpl.createThreadGroup(
          "Gateway Listener Group", getLogger());

      // Create the Executor
      ThreadFactory tf = new ThreadFactory() {
          public Thread newThread(Runnable command) {
            Thread thread =  new Thread(loggerGroup, command, "Queued Gateway Listener Thread");
            thread.setDaemon(true);
            return thread;
          }
        };
      LinkedBlockingQueue q = new LinkedBlockingQueue();
      this._executor = new ThreadPoolExecutor(1, 1/*max unused*/,
                                              120, TimeUnit.SECONDS, q, tf);
    }

    private void shutdownListenerExecutor()
    {
      if (this._executor != null) {
        this._executor.shutdown();
        this._executor = null;
      }
    }
  }

  /**
   * Class SecondaryGatewayListener is a
   * CacheListener that handles afterCreate events for the
   * RegionQueue. These afterCreate events are based on creates
   * from the primary Gateway queue. It also handles afterUpdate
   * which happens when the primary conflates a queue entry.
   *
   * @author Barry Oglesby
   *
   * @since 4.2
   */
  protected class SecondaryGatewayListener extends CacheListenerAdapter
   {

    /**
     * The GatewayEventProcessor used by this
     * CacheListener to process events.
     */
    private final GatewayEventProcessor _eventProcessor;

    protected SecondaryGatewayListener(GatewayEventProcessor eventProcessor) {
      this._eventProcessor = eventProcessor;
    }


    @Override
    public void afterCreate(EntryEvent event)
    {
      // fix bug 35730
      if (GatewayImpl.this.getPrimary()) {
        // The secondary has failed over to become the primary. There is a small
        // window where the secondary has become the primary, but the listener
        // is
        // still set. Ignore any updates to the map at this point. It is unknown
        // what the state of the map is. This may result in duplicate events
        // being sent.
        //GatewayImpl.this._logger.severe(GatewayImpl.this + ": XXXXXXXXX IS
        // PRIMARY BUT PROCESSING AFTER_DESTROY EVENT XXXXXXXXX: " + event);
        return;
      }

      // Send event to the event dispatcher
      GatewayEventImpl gatewayEvent = (GatewayEventImpl)event.getNewValue();
      this._eventProcessor.handlePrimaryEvent(gatewayEvent);
    }

    @Override
    public void afterDestroy(EntryEvent event) {
      // fix bug 37603
      if (GatewayImpl.this.getPrimary()) {
        return;
      }

      // Send event to the event dispatcher
      Object oldValue = event.getOldValue();
      if (oldValue instanceof GatewayEventImpl) {
        GatewayEventImpl gatewayEvent = (GatewayEventImpl)oldValue;
        this._eventProcessor.handlePrimaryDestroy(gatewayEvent);
      }
    }
  }
  /**
   * Has a reference to a GatewayEventImpl and has a timeout value.
   */
  private static class EventWrapper  {
    /**
     * Timeout events received from secondary after 5 minutes
     */
    static private final int EVENT_TIMEOUT
      = Integer.getInteger("Gateway.EVENT_TIMEOUT", 5 * 60 * 1000).intValue();
    public final long timeout;
    public final GatewayEventImpl event;
    public EventWrapper(GatewayEventImpl e) {
      this.event = e;
      this.timeout = System.currentTimeMillis() + EVENT_TIMEOUT;
    }
  }
}