All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher Maven / Gradle / Ivy

    There is a newer version: 9.1.7.Final
    Show newest version
    package org.infinispan.remoting.transport.jgroups;

import net.jcip.annotations.GuardedBy;
import org.infinispan.commands.FlagAffectedCommand;
import org.infinispan.commands.ReplicableCommand;
import org.infinispan.commands.remote.CacheRpcCommand;
import org.infinispan.commons.CacheException;
import org.infinispan.commons.util.Util;
import org.infinispan.context.Flag;
import org.infinispan.factories.GlobalComponentRegistry;
import org.infinispan.remoting.responses.SuccessfulResponse;
import org.infinispan.remoting.InboundInvocationHandler;
import org.infinispan.remoting.RpcException;
import org.infinispan.remoting.responses.ExceptionResponse;
import org.infinispan.remoting.responses.Response;
import org.infinispan.topology.CacheTopologyControlCommand;
import org.infinispan.topology.LocalTopologyManager;
import org.infinispan.util.TimeService;
import org.infinispan.util.concurrent.TimeoutException;
import org.infinispan.util.logging.Log;
import org.infinispan.util.logging.LogFactory;
import org.infinispan.xsite.BackupReceiver;
import org.infinispan.xsite.BackupReceiverRepository;
import org.infinispan.xsite.XSiteReplicateCommand;
import org.jgroups.Address;
import org.jgroups.AnycastAddress;
import org.jgroups.Channel;
import org.jgroups.Message;
import org.jgroups.SuspectedException;
import org.jgroups.UpHandler;
import org.jgroups.blocks.RequestOptions;
import org.jgroups.blocks.ResponseMode;
import org.jgroups.blocks.RpcDispatcher;
import org.jgroups.blocks.RspFilter;
import org.jgroups.blocks.mux.Muxer;
import org.jgroups.protocols.relay.SiteAddress;
import org.jgroups.util.Buffer;
import org.jgroups.util.FutureListener;
import org.jgroups.util.NotifyingFuture;
import org.jgroups.util.Rsp;
import org.jgroups.util.RspList;

import java.io.NotSerializableException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.infinispan.commons.util.Util.*;
import static org.infinispan.remoting.transport.jgroups.JGroupsTransport.fromJGroupsAddress;

/**
 * A JGroups RPC dispatcher that knows how to deal with {@link ReplicableCommand}s.
 *
 * @author Manik Surtani ([email protected])
 * @author Pedro Ruivo
 * @since 4.0
 */
public class CommandAwareRpcDispatcher extends RpcDispatcher {

   private final ExecutorService asyncExecutor;
   private final ExecutorService remoteCommandsExecutor;
   private final InboundInvocationHandler inboundInvocationHandler;
   private static final Log log = LogFactory.getLog(CommandAwareRpcDispatcher.class);
   private static final boolean trace = log.isTraceEnabled();
   private static final boolean FORCE_MCAST = Boolean.getBoolean("infinispan.unsafe.force_multicast");
   private final JGroupsTransport transport;
   private final GlobalComponentRegistry gcr;
   private final BackupReceiverRepository backupReceiverRepository;

   public CommandAwareRpcDispatcher(Channel channel,
                                    JGroupsTransport transport,
                                    ExecutorService asyncExecutor,
                                    ExecutorService remoteCommandsExecutor,
                                    InboundInvocationHandler inboundInvocationHandler,
                                    GlobalComponentRegistry gcr, BackupReceiverRepository backupReceiverRepository) {
      this.server_obj = transport;
      this.asyncExecutor = asyncExecutor;
      this.remoteCommandsExecutor = remoteCommandsExecutor;
      this.inboundInvocationHandler = inboundInvocationHandler;
      this.transport = transport;
      this.gcr = gcr;
      this.backupReceiverRepository = backupReceiverRepository;

      // MessageDispatcher superclass constructors will call start() so perform all init here
      this.setMembershipListener(transport);
      this.setChannel(channel);
      // If existing up handler is a muxing up handler, setChannel(..) will not have replaced it
      UpHandler handler = channel.getUpHandler();
      if (handler instanceof Muxer) {
         @SuppressWarnings("unchecked")
         Muxer mux = (Muxer) handler;
         mux.setDefaultHandler(this.prot_adapter);
      }
      channel.addChannelListener(this);
      asyncDispatching(true);
   }

   private boolean isValid(Message req) {
      if (req == null || req.getLength() == 0) {
         log.msgOrMsgBufferEmpty();
         return false;
      }

      return true;
   }

   /**
    * @param recipients Guaranteed not to be null.  Must not contain self.
    */
   public RspList invokeRemoteCommands(final List
     recipients, final ReplicableCommand command, final ResponseMode mode, final long timeout,
                                               final boolean oob, final RspFilter filter,
                                               boolean asyncMarshalling, final boolean ignoreLeavers, final boolean totalOrder) throws InterruptedException {
      if (asyncMarshalling) {
         asyncExecutor.submit(new Callable>() {
            @Override
            public RspList call() throws Exception {
               return processCalls(command, recipients == null, timeout, filter, recipients, mode,
                                   req_marshaller, CommandAwareRpcDispatcher.this, oob, ignoreLeavers, totalOrder);
            }
         });
         return null; // don't wait for a response!
      } else {
         RspList response;
         try {
            response = processCalls(command, recipients == null, timeout, filter, recipients, mode,
                                    req_marshaller, this, oob, ignoreLeavers, totalOrder);
         } catch (InterruptedException e) {
            throw e;
         } catch (SuspectedException e) {
            throw new SuspectException("One of the nodes " + recipients + " was suspected", e);
         } catch (org.jgroups.TimeoutException e) {
            throw new TimeoutException("One of the nodes " + recipients + " timed out", e);
         } catch (Exception e) {
            throw rewrapAsCacheException(e);
         }
         if (mode == ResponseMode.GET_NONE) return null; // "Traditional" async.
         if (response.isEmpty() || containsOnlyNulls(response))
            return null;
         else
            return response;
      }
   }

   public Response invokeRemoteCommand(final Address recipient, final ReplicableCommand command, final ResponseMode mode,
                                       final long timeout, final boolean oob,
                                       boolean asyncMarshalling) throws InterruptedException {
      if (asyncMarshalling) {
         asyncExecutor.submit(new Callable() {

            @Override
            public Response call() throws Exception {
               return processSingleCall(command, timeout, recipient, mode,
                                        req_marshaller, CommandAwareRpcDispatcher.this, oob, transport);
            }
         });
         return null; // don't wait for a response!
      } else {
         Response response;
         try {
            response = processSingleCall(command, timeout, recipient, mode,
                                         req_marshaller, this, oob, transport);
         } catch (InterruptedException e) {
            throw e;
         } catch (SuspectedException e) {
            throw new SuspectException("Node " + recipient + " was suspected", e);
         } catch (org.jgroups.TimeoutException e) {
            throw new TimeoutException("Node " + recipient + " timed out", e);
         } catch (Exception e) {
            throw rewrapAsCacheException(e);
         }
         if (mode == ResponseMode.GET_NONE) return null; // "Traditional" async.
         return response;
      }
   }

   public RspList broadcastRemoteCommands(ReplicableCommand command, ResponseMode mode, long timeout,
                                                  boolean oob, RspFilter filter,
                                                  boolean asyncMarshalling, boolean ignoreLeavers, boolean totalOrder)
         throws InterruptedException {
      return invokeRemoteCommands(null, command, mode, timeout, oob, filter, asyncMarshalling, ignoreLeavers, totalOrder);
   }

   private boolean containsOnlyNulls(RspList l) {
      for (Rsp r : l.values()) {
         if (r.getValue() != null || !r.wasReceived() || r.wasSuspected()) return false;
      }
      return true;
   }

   /**
    * Message contains a Command. Execute it against *this* object and return result.
    */
   @Override
   public void handle(Message req, org.jgroups.blocks.Response response) throws Exception {
      if (isValid(req)) {
         boolean preserveOrder = !req.isFlagSet(Message.Flag.OOB);
         ReplicableCommand cmd = null;
         try {
            cmd = (ReplicableCommand) req_marshaller.objectFromBuffer(req.getRawBuffer(), req.getOffset(), req.getLength());
            if (cmd == null) throw new NullPointerException("Unable to execute a null command!  Message was " + req);
            if (req.getSrc() instanceof SiteAddress) {
               executeCommandFromRemoteSite(cmd, (SiteAddress) req.getSrc(), response, preserveOrder);
            } else {
               executeCommandFromLocalCluster(cmd, req, response, preserveOrder);
            }
         } catch (InterruptedException e) {
            log.shutdownHandlingCommand(cmd);
            reply(response, new ExceptionResponse(new CacheException("Cache is shutting down")));
         } catch (Throwable x) {
            if (cmd == null)
               log.errorUnMarshallingCommand(x);
            else
               log.exceptionHandlingCommand(cmd, x);
            reply(response, new ExceptionResponse(new CacheException("Problems invoking command.", x)));
         }
      } else {
         reply(response, null);
      }
   }

   private void executeCommandFromRemoteSite(final ReplicableCommand cmd, final SiteAddress src,
                                             final org.jgroups.blocks.Response response, boolean preserveOrder) throws Throwable {
      if (!(cmd instanceof XSiteReplicateCommand)) {
         throw new IllegalStateException("Only XSiteReplicateCommand commands expected as a result of xsite calls but got "
                                               + cmd.getClass().getName());
      }
      if (trace) {
         log.tracef("Handling command %s from remote site %s", cmd, src);
      }

      ((XSiteReplicateCommand) cmd).setOriginSite(src.getSite());

      final BackupReceiver receiver = backupReceiverRepository.getBackupReceiver(src.getSite(),
                                                                                 ((XSiteReplicateCommand) cmd).getCacheName());
      if (preserveOrder) {
         reply(response, ((XSiteReplicateCommand) cmd).performInLocalSite(receiver));
         return;
      }

      //the remote site commands may need to be forwarded to the appropriate owners
      remoteCommandsExecutor.execute(new Runnable() {
         @Override
         public void run() {
            try {
               reply(response, ((XSiteReplicateCommand) cmd).performInLocalSite(receiver));
            } catch (InterruptedException e) {
               log.shutdownHandlingCommand(cmd);
               reply(response, new ExceptionResponse(new CacheException("Cache is shutting down")));
            } catch (Throwable throwable) {
               log.exceptionHandlingCommand(cmd, throwable);
               reply(response, new ExceptionResponse(new CacheException("Problems invoking command.", throwable)));
            }
         }
      });
   }

   private void executeCommandFromLocalCluster(final ReplicableCommand cmd, final Message req, final org.jgroups.blocks.Response response, boolean preserveOrder) throws Throwable {
      if (cmd instanceof CacheRpcCommand) {
         if (trace) log.tracef("Attempting to execute command: %s [sender=%s]", cmd, req.getSrc());
         inboundInvocationHandler.handle((CacheRpcCommand) cmd, fromJGroupsAddress(req.getSrc()), response, preserveOrder);
      } else {
         if (!isTotalOrderStateTransferCommand(cmd) && !preserveOrder && cmd.canBlock()) {
            remoteCommandsExecutor.execute(new Runnable() {
               @Override
               public void run() {
                  try {
                     if (trace)
                        log.tracef("Attempting to execute non-CacheRpcCommand command: %s [sender=%s]", cmd, req.getSrc());
                     gcr.wireDependencies(cmd);

                     Object retVal = cmd.perform(null);
                     if (retVal != null && !(retVal instanceof Response)) {
                        retVal = SuccessfulResponse.create(retVal);
                     }
                     reply(response, retVal);
                  } catch (InterruptedException e) {
                     log.shutdownHandlingCommand(cmd);
                     reply(response, new ExceptionResponse(new CacheException("Cache is shutting down")));
                  } catch (Throwable throwable) {
                     log.exceptionHandlingCommand(cmd, throwable);
                     reply(response, new ExceptionResponse(new CacheException("Problems invoking command.", throwable)));
                  }
               }
            });
         } else {
            if (trace) log.tracef("Attempting to execute non-CacheRpcCommand command: %s [sender=%s]", cmd, req.getSrc());
            gcr.wireDependencies(cmd);

            Object retVal = cmd.perform(null);
            if (retVal != null && !(retVal instanceof Response)) {
               retVal = SuccessfulResponse.create(retVal);
            }
            reply(response, retVal);
         }
      }
   }

   @Override
   public String toString() {
      return getClass().getSimpleName() + "[Outgoing marshaller: " + req_marshaller + "; incoming marshaller: " + rsp_marshaller + "]";
   }

   private void reply(org.jgroups.blocks.Response response, Object retVal) {
      if (response != null) {
         //exceptionThrown is always false because the exceptions are wrapped in an ExceptionResponse
         response.send(retVal, false);
      }
   }

   private boolean isTotalOrderStateTransferCommand(ReplicableCommand command) throws InterruptedException {
      boolean isTotalOrder = false;
      if (command instanceof CacheTopologyControlCommand) {
         CacheTopologyControlCommand controlCommand = (CacheTopologyControlCommand) command;
         switch (controlCommand.getType()) {
            case REBALANCE_START:
            case CH_UPDATE:
               LocalTopologyManager topologyManager = gcr.getComponent(LocalTopologyManager.class);
               isTotalOrder = topologyManager != null && topologyManager.isTotalOrderCache(controlCommand.getCacheName());
               break;
         }
      }
      return isTotalOrder;
   }

   protected static Message constructMessage(Buffer buf, Address recipient, boolean oob, ResponseMode mode, boolean rsvp,
                                             boolean totalOrder) {
      Message msg = new Message();
      msg.setBuffer(buf);
      if (oob) msg.setFlag(Message.Flag.OOB);
      //some issues with the new bundler. put back the DONT_BUNDLE flag.
      if (oob || mode != ResponseMode.GET_NONE) msg.setFlag(Message.Flag.DONT_BUNDLE);
      if (rsvp) msg.setFlag(Message.Flag.RSVP);

      //In total order protocol, the sequencer is in the protocol stack so we need to bypass the protocol
      if(!totalOrder) {
         msg.setFlag(Message.Flag.NO_TOTAL_ORDER);
      } else {
         msg.clearFlag(Message.Flag.OOB);
      }
      if (recipient != null) msg.setDest(recipient);
      return msg;
   }

   static Buffer marshallCall(Marshaller marshaller, ReplicableCommand command) {
      Buffer buf;
      try {
         buf = marshaller.objectToBuffer(command);
      } catch (Exception e) {
         throw new RuntimeException("Failure to marshal argument(s)", e);
      }
      return buf;
   }

   private static Response processSingleCall(ReplicableCommand command, long timeout,
                                             Address destination, ResponseMode mode,
                                             Marshaller marshaller, CommandAwareRpcDispatcher card, boolean oob,
                                             JGroupsTransport transport) throws Exception {
      if (trace) log.tracef("Replication task sending %s to single recipient %s with response mode %s", command, destination, mode);
      boolean rsvp = isRsvpCommand(command);

      // Replay capability requires responses from all members!
      Response retval;
      Buffer buf;
      buf = marshallCall(marshaller, command);
      retval = card.sendMessage(constructMessage(buf, destination, oob, mode, rsvp, false),
                                new RequestOptions(mode, timeout));

      // we only bother parsing responses if we are not in ASYNC mode.
      if (trace) log.tracef("Response: %s", retval);
      if (mode == ResponseMode.GET_NONE)
         return null;

      if (retval != null) {
         if (!transport.checkResponse(retval, fromJGroupsAddress(destination))) {
            if (trace) log.tracef("Invalid response from %s", destination);
            throw new TimeoutException("Received an invalid response " + retval + " from " + destination);
         }
      }

      return retval;
   }

   private static RspList processCalls(ReplicableCommand command, boolean broadcast, long timeout,
                                               RspFilter filter, List
     dests, ResponseMode mode,
                                               Marshaller marshaller, CommandAwareRpcDispatcher card,
                                               boolean oob, boolean ignoreLeavers, boolean totalOrder) throws Exception {
      if (trace) log.tracef("Replication task sending %s to addresses %s with response mode %s", command, dests, mode);
      boolean rsvp = isRsvpCommand(command);

      RspList retval = null;
      Buffer buf;
      if (totalOrder) {
         buf = marshallCall(marshaller, command);
         Message message = constructMessage(buf, null, oob, mode, rsvp, true);

         AnycastAddress address = new AnycastAddress(dests);
         message.setDest(address);

         retval = card.castMessage(dests, message, new RequestOptions(mode, timeout, false, filter));
      } else if (broadcast || FORCE_MCAST) {
         buf = marshallCall(marshaller, command);
         RequestOptions opts = new RequestOptions(mode, timeout, false, filter);

         //Only the commands in total order must be received...
         //For correctness, ispn doesn't need their own message, so add own address to exclusion list
         opts.setExclusionList(card.getChannel().getAddress());

         retval = card.castMessage(dests, constructMessage(buf, null, oob, mode, rsvp, false),opts);
      } else {
         RequestOptions opts = new RequestOptions(mode, timeout);

         //Only the commands in total order must be received...
         opts.setExclusionList(card.getChannel().getAddress());

         if (dests.isEmpty()) return new RspList();
         buf = marshallCall(marshaller, command);

         // if at all possible, try not to use JGroups' ANYCAST for now.  Multiple (parallel) UNICASTs are much faster.
         if (filter != null) {
            // This is possibly a remote GET.
            // These UNICASTs happen in parallel using sendMessageWithFuture.  Each future has a listener attached
            // (see FutureCollator) and the first successful response is used.
            FutureCollator futureCollator = new FutureCollator(filter, dests.size(), timeout, card.gcr.getTimeService());
            for (Address a : dests) {
               NotifyingFuture f = card.sendMessageWithFuture(constructMessage(buf, a, oob, mode, rsvp, false), opts);
               futureCollator.watchFuture(f, a);
            }
            retval = futureCollator.getResponseList();
         } else if (mode == ResponseMode.GET_ALL) {
            // A SYNC call that needs to go everywhere
            Map> futures = new HashMap>(dests.size());

            for (Address dest : dests)
               futures.put(dest, card.sendMessageWithFuture(constructMessage(buf, dest, oob, mode, rsvp, false), opts));

            retval = new RspList();

            // a get() on each future will block till that call completes.
            for (Map.Entry> entry : futures.entrySet()) {
               Address target = entry.getKey();
               try {
                  retval.addRsp(target, entry.getValue().get(timeout, MILLISECONDS));
               } catch (java.util.concurrent.TimeoutException te) {
                  throw new TimeoutException(formatString("Timed out after %s waiting for a response from %s",
                                                          prettyPrintTime(timeout), target));
               } catch (ExecutionException e) {
                  if (ignoreLeavers && e.getCause() instanceof SuspectedException) {
                     log.tracef(formatString("Ignoring node %s that left during the remote call", target));
                  } else {
                     throw wrapThrowableInException(e.getCause());
                  }
               }
            }
         } else if (mode == ResponseMode.GET_NONE) {
            // An ASYNC call.  We don't care about responses.
            for (Address dest : dests) card.sendMessage(constructMessage(buf, dest, oob, mode, rsvp, false), opts);
         }
      }

      // we only bother parsing responses if we are not in ASYNC mode.
      if (mode != ResponseMode.GET_NONE) {

         if (trace) log.tracef("Responses: %s", retval);

         // a null response is 99% likely to be due to a marshalling problem - we throw a NSE, this needs to be changed when
         // JGroups supports http://jira.jboss.com/jira/browse/JGRP-193
         // the serialization problem could be on the remote end and this is why we cannot catch this above, when marshalling.
         if (retval == null)
            throw new NotSerializableException("RpcDispatcher returned a null.  This is most often caused by args for "
                                                     + command.getClass().getSimpleName() + " not being serializable.");
      }

      return retval;
   }

   private static Exception wrapThrowableInException(Throwable t) {
      if (t instanceof Exception) {
         return (Exception) t;
      } else {
         return new CacheException(t);
      }
   }

   private static boolean isRsvpCommand(ReplicableCommand command) {
      return command instanceof FlagAffectedCommand
            && ((FlagAffectedCommand) command).hasFlag(Flag.GUARANTEED_DELIVERY);
   }

   static class SenderContainer {
      final Address address;
      volatile boolean processed = false;

      SenderContainer(Address address) {
         this.address = address;
      }

      @Override
      public String toString() {
         return "Sender{" +
               "address=" + address +
               ", responded=" + processed +
               '}';
      }
   }

   final static class FutureCollator implements FutureListener {
      final RspFilter filter;
      final Map, SenderContainer> futures = new HashMap, SenderContainer>(4);
      final long timeout;
      @GuardedBy("this")
      private RspList retval;
      @GuardedBy("this")
      private Exception exception;
      @GuardedBy("this")
      private int expectedResponses;
      private final TimeService timeService;

      FutureCollator(RspFilter filter, int expectedResponses, long timeout, TimeService timeService) {
         this.filter = filter;
         this.expectedResponses = expectedResponses;
         this.timeout = timeout;
         this.timeService = timeService;
      }

      public void watchFuture(NotifyingFuture f, Address address) {
         futures.put(f, new SenderContainer(address));
         f.setListener(this);
      }

      public synchronized RspList getResponseList() throws Exception {
         long expectedEndTime = timeService.expectedEndTime(timeout, MILLISECONDS);
         long waitingTime;
         while (expectedResponses > 0 && retval == null &&
               (waitingTime = timeService.remainingTime(expectedEndTime, MILLISECONDS)) > 0) {
            try {
               this.wait(waitingTime);
            } catch (InterruptedException e) {
               // reset interruption flag
               Thread.currentThread().interrupt();
               expectedResponses = -1;
            }
         }
         // Now we either have the response we need or aren't expecting any more responses - or have run out of time.
         if (retval != null)
            return retval;
         else if (exception != null)
            throw exception;
         else if (expectedResponses == 0)
            throw new RpcException(format("No more valid responses.  Received invalid responses from all of %s", futures.values()));
         else
            throw new TimeoutException(format("Timed out waiting for %s for valid responses from any of %s.", Util.prettyPrintTime(timeout), futures.values()));
      }

      @Override
      @SuppressWarnings("unchecked")
      public synchronized void futureDone(Future objectFuture) {
         SenderContainer sc = futures.get(objectFuture);
         if (sc.processed) {
            // This can happen - it is a race condition in JGroups' NotifyingFuture.setListener() where a listener
            // could be notified twice.
            if (trace) log.tracef("Not processing callback; already processed callback for sender %s", sc.address);
         } else {
            sc.processed = true;
            Address sender = sc.address;
            boolean done = false;
            try {
               if (retval == null) {
                  Object response = objectFuture.get();
                  if (trace) log.tracef("Received response: %s from %s", response, sender);
                  filter.isAcceptable(response, sender);
                  if (!filter.needMoreResponses()) {
                     retval = new RspList(Collections.singleton(new Rsp(sender, response)));
                     done = true;
                     //TODO cancel other tasks?
                  }
               } else {
                  if (trace) log.tracef("Skipping response from %s since a valid response for this request has already been received", sender);
               }
            } catch (InterruptedException e) {
               Thread.currentThread().interrupt();
            } catch (ExecutionException e) {
               exception = e;
               if (e.getCause() instanceof org.jgroups.TimeoutException)
                  exception = new TimeoutException("Timeout!", e);
               else if (e.getCause() instanceof Exception)
                  exception = (Exception) e.getCause();
               else
                  exception = new CacheException("Caught a throwable", e.getCause());

               if (log.isDebugEnabled())
                  log.debugf("Caught exception %s from sender %s.  Will skip this response.", exception.getClass().getName(), sender);
               log.trace("Exception caught: ", exception);
            } finally {
               expectedResponses--;
               if (expectedResponses == 0 || done) {
                  this.notify(); //make sure to awake waiting thread, but avoid unnecessary wakeups!
               }
            }
         }
      }
   }
}

    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy