org.infinispan.xsite.XSiteAdminOperations Maven / Gradle / Ivy

Go to download
package org.infinispan.xsite;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.infinispan.Cache;
import org.infinispan.commons.CacheException;
import org.infinispan.factories.annotations.Inject;
import org.infinispan.jmx.annotations.MBean;
import org.infinispan.jmx.annotations.ManagedOperation;
import org.infinispan.jmx.annotations.Parameter;
import org.infinispan.remoting.inboundhandler.DeliverOrder;
import org.infinispan.remoting.responses.CacheNotFoundResponse;
import org.infinispan.remoting.responses.Response;
import org.infinispan.remoting.responses.SuccessfulResponse;
import org.infinispan.remoting.rpc.ResponseMode;
import org.infinispan.remoting.rpc.RpcManager;
import org.infinispan.remoting.transport.Address;
import org.infinispan.util.ByteString;
import org.infinispan.util.logging.Log;
import org.infinispan.util.logging.LogFactory;
import org.infinispan.xsite.statetransfer.XSiteStateTransferManager;
import org.infinispan.xsite.status.CacheSiteStatusBuilder;
import org.infinispan.xsite.status.SiteStatus;

/**
 * Managed bean exposing sys admin operations for Cross-Site replication functionality.
 *
 * @author Mircea Markus
 * @since 5.2
 */
@MBean(objectName = "XSiteAdmin", description = "Exposes tooling for handling backing up data to remote sites.")
public class XSiteAdminOperations {

   public static final String ONLINE = "online";
   public static final String OFFLINE = "offline";
   public static final String SUCCESS = "ok";
   private static Log log = LogFactory.getLog(XSiteAdminOperations.class);
   private RpcManager rpcManager;
   private Cache cache;

   private volatile BackupSender backupSender;
   private XSiteStateTransferManager stateTransferManager;

   @Inject
   public void init(RpcManager rpcManager, BackupSender backupSender, Cache cache,
                    XSiteStateTransferManager stateTransferManager) {
      this.backupSender = backupSender;
      this.rpcManager = rpcManager;
      this.backupSender = backupSender;
      this.cache = cache;
      this.stateTransferManager = stateTransferManager;
   }

   public Map clusterStatus()  {
      Map localNodeStatus = backupSender.status();
      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), null, XSiteAdminCommand.AdminOperation.STATUS, null, null);
      Map responses = invokeRemotely(command);
      List errors = checkForErrors(responses);
      if (!errors.isEmpty()) {
         throw new CacheException("Unable to check cluster state for members: " + errors);
      }
      //site name => online/offline/mixed
      Map perSiteBuilder = new HashMap<>();
      for (Map.Entry entry : localNodeStatus.entrySet()) {
         CacheSiteStatusBuilder builder = new CacheSiteStatusBuilder();
         builder.addMember(rpcManager.getAddress(), entry.getValue());
         perSiteBuilder.put(entry.getKey(), builder);
      }
      for (Map.Entry entry : responses.entrySet()) {
         Response response = entry.getValue();
         if (response == CacheNotFoundResponse.INSTANCE) {
            continue; //shutting down.
         }
         if (!response.isSuccessful()) {
            throw new CacheException("Unsuccessful response received from. " + entry);
         }
         //noinspection unchecked
         Map sites = (Map) ((SuccessfulResponse) response).getResponseValue();
         for (Map.Entry site : sites.entrySet()) {
            CacheSiteStatusBuilder builder = perSiteBuilder.get(site.getKey());
            if (builder == null) {
               throw new IllegalStateException("Site " + entry.getKey() + " not defined in all the cluster members");
            }
            builder.addMember(entry.getKey(), site.getValue());
         }
      }

      Map result = new HashMap<>();
      perSiteBuilder.forEach((site, builder) -> result.put(site, builder.build()));
      return result;
   }

   @ManagedOperation(description = "Check whether the given backup site is offline or not.", displayName = "Check whether the given backup site is offline or not.")
   public String siteStatus(@Parameter(name = "site", description = "The name of the backup site") String site) {
      //also consider local node
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null)
         return "Incorrect site name: " + site;
      log.tracef("This node's status is %s", offlineStatus);

      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), site, XSiteAdminCommand.AdminOperation.SITE_STATUS, null, null);
      Map responses = invokeRemotely(command);
      List online = new ArrayList<>(responses.size());
      List offline = new ArrayList<>(responses.size());
      List failed = new ArrayList<>(responses.size());
      for (Map.Entry e : responses.entrySet()) {
         if (!e.getValue().isSuccessful() || !e.getValue().isValid()) {
            if (e.getValue() != CacheNotFoundResponse.INSTANCE) {
               //the node can be shutting down.
               failed.add(e.getKey());
            }
            continue;
         }
         SuccessfulResponse response = (SuccessfulResponse) e.getValue();
         log.tracef("Got status %s from node %s", response.getResponseValue(), e.getKey());
         if (response.getResponseValue() == XSiteAdminCommand.Status.OFFLINE) {
            offline.add(e.getKey());
         } else if (response.getResponseValue() == XSiteAdminCommand.Status.ONLINE) {
            online.add(e.getKey());
         } else {
            throw new IllegalStateException("Unknown response: " + response.getResponseValue());
         }
      }
      if (!failed.isEmpty()) {
         return rpcError(failed, "Could not query nodes ");
      }

      if (offlineStatus.isOffline()) {
         offline.add(rpcManager.getAddress());
      } else {
         online.add(rpcManager.getAddress());
      }

      if (offline.isEmpty()) {
         return ONLINE;
      }
      if (online.isEmpty()) {
         return OFFLINE;
      }
      return "Site appears online on nodes:" + online + " and offline on nodes: " + offline;
   }

   @ManagedOperation(description = "Returns the the status(offline/online) of all the configured backup sites.", displayName = "Returns the the status(offline/online) of all the configured backup sites.")
   public String status() {
      //also consider local node
      Map localNodeStatus = backupSender.status();
      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), null, XSiteAdminCommand.AdminOperation.STATUS, null, null);
      Map responses = invokeRemotely(command);
      List errors = checkForErrors(responses);
      if (!errors.isEmpty()) {
         return rpcError(errors, "Failure invoking 'status()' on nodes: ");
      }
      //
      Map> result = new HashMap<>();
      for (Map.Entry e : localNodeStatus.entrySet()) {
         ArrayList failedSites = new ArrayList<>();
         result.put(e.getKey(), failedSites);
         if (!e.getValue()) {
            failedSites.add(rpcManager.getAddress());
         }
      }
      for (Map.Entry response : responses.entrySet()) {
         @SuppressWarnings("unchecked")
         Map status = (Map) ((SuccessfulResponse) response.getValue()).getResponseValue();
         for (Map.Entry entry : status.entrySet()) {
            List addresses = result.get(entry.getKey());
            if (addresses == null)
               throw new IllegalStateException("All sites must be defined on all the nodes of the cluster!");
            if (!entry.getValue()) {
               addresses.add(rpcManager.getAddress());
            }
         }
      }

      int clusterSize = rpcManager.getTransport().getMembers().size();

      StringBuilder resultStr = new StringBuilder();
      //now generate the final response
      boolean first = true;
      for (Map.Entry> e : result.entrySet()) {
         if (!first) {
            resultStr.append("\n");
         } else first = false;
         resultStr.append(e.getKey()).append("[");
         List value = e.getValue();
         if (value.isEmpty()) {
            resultStr.append("ONLINE");
         } else if (value.size() == clusterSize) {
            resultStr.append("OFFLINE");
         } else {
            resultStr.append("MIXED, offline on nodes: ").append(value);
         }
         resultStr.append("]");
      }
      return resultStr.toString();
   }

   @ManagedOperation(description = "Takes this site offline in all nodes in the cluster.", displayName = "Takes this site offline in all nodes in the cluster.")
   public String takeSiteOffline(@Parameter(name = "site", description = "The name of the backup site") String site) {
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null)
         return incorrectSiteName(site);
      backupSender.takeSiteOffline(site);
      log.tracef("Is site offline in node %s? %s", rpcManager.getAddress(), offlineStatus.isOffline());

      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), site, XSiteAdminCommand.AdminOperation.TAKE_OFFLINE, null, null);
      Map responses = invokeRemotely(command);

      List failed = checkForErrors(responses);

      String prefix = "Could not take the site offline on nodes:";
      return returnFailureOrSuccess(failed, prefix);
   }

   @ManagedOperation(description = "Amends the values for 'afterFailures' for the 'TakeOffline' functionality on all the nodes in the cluster.", displayName = "Amends the values for 'TakeOffline.afterFailures' on all the nodes in the cluster.")
   public String setTakeOfflineAfterFailures(
         @Parameter(name = "site", description = "The name of the backup site") String site,
         @Parameter(name = "afterFailures", description = "The number of failures after which the site will be taken offline",
                    type = "integer") int afterFailures) {
      return takeOffline(site, afterFailures, null);
   }

   @ManagedOperation(description = "Amends the values for 'minTimeToWait' for the 'TakeOffline' functionality on all the nodes in the cluster.", displayName = "Amends the values for 'TakeOffline.minTimeToWait' on all the nodes in the cluster.")
   public String setTakeOfflineMinTimeToWait(
         @Parameter(name = "site", description = "The name of the backup site") String site,
         @Parameter(name = "minTimeToWait", description = "The minimum amount of time in milliseconds to wait before taking a site offline", type = "long") long minTimeToWait) {
      return takeOffline(site, null, minTimeToWait);
   }

   @ManagedOperation(description = "Amends the values for 'TakeOffline' functionality on all the nodes in the cluster.", displayName = "Amends the values for 'TakeOffline' functionality on all the nodes in the cluster.")
   public String amendTakeOffline(
         @Parameter(name = "site", description = "The name of the backup site") String site,
         @Parameter(name = "afterFailures", description = "The number of failures after which the site will be taken offline", type = "integer") int afterFailures,
         @Parameter(name = "minTimeToWait", description = "The minimum amount of time in milliseconds to wait before taking a site offline", type = "long") long minTimeToWait) {
      return takeOffline(site, afterFailures, minTimeToWait);
   }

   @ManagedOperation(description = "Returns the value of the 'minTimeToWait' for the 'TakeOffline' functionality.", displayName = "Returns the value of the 'minTimeToWait' for the 'TakeOffline' functionality.")
   public String getTakeOfflineMinTimeToWait(@Parameter(name = "site", description = "The name of the backup site") String site) {
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null) return incorrectSiteName(site);
      return String.valueOf(offlineStatus.getTakeOffline().minTimeToWait());
   }

   @ManagedOperation(description = "Returns the value of the 'afterFailures' for the 'TakeOffline' functionality.", displayName = "Returns the value of the 'afterFailures' for the 'TakeOffline' functionality.")
   public String getTakeOfflineAfterFailures(@Parameter(name = "site", description = "The name of the backup site") String site) {
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null) return incorrectSiteName(site);
      return String.valueOf(offlineStatus.getTakeOffline().afterFailures());
   }

   @ManagedOperation(description = "Brings the given site back online on all the cluster.", displayName = "Brings the given site back online on all the cluster.")
   public String bringSiteOnline(@Parameter(name = "site", description = "The name of the backup site") String site) {
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null)
         return "Incorrect site name: " + site;
      backupSender.bringSiteOnline(site);

      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), site, XSiteAdminCommand.AdminOperation.BRING_ONLINE, null, null);
      Map responses = invokeRemotely(command);

      List failed = checkForErrors(responses);

      return returnFailureOrSuccess(failed, "Could not take the site online on nodes:");
   }

   @ManagedOperation(displayName = "Push state to site",
                     description = "Pushes the state of this cache to the remote site. " +
                           "The remote site will be bring back online",
                     name = "pushState")
   public final String pushState(@Parameter(description = "The destination site name", name = "SiteName") String siteName) {
      String status = bringSiteOnline(siteName);
      if (!SUCCESS.equals(status)) {
         return String.format("Unable to pushState to '%s'. %s", siteName, status);
      }
      try {
         stateTransferManager.startPushState(siteName);
      } catch (Throwable throwable) {
         log.debugf(throwable, "Unable to pushState to '%s'.", siteName);
         return String.format("Unable to pushState to '%s'. %s", siteName, throwable.getLocalizedMessage());
      }
      return SUCCESS;
   }

   /**
    * for debug only!
    */
   public final List getRunningStateTransfer() {
      return stateTransferManager.getRunningStateTransfers();
   }

   @ManagedOperation(displayName = "Push State Status",
                     description = "Shows a map with destination site name and the state transfer status.",
                     name = "PushStateStatus")
   public final Map getPushStateStatus() {
      Map map = new HashMap<>();
      try {
         for (String siteName : getRunningStateTransfer()) {
            map.put(siteName, XSiteStateTransferManager.STATUS_SENDING);
         }
         map.putAll(stateTransferManager.getClusterStatus());
         return map;
      } catch (Exception e) {
         return Collections.singletonMap(XSiteStateTransferManager.STATUS_ERROR, e.getLocalizedMessage());
      }
   }

   @ManagedOperation(displayName = "Clear State Status",
                     description = "Clears the state transfer status.",
                     name = "ClearPushStateStatus")
   public final String clearPushStateStatus() {
      return performOperation(() -> stateTransferManager.clearClusterStatus());
   }

   @ManagedOperation(displayName = "Cancel Push Status",
                     description = "Cancels the push state to remote site.",
                     name = "CancelPushState")
   public final String cancelPushState(@Parameter(description = "The destination site name", name = "SiteName")
                                          final String siteName) {
      return performOperation(() -> stateTransferManager.cancelPushState(siteName));
   }

   @ManagedOperation(displayName = "Cancel Receive State",
                     description = "Cancels the push state to this site. All the state received from state transfer " +
                           "will be ignored.",
                     name = "CancelReceiveState")
   public final String cancelReceiveState(@Parameter(description = "The sending site name", name = "SiteName")
                                             final String siteName) {
      return performOperation(() -> stateTransferManager.cancelReceive(siteName));
   }

   @ManagedOperation(displayName = "Sending Site Name",
                     description = "Returns the site name from which this site is receiving state.",
                     name = "SendingSiteName")
   public final String getSendingSiteName() {
      return stateTransferManager.getSendingSiteName();
   }

   private static String performOperation(Operation operation) {
      try {
         operation.execute();
      } catch (Throwable t) {
         return String.format("Unable to perform operation. Error=%s", t.getLocalizedMessage());
      }
      return SUCCESS;
   }

   private List checkForErrors(Map responses) {
      List failed = new ArrayList<>(responses.size());
      for (Map.Entry e : responses.entrySet()) {
         if (e.getValue() != CacheNotFoundResponse.INSTANCE &&
               (e.getValue() == null || !e.getValue().isSuccessful() || !e.getValue().isValid())) {
            failed.add(e.getKey());
         }
      }
      return failed;
   }

   private String takeOffline(String site, Integer afterFailures, Long minTimeToWait) {
      OfflineStatus offlineStatus = backupSender.getOfflineStatus(site);
      if (offlineStatus == null)
         return incorrectSiteName(site);

      XSiteAdminCommand command = new XSiteAdminCommand(ByteString.fromString(cache.getName()), site, XSiteAdminCommand.AdminOperation.AMEND_TAKE_OFFLINE,
                                                        afterFailures, minTimeToWait);
      Map responses = invokeRemotely(command);

      //also amend locally
      offlineStatus.amend(afterFailures, minTimeToWait);

      List failed = checkForErrors(responses);

      return returnFailureOrSuccess(failed, "Could not amend for nodes:");
   }

   private String returnFailureOrSuccess(List failed, String prefix) {
      if (!failed.isEmpty()) {
         return rpcError(failed, prefix);
      }
      return SUCCESS;
   }

   private String rpcError(List failed, String prefix) {
      return prefix + failed.toString();
   }

   private String incorrectSiteName(String site) {
      return "Incorrect site name: " + site;
   }

   private Map invokeRemotely(XSiteAdminCommand command) {
      return rpcManager.invokeRemotely(null, command,
                                       rpcManager.getRpcOptionsBuilder(ResponseMode.SYNCHRONOUS_IGNORE_LEAVERS, DeliverOrder.NONE).build());
   }

   private interface Operation {
      void execute() throws Throwable;
   }
}