All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.infinispan.distexec.DefaultExecutorService Maven / Gradle / Ivy

There is a newer version: 9.1.7.Final
Show newest version
package org.infinispan.distexec;

import static org.infinispan.factories.KnownComponentNames.CACHE_MARSHALLER;

import java.io.Externalizable;
import java.io.NotSerializableException;
import java.io.Serializable;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;

import org.infinispan.AdvancedCache;
import org.infinispan.Cache;
import org.infinispan.commands.CancelCommand;
import org.infinispan.commands.CancellationService;
import org.infinispan.commands.CommandsFactory;
import org.infinispan.commands.read.DistributedExecuteCommand;
import org.infinispan.commons.marshall.Marshaller;
import org.infinispan.commons.marshall.StreamingMarshaller;
import org.infinispan.commons.util.InfinispanCollections;
import org.infinispan.commons.util.Util;
import org.infinispan.commons.util.concurrent.FutureListener;
import org.infinispan.commons.util.concurrent.NotifyingFuture;
import org.infinispan.commons.util.concurrent.NotifyingFutureImpl;
import org.infinispan.configuration.cache.Configuration;
import org.infinispan.distexec.spi.DistributedTaskLifecycleService;
import org.infinispan.distribution.DistributionManager;
import org.infinispan.factories.ComponentRegistry;
import org.infinispan.interceptors.InterceptorChain;
import org.infinispan.interceptors.locking.ClusteringDependentLogic;
import org.infinispan.lifecycle.ComponentStatus;
import org.infinispan.remoting.responses.Response;
import org.infinispan.remoting.responses.SuccessfulResponse;
import org.infinispan.remoting.rpc.ResponseMode;
import org.infinispan.remoting.rpc.RpcManager;
import org.infinispan.remoting.transport.Address;
import org.infinispan.remoting.transport.TopologyAwareAddress;
import org.infinispan.security.AuthorizationManager;
import org.infinispan.security.AuthorizationPermission;
import org.infinispan.util.TimeService;
import org.infinispan.util.logging.Log;
import org.infinispan.util.logging.LogFactory;

/**
 * Infinispan's implementation of an {@link ExecutorService} and {@link DistributedExecutorService}.
 * This ExecutorService provides methods to submit tasks for an execution on a cluster of Infinispan
 * nodes.
 * 

* * * Note that due to potential task migration to another nodes every {@link Callable}, * {@link Runnable} and/or {@link DistributedCallable} submitted must be either {@link Serializable} * or {@link Externalizable}. Also the value returned from a callable must be {@link Serializable} * or {@link Externalizable}. Unfortunately if the value returned is not serializable then a * {@link NotSerializableException} will be thrown. * * @author Vladimir Blagojevic * @since 5.0 * */ public class DefaultExecutorService extends AbstractExecutorService implements DistributedExecutorService { private static final NodeFilter SAME_MACHINE_FILTER = new NodeFilter(){ @Override public boolean include(TopologyAwareAddress thisAddress, TopologyAwareAddress otherAddress) { return thisAddress.isSameMachine(otherAddress); }; }; private static final NodeFilter SAME_RACK_FILTER = new NodeFilter(){ @Override public boolean include(TopologyAwareAddress thisAddress, TopologyAwareAddress otherAddress) { return thisAddress.isSameRack(otherAddress); }; }; private static final NodeFilter SAME_SITE_FILTER = new NodeFilter(){ @Override public boolean include(TopologyAwareAddress thisAddress, TopologyAwareAddress otherAddress) { return thisAddress.isSameSite(otherAddress); }; }; private static final NodeFilter ALL_FILTER = new NodeFilter(){ @Override public boolean include(TopologyAwareAddress thisAddress, TopologyAwareAddress otherAddress) { return true; }; }; public static final DistributedTaskFailoverPolicy NO_FAILOVER = new NoTaskFailoverPolicy(); public static final DistributedTaskFailoverPolicy RANDOM_NODE_FAILOVER = new RandomNodeTaskFailoverPolicy(); private static final Log log = LogFactory.getLog(DefaultExecutorService.class); private static final boolean trace = log.isTraceEnabled(); protected final AtomicBoolean isShutdown = new AtomicBoolean(false); protected final AdvancedCache cache; protected final RpcManager rpc; protected final InterceptorChain invoker; protected final CommandsFactory factory; protected final Marshaller marshaller; protected final ExecutorService localExecutorService; protected final CancellationService cancellationService; protected final ClusteringDependentLogic clusterDependentLogic; protected final boolean takeExecutorOwnership; private final TimeService timeService; /** * Creates a new DefaultExecutorService given a master cache node for local task execution. All * distributed task executions will be initiated from this Infinispan cache node * * @param masterCacheNode * Cache node initiating distributed task */ public DefaultExecutorService(Cache masterCacheNode) { this(masterCacheNode, Executors.newSingleThreadExecutor(), true); } /** * Creates a new DefaultExecutorService given a master cache node and an ExecutorService for * parallel execution of tasks ran on this node. All distributed task executions will be * initiated from this Infinispan cache node. *

* Note that DefaultExecutorService will not shutdown client supplied localExecutorService once * this DefaultExecutorService is shutdown. Lifecycle management of a supplied ExecutorService is * left to the client * * Also note that client supplied ExecutorService should not execute tasks in the caller's thread * ( i.e rejectionHandler of {@link ThreadPoolExecutor} configured with {link * {@link ThreadPoolExecutor.CallerRunsPolicy}) * * @param masterCacheNode * Cache node initiating distributed task * @param localExecutorService * ExecutorService to run local tasks */ public DefaultExecutorService(Cache masterCacheNode, ExecutorService localExecutorService) { this(masterCacheNode, localExecutorService, false); } /** * Creates a new DefaultExecutorService given a master cache node and an ExecutorService for * parallel execution of task ran on this node. All distributed task executions will be initiated * from this Infinispan cache node. * * @param masterCacheNode * Cache node initiating distributed task * @param localExecutorService * ExecutorService to run local tasks * @param takeExecutorOwnership * if true {@link DistributedExecutorService#shutdown()} and * {@link DistributedExecutorService#shutdownNow()} method will shutdown * localExecutorService as well * */ public DefaultExecutorService(Cache masterCacheNode, ExecutorService localExecutorService, boolean takeExecutorOwnership) { super(); if (masterCacheNode == null) throw new IllegalArgumentException("Can not use null cache for DefaultExecutorService"); else if (localExecutorService == null) throw new IllegalArgumentException("Can not use null instance of ExecutorService"); else if (localExecutorService.isShutdown()) throw new IllegalArgumentException("Can not use an instance of ExecutorService which is shutdown"); ensureAccessPermissions(masterCacheNode.getAdvancedCache()); ensureProperCacheState(masterCacheNode.getAdvancedCache()); this.cache = masterCacheNode.getAdvancedCache(); ComponentRegistry registry = SecurityActions.getCacheComponentRegistry(cache); this.rpc = SecurityActions.getCacheRpcManager(cache); this.invoker = registry.getComponent(InterceptorChain.class); this.factory = registry.getComponent(CommandsFactory.class); this.marshaller = registry.getComponent(StreamingMarshaller.class, CACHE_MARSHALLER); this.cancellationService = registry.getComponent(CancellationService.class); this.localExecutorService = localExecutorService; this.takeExecutorOwnership = takeExecutorOwnership; this.timeService = registry.getTimeService(); this.clusterDependentLogic = registry.getComponent(ClusteringDependentLogic.class); } @Override public DistributedTaskBuilder createDistributedTaskBuilder(Callable callable) { Configuration cacheConfiguration = SecurityActions.getCacheConfiguration(cache); long to = cacheConfiguration.clustering().sync().replTimeout(); DistributedTaskBuilder dtb = new DefaultDistributedTaskBuilder(to); dtb.callable(callable); return dtb; } @Override public NotifyingFuture submit(Runnable task, T result) { return (NotifyingFuture) super.submit(task, result); } @Override public NotifyingFuture submit(Callable task) { return (NotifyingFuture) super.submit(task); } @Override public void shutdown() { realShutdown(false); } protected List

getMembers() { if (rpc != null) { return rpc.getMembers(); } else { return Collections.singletonList(getAddress()); } } protected List
executionCandidates(DistributedTask task) { return filterMembers(task.getTaskExecutionPolicy(), getMembers()); } private Address getAddress() { return clusterDependentLogic.getAddress(); } private List realShutdown(boolean interrupt) { isShutdown.set(true); // TODO cancel all tasks if (takeExecutorOwnership) { if (interrupt) localExecutorService.shutdownNow(); else localExecutorService.shutdown(); } return InfinispanCollections.emptyList(); } @Override public List shutdownNow() { return realShutdown(true); } @Override public boolean isShutdown() { return isShutdown.get(); } @Override public boolean isTerminated() { if (isShutdown.get()) { // TODO account for all tasks return true; } return false; } @Override public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { //long nanoTimeWait = unit.toNanos(timeout); // TODO wait for all tasks to finish return true; } @Override public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { try { return doInvokeAny(tasks, false, 0); } catch (TimeoutException cannotHappen) { assert false; return null; } } @Override public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { return doInvokeAny(tasks, true, unit.toNanos(timeout)); } /** * the main mechanics of invokeAny. This was essentially copied from * {@link AbstractExecutorService} doInvokeAny except that we replaced the * {@link ExecutorCompletionService} with our {@link DistributedExecutionCompletionService}. */ private T doInvokeAny(Collection> tasks, boolean timed, long nanos) throws InterruptedException, ExecutionException, TimeoutException { if (tasks == null) throw new NullPointerException(); int ntasks = tasks.size(); if (ntasks == 0) throw new IllegalArgumentException(); List> futures = new ArrayList>(ntasks); CompletionService ecs = new DistributedExecutionCompletionService(this); // For efficiency, especially in executors with limited // parallelism, check to see if previously submitted tasks are // done before submitting more of them. This interleaving // plus the exception mechanics account for messiness of main // loop. try { // Record exceptions so that if we fail to obtain any // result, we can throw the last exception we got. ExecutionException ee = null; long lastTime = (timed) ? timeService.time() : 0; Iterator> it = tasks.iterator(); // Start one task for sure; the rest incrementally futures.add(ecs.submit(it.next())); --ntasks; int active = 1; for (;;) { Future f = ecs.poll(); if (f == null) { if (ntasks > 0) { --ntasks; futures.add(ecs.submit(it.next())); ++active; } else if (active == 0) break; else if (timed) { f = ecs.poll(nanos, TimeUnit.NANOSECONDS); if (f == null) throw new TimeoutException(); long now = timeService.time(); nanos -= timeService.timeDuration(lastTime, now, TimeUnit.NANOSECONDS); lastTime = now; } else f = ecs.take(); } if (f != null) { --active; try { return f.get(); } catch (InterruptedException ie) { throw ie; } catch (ExecutionException eex) { ee = eex; } catch (RuntimeException rex) { ee = new ExecutionException(rex); } } } if (ee == null) ee = new ExecutionException() { private static final long serialVersionUID = 200818694545553992L; }; throw ee; } finally { for (Future f : futures) f.cancel(true); } } @Override public void execute(Runnable command) { if (!isShutdown.get()) { DistributedTaskPart cmd; if (command instanceof DistributedTaskPart) { cmd = (DistributedTaskPart) command; } else if (command instanceof Serializable) { cmd = (DistributedTaskPart) newTaskFor(command, null); } else { throw new IllegalArgumentException("Runnable command is not Serializable " + command); } cmd.execute(); } else { throw new RejectedExecutionException(); } } @Override protected RunnableFuture newTaskFor(Runnable runnable, T value) { if (runnable == null) throw new NullPointerException(); RunnableAdapter adapter = new RunnableAdapter(runnable, value); return newTaskFor(adapter); } @Override protected RunnableFuture newTaskFor(Callable callable) { if (callable == null) throw new NullPointerException(); DistributedTaskBuilder distributedTaskBuilder = createDistributedTaskBuilder(callable); DistributedTask task = distributedTaskBuilder.build(); DistributedExecuteCommand executeCommand = factory.buildDistributedExecuteCommand( callable, getAddress(), null); return createDistributedTaskPart(task, executeCommand, selectExecutionNode(task), 0); } @Override public NotifyingFuture submit(Address target, Callable task) { DistributedTaskBuilder distributedTaskBuilder = createDistributedTaskBuilder(task); DistributedTask distributedTask = distributedTaskBuilder.build(); return submit(target, distributedTask); } @Override public NotifyingFuture submit(Address target, DistributedTask task) { if (task == null) throw new NullPointerException(); if (target == null) throw new NullPointerException(); List
members = getMembers(); if (!members.contains(target)) { throw new IllegalArgumentException("Target node " + target + " is not a cluster member, members are " + members); } Address me = getAddress(); DistributedExecuteCommand c = null; if (target.equals(me)) { c = factory.buildDistributedExecuteCommand(clone(task.getCallable()), me, null); } else { c = factory.buildDistributedExecuteCommand(task.getCallable(), me, null); } DistributedTaskPart part = createDistributedTaskPart(task, c, target, 0); part.execute(); return part; } @Override public NotifyingFuture submit(Callable task, K... input) { DistributedTaskBuilder distributedTaskBuilder = createDistributedTaskBuilder(task); DistributedTask distributedTask = distributedTaskBuilder.build(); return submit(distributedTask, input); } @Override public NotifyingFuture submit(DistributedTask task, K... input) { if (task == null) throw new NullPointerException(); if(inputKeysSpecified(input)){ Map> nodesKeysMap = keysToExecutionNodes(task.getTaskExecutionPolicy(), input); checkExecutionPolicy(task, nodesKeysMap, input); Address me = getAddress(); DistributedExecuteCommand c = factory.buildDistributedExecuteCommand(task.getCallable(), me, Arrays.asList(input)); ArrayList
nodes = new ArrayList
(nodesKeysMap.keySet()); DistributedTaskPart part = createDistributedTaskPart(task, c, Arrays.asList(input), selectExecutionNode(nodes), 0); part.execute(); return part; } else { return submit(task.getCallable()); } } @Override public List> submitEverywhere(Callable task) { DistributedTaskBuilder distributedTaskBuilder = createDistributedTaskBuilder(task); DistributedTask distributedTask = distributedTaskBuilder.build(); return submitEverywhere(distributedTask); } @Override public List> submitEverywhere(DistributedTask task) { if (task == null) throw new NullPointerException(); List
members = executionCandidates(task); List> futures = new ArrayList>(members.size()); Address me = getAddress(); for (Address target : members) { DistributedExecuteCommand c = null; if (target.equals(me)) { c = factory.buildDistributedExecuteCommand(clone(task.getCallable()), me, null); } else { c = factory.buildDistributedExecuteCommand(task.getCallable(), me, null); } DistributedTaskPart part = createDistributedTaskPart(task, c, target, 0); futures.add(part); part.execute(); } return futures; } @Override public List> submitEverywhere(Callable task, K... input) { DistributedTaskBuilder distributedTaskBuilder = createDistributedTaskBuilder(task); DistributedTask distributedTask = distributedTaskBuilder.build(); return submitEverywhere(distributedTask, input); } @Override public List> submitEverywhere(DistributedTask task, K... input) { if (task == null) throw new NullPointerException(); if(inputKeysSpecified(input)) { List> futures = new ArrayList>(input.length * 2); Address me = getAddress(); Map> nodesKeysMap = keysToExecutionNodes(task.getTaskExecutionPolicy(), input); checkExecutionPolicy(task, nodesKeysMap, input); for (Entry> e : nodesKeysMap.entrySet()) { Address target = e.getKey(); DistributedExecuteCommand c = null; if (target.equals(me)) { c = factory.buildDistributedExecuteCommand(clone(task.getCallable()), me, e.getValue()); } else { c = factory.buildDistributedExecuteCommand(task.getCallable(), me, e.getValue()); } DistributedTaskPart part = createDistributedTaskPart(task, c, e.getValue(), target, 0); futures.add(part); part.execute(); } return futures; } else { return submitEverywhere(task); } } protected Callable clone(Callable task){ return Util.cloneWithMarshaller(marshaller, task); } protected DistributedTaskPart createDistributedTaskPart(DistributedTask task, DistributedExecuteCommand c, List inputKeys, Address target, int failoverCount) { return getAddress().equals(target) ? new LocalDistributedTaskPart(task, c, (List) inputKeys, failoverCount) : new RemoteDistributedTaskPart(task, c, (List) inputKeys, target, failoverCount); } protected DistributedTaskPart createDistributedTaskPart(DistributedTask task, DistributedExecuteCommand c, Address target, int failoverCount) { return createDistributedTaskPart(task, c, Collections.emptyList(), target, failoverCount); } private void checkExecutionPolicy(DistributedTask task, Map> nodesKeysMap, K... input) { if (nodesKeysMap == null || nodesKeysMap.isEmpty()) { throw new IllegalStateException("DistributedTaskExecutionPolicy " + task.getTaskExecutionPolicy() + " for task " + task + " returned invalid keysToExecutionNodes " + nodesKeysMap + " execution policy plan for a given input " + Arrays.toString(input)); } } private boolean inputKeysSpecified(K...input){ return input != null && input.length > 0; } protected Address selectExecutionNode(List
candidates) { List
list = randomClusterMembers(candidates,1); return list.get(0); } protected Address selectExecutionNode(DistributedTask task) { return selectExecutionNode(executionCandidates(task)); } protected List
randomClusterMembers(final List
members, int numNeeded) { if(members == null || members.isEmpty()) throw new IllegalArgumentException("Invalid member list " + members); if (members.size() < numNeeded) { log.cannotSelectRandomMembers(numNeeded, members); numNeeded = members.size(); } List
membersCopy = new ArrayList
(members); List
chosen = new ArrayList
(numNeeded); Random r = new Random(); while (!membersCopy.isEmpty() && numNeeded >= chosen.size()) { int count = membersCopy.size(); Address address = membersCopy.remove(r.nextInt(count)); chosen.add(address); } return chosen; } protected Map> keysToExecutionNodes(DistributedTaskExecutionPolicy policy, K... input) { DistributionManager dm = cache.getDistributionManager(); Map> addressToKey = new HashMap>(input.length * 2); boolean usingREPLMode = dm == null; for (K key : input) { Address ownerOfKey = null; if (usingREPLMode) { List
members = new ArrayList
(getMembers()); members = filterMembers(policy, members); // using REPL mode https://issues.jboss.org/browse/ISPN-1886 // since keys and values are on all nodes, lets just pick randomly Collections.shuffle(members); ownerOfKey = members.get(0); } else { // DIST mode List
owners = dm.locate(key); List
filtered = filterMembers(policy, owners); if(!filtered.isEmpty()){ ownerOfKey = filtered.get(0); } else { ownerOfKey = owners.get(0); } } List keysAtNode = addressToKey.get(ownerOfKey); if (keysAtNode == null) { keysAtNode = new LinkedList(); addressToKey.put(ownerOfKey, keysAtNode); } keysAtNode.add(key); } return addressToKey; } private List
filterMembers(DistributedTaskExecutionPolicy policy, List
members) { NodeFilter filter = null; switch (policy) { case SAME_MACHINE: filter = SAME_MACHINE_FILTER; break; case SAME_SITE: filter = SAME_SITE_FILTER; break; case SAME_RACK: filter = SAME_RACK_FILTER; break; case ALL: filter = ALL_FILTER; break; default: filter = ALL_FILTER; break; } List
result = new ArrayList
(); for (Address address : members) { if(address instanceof TopologyAwareAddress){ TopologyAwareAddress taa = (TopologyAwareAddress)address; if(filter.include(taa, (TopologyAwareAddress)getAddress())){ result.add(address); } } else { result.add(address); } } return result; } private void ensureAccessPermissions(final AdvancedCache cache) { AuthorizationManager authorizationManager = SecurityActions.getCacheAuthorizationManager(cache); if (authorizationManager != null) { authorizationManager.checkPermission(AuthorizationPermission.EXEC); } } private void ensureProperCacheState(AdvancedCache cache) throws NullPointerException, IllegalStateException { // We allow for INITIALIZING state so the ExecutorService can be used by components defining a method with // {@link Start} annotation if (cache.getStatus() != ComponentStatus.RUNNING && cache.getStatus() != ComponentStatus.INITIALIZING) throw new IllegalStateException("Invalid cache state " + cache.getStatus()); } private static class RandomNodeTaskFailoverPolicy implements DistributedTaskFailoverPolicy { public RandomNodeTaskFailoverPolicy() { super(); } @Override public Address failover(FailoverContext fc) { return randomNode(fc.executionCandidates(),fc.executionFailureLocation()); } protected Address randomNode(List
candidates, Address failedExecutionLocation){ Random r = new Random(); candidates.remove(failedExecutionLocation); if (candidates.isEmpty()) throw new IllegalStateException("There are no candidates for failover: " + candidates); int tIndex = r.nextInt(candidates.size()); return candidates.get(tIndex); } @Override public int maxFailoverAttempts() { return 1; } } private static class NoTaskFailoverPolicy implements DistributedTaskFailoverPolicy { public NoTaskFailoverPolicy() { super(); } @Override public Address failover(FailoverContext fc) { return fc.executionFailureLocation(); } @Override public int maxFailoverAttempts() { return 0; } } /** * NodeFilter allows selection of nodes according to {@link DistributedTaskExecutionPolicy} */ interface NodeFilter { boolean include(TopologyAwareAddress thisAddress, TopologyAwareAddress otherAddress); } private class DefaultDistributedTaskBuilder implements DistributedTaskBuilder, DistributedTask{ private Callable callable; private long timeout; private DistributedTaskExecutionPolicy executionPolicy = DistributedTaskExecutionPolicy.ALL; private DistributedTaskFailoverPolicy failoverPolicy = NO_FAILOVER; public DefaultDistributedTaskBuilder(long taskTimeout) { this.timeout = taskTimeout; } @Override public DistributedTaskBuilder callable(Callable callable) { if (callable == null) throw new IllegalArgumentException("Callable cannot be null"); this.callable = callable; return this; } @Override public DistributedTaskBuilder timeout(long t, TimeUnit tu) { timeout = TimeUnit.MILLISECONDS.convert(t, tu); return this; } @Override public DistributedTaskBuilder executionPolicy(DistributedTaskExecutionPolicy policy) { if (policy == null) throw new IllegalArgumentException("DistributedTaskExecutionPolicy cannot be null"); this.executionPolicy = policy; return this; } @Override public DistributedTaskBuilder failoverPolicy(DistributedTaskFailoverPolicy policy) { if (policy == null) { this.failoverPolicy = NO_FAILOVER; } else { this.failoverPolicy = policy; } return this; } @Override public DistributedTask build() { DefaultDistributedTaskBuilder task = new DefaultDistributedTaskBuilder(timeout); task.callable(callable); task.executionPolicy(executionPolicy); task.failoverPolicy(failoverPolicy); return task; } @Override public long timeout() { return timeout; } @Override public DistributedTaskExecutionPolicy getTaskExecutionPolicy() { return executionPolicy; } @Override public DistributedTaskFailoverPolicy getTaskFailoverPolicy() { return failoverPolicy; } @Override public Callable getCallable() { return callable; } } /** * DistributedTaskPart represents a unit of work sent to remote VM and executed there * * * @author Mircea Markus * @author Vladimir Blagojevic */ private abstract class DistributedTaskPart implements NotifyingFuture, RunnableFuture { protected final DistributedExecuteCommand distCommand; private final List inputKeys; private final DistributedTask owningTask; private int failedOverCount; private volatile boolean cancelled; protected DistributedTaskPart(List inputKeys, DistributedExecuteCommand command, DistributedTask task, int failedOverCount) { this.inputKeys = inputKeys; this.distCommand = command; this.owningTask = task; this.failedOverCount = failedOverCount; } public List getInputKeys() { return inputKeys; } public DistributedExecuteCommand getCommand() { return distCommand; } public DistributedTask getOwningTask() { return owningTask; } public abstract Address getExecutionTarget(); private DefaultExecutorService getOuterType() { return DefaultExecutorService.this; } public abstract void execute(); @Override public void run() { //intentionally empty } @Override public boolean isCancelled() { return cancelled; } @Override public V get() throws InterruptedException, ExecutionException { try { return innerGet(0, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { throw new ExecutionException(e); } } @Override public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { return innerGet(timeout, unit); } protected V innerGet(long timeout, TimeUnit unit) throws ExecutionException, TimeoutException, InterruptedException { if (isCancelled()) throw new CancellationException("Task already cancelled"); long timeoutNanos = computeTimeoutNanos(timeout, unit); long endNanos = timeService.expectedEndTime(timeoutNanos, TimeUnit.NANOSECONDS); try { return getResult(timeoutNanos); } catch (TimeoutException te) { throw te; } catch (Exception e) { // The RPC could have finished with a org.infinispan.util.concurrent.TimeoutException right before // the Future.get timeout expired. If that's the case, we want to throw a TimeoutException. long remainingNanos = timeoutNanos > 0 ? timeService.remainingTime(endNanos, TimeUnit.NANOSECONDS) : timeoutNanos; if (timeoutNanos > 0 && remainingNanos <= 0) { if (trace) log.tracef("Distributed task timed out, throwing a TimeoutException and ignoring exception", e); throw new TimeoutException(); } boolean canFailover = failedOverCount++ < getOwningTask().getTaskFailoverPolicy().maxFailoverAttempts(); if (canFailover) { try { return failoverExecution(e, timeoutNanos, TimeUnit.NANOSECONDS); } catch (Exception failedOver) { throw wrapIntoExecutionException(failedOver); } } else { throw wrapIntoExecutionException(e); } } } protected abstract V getResult(long timeoutNanos) throws Exception; protected long computeTimeoutNanos(long timeout, TimeUnit unit) { long taskTimeout = TimeUnit.MILLISECONDS.toNanos(getOwningTask().timeout()); long futureTimeout = TimeUnit.NANOSECONDS.convert(timeout, unit); long actualTimeout; if (taskTimeout > 0 && futureTimeout > 0) { actualTimeout = Math.min(taskTimeout, futureTimeout); } else { actualTimeout = Math.max(taskTimeout, futureTimeout); } return actualTimeout; } protected ExecutionException wrapIntoExecutionException(Exception e){ if (e instanceof ExecutionException) { return (ExecutionException) e; } else { return new ExecutionException(e); } } protected V failoverExecution(final Exception cause, long timeout, TimeUnit unit) throws Exception { final List
executionCandidates = executionCandidates(getOwningTask()); FailoverContext fc = new FailoverContext() { @Override public List inputKeys() { return (List) getInputKeys(); } @Override public Address executionFailureLocation() { return getExecutionTarget(); } @Override public List
executionCandidates() { return executionCandidates; } @Override public Throwable cause() { return cause; } }; Address target = getOwningTask().getTaskFailoverPolicy().failover(fc); DistributedTaskPart part = createDistributedTaskPart(owningTask, distCommand, getInputKeys(), target, failedOverCount); part.execute(); return part.get(timeout, unit); } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + getOuterType().hashCode(); result = prime * result + ((distCommand == null) ? 0 : distCommand.hashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof DistributedTaskPart)) { return false; } DistributedTaskPart other = (DistributedTaskPart) obj; if (!getOuterType().equals(other.getOuterType())) { return false; } if (distCommand == null) { if (other.distCommand != null) { return false; } } else if (!distCommand.equals(other.distCommand)) { return false; } return true; } protected void setCancelled() { cancelled = true; } } private class RemoteDistributedTaskPart extends DistributedTaskPart { private final Address executionTarget; private final NotifyingFutureImpl future = new NotifyingFutureImpl(); public RemoteDistributedTaskPart(DistributedTask task, DistributedExecuteCommand command, List inputKeys, Address executionTarget, int failoverCount) { super(inputKeys, command, task, failoverCount); if (getAddress().equals(executionTarget)) { throw new IllegalArgumentException("This task should be executed as local."); } this.executionTarget = executionTarget; } @Override public Address getExecutionTarget() { return executionTarget; } @Override public void execute() { if (trace) log.tracef("Sending %s to remote execution at node %s", this, getExecutionTarget()); try { rpc.invokeRemotelyInFuture(Collections.singletonList(getExecutionTarget()), getCommand(), rpc.getRpcOptionsBuilder(ResponseMode.SYNCHRONOUS) .timeout(getOwningTask().timeout(), TimeUnit.MILLISECONDS).build(), future); } catch (Throwable e) { log.remoteExecutionFailed(getExecutionTarget(), e); } } @Override public boolean cancel(boolean mayInterruptIfRunning) { if (!isCancelled()) { CancelCommand ccc = factory.buildCancelCommandCommand(distCommand.getUUID()); rpc.invokeRemotely(Collections.singletonList(getExecutionTarget()), ccc, rpc.getDefaultRpcOptions(true)); setCancelled(); return future.cancel(true); } else { //already cancelled return false; } } @Override public boolean isDone() { return future.isDone(); } @Override protected V getResult(long timeoutNanos) throws Exception { if (timeoutNanos > 0) { return retrieveResult(future.get(timeoutNanos, TimeUnit.NANOSECONDS)); } else { return retrieveResult(future.get()); } } private V retrieveResult(Object response) throws Exception { V result = null; //this is application level Exception that was raised in execution //simply rethrow it (might be good candidate for failover) if (response instanceof Exception) { throw ((Exception) response); } //these two should never happen, mark them with IllegalStateException if (response == null || !(response instanceof Map)) { throw new IllegalStateException("Invalid response received " + response); } Map mapResult = (Map) response; if (mapResult.size() == 1) { for (Entry e : mapResult.entrySet()) { Response value = e.getValue(); if (value instanceof SuccessfulResponse) { result = (V) ((SuccessfulResponse) value).getResponseValue(); } } } else { //should never happen as we send DistributedTaskPart to one node for //execution only, therefore we should get only one response throw new IllegalStateException("Invalid response " + response); } return result; } @Override public NotifyingFuture attachListener(final FutureListener listener) { future.attachListener(new FutureListener() { @Override public void futureDone(Future future) { listener.futureDone(RemoteDistributedTaskPart.this); } }); return this; } } private class LocalDistributedTaskPart extends DistributedTaskPart { private final NotifyingFutureImpl future = new NotifyingFutureImpl(); public LocalDistributedTaskPart(DistributedTask task, DistributedExecuteCommand command, List inputKeys, int failoverCount) { super(inputKeys, command, task, failoverCount); } @Override public boolean isDone() { return future.isDone(); } @Override public synchronized boolean cancel(boolean mayInterruptIfRunning) { if (!isCancelled()) { CancelCommand ccc = factory.buildCancelCommandCommand(distCommand.getUUID()); ccc.init(cancellationService); try { ccc.perform(null); } catch (Throwable e) { log.couldNotExecuteCancellationLocally(e.getLocalizedMessage()); } setCancelled(); return future.cancel(true); } else { //already cancelled return false; } } @Override protected V getResult(long timeoutNanos) throws Exception { if (timeoutNanos > 0) { return future.get(timeoutNanos, TimeUnit.NANOSECONDS); } else { return future.get(); } } @Override public Address getExecutionTarget() { return getAddress(); } @Override public void execute() { log.debugf("Sending %s to self", this); try { Callable call = new Callable() { @Override public V call() throws Exception { return doLocalInvoke(); } private V doLocalInvoke() throws Exception { getCommand().init(cache); DistributedTaskLifecycleService lifecycle = DistributedTaskLifecycleService.getInstance(); try { // hook into lifecycle lifecycle.onPreExecute(getCommand().getCallable(), cache); cancellationService.register(Thread.currentThread(), getCommand().getUUID()); V result = getCommand().perform(null); future.notifyDone(result); return result; } catch (Exception e) { future.notifyException(e); throw e; } finally { // hook into lifecycle lifecycle.onPostExecute(getCommand().getCallable()); cancellationService.unregister(getCommand().getUUID()); } } }; future.setFuture(localExecutorService.submit(call)); } catch (Throwable e1) { log.localExecutionFailed(e1); } } @Override public NotifyingFuture attachListener(final FutureListener listener) { future.attachListener(listener); return this; } } private static final class RunnableAdapter implements Callable, Serializable { /** The serialVersionUID */ private static final long serialVersionUID = 6629286923873531028L; protected Runnable task; protected T result; protected RunnableAdapter() { } protected RunnableAdapter(Runnable task, T result) { this.task = task; this.result = result; } @Override public T call() { task.run(); return result; } } }