All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.redisson.RedissonExecutorService Maven / Gradle / Ivy

/**
 * Copyright 2016 Nikita Koksharov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.redisson;

import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.ref.ReferenceQueue;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;

import org.redisson.api.CronSchedule;
import org.redisson.api.RAtomicLong;
import org.redisson.api.RExecutorBatchFuture;
import org.redisson.api.RExecutorFuture;
import org.redisson.api.RFuture;
import org.redisson.api.RRemoteService;
import org.redisson.api.RScheduledExecutorService;
import org.redisson.api.RScheduledFuture;
import org.redisson.api.RSemaphore;
import org.redisson.api.RTopic;
import org.redisson.api.RemoteInvocationOptions;
import org.redisson.api.listener.MessageListener;
import org.redisson.client.codec.Codec;
import org.redisson.client.codec.LongCodec;
import org.redisson.client.protocol.RedisCommands;
import org.redisson.command.CommandExecutor;
import org.redisson.connection.ConnectionManager;
import org.redisson.executor.RedissonExecutorBatchFuture;
import org.redisson.executor.RedissonExecutorFuture;
import org.redisson.executor.RedissonExecutorFutureReference;
import org.redisson.executor.RedissonScheduledFuture;
import org.redisson.executor.RemoteExecutorService;
import org.redisson.executor.RemoteExecutorServiceAsync;
import org.redisson.executor.RemotePromise;
import org.redisson.executor.ScheduledTasksService;
import org.redisson.executor.TasksBatchService;
import org.redisson.executor.TasksRunnerService;
import org.redisson.executor.TasksService;
import org.redisson.misc.Injector;
import org.redisson.misc.PromiseDelegator;
import org.redisson.misc.RPromise;
import org.redisson.misc.RedissonPromise;
import org.redisson.remote.RequestId;
import org.redisson.remote.ResponseEntry;
import org.redisson.remote.ResponseEntry.Result;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import io.netty.util.concurrent.FutureListener;
import io.netty.util.internal.PlatformDependent;
import io.netty.util.internal.ThreadLocalRandom;

/**
 * 
 * @author Nikita Koksharov
 *
 */
public class RedissonExecutorService implements RScheduledExecutorService {

    private static final Logger log = LoggerFactory.getLogger(RedissonExecutorService.class);
    
    public static final int SHUTDOWN_STATE = 1;
    public static final int TERMINATED_STATE = 2;
    
    private final CommandExecutor commandExecutor;
    private final ConnectionManager connectionManager;
    private final Codec codec;
    private final Redisson redisson;
    
    private final String tasksName;
    private final String schedulerQueueName;
    private final String schedulerChannelName;
    
    private final String workersChannelName;
    private final String workersSemaphoreName;
    private final String workersCounterName;
    
    private final String tasksCounterName;
    private final String statusName;
    private final RTopic terminationTopic;
    private final RRemoteService remoteService;
    private final RTopic workersTopic;
    private int workersGroupListenerId;

    private final RemoteExecutorServiceAsync asyncScheduledService;
    private final RemoteExecutorServiceAsync asyncScheduledServiceAtFixed;
    private final RemoteExecutorServiceAsync asyncService;
    private final RemoteExecutorServiceAsync asyncServiceWithoutResult;
    
    private final ScheduledTasksService scheduledRemoteService;
    
    private final Map, byte[]> class2bytes = PlatformDependent.newConcurrentHashMap();

    private final String name;
    private final String requestQueueName;
    private final String responseQueueName;
    private final QueueTransferService queueTransferService;
    private final String executorId;
    private final ConcurrentMap responses;

    private final ReferenceQueue> referenceDueue = new ReferenceQueue>();
    private final Collection references = Collections.newSetFromMap(PlatformDependent.newConcurrentHashMap());
    
    public RedissonExecutorService(Codec codec, CommandExecutor commandExecutor, Redisson redisson, String name, QueueTransferService queueTransferService, ConcurrentMap responses, String redissonId) {
        super();
        this.codec = codec;
        this.commandExecutor = commandExecutor;
        this.connectionManager = commandExecutor.getConnectionManager();
        this.name = name;
        this.redisson = redisson;
        this.queueTransferService = queueTransferService;
        this.responses = responses;

        if (codec == connectionManager.getCodec()) {
            this.executorId = redissonId;
        } else {
            this.executorId = redissonId + ":" + RemoteExecutorServiceAsync.class.getName() + ":" + name;
        }
        
        remoteService = redisson.getRemoteService(name, codec);
        requestQueueName = ((RedissonRemoteService)remoteService).getRequestQueueName(RemoteExecutorService.class);
        responseQueueName = ((RedissonRemoteService)remoteService).getResponseQueueName(executorId);
        String objectName = requestQueueName;
        tasksCounterName = objectName + ":counter";
        tasksName = objectName + ":tasks";
        statusName = objectName + ":status";
        terminationTopic = redisson.getTopic(objectName + ":termination-topic", codec);

        
        schedulerChannelName = objectName + ":scheduler-channel";
        schedulerQueueName = objectName + ":scheduler";
        
        workersChannelName = objectName + ":workers-channel";
        workersSemaphoreName = objectName + ":workers-semaphore";
        workersCounterName = objectName + ":workers-counter";
        
        workersTopic = redisson.getTopic(workersChannelName);
        
        TasksService executorRemoteService = new TasksService(codec, redisson, name, commandExecutor, executorId, responses);
        executorRemoteService.setTerminationTopicName(terminationTopic.getChannelNames().get(0));
        executorRemoteService.setTasksCounterName(tasksCounterName);
        executorRemoteService.setStatusName(statusName);
        executorRemoteService.setTasksName(tasksName);
        asyncService = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        asyncServiceWithoutResult = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().noResult());
        
        scheduledRemoteService = new ScheduledTasksService(codec, redisson, name, commandExecutor, executorId, responses);
        scheduledRemoteService.setTerminationTopicName(terminationTopic.getChannelNames().get(0));
        scheduledRemoteService.setTasksCounterName(tasksCounterName);
        scheduledRemoteService.setStatusName(statusName);
        scheduledRemoteService.setSchedulerQueueName(schedulerQueueName);
        scheduledRemoteService.setSchedulerChannelName(schedulerChannelName);
        scheduledRemoteService.setTasksName(tasksName);
        asyncScheduledService = scheduledRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        asyncScheduledServiceAtFixed = scheduledRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().noResult());
    }
    
    protected String generateRequestId() {
        byte[] id = new byte[16];
        // TODO JDK UPGRADE replace to native ThreadLocalRandom
        ThreadLocalRandom.current().nextBytes(id);
        return ByteBufUtil.hexDump(id);
    }
    
    @Override
    public int countActiveWorkers() {
        String id = generateRequestId();
        int subscribers = (int) workersTopic.publish(id);
        RSemaphore semaphore = redisson.getSemaphore(workersSemaphoreName + ":" + id);
        try {
            semaphore.tryAcquire(subscribers, 10, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
        RAtomicLong atomicLong = redisson.getAtomicLong(workersCounterName + ":" + id);
        long result = atomicLong.get();
        redisson.getKeys().delete(semaphore, atomicLong);
        return (int) result;
    }
    
    @Override
    public void registerWorkers(int workers) {
        registerWorkers(workers, commandExecutor.getConnectionManager().getExecutor());
    }
    
    @Override
    public void registerWorkers(final int workers, ExecutorService executor) {
        QueueTransferTask task = new QueueTransferTask(connectionManager) {
            @Override
            protected RTopic getTopic() {
                return new RedissonTopic(LongCodec.INSTANCE, commandExecutor, schedulerChannelName);
            }

            @Override
            protected RFuture pushTaskAsync() {
                return commandExecutor.evalWriteAsync(name, LongCodec.INSTANCE, RedisCommands.EVAL_LONG,
                        "local expiredTaskIds = redis.call('zrangebyscore', KEYS[2], 0, ARGV[1], 'limit', 0, ARGV[2]); "
                      + "if #expiredTaskIds > 0 then "
                          + "redis.call('zrem', KEYS[2], unpack(expiredTaskIds));"
                          + "redis.call('rpush', KEYS[1], unpack(expiredTaskIds));"
                      + "end; "
                        // get startTime from scheduler queue head task
                      + "local v = redis.call('zrange', KEYS[2], 0, 0, 'WITHSCORES'); "
                      + "if v[1] ~= nil then "
                         + "return v[2]; "
                      + "end "
                      + "return nil;",
                      Arrays.asList(requestQueueName, schedulerQueueName), 
                      System.currentTimeMillis(), 100);
            }
        };
        queueTransferService.schedule(getName(), task);
        
        TasksRunnerService service = 
                new TasksRunnerService(commandExecutor, redisson, codec, requestQueueName, responses);
        service.setStatusName(statusName);
        service.setTasksCounterName(tasksCounterName);
        service.setTasksName(tasksName);
        service.setTerminationTopicName(terminationTopic.getChannelNames().get(0));
        service.setSchedulerChannelName(schedulerChannelName);
        service.setSchedulerQueueName(schedulerQueueName);
        
        remoteService.register(RemoteExecutorService.class, service, workers, executor);
        workersGroupListenerId = workersTopic.addListener(new MessageListener() {
            @Override
            public void onMessage(String channel, String id) {
                redisson.getAtomicLong(workersCounterName + ":" + id).getAndAdd(workers);
                redisson.getSemaphore(workersSemaphoreName + ":" + id).release();
            }
        });
    }

    @Override
    public void execute(Runnable task) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        RemotePromise promise = (RemotePromise)asyncServiceWithoutResult.executeRunnable(task.getClass().getName(), classBody, state, null);
        syncExecute(promise);
    }
    
    @Override
    public void execute(Runnable ...tasks) {
        if (tasks.length == 0) {
            throw new NullPointerException("Tasks are not defined");
        }

        TasksBatchService executorRemoteService = createBatchService();
        RemoteExecutorServiceAsync asyncServiceWithoutResult = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().noResult());
        for (Runnable task : tasks) {
            check(task);
            byte[] classBody = getClassBody(task);
            byte[] state = encode(task);
            asyncServiceWithoutResult.executeRunnable(task.getClass().getName(), classBody, state, null);
        }
        
        List result = (List) executorRemoteService.executeAdd();
        if (!result.get(0)) {
            throw new RejectedExecutionException("Tasks have been rejected. ExecutorService is in shutdown state");
        }
    }

    private TasksBatchService createBatchService() {
        TasksBatchService executorRemoteService = new TasksBatchService(codec, redisson, name, commandExecutor, executorId, responses);
        executorRemoteService.setTerminationTopicName(terminationTopic.getChannelNames().get(0));
        executorRemoteService.setTasksCounterName(tasksCounterName);
        executorRemoteService.setStatusName(statusName);
        executorRemoteService.setTasksName(tasksName);
        return executorRemoteService;
    }
    
    private byte[] encode(Object task) {
        // erase RedissonClient field to avoid its serialization
        Injector.inject(task, null);
        
        ByteBuf buf = null;
        try {
            buf = codec.getValueEncoder().encode(task);
            byte[] dst = new byte[buf.readableBytes()];
            buf.readBytes(dst);
            return dst;
        } catch (IOException e) {
            throw new IllegalArgumentException(e);
        } finally {
            if (buf != null) {
                buf.release();
            }
        }
    }

    private byte[] getClassBody(Object task) {
        Class c = task.getClass();
        byte[] classBody = class2bytes.get(c);
        if (classBody == null) {
            String className = c.getName();
            String classAsPath = className.replace('.', '/') + ".class";
            InputStream classStream = c.getClassLoader().getResourceAsStream(classAsPath);
            
            DataInputStream s = new DataInputStream(classStream);
            try {
                classBody = new byte[s.available()];
                s.readFully(classBody);
            } catch (IOException e) {
                throw new IllegalArgumentException(e);
            }
            
            class2bytes.put(c, classBody);
        }
        return classBody;
    }

    @Override
    public void shutdown() {
        queueTransferService.remove(getName());
        remoteService.deregister(RemoteExecutorService.class);
        workersTopic.removeListener(workersGroupListenerId);
        
        commandExecutor.evalWrite(getName(), LongCodec.INSTANCE, RedisCommands.EVAL_VOID,
                "if redis.call('exists', KEYS[2]) == 0 then "
                     + "if redis.call('get', KEYS[1]) == '0' or redis.call('exists', KEYS[1]) == 0 then "
                        + "redis.call('set', KEYS[2], ARGV[2]);"
                        + "redis.call('publish', KEYS[3], ARGV[2]);"
                     + "else "
                        + "redis.call('set', KEYS[2], ARGV[1]);"
                     + "end;"
                + "end;", 
                Arrays.asList(tasksCounterName, statusName, terminationTopic.getChannelNames().get(0)),
                SHUTDOWN_STATE, TERMINATED_STATE);
    }

    @Override
    public String getName() {
        return name;
    }
    
    @Override
    public boolean delete() {
        return commandExecutor.get(deleteAsync());
    }
    
    @Override
    public RFuture deleteAsync() {
        final RPromise result = new RedissonPromise();
        RFuture deleteFuture = redisson.getKeys().deleteAsync(
                requestQueueName, statusName, tasksCounterName, schedulerQueueName, tasksName);
        deleteFuture.addListener(new FutureListener() {
            @Override
            public void operationComplete(io.netty.util.concurrent.Future future) throws Exception {
                if (!future.isSuccess()) {
                    result.tryFailure(future.cause());
                    return;
                }
                
                result.trySuccess(future.getNow() > 0);
            }
        });
        return result;
    }
    
    @Override
    public List shutdownNow() {
        throw new UnsupportedOperationException();
    }

    @Override
    public boolean isShutdown() {
        return checkState(SHUTDOWN_STATE);
    }

    private boolean checkState(int state) {
        return commandExecutor.evalWrite(getName(), codec, RedisCommands.EVAL_BOOLEAN,
                "if redis.call('exists', KEYS[1]) == 1 and tonumber(redis.call('get', KEYS[1])) >= tonumber(ARGV[1]) then "
                + "return 1;"
            + "end;"
            + "return 0;", 
                Arrays.asList(statusName),
                state);
    }

    @Override
    public boolean isTerminated() {
        return checkState(TERMINATED_STATE);
    }

    @Override
    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
        if (isTerminated()) {
            return true;
        }
        
        final CountDownLatch latch = new CountDownLatch(1);
        MessageListener listener = new MessageListener() {
            @Override
            public void onMessage(String channel, Integer msg) {
                if (msg == TERMINATED_STATE) {
                    latch.countDown();
                }
            }
        };
        int listenerId = terminationTopic.addListener(listener);

        if (isTerminated()) {
            terminationTopic.removeListener(listenerId);
            return true;
        }
        
        boolean res = latch.await(timeout, unit);
        terminationTopic.removeListener(listenerId);
        return res;
    }

    @Override
    public  RExecutorFuture submit(Callable task) {
        RemotePromise promise = (RemotePromise) ((PromiseDelegator) submitAsync(task)).getInnerPromise();
        syncExecute(promise);
        return createFuture(promise);
    }
    
    @Override
    public  RExecutorFuture submitAsync(Callable task) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        RemotePromise result = (RemotePromise) asyncService.executeCallable(task.getClass().getName(), classBody, state, null);
        addListener(result);
        return createFuture(result);
    }
    
    @Override
    public RExecutorBatchFuture submit(Callable ...tasks) {
        if (tasks.length == 0) {
            throw new NullPointerException("Tasks are not defined");
        }

        List> result = new ArrayList>();
        TasksBatchService executorRemoteService = createBatchService();
        RemoteExecutorServiceAsync asyncService = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        for (Callable task : tasks) {
            check(task);
            byte[] classBody = getClassBody(task);
            byte[] state = encode(task);
            RemotePromise promise = (RemotePromise)asyncService.executeCallable(task.getClass().getName(), classBody, state, null);
            RedissonExecutorFuture executorFuture = new RedissonExecutorFuture(promise);
            result.add(executorFuture);
        }
        
        List addResult = (List) executorRemoteService.executeAdd();
        if (!addResult.get(0)) {
            throw new RejectedExecutionException("Tasks have been rejected. ExecutorService is in shutdown state");
        }
        
        return new RedissonExecutorBatchFuture(result);
    }
    
    @Override
    public RExecutorBatchFuture submitAsync(Callable ...tasks) {
        if (tasks.length == 0) {
            throw new NullPointerException("Tasks are not defined");
        }

        TasksBatchService executorRemoteService = createBatchService();
        RemoteExecutorServiceAsync asyncService = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        final List> result = new ArrayList>();
        for (Callable task : tasks) {
            check(task);
            byte[] classBody = getClassBody(task);
            byte[] state = encode(task);
            RemotePromise promise = (RemotePromise)asyncService.executeCallable(task.getClass().getName(), classBody, state, null);
            RedissonExecutorFuture executorFuture = new RedissonExecutorFuture(promise);
            result.add(executorFuture);
        }
        
        executorRemoteService.executeAddAsync().addListener(new FutureListener>() {

            @Override
            public void operationComplete(io.netty.util.concurrent.Future> future) throws Exception {
                if (!future.isSuccess()) {
                    for (RExecutorFuture executorFuture : result) {
                        ((RPromise)executorFuture).tryFailure(future.cause());
                    }
                    return;
                }
                
                for (Boolean bool : future.getNow()) {
                    if (!bool) {
                        RejectedExecutionException ex = new RejectedExecutionException("Task rejected. ExecutorService is in shutdown state");
                        for (RExecutorFuture executorFuture : result) {
                            ((RPromise)executorFuture).tryFailure(ex);
                        }
                        break;
                    }
                }
            }
        });

        return new RedissonExecutorBatchFuture(result);
    }


    private  void addListener(final RemotePromise result) {
        result.getAddFuture().addListener(new FutureListener() {

            @Override
            public void operationComplete(io.netty.util.concurrent.Future future) throws Exception {
                if (!future.isSuccess()) {
                    result.tryFailure(future.cause());
                    return;
                }
                
                if (!future.getNow()) {
                    result.tryFailure(new RejectedExecutionException("Task rejected. ExecutorService is in shutdown state"));
                }
                
            }
        });
    }
    
    private void check(Object task) {
        if (task == null) {
            throw new NullPointerException("Task is not defined");
        }
        if (task.getClass().isAnonymousClass()) {
            throw new IllegalArgumentException("Task can't be created using anonymous class");
        }
        if (task.getClass().isMemberClass()
                && !Modifier.isStatic(task.getClass().getModifiers())) {
            throw new IllegalArgumentException("Task class is an inner class and it should be static");
        }
    }

    private  void syncExecute(RemotePromise promise) {
        RFuture addFuture = promise.getAddFuture();
        addFuture.syncUninterruptibly();
        Boolean res = addFuture.getNow();
        if (!res) {
            throw new RejectedExecutionException("Task rejected. ExecutorService is in shutdown state");
        }
    }

    @Override
    public  RExecutorFuture submit(Runnable task, final T result) {
        final RPromise resultFuture = new RedissonPromise();
        RemotePromise future = (RemotePromise) ((PromiseDelegator) submit(task)).getInnerPromise();
        future.addListener(new FutureListener() {
            @Override
            public void operationComplete(io.netty.util.concurrent.Future future) throws Exception {
                if (!future.isSuccess()) {
                    resultFuture.tryFailure(future.cause());
                    return;
                }
                resultFuture.trySuccess(result);
            }
        });
        return new RedissonExecutorFuture(resultFuture, future.getRequestId());
    }

    @Override
    public RExecutorBatchFuture submit(Runnable ...tasks) {
        if (tasks.length == 0) {
            throw new NullPointerException("Tasks are not defined");
        }

        List> result = new ArrayList>();
        TasksBatchService executorRemoteService = createBatchService();
        RemoteExecutorServiceAsync asyncService = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        for (Runnable task : tasks) {
            check(task);
            byte[] classBody = getClassBody(task);
            byte[] state = encode(task);
            RemotePromise promise = (RemotePromise)asyncService.executeRunnable(task.getClass().getName(), classBody, state, null);
            RedissonExecutorFuture executorFuture = new RedissonExecutorFuture(promise);
            result.add(executorFuture);
        }
        
        List addResult = (List) executorRemoteService.executeAdd();
        if (!addResult.get(0)) {
            throw new RejectedExecutionException("Tasks have been rejected. ExecutorService is in shutdown state");
        }
        
        return new RedissonExecutorBatchFuture(result);
    }
    
    @Override
    public RExecutorBatchFuture submitAsync(Runnable ...tasks) {
        if (tasks.length == 0) {
            throw new NullPointerException("Tasks are not defined");
        }

        TasksBatchService executorRemoteService = createBatchService();
        RemoteExecutorServiceAsync asyncService = executorRemoteService.get(RemoteExecutorServiceAsync.class, RemoteInvocationOptions.defaults().noAck().expectResultWithin(1, TimeUnit.DAYS));
        final List> result = new ArrayList>();
        for (Runnable task : tasks) {
            check(task);
            byte[] classBody = getClassBody(task);
            byte[] state = encode(task);
            RemotePromise promise = (RemotePromise)asyncService.executeRunnable(task.getClass().getName(), classBody, state, null);
            RedissonExecutorFuture executorFuture = new RedissonExecutorFuture(promise);
            result.add(executorFuture);
        }
        
        executorRemoteService.executeAddAsync().addListener(new FutureListener>() {

            @Override
            public void operationComplete(io.netty.util.concurrent.Future> future) throws Exception {
                if (!future.isSuccess()) {
                    for (RExecutorFuture executorFuture : result) {
                        ((RPromise)executorFuture).tryFailure(future.cause());
                    }
                    return;
                }
                
                for (Boolean bool : future.getNow()) {
                    if (!bool) {
                        RejectedExecutionException ex = new RejectedExecutionException("Task rejected. ExecutorService is in shutdown state");
                        for (RExecutorFuture executorFuture : result) {
                            ((RPromise)executorFuture).tryFailure(ex);
                        }
                        break;
                    }
                }
            }
        });

        return new RedissonExecutorBatchFuture(result);
    }

    
    @Override
    public RExecutorFuture submit(Runnable task) {
        RemotePromise promise = (RemotePromise) ((PromiseDelegator) submitAsync(task)).getInnerPromise();
        syncExecute(promise);
        return createFuture(promise);
    }
    
    @Override
    public RExecutorFuture submitAsync(Runnable task) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        RemotePromise result = (RemotePromise) asyncService.executeRunnable(task.getClass().getName(), classBody, state, null);
        addListener(result);
        return createFuture(result);
    }
    
    private void cancelResponseHandling(RequestId requestId) {
        synchronized (responses) {
            ResponseEntry entry = responses.get(responseQueueName);
            if (entry == null) {
                return;
            }
            
            List list = entry.getResponses().remove(requestId);
            if (list != null) {
                for (Result result : list) {
                    result.getScheduledFuture().cancel(true);
                }
            }
            if (entry.getResponses().isEmpty()) {
                responses.remove(responseQueueName, entry);
            }
        }
    }
    
    @Override
    public RScheduledFuture schedule(Runnable task, long delay, TimeUnit unit) {
        RedissonScheduledFuture future = (RedissonScheduledFuture) scheduleAsync(task, delay, unit);
        RemotePromise rp = (RemotePromise)future.getInnerPromise();
        syncExecute(rp);
        storeReference(future, rp.getRequestId());
        return future;
    }

    private  RExecutorFuture createFuture(RemotePromise promise) {
        RExecutorFuture f = new RedissonExecutorFuture(promise);
        storeReference(f, promise.getRequestId());
        return f;
    }
    
    private  RScheduledFuture createFuture(RemotePromise promise, long scheduledExecutionTime) {
        RedissonScheduledFuture f = new RedissonScheduledFuture(promise, scheduledExecutionTime);
        storeReference(f, promise.getRequestId());
        return f;
    }
    
    private void storeReference(RExecutorFuture future, RequestId requestId) {
        while (true) {
            RedissonExecutorFutureReference r = (RedissonExecutorFutureReference) referenceDueue.poll();
            if (r == null) {
                break;
            }
            references.remove(r);
            
            if (!r.getPromise().hasListeners()) {
                cancelResponseHandling(r.getRequestId());
            }
        }
        
        RPromise promise = ((PromiseDelegator) future).getInnerPromise();
        RedissonExecutorFutureReference reference = new RedissonExecutorFutureReference(requestId, future, referenceDueue, promise);
        references.add(reference);
    }
    
    @Override
    public RScheduledFuture scheduleAsync(Runnable task, long delay, TimeUnit unit) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        long startTime = System.currentTimeMillis() + unit.toMillis(delay);
        RemotePromise result = (RemotePromise) asyncScheduledService.scheduleRunnable(task.getClass().getName(), classBody, state, startTime, null);
        addListener(result);
        
        return createFuture(result, startTime);
    }
    
    @Override
    public  RScheduledFuture schedule(Callable task, long delay, TimeUnit unit) {
        RedissonScheduledFuture future = (RedissonScheduledFuture) scheduleAsync(task, delay, unit);
        RemotePromise rp = (RemotePromise)future.getInnerPromise();
        syncExecute(rp);
        storeReference(future, rp.getRequestId());
        return future;
    }
    
    @Override
    public  RScheduledFuture scheduleAsync(Callable task, long delay, TimeUnit unit) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        long startTime = System.currentTimeMillis() + unit.toMillis(delay);
        RemotePromise result = (RemotePromise) asyncScheduledService.scheduleCallable(task.getClass().getName(), classBody, state, startTime, null);
        addListener(result);
        return createFuture(result, startTime);
    }
    
    @Override
    public RScheduledFuture scheduleAtFixedRate(Runnable task, long initialDelay, long period, TimeUnit unit) {
        RedissonScheduledFuture future = (RedissonScheduledFuture) scheduleAtFixedRateAsync(task, initialDelay, period, unit);
        RemotePromise rp = (RemotePromise)future.getInnerPromise();
        syncExecute(rp);
        storeReference(future, rp.getRequestId());
        return future;
    }
    
    @Override
    public RScheduledFuture scheduleAtFixedRateAsync(Runnable task, long initialDelay, long period, TimeUnit unit) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        long startTime = System.currentTimeMillis() + unit.toMillis(initialDelay);
        RemotePromise result = (RemotePromise) asyncScheduledServiceAtFixed.scheduleAtFixedRate(task.getClass().getName(), classBody, state, startTime, unit.toMillis(period), executorId, null);
        addListener(result);
        return createFuture(result, startTime);
    }

    @Override
    public RScheduledFuture schedule(Runnable task, CronSchedule cronSchedule) {
        RedissonScheduledFuture future = (RedissonScheduledFuture) scheduleAsync(task, cronSchedule);
        RemotePromise rp = (RemotePromise)future.getInnerPromise();
        syncExecute(rp);
        storeReference(future, rp.getRequestId());
        return future;
    }
    
    @Override
    public RScheduledFuture scheduleAsync(Runnable task, CronSchedule cronSchedule) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        final Date startDate = cronSchedule.getExpression().getNextValidTimeAfter(new Date());
        long startTime = startDate.getTime();
        RemotePromise result = (RemotePromise) asyncScheduledServiceAtFixed.schedule(task.getClass().getName(), classBody, state, startTime, cronSchedule.getExpression().getCronExpression(), executorId, null);
        addListener(result);
        RedissonScheduledFuture f = new RedissonScheduledFuture(result, startTime) {
            public long getDelay(TimeUnit unit) {
                return unit.convert(startDate.getTime() - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
            };
        };
        storeReference(f, result.getRequestId());
        return f;
    }
    
    @Override
    public RScheduledFuture scheduleWithFixedDelay(Runnable task, long initialDelay, long delay, TimeUnit unit) {
        RedissonScheduledFuture future = (RedissonScheduledFuture) scheduleWithFixedDelayAsync(task, initialDelay, delay, unit);
        RemotePromise rp = (RemotePromise)future.getInnerPromise();
        syncExecute(rp);
        storeReference(future, rp.getRequestId());
        return future;
    }
    
    @Override
    public RScheduledFuture scheduleWithFixedDelayAsync(Runnable task, long initialDelay, long delay, TimeUnit unit) {
        check(task);
        byte[] classBody = getClassBody(task);
        byte[] state = encode(task);
        long startTime = System.currentTimeMillis() + unit.toMillis(initialDelay);
        RemotePromise result = (RemotePromise) asyncScheduledServiceAtFixed.scheduleWithFixedDelay(task.getClass().getName(), classBody, state, startTime, unit.toMillis(delay), executorId, null);
        addListener(result);
        return createFuture(result, startTime);
    }

    @Override
    public boolean cancelScheduledTask(String taskId) {
        return cancelTask(taskId);
    }
    
    @Override
    public boolean cancelTask(String taskId) {
        RFuture scheduledFuture = scheduledRemoteService.cancelExecutionAsync(new RequestId(taskId));
        return commandExecutor.get(scheduledFuture);
    }

    private  T doInvokeAny(Collection> tasks,
                            boolean timed, long millis) throws InterruptedException, ExecutionException, TimeoutException {
        if (tasks == null) {
            throw new NullPointerException();
        }

        int ntasks = tasks.size();
        if (ntasks == 0) {
            throw new IllegalArgumentException();
        }

        List> futures = new ArrayList>(ntasks);

        try {
            ExecutionException ee = null;
            long lastTime = timed ? System.currentTimeMillis() : 0;
            Iterator> it = tasks.iterator();

            // Start one task for sure; the rest incrementally
            futures.add(submit(it.next()));
            --ntasks;
            int active = 1;

            for (;;) {
                Future f = poll(futures);
                if (f == null) {
                    if (ntasks > 0) {
                        --ntasks;
                        futures.add(submit(it.next()));
                        ++active;
                    }
                    else if (active == 0)
                        break;
                    else if (timed) {
                        f = poll(futures, millis, TimeUnit.MILLISECONDS);
                        if (f == null)
                            throw new TimeoutException();
                        long now = System.currentTimeMillis();
                        millis -= now - lastTime;
                        lastTime = now;
                    }
                    else
                        f = poll(futures, -1, null);
                }
                if (f != null) {
                    --active;
                    try {
                        return f.get();
                    } catch (ExecutionException eex) {
                        ee = eex;
                    } catch (RuntimeException rex) {
                        ee = new ExecutionException(rex);
                    }
                }
            }

            if (ee == null)
                ee = new ExecutionException("No tasks were finised", null);
            throw ee;

        } finally {
            for (Future f : futures)
                f.cancel(true);
        }
    }
    
    private  Future poll(List> futures, long timeout, TimeUnit timeUnit) throws InterruptedException {
        final CountDownLatch latch = new CountDownLatch(1);
        final AtomicReference> result = new AtomicReference>();
        FutureListener listener = new FutureListener() {
            @Override
            public void operationComplete(io.netty.util.concurrent.Future future) throws Exception {
                latch.countDown();
                result.compareAndSet(null, future);
            }
        };
        for (Future future : futures) {
            RFuture f = (RFuture) future;
            f.addListener(listener);
        }
        
        if (timeout == -1) {
            latch.await();
        } else {
            latch.await(timeout, timeUnit);
        }
        
        for (Future future : futures) {
            RFuture f = (RFuture) future;
            f.removeListener(listener);
        }

        return result.get();
    }
    
    private  Future poll(List> futures) {
        for (Future future : futures) {
            if (future.isDone()) {
                return future;
            }
        }
        return null;
    }

    @Override
    public  T invokeAny(Collection> tasks)
        throws InterruptedException, ExecutionException {
        try {
            return doInvokeAny(tasks, false, 0);
        } catch (TimeoutException cannotHappen) {
            return null;
        }
    }

    @Override
    public  T invokeAny(Collection> tasks,
                           long timeout, TimeUnit unit)
        throws InterruptedException, ExecutionException, TimeoutException {
        return doInvokeAny(tasks, true, unit.toMillis(timeout));
    }

    @Override
    public  List> invokeAll(Collection> tasks) throws InterruptedException {
        if (tasks == null) {
            throw new NullPointerException();
        }
        
        List> futures = new ArrayList>(tasks.size());
        boolean done = false;
        try {
            for (Callable t : tasks) {
                Future future = submit(t);
                futures.add(future);
            }
            for (Future f : futures) {
                if (!f.isDone()) {
                    try {
                        f.get();
                    } catch (CancellationException ignore) {
                    } catch (ExecutionException ignore) {
                    }
                }
            }
            done = true;
            return futures;
        } finally {
            if (!done)
                for (Future f : futures)
                    f.cancel(true);
        }
    }

    @Override
    public  List> invokeAll(Collection> tasks,
                                         long timeout, TimeUnit unit) throws InterruptedException {
        if (tasks == null || unit == null) {
            throw new NullPointerException();
        }
        
        long millis = unit.toMillis(timeout);
        List> futures = new ArrayList>(tasks.size());
        boolean done = false;
        
        try {
            long lastTime = System.currentTimeMillis();

            for (Callable task : tasks) {
                Future future = submit(task);
                futures.add(future);
                
                long now = System.currentTimeMillis();
                millis -= now - lastTime;
                lastTime = now;
                if (millis <= 0) {
                    int remainFutures = tasks.size() - futures.size();
                    for (int i = 0; i < remainFutures; i++) {
                        RPromise cancelledFuture = new RedissonPromise();
                        cancelledFuture.cancel(true);
                        futures.add(cancelledFuture);
                        
                    }
                    return futures;
                }
            }

            for (Future f : futures) {
                if (!f.isDone()) {
                    if (millis <= 0)
                        return futures;
                    try {
                        f.get(millis, TimeUnit.MILLISECONDS);
                    } catch (CancellationException ignore) {
                    } catch (ExecutionException ignore) {
                    } catch (TimeoutException toe) {
                        return futures;
                    }
                    long now = System.currentTimeMillis();
                    millis -= now - lastTime;
                    lastTime = now;
                }
            }
            done = true;
            return futures;
        } finally {
            if (!done)
                for (Future f : futures)
                    f.cancel(true);
        }
    }

}