
groovyx.gpars.dataflow.DataflowQueue Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of gpars Show documentation
Show all versions of gpars Show documentation
The Groovy and Java high-level concurrency library offering actors, dataflow, CSP, agents, parallel collections, fork/join and more
// GPars - Groovy Parallel Systems
//
// Copyright © 2008-2012 The original author or authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package groovyx.gpars.dataflow;
import groovy.lang.Closure;
import groovyx.gpars.actor.impl.MessageStream;
import groovyx.gpars.dataflow.expression.DataflowExpression;
import groovyx.gpars.dataflow.impl.DataflowChannelEventListenerManager;
import groovyx.gpars.dataflow.impl.DataflowChannelEventOrchestrator;
import groovyx.gpars.dataflow.impl.ThenMessagingRunnable;
import groovyx.gpars.dataflow.operator.BinaryChoiceClosure;
import groovyx.gpars.dataflow.operator.ChainWithClosure;
import groovyx.gpars.dataflow.operator.ChoiceClosure;
import groovyx.gpars.dataflow.operator.CopyChannelsClosure;
import groovyx.gpars.dataflow.operator.FilterClosure;
import groovyx.gpars.dataflow.operator.SeparationClosure;
import groovyx.gpars.group.DefaultPGroup;
import groovyx.gpars.group.PGroup;
import groovyx.gpars.scheduler.Pool;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import static java.util.Arrays.asList;
/**
* Represents a thread-safe data flow stream. Values or DataflowVariables are added using the '<<' operator
* and safely read once available using the 'val' property.
* The iterative methods like each(), collect(), iterator(), any(), all() or the for loops work with snapshots
* of the stream at the time of calling the particular method.
* For actors and Dataflow Operators the asynchronous non-blocking variants of the getValAsync() methods can be used.
* They register the request to read a value and will send a message to the actor or operator once the value is available.
*
* @author Vaclav Pech
* Date: Jun 5, 2009
*/
@SuppressWarnings({"ClassWithTooManyMethods", "unchecked"})
public class DataflowQueue implements DataflowChannel {
/**
* Internal lock
*/
private final Object queueLock = new Object();
/**
* Stores the received DataflowVariables in the buffer.
*/
private final LinkedBlockingQueue> queue = new LinkedBlockingQueue>();
/**
* Stores unsatisfied requests for values
*/
private final Queue> requests = new LinkedList>();
/**
* A collection of listeners who need to be informed each time the stream is bound to a value
*/
private final Collection wheneverBoundListeners = new CopyOnWriteArrayList();
/**
* Adds a DataflowVariable to the buffer.
* Implementation detail - in fact another DFV is added to the buffer and an asynchronous 'whenBound' handler
* is registered with the supplied DFV to update the one stored in the buffer.
*
* @param ref The DFV to add to the stream
*/
@Override
@SuppressWarnings("unchecked")
public final DataflowWriteChannel leftShift(final DataflowReadChannel ref) {
final DataflowVariable originalRef = retrieveForBind();
hookWheneverBoundListeners(originalRef);
ref.getValAsync(new MessageStream() {
private static final long serialVersionUID = -4966523895011173569L;
@Override
public MessageStream send(final Object message) {
originalRef.bind((T) message);
fireOnMessage((T) message);
return this;
}
});
return this;
}
/**
* Adds a DataflowVariable representing the passed in value to the buffer.
*
* @param value The value to bind to the head of the stream
*/
@Override
public final DataflowWriteChannel leftShift(final T value) {
hookWheneverBoundListeners(retrieveForBind()).bind(value);
fireOnMessage(value);
return this;
}
/**
* Adds a DataflowVariable representing the passed in value to the buffer.
*
* @param value The value to bind to the head of the stream
*/
@Override
public final void bind(final T value) {
hookWheneverBoundListeners(retrieveForBind()).bind(value);
fireOnMessage(value);
}
/**
* Hooks the registered when bound handlers to the supplied dataflow expression
*
* @param expr The expression to hook all the when bound listeners to
* @return The supplied expression handler to allow method chaining
*/
private DataflowExpression hookWheneverBoundListeners(final DataflowExpression expr) {
for (final MessageStream listener : wheneverBoundListeners) {
expr.whenBound(listener);
}
return expr;
}
/**
* Takes the first unsatisfied value request and binds a value on it.
* If there are no unsatisfied value requests, a new DFV is stored in the queue.
*
* @return The DFV to bind the value on
*/
private DataflowVariable retrieveForBind() {
return copyDFV(requests, queue);
}
private DataflowVariable copyDFV(final Queue> from, final Queue> to) {
DataflowVariable ref;
synchronized (queueLock) {
ref = from.poll();
if (ref == null) {
ref = createVariable();
to.offer(ref);
}
}
return ref;
}
/**
* Creates a new variable to perform the next data exchange
*
* @return The newly created DataflowVariable instance
*/
protected DataflowVariable createVariable() {
return new DataflowVariable();
}
/**
* Retrieves the value at the head of the buffer. Blocks until a value is available.
*
* @return The value bound to the DFV at the head of the stream
* @throws InterruptedException If the current thread is interrupted
*/
@Override
public final T getVal() throws InterruptedException {
return retrieveOrCreateVariable().getVal();
}
/**
* Retrieves the value at the head of the buffer. Blocks until a value is available.
*
* @param timeout The timeout value
* @param units Units for the timeout
* @return The value bound to the DFV at the head of the stream
* @throws InterruptedException If the current thread is interrupted
*/
@Override
public final T getVal(final long timeout, final TimeUnit units) throws InterruptedException {
final DataflowVariable variable = retrieveOrCreateVariable();
variable.getVal(timeout, units);
synchronized (queueLock) {
if (!variable.isBound()) {
requests.remove(variable);
return null;
}
}
return variable.getVal();
}
/**
* Retrieves the value at the head of the buffer. Returns null, if no value is available.
*
* @return The value bound to the DFV at the head of the stream or null
*/
@Override
public final DataflowExpression poll() {
synchronized (queueLock) {
final DataflowVariable df = queue.peek();
if (df != null && df.isBound()) {
queue.poll();
return df;
}
return null;
}
}
/**
* Asynchronously retrieves the value at the head of the buffer. Sends the actual value of the variable as a message
* back the the supplied actor once the value has been bound.
* The actor can perform other activities or release a thread back to the pool by calling react() waiting for the message
* with the value of the Dataflow Variable.
*
* @param callback The actor to notify when a value is bound
*/
@Override
public final void getValAsync(final MessageStream callback) {
getValAsync(null, callback);
}
/**
* Asynchronously retrieves the value at the head of the buffer. Sends a message back the the supplied actor / operator
* with a map holding the supplied index under the 'index' key and the actual value of the variable under
* the 'result' key once the value has been bound.
* The actor/operator can perform other activities or release a thread back to the pool by calling react() waiting for the message
* with the value of the Dataflow Variable.
*
* @param attachment An arbitrary value to identify operator channels and so match requests and replies
* @param callback The actor / operator to notify when a value is bound
*/
@Override
public final void getValAsync(final Object attachment, final MessageStream callback) {
retrieveOrCreateVariable().getValAsync(attachment, callback);
}
/**
* Schedule closure to be executed by pooled actor after data became available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled
*
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
*/
@Override
public final Promise rightShift(final Closure closure) {
return then(closure);
}
/**
* Schedule closure to be executed by pooled actor after the next data becomes available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled.
*
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
*/
@Override
public final void whenBound(final Closure closure) {
getValAsync(new DataCallback(closure, Dataflow.retrieveCurrentDFPGroup()));
}
/**
* Schedule closure to be executed by pooled actor after data becomes available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled.
*
* @param pool The thread pool to use for task scheduling for asynchronous message delivery
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
*/
@Override
public final void whenBound(final Pool pool, final Closure closure) {
getValAsync(new DataCallbackWithPool(pool, closure));
}
@Override
public void whenBound(final PGroup group, final Closure closure) {
getValAsync(new DataCallback(closure, group));
}
/**
* Send the next bound piece of data to the provided stream when it becomes available.
*
* @param stream stream where to send result
*/
@Override
public final void whenBound(final MessageStream stream) {
getValAsync(stream);
}
/**
* Schedule closure to be executed after data became available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled
*
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
* @return A promise for the results of the supplied closure. This allows for chaining of then() method calls.
*/
@Override
public final Promise then(final Closure closure) {
final DataflowVariable result = new DataflowVariable();
whenBound(new ThenMessagingRunnable(result, closure));
return result;
}
/**
* Schedule closure to be executed after data becomes available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled.
*
* @param pool The thread pool to use for task scheduling for asynchronous message delivery
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
* @return A promise for the results of the supplied closure. This allows for chaining of then() method calls.
*/
@Override
public Promise then(final Pool pool, final Closure closure) {
final DataflowVariable result = new DataflowVariable();
whenBound(pool, new ThenMessagingRunnable(result, closure));
return result;
}
/**
* Schedule closure to be executed after data becomes available.
* It is important to notice that even if the expression is already bound the execution of closure
* will not happen immediately but will be scheduled.
*
* @param group The PGroup to use for task scheduling for asynchronous message delivery
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
* @return A promise for the results of the supplied closure. This allows for chaining of then() method calls.
*/
@Override
public Promise then(final PGroup group, final Closure closure) {
final DataflowVariable result = new DataflowVariable();
whenBound(group, new ThenMessagingRunnable(result, closure));
return result;
}
/**
* Send all pieces of data bound in the future to the provided stream when it becomes available. *
*
* @param closure closure to execute when data becomes available. The closure should take at most one argument.
*/
@Override
public final void wheneverBound(final Closure closure) {
wheneverBoundListeners.add(new DataCallback(closure, Dataflow.retrieveCurrentDFPGroup()));
}
/**
* Send all pieces of data bound in the future to the provided stream when it becomes available.
*
* @param stream stream where to send result
*/
@Override
public final void wheneverBound(final MessageStream stream) {
wheneverBoundListeners.add(stream);
}
@Override
public final DataflowReadChannel chainWith(final Closure closure) {
return chainWith(Dataflow.retrieveCurrentDFPGroup(), closure);
}
@Override
public final DataflowReadChannel chainWith(final Pool pool, final Closure closure) {
return chainWith(new DefaultPGroup(pool), closure);
}
@Override
public DataflowReadChannel chainWith(final PGroup group, final Closure closure) {
final DataflowQueue result = new DataflowQueue();
group.operator(this, result, new ChainWithClosure(closure));
return result;
}
@Override
public final DataflowReadChannel chainWith(final Map params, final Closure closure) {
return chainWith(Dataflow.retrieveCurrentDFPGroup(), params, closure);
}
@Override
public final DataflowReadChannel chainWith(final Pool pool, final Map params, final Closure closure) {
return chainWith(new DefaultPGroup(pool), params, closure);
}
@Override
public DataflowReadChannel chainWith(final PGroup group, final Map params, final Closure closure) {
final DataflowQueue result = new DataflowQueue();
final Map parameters = new HashMap(params);
parameters.put("inputs", asList(this));
parameters.put("outputs", asList(asList(result)));
group.operator(parameters, new ChainWithClosure(closure));
return result;
}
@Override
public DataflowReadChannel or(final Closure closure) {
return chainWith(closure);
}
@Override
public DataflowReadChannel filter(final Closure closure) {
return chainWith(new FilterClosure(closure));
}
@Override
public DataflowReadChannel filter(final Pool pool, final Closure closure) {
return chainWith(pool, new FilterClosure(closure));
}
@Override
public DataflowReadChannel filter(final PGroup group, final Closure closure) {
return chainWith(group, new FilterClosure(closure));
}
@Override
public DataflowReadChannel filter(final Map params, final Closure closure) {
return chainWith(params, new FilterClosure(closure));
}
@Override
public DataflowReadChannel filter(final Pool pool, final Map params, final Closure closure) {
return chainWith(pool, params, new FilterClosure(closure));
}
@Override
public DataflowReadChannel filter(final PGroup group, final Map params, final Closure closure) {
return chainWith(group, params, new FilterClosure(closure));
}
@Override
public void into(final DataflowWriteChannel target) {
into(Dataflow.retrieveCurrentDFPGroup(), target);
}
@Override
public void into(final Pool pool, final DataflowWriteChannel target) {
into(new DefaultPGroup(pool), target);
}
@Override
public void into(final PGroup group, final DataflowWriteChannel target) {
group.operator(this, target, new ChainWithClosure(new CopyChannelsClosure()));
}
@Override
public void into(final Map params, final DataflowWriteChannel target) {
into(Dataflow.retrieveCurrentDFPGroup(), params, target);
}
@Override
public void into(final Pool pool, final Map params, final DataflowWriteChannel target) {
into(new DefaultPGroup(pool), params, target);
}
@Override
public void into(final PGroup group, final Map params, final DataflowWriteChannel target) {
final Map parameters = new HashMap(params);
parameters.put("inputs", asList(this));
parameters.put("outputs", asList(asList(target)));
group.operator(parameters, new ChainWithClosure(new CopyChannelsClosure()));
}
@Override
public void or(final DataflowWriteChannel target) {
into(target);
}
@Override
public void split(final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(Dataflow.retrieveCurrentDFPGroup(), target1, target2);
}
@Override
public void split(final Pool pool, final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(new DefaultPGroup(pool), target1, target2);
}
@Override
public void split(final PGroup group, final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(group, asList(target1, target2));
}
@Override
public void split(final List> targets) {
split(Dataflow.retrieveCurrentDFPGroup(), targets);
}
@Override
public void split(final Pool pool, final List> targets) {
split(new DefaultPGroup(pool), targets);
}
@Override
public void split(final PGroup group, final List> targets) {
group.operator(asList(this), targets, new ChainWithClosure(new CopyChannelsClosure()));
}
@Override
public void split(final Map params, final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(Dataflow.retrieveCurrentDFPGroup(), params, target1, target2);
}
@Override
public void split(final Pool pool, final Map params, final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(new DefaultPGroup(pool), params, target1, target2);
}
@Override
public void split(final PGroup group, final Map params, final DataflowWriteChannel target1, final DataflowWriteChannel target2) {
split(group, params, asList(target1, target2));
}
@Override
public void split(final Map params, final List> targets) {
split(Dataflow.retrieveCurrentDFPGroup(), params, targets);
}
@Override
public void split(final Pool pool, final Map params, final List> targets) {
split(new DefaultPGroup(pool), params, targets);
}
@Override
public void split(final PGroup group, final Map params, final List> targets) {
final Map parameters = new HashMap(params);
parameters.put("inputs", asList(this));
parameters.put("outputs", asList(asList(targets)));
group.operator(parameters, new ChainWithClosure(new CopyChannelsClosure()));
}
@Override
public DataflowReadChannel tap(final DataflowWriteChannel target) {
return tap(Dataflow.retrieveCurrentDFPGroup(), target);
}
@Override
public DataflowReadChannel tap(final Pool pool, final DataflowWriteChannel target) {
return tap(new DefaultPGroup(pool), target);
}
@Override
public DataflowReadChannel tap(final PGroup group, final DataflowWriteChannel target) {
final DataflowQueue result = new DataflowQueue();
group.operator(asList(this), asList(result, target), new ChainWithClosure(new CopyChannelsClosure()));
return result;
}
@Override
public DataflowReadChannel tap(final Map params, final DataflowWriteChannel target) {
return tap(Dataflow.retrieveCurrentDFPGroup(), params, target);
}
@Override
public DataflowReadChannel tap(final Pool pool, final Map params, final DataflowWriteChannel target) {
return tap(new DefaultPGroup(pool), params, target);
}
@Override
public DataflowReadChannel tap(final PGroup group, final Map params, final DataflowWriteChannel target) {
final DataflowQueue result = new DataflowQueue();
final Map parameters = new HashMap(params);
parameters.put("inputs", asList(this));
parameters.put("outputs", asList(asList(result, target)));
group.operator(parameters, new ChainWithClosure(new CopyChannelsClosure()));
return result;
}
@Override
public DataflowReadChannel merge(final DataflowReadChannel