com.gs.collections.impl.lazy.parallel.AbstractParallelIterable Maven / Gradle / Ivy
Show all versions of gs-collections Show documentation
/*
* Copyright 2014 Goldman Sachs.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.gs.collections.impl.lazy.parallel;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.gs.collections.api.LazyIterable;
import com.gs.collections.api.ParallelIterable;
import com.gs.collections.api.annotation.Beta;
import com.gs.collections.api.bag.MutableBag;
import com.gs.collections.api.bag.sorted.MutableSortedBag;
import com.gs.collections.api.block.function.Function;
import com.gs.collections.api.block.function.Function0;
import com.gs.collections.api.block.function.Function2;
import com.gs.collections.api.block.function.primitive.DoubleFunction;
import com.gs.collections.api.block.function.primitive.FloatFunction;
import com.gs.collections.api.block.function.primitive.IntFunction;
import com.gs.collections.api.block.function.primitive.LongFunction;
import com.gs.collections.api.block.predicate.Predicate;
import com.gs.collections.api.block.predicate.Predicate2;
import com.gs.collections.api.block.procedure.Procedure;
import com.gs.collections.api.block.procedure.Procedure2;
import com.gs.collections.api.list.MutableList;
import com.gs.collections.api.map.MapIterable;
import com.gs.collections.api.map.MutableMap;
import com.gs.collections.api.map.sorted.MutableSortedMap;
import com.gs.collections.api.set.MutableSet;
import com.gs.collections.api.set.sorted.MutableSortedSet;
import com.gs.collections.impl.Counter;
import com.gs.collections.impl.bag.mutable.HashBag;
import com.gs.collections.impl.bag.sorted.mutable.TreeBag;
import com.gs.collections.impl.block.factory.Comparators;
import com.gs.collections.impl.block.factory.Functions2;
import com.gs.collections.impl.block.factory.Predicates;
import com.gs.collections.impl.block.factory.Procedures;
import com.gs.collections.impl.block.procedure.CollectionAddProcedure;
import com.gs.collections.impl.block.procedure.DoubleSumResultHolder;
import com.gs.collections.impl.block.procedure.MapCollectProcedure;
import com.gs.collections.impl.block.procedure.MutatingAggregationProcedure;
import com.gs.collections.impl.block.procedure.NonMutatingAggregationProcedure;
import com.gs.collections.impl.block.procedure.checked.CheckedProcedure2;
import com.gs.collections.impl.list.mutable.CompositeFastList;
import com.gs.collections.impl.list.mutable.FastList;
import com.gs.collections.impl.map.mutable.ConcurrentHashMap;
import com.gs.collections.impl.map.mutable.ConcurrentHashMapUnsafe;
import com.gs.collections.impl.map.mutable.UnifiedMap;
import com.gs.collections.impl.map.sorted.mutable.TreeSortedMap;
import com.gs.collections.impl.set.mutable.SetAdapter;
import com.gs.collections.impl.set.mutable.UnifiedSet;
import com.gs.collections.impl.set.sorted.mutable.TreeSortedSet;
@Beta
public abstract class AbstractParallelIterable> implements ParallelIterable
{
protected static void forEach(final AbstractParallelIterable> parallelIterable, final Procedure procedure)
{
LazyIterable> futures = parallelIterable.split().collect(new Function, Future>()
{
public Future valueOf(final RootBatch chunk)
{
return parallelIterable.getExecutorService().submit(new Runnable()
{
public void run()
{
chunk.forEach(procedure);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
future.get();
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
protected static boolean anySatisfy(AbstractParallelIterable> parallelIterable, final Predicate predicate)
{
final CompletionService completionService = new ExecutorCompletionService(parallelIterable.getExecutorService());
MutableSet> futures = parallelIterable.split().collect(new Function, Future>()
{
public Future valueOf(final RootBatch batch)
{
return completionService.submit(new Callable()
{
public Boolean call()
{
return batch.anySatisfy(predicate);
}
});
}
}, UnifiedSet.>newSet());
while (futures.notEmpty())
{
try
{
Future future = completionService.take();
if (future.get())
{
for (Future eachFuture : futures)
{
eachFuture.cancel(true);
}
return true;
}
futures.remove(future);
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return false;
}
protected static boolean allSatisfy(AbstractParallelIterable> parallelIterable, final Predicate predicate)
{
final CompletionService completionService = new ExecutorCompletionService(parallelIterable.getExecutorService());
MutableSet> futures = parallelIterable.split().collect(new Function, Future>()
{
public Future valueOf(final RootBatch batch)
{
return completionService.submit(new Callable()
{
public Boolean call()
{
return batch.allSatisfy(predicate);
}
});
}
}, UnifiedSet.>newSet());
while (futures.notEmpty())
{
try
{
Future future = completionService.take();
if (!future.get())
{
for (Future eachFuture : futures)
{
eachFuture.cancel(true);
}
return false;
}
futures.remove(future);
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return true;
}
protected static T detect(final AbstractParallelIterable> parallelIterable, final Predicate predicate)
{
LazyIterable> chunks = parallelIterable.split();
LazyIterable> futures = chunks.collect(new Function, Future>()
{
public Future valueOf(final RootBatch chunk)
{
return parallelIterable.getExecutorService().submit(new Callable()
{
public T call()
{
return chunk.detect(predicate);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
T eachResult = future.get();
if (eachResult != null)
{
for (Future eachFutureToCancel : futuresList)
{
eachFutureToCancel.cancel(true);
}
return eachResult;
}
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return null;
}
public abstract ExecutorService getExecutorService();
public abstract int getBatchSize();
public abstract LazyIterable split();
protected abstract boolean isOrdered();
protected void collectCombine(Function, V> function, Procedure2 combineProcedure, S state)
{
if (this.isOrdered())
{
this.collectCombineOrdered(function, combineProcedure, state);
}
else
{
this.collectCombineUnordered(function, combineProcedure, state);
}
}
private void collectCombineOrdered(final Function, V> function, Procedure2 combineProcedure, S state)
{
LazyIterable> chunks = this.split();
LazyIterable> futures = chunks.collect(new Function, Future>()
{
public Future valueOf(final Batch chunk)
{
return AbstractParallelIterable.this.getExecutorService().submit(new Callable()
{
public V call()
{
return function.valueOf(chunk);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
combineProcedure.value(state, future.get());
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
private void collectCombineUnordered(final Function, V> function, Procedure2 combineProcedure, S state)
{
LazyIterable> chunks = this.split();
MutableList> callables = chunks.collect(new Function, Callable>()
{
public Callable valueOf(final Batch chunk)
{
return new Callable()
{
public V call()
{
return function.valueOf(chunk);
}
};
}
}).toList();
final ExecutorCompletionService completionService = new ExecutorCompletionService(this.getExecutorService());
callables.forEach(new Procedure>()
{
public void value(Callable callable)
{
completionService.submit(callable);
}
});
int numTasks = callables.size();
while (numTasks > 0)
{
try
{
Future future = completionService.take();
combineProcedure.value(state, future.get());
numTasks--;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
private T collectReduce(Function, T> map, Function2 function2)
{
return this.isOrdered()
? this.collectReduceOrdered(map, function2)
: this.collectReduceUnordered(map, function2);
}
private T collectReduceOrdered(final Function, T> map, Function2 function2)
{
LazyIterable> chunks = this.split();
LazyIterable> futures = chunks.collect(new Function, Future>()
{
public Future valueOf(final Batch chunk)
{
return AbstractParallelIterable.this.getExecutorService().submit(new Callable()
{
public T call()
{
return map.valueOf(chunk);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
T result = futuresList.getFirst().get();
for (int i = 1; i < futuresList.size(); i++)
{
T next = futuresList.get(i).get();
if (next != null)
{
if (result == null)
{
result = next;
}
else
{
result = function2.value(result, next);
}
}
}
if (result == null)
{
throw new NoSuchElementException();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
if (e.getCause() instanceof NullPointerException)
{
throw (NullPointerException) e.getCause();
}
throw new RuntimeException(e);
}
}
private T collectReduceUnordered(final Function, T> map, Function2 function2)
{
LazyIterable> chunks = this.split();
MutableList> callables = chunks.collect(new Function, Callable>()
{
public Callable valueOf(final Batch chunk)
{
return new Callable()
{
public T call()
{
return map.valueOf(chunk);
}
};
}
}).toList();
final ExecutorCompletionService completionService = new ExecutorCompletionService(this.getExecutorService());
callables.forEach(new Procedure>()
{
public void value(Callable callable)
{
completionService.submit(callable);
}
});
try
{
T result = completionService.take().get();
int numTasks = callables.size() - 1;
while (numTasks > 0)
{
T next = completionService.take().get();
if (next != null)
{
if (result == null)
{
result = next;
}
else
{
result = function2.value(result, next);
}
}
numTasks--;
}
if (result == null)
{
throw new NoSuchElementException();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
if (e.getCause() instanceof NullPointerException)
{
throw (NullPointerException) e.getCause();
}
throw new RuntimeException(e);
}
}
@Override
public String toString()
{
return this.makeString("[", ", ", "]");
}
public String makeString()
{
return this.makeString(", ");
}
public String makeString(String separator)
{
return this.makeString("", separator, "");
}
public String makeString(String start, String separator, String end)
{
Appendable stringBuilder = new StringBuilder();
this.appendString(stringBuilder, start, separator, end);
return stringBuilder.toString();
}
public void appendString(Appendable appendable)
{
this.appendString(appendable, ", ");
}
public void appendString(Appendable appendable, String separator)
{
this.appendString(appendable, "", separator, "");
}
public void appendString(final Appendable appendable, String start, final String separator, String end)
{
try
{
appendable.append(start);
Function, String> map = new Function, String>()
{
public String valueOf(Batch batch)
{
return batch.makeString(separator);
}
};
Procedure2 reduce = new CheckedProcedure2()
{
private boolean first = true;
public void safeValue(Appendable accumulator, String each) throws IOException
{
if ("".equals(each))
{
return;
}
if (this.first)
{
this.first = false;
}
else
{
appendable.append(separator);
}
appendable.append(each);
}
};
this.collectCombine(map, reduce, appendable);
appendable.append(end);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
public void forEachWith(Procedure2 procedure, P parameter)
{
this.forEach(Procedures.bind(procedure, parameter));
}
public
boolean anySatisfyWith(Predicate2 predicate, P parameter)
{
return this.anySatisfy(Predicates.bind(predicate, parameter));
}
public
boolean allSatisfyWith(Predicate2 predicate, P parameter)
{
return this.allSatisfy(Predicates.bind(predicate, parameter));
}
public boolean noneSatisfy(Predicate predicate)
{
return this.allSatisfy(Predicates.not(predicate));
}
public
boolean noneSatisfyWith(Predicate2 predicate, P parameter)
{
return this.noneSatisfy(Predicates.bind(predicate, parameter));
}
public
T detectWith(Predicate2 predicate, P parameter)
{
return this.detect(Predicates.bind(predicate, parameter));
}
public T detectIfNone(Predicate predicate, Function0 function)
{
T result = this.detect(predicate);
return result == null ? function.value() : result;
}
public
T detectWithIfNone(Predicate2 predicate, P parameter, Function0 function)
{
return this.detectIfNone(Predicates.bind(predicate, parameter), function);
}
public Object[] toArray()
{
throw new UnsupportedOperationException(this.getClass().getSimpleName() + ".toArray() not implemented yet");
}
public E[] toArray(E[] array)
{
throw new UnsupportedOperationException(this.getClass().getSimpleName() + ".toArray() not implemented yet");
}
public MutableList toList()
{
Function, FastList> map = new Function, FastList>()
{
public FastList valueOf(Batch batch)
{
FastList list = FastList.newList();
batch.forEach(CollectionAddProcedure.on(list));
return list;
}
};
Procedure2, FastList> reduce = new Procedure2, FastList>()
{
public void value(MutableList accumulator, FastList each)
{
accumulator.addAll(each);
}
};
MutableList state = new CompositeFastList();
this.collectCombine(map, reduce, state);
return state;
}
public MutableList toSortedList()
{
return this.toList().toSortedList();
}
public MutableList toSortedList(Comparator comparator)
{
return this.toList().toSortedList(comparator);
}
public > MutableList toSortedListBy(Function function)
{
return this.toSortedList(Comparators.byFunction(function));
}
public MutableSet toSet()
{
ConcurrentHashMapUnsafe map = ConcurrentHashMapUnsafe.newMap();
Set result = Collections.newSetFromMap(map);
this.forEach(CollectionAddProcedure.on(result));
return SetAdapter.adapt(map.keySet());
}
public MutableSortedSet toSortedSet()
{
MutableSortedSet result = TreeSortedSet.newSet().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
public > MutableSortedSet toSortedSetBy(Function function)
{
return this.toSortedSet(Comparators.byFunction(function));
}
public MutableBag toBag()
{
MutableBag result = HashBag.newBag().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
public MutableSortedBag toSortedBag()
{
MutableSortedBag result = TreeBag.newBag().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
public MutableSortedBag toSortedBag(Comparator comparator)
{
MutableSortedBag result = TreeBag.newBag(comparator).asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
public > MutableSortedBag toSortedBagBy(Function function)
{
return this.toSortedBag(Comparators.byFunction(function));
}
public MutableSortedSet toSortedSet(Comparator comparator)
{
MutableSortedSet result = TreeSortedSet.newSet(comparator).asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
public MutableMap toMap(
Function keyFunction,
Function valueFunction)
{
MutableMap map = UnifiedMap.newMap().asSynchronized();
this.forEach(new MapCollectProcedure(map, keyFunction, valueFunction));
return map;
}
public MutableSortedMap toSortedMap(
Function keyFunction,
Function valueFunction)
{
MutableSortedMap sortedMap = TreeSortedMap.newMap().asSynchronized();
this.forEach(new MapCollectProcedure(sortedMap, keyFunction, valueFunction));
return sortedMap;
}
public MutableSortedMap toSortedMap(Comparator comparator,
Function keyFunction,
Function valueFunction)
{
MutableSortedMap sortedMap = TreeSortedMap.newMap(comparator).asSynchronized();
this.forEach(new MapCollectProcedure(sortedMap, keyFunction, valueFunction));
return sortedMap;
}
public MapIterable aggregateBy(
Function groupBy,
Function0 zeroValueFactory,
Function2 nonMutatingAggregator)
{
MutableMap map = ConcurrentHashMapUnsafe.newMap();
this.forEach(new NonMutatingAggregationProcedure(map, groupBy, zeroValueFactory, nonMutatingAggregator));
return map;
}
public MapIterable aggregateInPlaceBy(
Function groupBy,
Function0 zeroValueFactory,
Procedure2 mutatingAggregator)
{
MutableMap map = ConcurrentHashMapUnsafe.newMap();
this.forEach(new MutatingAggregationProcedure(map, groupBy, zeroValueFactory, mutatingAggregator));
return map;
}
public int count(final Predicate predicate)
{
Function, Integer> map = new Function, Integer>()
{
public Integer valueOf(Batch batch)
{
return batch.count(predicate);
}
};
Procedure2 combineProcedure = new Procedure2()
{
public void value(Counter counter, Integer eachCount)
{
counter.add(eachCount);
}
};
Counter state = new Counter();
this.collectCombineUnordered(map, combineProcedure, state);
return state.getCount();
}
public int countWith(Predicate2 predicate, P parameter)
{
return this.count(Predicates.bind(predicate, parameter));
}
public T min(final Comparator comparator)
{
Function, T> map = new Function, T>()
{
public T valueOf(Batch batch)
{
return batch.min(comparator);
}
};
return this.collectReduce(map, Functions2.min(comparator));
}
public T max(final Comparator comparator)
{
Function, T> map = new Function, T>()
{
public T valueOf(Batch batch)
{
return batch.max(comparator);
}
};
return this.collectReduce(map, Functions2.max(comparator));
}
public T min()
{
return this.min(Comparators.naturalOrder());
}
public T max()
{
return this.max(Comparators.naturalOrder());
}
public > T minBy(final Function function)
{
Function, T> map = new Function, T>()
{
public T valueOf(Batch batch)
{
return batch.minBy(function);
}
};
return this.collectReduce(map, Functions2.minBy(function));
}
public > T maxBy(final Function function)
{
Function, T> map = new Function, T>()
{
public T valueOf(Batch batch)
{
return batch.maxBy(function);
}
};
return this.collectReduce(map, Functions2.maxBy(function));
}
public long sumOfInt(final IntFunction function)
{
LongFunction> map = new LongFunction>()
{
public long longValueOf(Batch batch)
{
return batch.sumOfInt(function);
}
};
return this.sumOfLongOrdered(map);
}
public double sumOfFloat(final FloatFunction function)
{
Function, DoubleSumResultHolder> map = new Function, DoubleSumResultHolder>()
{
public DoubleSumResultHolder valueOf(Batch batch)
{
return batch.sumOfFloat(function);
}
};
return this.sumOfDoubleOrdered(map);
}
public long sumOfLong(final LongFunction function)
{
LongFunction> map = new LongFunction>()
{
public long longValueOf(Batch batch)
{
return batch.sumOfLong(function);
}
};
return this.sumOfLongOrdered(map);
}
public double sumOfDouble(final DoubleFunction function)
{
Function, DoubleSumResultHolder> map = new Function, DoubleSumResultHolder>()
{
public DoubleSumResultHolder valueOf(Batch batch)
{
return batch.sumOfDouble(function);
}
};
return this.sumOfDoubleOrdered(map);
}
private long sumOfLongOrdered(final LongFunction> map)
{
LazyIterable> chunks = this.split();
LazyIterable> futures = chunks.collect(new Function, Future>()
{
public Future valueOf(final Batch chunk)
{
return AbstractParallelIterable.this.getExecutorService().submit(new Callable()
{
public Long call()
{
return map.longValueOf(chunk);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
long result = 0;
for (int i = 0; i < futuresList.size(); i++)
{
result += futuresList.get(i).get();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
private double sumOfDoubleOrdered(final Function, DoubleSumResultHolder> map)
{
LazyIterable> chunks = this.split();
LazyIterable> futures = chunks.collect(new Function, Future>()
{
public Future valueOf(final Batch chunk)
{
return AbstractParallelIterable.this.getExecutorService().submit(new Callable()
{
public DoubleSumResultHolder call()
{
return map.valueOf(chunk);
}
});
}
});
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
double sum = 0.0d;
double compensation = 0.0d;
for (int i = 0; i < futuresList.size(); i++)
{
compensation += futuresList.get(i).get().getCompensation();
double adjustedValue = futuresList.get(i).get().getResult() - compensation;
double nextSum = sum + adjustedValue;
compensation = nextSum - sum - adjustedValue;
sum = nextSum;
}
return sum;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
public MapIterable groupByUniqueKey(final Function function)
{
final MutableMap result = ConcurrentHashMap.newMap();
this.forEach(new Procedure()
{
public void value(T value)
{
V key = function.valueOf(value);
if (result.put(key, value) != null)
{
throw new IllegalStateException("Key " + key + " already exists in map!");
}
}
});
return result;
}
}