org.eclipse.collections.impl.lazy.parallel.AbstractParallelIterable Maven / Gradle / Ivy
Show all versions of eclipse-collections Show documentation
/*
* Copyright (c) 2022 Goldman Sachs and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* and Eclipse Distribution License v. 1.0 which accompany this distribution.
* The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
* and the Eclipse Distribution License is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*/
package org.eclipse.collections.impl.lazy.parallel;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.eclipse.collections.api.LazyIterable;
import org.eclipse.collections.api.ParallelIterable;
import org.eclipse.collections.api.annotation.Beta;
import org.eclipse.collections.api.bag.MutableBag;
import org.eclipse.collections.api.bag.sorted.MutableSortedBag;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.block.function.Function0;
import org.eclipse.collections.api.block.function.Function2;
import org.eclipse.collections.api.block.function.primitive.DoubleFunction;
import org.eclipse.collections.api.block.function.primitive.FloatFunction;
import org.eclipse.collections.api.block.function.primitive.IntFunction;
import org.eclipse.collections.api.block.function.primitive.LongFunction;
import org.eclipse.collections.api.block.predicate.Predicate;
import org.eclipse.collections.api.block.predicate.Predicate2;
import org.eclipse.collections.api.block.procedure.Procedure;
import org.eclipse.collections.api.block.procedure.Procedure2;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.factory.SortedBags;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.map.MapIterable;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.map.sorted.MutableSortedMap;
import org.eclipse.collections.api.set.MutableSet;
import org.eclipse.collections.api.set.sorted.MutableSortedSet;
import org.eclipse.collections.impl.Counter;
import org.eclipse.collections.impl.bag.mutable.HashBag;
import org.eclipse.collections.impl.bag.sorted.mutable.TreeBag;
import org.eclipse.collections.impl.block.factory.Comparators;
import org.eclipse.collections.impl.block.factory.Functions2;
import org.eclipse.collections.impl.block.factory.Predicates;
import org.eclipse.collections.impl.block.factory.Procedures;
import org.eclipse.collections.impl.block.procedure.CollectionAddProcedure;
import org.eclipse.collections.impl.block.procedure.DoubleSumResultHolder;
import org.eclipse.collections.impl.block.procedure.MapCollectProcedure;
import org.eclipse.collections.impl.block.procedure.MutatingAggregationProcedure;
import org.eclipse.collections.impl.block.procedure.NonMutatingAggregationProcedure;
import org.eclipse.collections.impl.block.procedure.checked.CheckedProcedure2;
import org.eclipse.collections.impl.list.mutable.CompositeFastList;
import org.eclipse.collections.impl.list.mutable.FastList;
import org.eclipse.collections.impl.map.mutable.ConcurrentHashMap;
import org.eclipse.collections.impl.map.mutable.ConcurrentHashMapUnsafe;
import org.eclipse.collections.impl.map.mutable.UnifiedMap;
import org.eclipse.collections.impl.map.sorted.mutable.TreeSortedMap;
import org.eclipse.collections.impl.set.mutable.SetAdapter;
import org.eclipse.collections.impl.set.sorted.mutable.TreeSortedSet;
@Beta
public abstract class AbstractParallelIterable> implements ParallelIterable
{
protected static void forEach(AbstractParallelIterable> parallelIterable, Procedure procedure)
{
LazyIterable> futures =
parallelIterable.split().collect(chunk -> parallelIterable.getExecutorService().submit(() -> chunk.forEach(procedure)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
future.get();
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
protected static boolean anySatisfy(AbstractParallelIterable> parallelIterable, Predicate predicate)
{
CompletionService completionService = new ExecutorCompletionService<>(parallelIterable.getExecutorService());
MutableSet> futures =
parallelIterable.split().collect(batch -> completionService.submit(() -> batch.anySatisfy(predicate)), Sets.mutable.empty());
while (futures.notEmpty())
{
try
{
Future future = completionService.take();
if (future.get())
{
for (Future eachFuture : futures)
{
eachFuture.cancel(true);
}
return true;
}
futures.remove(future);
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return false;
}
protected static boolean allSatisfy(AbstractParallelIterable> parallelIterable, Predicate predicate)
{
CompletionService completionService = new ExecutorCompletionService<>(parallelIterable.getExecutorService());
MutableSet> futures =
parallelIterable.split().collect(batch -> completionService.submit(() -> batch.allSatisfy(predicate)), Sets.mutable.empty());
while (futures.notEmpty())
{
try
{
Future future = completionService.take();
if (!future.get())
{
for (Future eachFuture : futures)
{
eachFuture.cancel(true);
}
return false;
}
futures.remove(future);
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return true;
}
protected static T detect(AbstractParallelIterable> parallelIterable, Predicate predicate)
{
LazyIterable> chunks = parallelIterable.split();
LazyIterable> futures =
chunks.collect(chunk -> parallelIterable.getExecutorService().submit(() -> chunk.detect(predicate)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
T eachResult = future.get();
if (eachResult != null)
{
for (Future eachFutureToCancel : futuresList)
{
eachFutureToCancel.cancel(true);
}
return eachResult;
}
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
return null;
}
public abstract ExecutorService getExecutorService();
public abstract int getBatchSize();
public abstract LazyIterable split();
protected abstract boolean isOrdered();
protected void collectCombine(Function, V> function, Procedure2 combineProcedure, S state)
{
if (this.isOrdered())
{
this.collectCombineOrdered(function, combineProcedure, state);
}
else
{
this.collectCombineUnordered(function, combineProcedure, state);
}
}
private void collectCombineOrdered(Function, V> function, Procedure2 combineProcedure, S state)
{
LazyIterable> chunks = this.split();
LazyIterable> futures =
chunks.collect(chunk -> this.getExecutorService().submit(() -> function.valueOf(chunk)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
for (Future future : futuresList)
{
try
{
combineProcedure.value(state, future.get());
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
private void collectCombineUnordered(Function, V> function, Procedure2 combineProcedure, S state)
{
LazyIterable> chunks = this.split();
MutableList> callables = chunks.collect((Function, Callable>) chunk -> () -> function.valueOf(chunk)).toList();
ExecutorCompletionService completionService = new ExecutorCompletionService<>(this.getExecutorService());
callables.each(completionService::submit);
int numTasks = callables.size();
while (numTasks > 0)
{
try
{
Future future = completionService.take();
combineProcedure.value(state, future.get());
numTasks--;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
}
private T collectReduce(Function, T> map, Function2 function2)
{
return this.isOrdered()
? this.collectReduceOrdered(map, function2)
: this.collectReduceUnordered(map, function2);
}
private T collectReduceOrdered(Function, T> map, Function2 function2)
{
LazyIterable> chunks = this.split();
LazyIterable> futures = chunks.collect(chunk -> this.getExecutorService().submit(() -> map.valueOf(chunk)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
T result = futuresList.getFirst().get();
for (int i = 1; i < futuresList.size(); i++)
{
T next = futuresList.get(i).get();
if (next != null)
{
if (result == null)
{
result = next;
}
else
{
result = function2.value(result, next);
}
}
}
if (result == null)
{
throw new NoSuchElementException();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
if (e.getCause() instanceof NullPointerException)
{
throw (NullPointerException) e.getCause();
}
throw new RuntimeException(e);
}
}
private T collectReduceUnordered(Function, T> map, Function2 function2)
{
LazyIterable> chunks = this.split();
MutableList> callables =
chunks.collect((Function, Callable>) chunk -> () -> map.valueOf(chunk)).toList();
ExecutorCompletionService completionService = new ExecutorCompletionService<>(this.getExecutorService());
callables.each(completionService::submit);
try
{
T result = completionService.take().get();
int numTasks = callables.size() - 1;
while (numTasks > 0)
{
T next = completionService.take().get();
if (next != null)
{
if (result == null)
{
result = next;
}
else
{
result = function2.value(result, next);
}
}
numTasks--;
}
if (result == null)
{
throw new NoSuchElementException();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
if (e.getCause() instanceof NullPointerException)
{
throw (NullPointerException) e.getCause();
}
throw new RuntimeException(e);
}
}
@Override
public String toString()
{
return this.makeString("[", ", ", "]");
}
@Override
public void appendString(Appendable appendable, String start, String separator, String end)
{
try
{
appendable.append(start);
Function, String> map = batch -> batch.makeString(separator);
Procedure2 reduce = new CheckedProcedure2()
{
private boolean first = true;
public void safeValue(Appendable accumulator, String each) throws IOException
{
if ("".equals(each))
{
return;
}
if (this.first)
{
this.first = false;
}
else
{
appendable.append(separator);
}
appendable.append(each);
}
};
this.collectCombine(map, reduce, appendable);
appendable.append(end);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
@Override
public void forEachWith(Procedure2 procedure, P parameter)
{
this.forEach(Procedures.bind(procedure, parameter));
}
@Override
public
boolean anySatisfyWith(Predicate2 predicate, P parameter)
{
return this.anySatisfy(Predicates.bind(predicate, parameter));
}
@Override
public
boolean allSatisfyWith(Predicate2 predicate, P parameter)
{
return this.allSatisfy(Predicates.bind(predicate, parameter));
}
@Override
public boolean noneSatisfy(Predicate predicate)
{
return this.allSatisfy(Predicates.not(predicate));
}
@Override
public
boolean noneSatisfyWith(Predicate2 predicate, P parameter)
{
return this.noneSatisfy(Predicates.bind(predicate, parameter));
}
@Override
public
T detectWith(Predicate2 predicate, P parameter)
{
return this.detect(Predicates.bind(predicate, parameter));
}
@Override
public T detectIfNone(Predicate predicate, Function0 function)
{
T result = this.detect(predicate);
return result == null ? function.value() : result;
}
@Override
public
T detectWithIfNone(Predicate2 predicate, P parameter, Function0 function)
{
return this.detectIfNone(Predicates.bind(predicate, parameter), function);
}
@Override
public E[] toArray(E[] array)
{
throw new UnsupportedOperationException(this.getClass().getSimpleName() + ".toArray() not implemented yet");
}
@Override
public MutableList toList()
{
Function, FastList> map = batch -> {
FastList list = FastList.newList();
batch.forEach(CollectionAddProcedure.on(list));
return list;
};
MutableList state = new CompositeFastList<>();
this.collectCombine(map, MutableList::addAll, state);
return state;
}
@Override
public MutableList toSortedList(Comparator comparator)
{
return this.toList().toSortedList(comparator);
}
@Override
public > MutableList toSortedListBy(Function function)
{
return this.toSortedList(Comparators.byFunction(function));
}
@Override
public MutableSet toSet()
{
ConcurrentHashMapUnsafe map = ConcurrentHashMapUnsafe.newMap();
Set result = Collections.newSetFromMap(map);
this.forEach(CollectionAddProcedure.on(result));
return SetAdapter.adapt(map.keySet());
}
@Override
public MutableSortedSet toSortedSet()
{
MutableSortedSet result = TreeSortedSet.newSet().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
@Override
public > MutableSortedSet toSortedSetBy(Function function)
{
return this.toSortedSet(Comparators.byFunction(function));
}
@Override
public MutableBag toBag()
{
MutableBag result = HashBag.newBag().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
@Override
public MutableSortedBag toSortedBag()
{
MutableSortedBag result = TreeBag.newBag().asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
@Override
public MutableSortedBag toSortedBag(Comparator comparator)
{
MutableSortedBag result = SortedBags.mutable.empty(comparator);
result = result.asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
@Override
public > MutableSortedBag toSortedBagBy(Function function)
{
return this.toSortedBag(Comparators.byFunction(function));
}
@Override
public MutableSortedSet toSortedSet(Comparator comparator)
{
MutableSortedSet result = TreeSortedSet.newSet(comparator);
result = result.asSynchronized();
this.forEach(CollectionAddProcedure.on(result));
return result;
}
@Override
public MutableMap toMap(
Function keyFunction,
Function valueFunction)
{
MutableMap map = UnifiedMap.newMap().asSynchronized();
this.forEach(new MapCollectProcedure<>(map, keyFunction, valueFunction));
return map;
}
@Override
public MutableSortedMap toSortedMap(
Function keyFunction,
Function valueFunction)
{
MutableSortedMap sortedMap = TreeSortedMap.newMap().asSynchronized();
this.forEach(new MapCollectProcedure<>(sortedMap, keyFunction, valueFunction));
return sortedMap;
}
@Override
public MutableSortedMap toSortedMap(
Comparator comparator,
Function keyFunction,
Function valueFunction)
{
MutableSortedMap sortedMap = TreeSortedMap.newMap(comparator).asSynchronized();
this.forEach(new MapCollectProcedure<>(sortedMap, keyFunction, valueFunction));
return sortedMap;
}
@Override
public MapIterable aggregateBy(
Function groupBy,
Function0 zeroValueFactory,
Function2 nonMutatingAggregator)
{
MutableMap map = ConcurrentHashMapUnsafe.newMap();
this.forEach(new NonMutatingAggregationProcedure<>(map, groupBy, zeroValueFactory, nonMutatingAggregator));
return map;
}
@Override
public MapIterable aggregateInPlaceBy(
Function groupBy,
Function0 zeroValueFactory,
Procedure2 mutatingAggregator)
{
MutableMap map = ConcurrentHashMapUnsafe.newMap();
this.forEach(new MutatingAggregationProcedure<>(map, groupBy, zeroValueFactory, mutatingAggregator));
return map;
}
@Override
public int count(Predicate predicate)
{
Function, Integer> map = batch -> batch.count(predicate);
Counter state = new Counter();
this.collectCombineUnordered(map, Counter::add, state);
return state.getCount();
}
@Override
public int countWith(Predicate2 predicate, P parameter)
{
return this.count(Predicates.bind(predicate, parameter));
}
@Override
public T min(Comparator comparator)
{
Function, T> map = batch -> batch.min(comparator);
return this.collectReduce(map, Functions2.min(comparator));
}
@Override
public T max(Comparator comparator)
{
Function, T> map = batch -> batch.max(comparator);
return this.collectReduce(map, Functions2.max(comparator));
}
@Override
public T min()
{
return this.min(Comparators.naturalOrder());
}
@Override
public T max()
{
return this.max(Comparators.naturalOrder());
}
@Override
public > T minBy(Function function)
{
Function, T> map = batch -> batch.minBy(function);
return this.collectReduce(map, Functions2.minBy(function));
}
@Override
public > T maxBy(Function function)
{
Function, T> map = batch -> batch.maxBy(function);
return this.collectReduce(map, Functions2.maxBy(function));
}
@Override
public long sumOfInt(IntFunction function)
{
return this.sumOfLongOrdered(batch -> batch.sumOfInt(function));
}
@Override
public double sumOfFloat(FloatFunction function)
{
return this.sumOfDoubleOrdered(batch -> batch.sumOfFloat(function));
}
@Override
public long sumOfLong(LongFunction function)
{
return this.sumOfLongOrdered(batch -> batch.sumOfLong(function));
}
@Override
public double sumOfDouble(DoubleFunction function)
{
return this.sumOfDoubleOrdered(batch -> batch.sumOfDouble(function));
}
private long sumOfLongOrdered(LongFunction> map)
{
LazyIterable> chunks = this.split();
LazyIterable> futures =
chunks.collect(chunk -> this.getExecutorService().submit(() -> map.longValueOf(chunk)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
long result = 0;
for (int i = 0; i < futuresList.size(); i++)
{
result += futuresList.get(i).get();
}
return result;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
private double sumOfDoubleOrdered(Function, DoubleSumResultHolder> map)
{
LazyIterable> chunks = this.split();
LazyIterable> futures =
chunks.collect(chunk -> this.getExecutorService().submit(() -> map.valueOf(chunk)));
// The call to toList() is important to stop the lazy evaluation and force all the Runnables to start executing.
MutableList> futuresList = futures.toList();
try
{
double sum = 0.0d;
double compensation = 0.0d;
for (int i = 0; i < futuresList.size(); i++)
{
compensation += futuresList.get(i).get().getCompensation();
double adjustedValue = futuresList.get(i).get().getResult() - compensation;
double nextSum = sum + adjustedValue;
compensation = nextSum - sum - adjustedValue;
sum = nextSum;
}
return sum;
}
catch (InterruptedException e)
{
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
}
@Override
public MapIterable groupByUniqueKey(Function function)
{
MutableMap result = ConcurrentHashMap.newMap(this.getBatchSize());
this.forEach(value -> {
V key = function.valueOf(value);
if (result.put(key, value) != null)
{
throw new IllegalStateException("Key " + key + " already exists in map!");
}
});
return result;
}
}