picard.vcf.processor.VariantAccumulatorExecutor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of picard Show documentation
Show all versions of picard Show documentation
A set of command line tools (in Java) for manipulating high-throughput sequencing (HTS) data and formats such as SAM/BAM/CRAM and VCF.
/*
* The MIT License
*
* Copyright (c) 2015 The Broad Institute
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package picard.vcf.processor;
import com.google.common.base.Joiner;
import com.google.common.collect.FluentIterable;
import picard.util.AtomicIterator;
import picard.util.Iterators;
import htsjdk.samtools.util.Log;
import htsjdk.samtools.util.CloseableIterator;
import htsjdk.variant.variantcontext.VariantContext;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* Describes the functionality for an executor that manages the delegation of work to {@link VariantProcessor.Accumulator}s.
*
* @author mccowan
*/
public interface VariantAccumulatorExecutor, RESULT> {
/** Starts the work of the executor, returning immediately. */
void start();
/** Blocks until the work is complete. */
void awaitCompletion() throws InterruptedException;
/** Returns the {@link VariantProcessor.Accumulator}s associated with this executor. */
Collection accumulators();
/**
* A {@link VariantAccumulatorExecutor} that breaks down work into chunks described by the provided {@link VariantIteratorProducer} and
* spreads them over the indicated number of threads.
*
* @author mccowan
*/
class MultiThreadedChunkBased, R> implements VariantAccumulatorExecutor {
private static final Log LOG = Log.getInstance(MultiThreadedChunkBased.class);
final AtomicIterator> vcIterators;
final ExecutorService executor;
final Collection accumulators = Collections.synchronizedCollection(new ArrayList());
/** Signals whether or not this executor is started. */
volatile boolean started = false;
final int numThreads;
private final List childrenErrors = Collections.synchronizedList(new ArrayList());
final VariantProcessor.AccumulatorGenerator accumulatorGenerator;
public MultiThreadedChunkBased(
final int numThreads,
final VariantIteratorProducer vcIteratorProducer,
final VariantProcessor.AccumulatorGenerator accumulatorGenerator
) {
this.executor = Executors.newFixedThreadPool(numThreads);
this.vcIterators = Iterators.atomicIteratorOf(vcIteratorProducer.iterators());
this.numThreads = numThreads;
this.accumulatorGenerator = accumulatorGenerator;
}
@Override
public synchronized void start() {
started = true;
for (int i = 0; i < numThreads; i++) {
final A accumulator = accumulatorGenerator.build();
accumulators.add(accumulator);
executor.submit(new Worker(accumulator));
}
executor.shutdown();
}
public synchronized Collection accumulators() {
return Collections.unmodifiableCollection(accumulators);
}
@Override
public void awaitCompletion() throws InterruptedException {
if (!started) {
throw new IllegalStateException("This method can be called only after the executor has been started.");
} else {
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
if (!childrenErrors.isEmpty()) {
throw new MultiException(childrenErrors);
}
}
}
static class MultiException extends RuntimeException {
final List childrenExceptions;
public MultiException(final List childrenExceptions) {
this.childrenExceptions = childrenExceptions;
}
@Override
public String getMessage() {
return "Children threads encountered exceptions:\n" + Joiner.on("\n\t").join(FluentIterable.from(childrenExceptions).transform
(throwable -> throwable.getMessage()));
}
}
/** Continually requests and exhausts variant context iterators, delegating each to the child {@link Worker#processor}. */
class Worker implements Runnable {
final VariantProcessor.Accumulator processor;
Worker(final VariantProcessor.Accumulator processor) {
this.processor = processor;
}
@Override
public void run() {
try {
Optional> readerMaybe;
while ((readerMaybe = vcIterators.next()).isPresent()) {
final CloseableIterator reader = readerMaybe.get();
while (reader.hasNext()) processor.accumulate(reader.next());
reader.close();
if (!childrenErrors.isEmpty()) {
LOG.error(Thread.currentThread() + " aborting: observed error in another child thread.");
break;
}
}
} catch (final Throwable e) {
childrenErrors.add(e);
LOG.error(e, "Unexpected exception encountered in child thread.");
} finally {
LOG.debug(String.format("Thread %s is finishing.", Thread.currentThread()));
}
}
}
}
}