org.sweble.wikitext.articlecruncher.pnodes.LocalProcessingNode Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of swc-article-cruncher Show documentation
A framework for multi-threaded processing of lots of articles.
There is a newer version: 3.1.9
/**
 * Copyright 2011 The Open Source Research Group,
 *                University of Erlangen-Nürnberg
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sweble.wikitext.articlecruncher.pnodes;

import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletionService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;

import org.sweble.wikitext.articlecruncher.Job;
import org.sweble.wikitext.articlecruncher.ProcessingNode;
import org.sweble.wikitext.articlecruncher.WorkerInstantiator;
import org.sweble.wikitext.articlecruncher.utils.AbortHandler;
import org.sweble.wikitext.articlecruncher.utils.ExecutorType;
import org.sweble.wikitext.articlecruncher.utils.MyExecutorService;
import org.sweble.wikitext.articlecruncher.utils.WorkerBase;
import org.sweble.wikitext.articlecruncher.utils.WorkerLauncher;
import org.sweble.wikitext.articlecruncher.utils.WorkerSynchronizer;

public class LocalProcessingNode
		extends
			ProcessingNode
{
	private final BlockingQueue inTray;

	private final BlockingQueue processedJobs;

	private final int numWorkers;

	private final LpnJobProcessorFactory jobProcessorFactory;

	private MyExecutorService executor;

	private Semaphore backPressure;

	private WorkerLauncher distributor;

	private WorkerLauncher gatherer;

	// =========================================================================

	public LocalProcessingNode(
			AbortHandler abortHandler,
			BlockingQueue inTray,
			BlockingQueue processedJobs,
			LpnJobProcessorFactory jobProcessorFactory,
			int numWorkers)
	{
		super(getClassName(), abortHandler);

		Thread.currentThread().setName(getClassName());

		this.inTray = inTray;
		this.processedJobs = processedJobs;
		this.jobProcessorFactory = jobProcessorFactory;
		this.numWorkers = numWorkers;
	}

	private static String getClassName()
	{
		return LocalProcessingNode.class.getSimpleName();
	}

	// =========================================================================

	@Override
	protected void work() throws Throwable
	{
		try
		{
			executor = new MyExecutorService(
					ExecutorType.CACHED_THREAD_POOL,
					getLogger());

			final AbortHandler abortHandler = new AbortHandler()
			{
				@Override
				public void notify(Throwable t)
				{
					stop();
				}
			};

			WorkerSynchronizer synchronizer = new WorkerSynchronizer();

			backPressure = new Semaphore(numWorkers);

			final BlockingQueue> ecsQueue =
					new LinkedBlockingQueue>();

			distributor = new WorkerLauncher(new WorkerInstantiator()
			{
				@Override
				public WorkerBase instantiate()
				{
					LpnDistributor d = new LpnDistributor(
							abortHandler,
							inTray,
							executor.getThreadGroup(),
							numWorkers,
							jobProcessorFactory,
							backPressure);
					ecsQueue.add(d.getEcs());
					return d;
				}
			}, abortHandler);

			distributor.start(executor, synchronizer);

			final CompletionService ecs = ecsQueue.take();

			gatherer = new WorkerLauncher(new WorkerInstantiator()
			{
				@Override
				public WorkerBase instantiate()
				{
					return new LpnGatherer(
							abortHandler,
							ecs,
							processedJobs,
							backPressure);
				}
			}, abortHandler);

			gatherer.start(executor, synchronizer);

			synchronizer.waitForAny();
		}
		finally
		{
			info("Sending kill signal to workers");

			if (distributor != null)
				distributor.stop();

			if (gatherer != null)
				gatherer.stop();

			if (executor != null)
				executor.shutdownAndAwaitTermination();
		}
	}
}