org.apache.manifoldcf.crawler.system.CrawlerAgent Maven / Gradle / Ivy
/* $Id: CrawlerAgent.java 988245 2010-08-23 18:39:35Z kwright $ */
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.system;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import java.util.*;
/** This is the main agent class for the crawler.
*/
public class CrawlerAgent implements IAgent
{
public static final String _rcsid = "@(#)$Id: CrawlerAgent.java 988245 2010-08-23 18:39:35Z kwright $";
// Thread objects.
// These get filled in as threads are created.
protected JobStartThread jobStartThread = null;
protected StufferThread stufferThread = null;
protected FinisherThread finisherThread = null;
protected JobNotificationThread notificationThread = null;
protected StartupThread startupThread = null;
protected StartDeleteThread startDeleteThread = null;
protected JobDeleteThread jobDeleteThread = null;
protected WorkerThread[] workerThreads = null;
protected ExpireStufferThread expireStufferThread = null;
protected ExpireThread[] expireThreads = null;
protected DocumentDeleteStufferThread deleteStufferThread = null;
protected DocumentDeleteThread[] deleteThreads = null;
protected DocumentCleanupStufferThread cleanupStufferThread = null;
protected DocumentCleanupThread[] cleanupThreads = null;
protected JobResetThread jobResetThread = null;
protected SeedingThread seedingThread = null;
protected IdleCleanupThread idleCleanupThread = null;
protected SetPriorityThread setPriorityThread = null;
protected HistoryCleanupThread historyCleanupThread = null;
protected AssessmentThread assessmentThread = null;
// Reset managers
/** Worker thread pool reset manager */
protected WorkerResetManager workerResetManager = null;
/** Delete thread pool reset manager */
protected DocDeleteResetManager docDeleteResetManager = null;
/** Cleanup thread pool reset manager */
protected DocCleanupResetManager docCleanupResetManager = null;
// Number of worker threads
protected int numWorkerThreads = 0;
// Number of delete threads
protected int numDeleteThreads = 0;
// Number of cleanup threads
protected int numCleanupThreads = 0;
// Number of expiration threads
protected int numExpireThreads = 0;
// Factor for low water level in queueing
protected float lowWaterFactor = 5.0f;
// Factor in amount to stuff
protected float stuffAmtFactor = 0.5f;
/** Process identifier for this agent */
protected String processID = null;
/** Constructor.
*/
public CrawlerAgent()
throws ManifoldCFException
{
}
/** Initialize agent environment.
* This is called before any of the other operations are called, and is meant to insure that
* the environment is properly initialized.
*/
public void initialize(IThreadContext threadContext)
throws ManifoldCFException
{
org.apache.manifoldcf.authorities.system.ManifoldCF.localInitialize(threadContext);
org.apache.manifoldcf.crawler.system.ManifoldCF.localInitialize(threadContext);
}
/** Tear down agent environment.
* This is called after all the other operations are completed, and is meant to allow
* environment resources to be freed.
*/
public void cleanUp(IThreadContext threadContext)
throws ManifoldCFException
{
org.apache.manifoldcf.crawler.system.ManifoldCF.localCleanup(threadContext);
org.apache.manifoldcf.authorities.system.ManifoldCF.localCleanup(threadContext);
}
/** Install agent. This usually installs the agent's database tables etc.
*/
@Override
public void install(IThreadContext threadContext)
throws ManifoldCFException
{
// Install the system tables for the crawler.
ManifoldCF.installSystemTables(threadContext);
}
/** Uninstall agent. This must clean up everything the agent is responsible for.
*/
@Override
public void deinstall(IThreadContext threadContext)
throws ManifoldCFException
{
ManifoldCF.deinstallSystemTables(threadContext);
}
/** Called ONLY when no other active services of this kind are running. Meant to be
* used after the cluster has been down for an indeterminate period of time.
*/
@Override
public void clusterInit(IThreadContext threadContext)
throws ManifoldCFException
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.prepareForClusterStart();
}
/** Cleanup after ALL agents processes.
* Call this method to clean up dangling persistent state when a cluster is just starting
* to come up. This method CANNOT be called when there are any active agents
* processes at all.
*@param currentProcessID is the current process ID.
*/
@Override
public void cleanUpAllAgentData(IThreadContext threadContext, String currentProcessID)
throws ManifoldCFException
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.cleanupProcessData();
// What kind of reprioritization should be done here?
// Answer: since we basically keep everything in the database now, the only kind of reprioritization we need
// to take care of are dangling ones that won't get done because the process that was doing them went
// away. BUT: somebody may have blown away lock info, in which case we won't know anything at all.
// So we do everything in that case.
ManifoldCF.resetAllDocumentPriorities(threadContext,currentProcessID);
}
/** Cleanup after agents process.
* Call this method to clean up dangling persistent state after agent has been stopped.
* This method CANNOT be called when the agent is active, but it can
* be called at any time and by any process in order to guarantee that a terminated
* agent does not block other agents from completing their tasks.
*@param currentProcessID is the current process ID.
*@param cleanupProcessID is the process ID of the agent to clean up after.
*/
@Override
public void cleanUpAgentData(IThreadContext threadContext, String currentProcessID, String cleanupProcessID)
throws ManifoldCFException
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.cleanupProcessData(cleanupProcessID);
// If one agents process was starting a reprioritization, it could have started the reprioritization sequence, but
// failed to complete it. If so, we may need to reset/complete the reprioritization sequence, which is defined as:
// - Resetting prioritization parameters
// - Removing all existing document priorities
// These must go together in order for the reset to be correct.
IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
String reproID = rt.isSpecifiedProcessReprioritizing(cleanupProcessID);
if (reproID != null)
{
// We have to take over the prioritization for the process, which apparently died
// in the middle.
jobManager.clearAllDocumentPriorities();
/*
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
// Reprioritize all documents in the jobqueue, 1000 at a time
Map connectionMap = new HashMap();
Map jobDescriptionMap = new HashMap();
// Do the 'not yet processed' documents only. Documents that are queued for reprocessing will be assigned
// new priorities. Already processed documents won't. This guarantees that our bins are appropriate for current thread
// activity.
// In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
// priorities.
// ??? -- start the process of reprioritization ONLY; don't do the whole thing.
while (true)
{
long startTime = System.currentTimeMillis();
Long currentTimeValue = rt.checkReprioritizationInProgress();
if (currentTimeValue == null)
{
// Some other process or thread superceded us.
return;
}
long updateTime = currentTimeValue.longValue();
DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(10000);
if (docs.length == 0)
break;
// Calculate new priorities for all these documents
ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
*/
rt.doneReprioritization(reproID);
}
}
/** Start the agent. This method should spin up the agent threads, and
* then return.
*/
@Override
public void startAgent(IThreadContext threadContext, String processID)
throws ManifoldCFException
{
this.processID = processID;
startSystem(threadContext);
}
/** Stop the agent. This should shut down the agent threads etc.
*/
@Override
public void stopAgent(IThreadContext threadContext)
throws ManifoldCFException
{
stopSystem(threadContext);
}
/** Request permission from agent to delete an output connection.
*@param connName is the name of the output connection.
*@return true if the connection is in use, false otherwise.
*/
@Override
public boolean isOutputConnectionInUse(IThreadContext threadContext, String connName)
throws ManifoldCFException
{
// Check with job manager.
IJobManager jobManager = JobManagerFactory.make(threadContext);
return jobManager.checkIfOutputReference(connName);
}
/** Note the deregistration of a set of output connections.
*@param connectionNames are the names of the connections being deregistered.
*/
@Override
public void noteOutputConnectorDeregistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectorDeregistration(connectionNames);
}
/** Note the registration of a set of output connections.
*@param connectionNames are the names of the connections being registered.
*/
@Override
public void noteOutputConnectorRegistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectorRegistration(connectionNames);
}
/** Note a change in configuration for an output connection.
*@param connectionName is the name of the connections being changed.
*/
@Override
public void noteOutputConnectionChange(IThreadContext threadContext, String connectionName)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteOutputConnectionChange(connectionName);
}
/** Request permission from agent to delete a transformation connection.
*@param connName is the name of the transformation connection.
*@return true if the connection is in use, false otherwise.
*/
@Override
public boolean isTransformationConnectionInUse(IThreadContext threadContext, String connName)
throws ManifoldCFException
{
// Check with job manager.
IJobManager jobManager = JobManagerFactory.make(threadContext);
return jobManager.checkIfTransformationReference(connName);
}
/** Note the deregistration of a set of transformation connections.
*@param connectionNames are the names of the connections being deregistered.
*/
@Override
public void noteTransformationConnectorDeregistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteTransformationConnectorDeregistration(connectionNames);
}
/** Note the registration of a set of transformation connections.
*@param connectionNames are the names of the connections being registered.
*/
@Override
public void noteTransformationConnectorRegistration(IThreadContext threadContext, String[] connectionNames)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteTransformationConnectorRegistration(connectionNames);
}
/** Note a change in configuration for a transformation connection.
*@param connectionName is the name of the connection being changed.
*/
@Override
public void noteTransformationConnectionChange(IThreadContext threadContext, String connectionName)
throws ManifoldCFException
{
// Notify job manager
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteTransformationConnectionChange(connectionName);
}
/** Start everything.
*/
public void startSystem(IThreadContext threadContext)
throws ManifoldCFException
{
Logging.root.info("Starting up pull-agent...");
// Now, start all the threads
numWorkerThreads = ManifoldCF.getMaxWorkerThreads(threadContext);
if (numWorkerThreads < 1 || numWorkerThreads > 300)
throw new ManifoldCFException("Illegal value for the number of worker threads", ManifoldCFException.SETUP_ERROR);
numDeleteThreads = ManifoldCF.getMaxDeleteThreads(threadContext);
numCleanupThreads = ManifoldCF.getMaxCleanupThreads(threadContext);
numExpireThreads = ManifoldCF.getMaxExpireThreads(threadContext);
if (numDeleteThreads < 1 || numDeleteThreads > 300)
throw new ManifoldCFException("Illegal value for the number of delete threads", ManifoldCFException.SETUP_ERROR);
if (numCleanupThreads < 1 || numCleanupThreads > 300)
throw new ManifoldCFException("Illegal value for the number of cleanup threads", ManifoldCFException.SETUP_ERROR);
if (numExpireThreads < 1 || numExpireThreads > 300)
throw new ManifoldCFException("Illegal value for the number of expire threads", ManifoldCFException.SETUP_ERROR);
lowWaterFactor = (float)LockManagerFactory.getDoubleProperty(threadContext,ManifoldCF.lowWaterFactorProperty,5.0);
if (lowWaterFactor < 1.0 || lowWaterFactor > 1000.0)
throw new ManifoldCFException("Illegal value for the low water factor", ManifoldCFException.SETUP_ERROR);
stuffAmtFactor = (float)LockManagerFactory.getDoubleProperty(threadContext,ManifoldCF.stuffAmtFactorProperty,2.0);
if (stuffAmtFactor < 0.1 || stuffAmtFactor > 1000.0)
throw new ManifoldCFException("Illegal value for the stuffing amount factor", ManifoldCFException.SETUP_ERROR);
// Create the threads and objects. This MUST be completed before there is any chance of "shutdownSystem" getting called.
QueueTracker queueTracker = new QueueTracker();
DocumentQueue documentQueue = new DocumentQueue();
DocumentDeleteQueue documentDeleteQueue = new DocumentDeleteQueue();
DocumentCleanupQueue documentCleanupQueue = new DocumentCleanupQueue();
DocumentCleanupQueue expireQueue = new DocumentCleanupQueue();
BlockingDocuments blockingDocuments = new BlockingDocuments();
workerResetManager = new WorkerResetManager(documentQueue,expireQueue,processID);
docDeleteResetManager = new DocDeleteResetManager(documentDeleteQueue,processID);
docCleanupResetManager = new DocCleanupResetManager(documentCleanupQueue,processID);
jobStartThread = new JobStartThread(processID);
startupThread = new StartupThread(new StartupResetManager(processID),processID);
startDeleteThread = new StartDeleteThread(new DeleteStartupResetManager(processID),processID);
finisherThread = new FinisherThread(processID);
notificationThread = new JobNotificationThread(new NotificationResetManager(processID),processID);
jobDeleteThread = new JobDeleteThread(processID);
stufferThread = new StufferThread(documentQueue,numWorkerThreads,workerResetManager,queueTracker,blockingDocuments,lowWaterFactor,stuffAmtFactor,processID);
expireStufferThread = new ExpireStufferThread(expireQueue,numExpireThreads,workerResetManager,processID);
setPriorityThread = new SetPriorityThread(numWorkerThreads,blockingDocuments,processID);
historyCleanupThread = new HistoryCleanupThread(processID);
workerThreads = new WorkerThread[numWorkerThreads];
int i = 0;
while (i < numWorkerThreads)
{
workerThreads[i] = new WorkerThread(Integer.toString(i),documentQueue,workerResetManager,queueTracker,processID);
i++;
}
expireThreads = new ExpireThread[numExpireThreads];
i = 0;
while (i < numExpireThreads)
{
expireThreads[i] = new ExpireThread(Integer.toString(i),expireQueue,workerResetManager,processID);
i++;
}
deleteStufferThread = new DocumentDeleteStufferThread(documentDeleteQueue,numDeleteThreads,docDeleteResetManager,processID);
deleteThreads = new DocumentDeleteThread[numDeleteThreads];
i = 0;
while (i < numDeleteThreads)
{
deleteThreads[i] = new DocumentDeleteThread(Integer.toString(i),documentDeleteQueue,docDeleteResetManager,processID);
i++;
}
cleanupStufferThread = new DocumentCleanupStufferThread(documentCleanupQueue,numCleanupThreads,docCleanupResetManager,processID);
cleanupThreads = new DocumentCleanupThread[numCleanupThreads];
i = 0;
while (i < numCleanupThreads)
{
cleanupThreads[i] = new DocumentCleanupThread(Integer.toString(i),documentCleanupQueue,docCleanupResetManager,processID);
i++;
}
jobResetThread = new JobResetThread(processID);
seedingThread = new SeedingThread(new SeedingResetManager(processID),processID);
idleCleanupThread = new IdleCleanupThread(processID);
assessmentThread = new AssessmentThread(processID);
// Start all the threads
jobStartThread.start();
startupThread.start();
startDeleteThread.start();
finisherThread.start();
notificationThread.start();
jobDeleteThread.start();
stufferThread.start();
expireStufferThread.start();
setPriorityThread.start();
historyCleanupThread.start();
i = 0;
while (i < numWorkerThreads)
{
workerThreads[i].start();
i++;
}
i = 0;
while (i < numExpireThreads)
{
expireThreads[i].start();
i++;
}
cleanupStufferThread.start();
i = 0;
while (i < numCleanupThreads)
{
cleanupThreads[i].start();
i++;
}
deleteStufferThread.start();
i = 0;
while (i < numDeleteThreads)
{
deleteThreads[i].start();
i++;
}
jobResetThread.start();
seedingThread.start();
idleCleanupThread.start();
assessmentThread.start();
Logging.root.info("Pull-agent started");
}
/** Stop the system.
*/
public void stopSystem(IThreadContext threadContext)
throws ManifoldCFException
{
Logging.root.info("Shutting down pull-agent...");
while (jobDeleteThread != null || startupThread != null || startDeleteThread != null ||
jobStartThread != null || stufferThread != null ||
finisherThread != null || notificationThread != null || workerThreads != null || expireStufferThread != null || expireThreads != null ||
deleteStufferThread != null || deleteThreads != null ||
cleanupStufferThread != null || cleanupThreads != null ||
jobResetThread != null || seedingThread != null || idleCleanupThread != null || assessmentThread != null || setPriorityThread != null || historyCleanupThread != null)
{
// Send an interrupt to all threads that are still there.
// In theory, this only needs to be done once. In practice, I have seen cases where the thread loses track of the fact that it has been
// interrupted (which may be a JVM bug - who knows?), but in any case there's no harm in doing it again.
if (historyCleanupThread != null)
{
historyCleanupThread.interrupt();
}
if (setPriorityThread != null)
{
setPriorityThread.interrupt();
}
if (jobStartThread != null)
{
jobStartThread.interrupt();
}
if (jobDeleteThread != null)
{
jobDeleteThread.interrupt();
}
if (startupThread != null)
{
startupThread.interrupt();
}
if (startDeleteThread != null)
{
startDeleteThread.interrupt();
}
if (stufferThread != null)
{
stufferThread.interrupt();
}
if (expireStufferThread != null)
{
expireStufferThread.interrupt();
}
if (finisherThread != null)
{
finisherThread.interrupt();
}
if (notificationThread != null)
{
notificationThread.interrupt();
}
if (workerThreads != null)
{
int i = 0;
while (i < workerThreads.length)
{
Thread workerThread = workerThreads[i++];
if (workerThread != null)
workerThread.interrupt();
}
}
if (expireThreads != null)
{
int i = 0;
while (i < expireThreads.length)
{
Thread expireThread = expireThreads[i++];
if (expireThread != null)
expireThread.interrupt();
}
}
if (cleanupStufferThread != null)
{
cleanupStufferThread.interrupt();
}
if (cleanupThreads != null)
{
int i = 0;
while (i < cleanupThreads.length)
{
Thread cleanupThread = cleanupThreads[i++];
if (cleanupThread != null)
cleanupThread.interrupt();
}
}
if (deleteStufferThread != null)
{
deleteStufferThread.interrupt();
}
if (deleteThreads != null)
{
int i = 0;
while (i < deleteThreads.length)
{
Thread deleteThread = deleteThreads[i++];
if (deleteThread != null)
deleteThread.interrupt();
}
}
if (jobResetThread != null)
{
jobResetThread.interrupt();
}
if (seedingThread != null)
{
seedingThread.interrupt();
}
if (idleCleanupThread != null)
{
idleCleanupThread.interrupt();
}
if (assessmentThread != null)
{
assessmentThread.interrupt();
}
// Now, wait for all threads to die.
try
{
ManifoldCF.sleep(1000L);
}
catch (InterruptedException e)
{
}
// Check to see which died.
if (historyCleanupThread != null)
{
if (!historyCleanupThread.isAlive())
historyCleanupThread = null;
}
if (setPriorityThread != null)
{
if (!setPriorityThread.isAlive())
setPriorityThread = null;
}
if (jobDeleteThread != null)
{
if (!jobDeleteThread.isAlive())
jobDeleteThread = null;
}
if (startupThread != null)
{
if (!startupThread.isAlive())
startupThread = null;
}
if (startDeleteThread != null)
{
if (!startDeleteThread.isAlive())
startDeleteThread = null;
}
if (jobStartThread != null)
{
if (!jobStartThread.isAlive())
jobStartThread = null;
}
if (stufferThread != null)
{
if (!stufferThread.isAlive())
stufferThread = null;
}
if (expireStufferThread != null)
{
if (!expireStufferThread.isAlive())
expireStufferThread = null;
}
if (finisherThread != null)
{
if (!finisherThread.isAlive())
finisherThread = null;
}
if (notificationThread != null)
{
if (!notificationThread.isAlive())
notificationThread = null;
}
if (workerThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < workerThreads.length)
{
Thread workerThread = workerThreads[i];
if (workerThread != null)
{
if (!workerThread.isAlive())
workerThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
workerThreads = null;
}
if (expireThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < expireThreads.length)
{
Thread expireThread = expireThreads[i];
if (expireThread != null)
{
if (!expireThread.isAlive())
expireThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
expireThreads = null;
}
if (cleanupStufferThread != null)
{
if (!cleanupStufferThread.isAlive())
cleanupStufferThread = null;
}
if (cleanupThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < cleanupThreads.length)
{
Thread cleanupThread = cleanupThreads[i];
if (cleanupThread != null)
{
if (!cleanupThread.isAlive())
cleanupThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
cleanupThreads = null;
}
if (deleteStufferThread != null)
{
if (!deleteStufferThread.isAlive())
deleteStufferThread = null;
}
if (deleteThreads != null)
{
int i = 0;
boolean isAlive = false;
while (i < deleteThreads.length)
{
Thread deleteThread = deleteThreads[i];
if (deleteThread != null)
{
if (!deleteThread.isAlive())
deleteThreads[i] = null;
else
isAlive = true;
}
i++;
}
if (!isAlive)
deleteThreads = null;
}
if (jobResetThread != null)
{
if (!jobResetThread.isAlive())
jobResetThread = null;
}
if (seedingThread != null)
{
if (!seedingThread.isAlive())
seedingThread = null;
}
if (idleCleanupThread != null)
{
if (!idleCleanupThread.isAlive())
idleCleanupThread = null;
}
if (assessmentThread != null)
{
if (!assessmentThread.isAlive())
assessmentThread = null;
}
}
// Threads are down; release connectors
RepositoryConnectorPoolFactory.make(threadContext).flushUnusedConnectors();
NotificationConnectorPoolFactory.make(threadContext).flushUnusedConnectors();
numWorkerThreads = 0;
numDeleteThreads = 0;
numExpireThreads = 0;
Logging.root.info("Pull-agent successfully shut down");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy