org.pageseeder.flint.indexing.IndexJobQueue Maven / Gradle / Ivy
/*
* Copyright 2015 Allette Systems (Australia)
* http://www.allette.com.au
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pageseeder.flint.indexing;
import org.pageseeder.flint.Index;
import org.pageseeder.flint.Requester;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.PriorityBlockingQueue;
/**
* The queue containing index jobs.
*
* This class uses a {@link PriorityBlockingQueue} so that when the queue is empty,
* the calling thread will be delayed.
*
* @author Jean-Baptiste Reure
* @author Christophe Lauret
*
* @version 28 February 2013
*/
public final class IndexJobQueue {
/**
* The actual queue.
*/
private final PriorityBlockingQueue _queue;
/**
* The single thread queue.
*/
private final PriorityBlockingQueue _singleThreadQueue;
/**
* Simple Constructor.
*
* @param withSingleThreadQueue if there's only one queue
*/
public IndexJobQueue(boolean withSingleThreadQueue) {
this._queue = new PriorityBlockingQueue<>();
this._singleThreadQueue = withSingleThreadQueue ? new PriorityBlockingQueue<>() : null;
}
// public external methods
// ----------------------------------------------------------------------------------------------
/**
* Add a new update job to the indexing queue.
*
* @param job The job to add to this queue.
*/
public void addSingleThreadJob(IndexJob job) {
addJob(job, true);
}
/**
* Add a new update job to the indexing queue.
*
* @param job The job to add to this queue.
*/
public void addMultiThreadJob(IndexJob job) {
addJob(job, false);
}
/**
* Returns the list of jobs for the specified requester.
*
* Note that by the time each job is checked, they might have run already so the method
* {@link IndexJob#isFinished()} should be called before parsing the job.
*
*
The list will never be null.
*
* @param requester the Requester
* @return the list of jobs (never null)
*/
public List getJobsForRequester(Requester requester) {
if (requester == null) return getAllJobs();
List jobs = new ArrayList<>();
for (IndexJob job : this._queue) {
if (job.isForRequester(requester)) {
jobs.add(job);
}
}
if (this._singleThreadQueue != null) {
for (IndexJob job : this._singleThreadQueue) {
if (job.isForRequester(requester)) {
jobs.add(job);
}
}
}
return jobs;
}
/**
* Returns the number of jobs for the specified requester.
*
* Note that some jobs may have been processed by the time this method returns.
*
* @param requester the requester
*
* @return the number of jobs for the specified requester.
*/
public int countJobsForRequester(Requester requester) {
if (requester == null) return this._queue.size();
int count = 0;
for (IndexJob job : this._queue) {
if (job.isForRequester(requester)) {
count++;
}
}
if (this._singleThreadQueue != null) {
for (IndexJob job : this._singleThreadQueue) {
if (job.isForRequester(requester)) {
count++;
}
}
}
return count;
}
/**
* Removes the jobs for the index provided.
*
* @param index the index
*/
public void clearJobsForIndex(Index index) {
if (index == null) return;
List jobs = new ArrayList<>();
for (IndexJob job : this._queue) {
if (job.isForIndex(index)) {
jobs.add(job);
}
}
this._queue.removeAll(jobs);
if (this._singleThreadQueue != null) {
jobs.clear();
for (IndexJob job : this._singleThreadQueue) {
if (job.isForIndex(index)) {
jobs.add(job);
}
}
this._singleThreadQueue.removeAll(jobs);
}
}
/**
* Returns the list of jobs for the index provided.
*
* Note that by the time each job is checked, they might have run already so the method
* {@link IndexJob#isFinished()} should be called before parsing the job.
*
*
The list will never be null.
*
* @param index the index
* @return the list of jobs (never null)
*/
public List getJobsForIndex(Index index) {
if (index == null) return getAllJobs();
List jobs = new ArrayList<>();
for (IndexJob job : this._queue) {
if (job.isForIndex(index)) {
jobs.add(job);
}
}
if (this._singleThreadQueue != null) {
for (IndexJob job : this._singleThreadQueue) {
if (job.isForIndex(index)) {
jobs.add(job);
}
}
}
return jobs;
}
/**
* Note that some jobs may have been processed by the time this method returns.
*
* @param index the index
* @return true if there is at least one job for the index provided.
*/
public boolean hasJobsForIndex(Index index) {
if (index != null) {
for (IndexJob job : this._queue) {
if (job.isForIndex(index)) return true;
}
if (this._singleThreadQueue != null) {
for (IndexJob job : this._singleThreadQueue) {
if (job.isForIndex(index)) return true;
}
}
}
return false;
}
/**
* Returns the number of jobs for the specified index provided.
*
*
Note that some jobs may have been processed by the time this method returns.
*
* @param index the index
* @return the number of jobs for the specified provided.
*/
public int countJobsForIndex(Index index) {
if (index == null) return this._queue.size();
int count = 0;
for (IndexJob job : this._queue) {
if (job.isForIndex(index)) {
count++;
}
}
if (this._singleThreadQueue != null) {
for (IndexJob job : this._singleThreadQueue) {
if (job.isForIndex(index)) {
count++;
}
}
}
return count;
}
/**
* Returns the complete list of jobs.
*
*
Note that by the time each job is checked, they might have run already so the method
* {@link IndexJob#isFinished()} should be called before parsing the job.
*
*
The list will never be null.
*
* @return the list of jobs waiting (never null)
*/
public List getAllJobs() {
ArrayList list = new ArrayList<>(this._queue);
if (this._singleThreadQueue != null)
list.addAll(this._singleThreadQueue);
return list;
}
/**
* Poll the next job in the queue (null if the queue is currently empty).
*
* @return the next job in the queue (null if the queue is currently empty).
*
* @throws InterruptedException if the thread was interrupted when waiting for the next job
*/
public IndexJob nextMultiThreadJob() throws InterruptedException {
return this._queue.take();
}
/**
* Poll the next job in the queue (null if the queue is currently empty).
*
* @return the next job in the queue (null if the queue is currently empty).
*
* @throws InterruptedException if the thread was interrupted when waiting for the next job
*/
public IndexJob nextSingleThreadJob() throws InterruptedException {
return this._singleThreadQueue != null ? this._singleThreadQueue.take() : null;
}
/**
* Indicates whether the queue is currently empty.
*
* @return true if there are currently no jobs;
* false otherwise.
*/
public boolean isMultiThreadsEmpty() {
return this._queue.isEmpty();
}
/**
* Indicates whether the queue is currently empty.
*
* @return true if there are currently no jobs;
* false otherwise.
*/
public boolean isSingleThreadEmpty() {
return this._singleThreadQueue == null || this._singleThreadQueue.isEmpty();
}
/**
* clear all queues
*/
public void clear() {
this._queue.clear();
if (this._singleThreadQueue != null)
this._singleThreadQueue.clear();
}
// -----------------------------------------------------------------------
// private methods
// -----------------------------------------------------------------------
private void addJob(IndexJob job, boolean singleThread) {
// check if similar job already there
IndexJob existing;
synchronized (this._queue) {
existing = this._queue.stream().filter(ajob -> ajob.isSimilar(job)).findFirst().orElse(null);
}
// in the other queue?
boolean foundInSingleQueue = false;
if (existing == null && this._singleThreadQueue != null) {
synchronized (this._singleThreadQueue) {
existing = this._singleThreadQueue.stream().filter(ajob -> ajob.isSimilar(job)).findFirst().orElse(null);
foundInSingleQueue = existing != null;
}
}
// if there is one similar, and this one has higher priority, add this one and remove the old one
boolean higherPriority = existing != null && existing.getPriority() == IndexJob.Priority.LOW && job.getPriority() == IndexJob.Priority.HIGH;
// force job if in a batch with a clear job
boolean force = job.isBatch() && job.getBatch().hasClearJob();
// don't remove existing if in batch with a clear job
boolean existingHasPriority = existing != null && existing.isBatch() && existing.getBatch().hasClearJob();
if (!existingHasPriority && (existing == null || force || higherPriority)) {
if (singleThread && this._singleThreadQueue != null) {
synchronized (this._singleThreadQueue) {
if (existing != null && !force)
this.removeExistingJob(job, existing, foundInSingleQueue);
this._singleThreadQueue.put(job);
}
} else {
synchronized (this._queue) {
if (existing != null && !force)
this.removeExistingJob(job, existing, foundInSingleQueue);
this._queue.put(job);
}
}
} else {
// if a batch, decrease total except if last job
IndexBatch batch = job.getBatch();
if (batch != null && batch.getCurrentCount() != batch.getTotalDocuments() - 1)
batch.remove(1);
}
}
private void removeExistingJob(IndexJob job, IndexJob existing, boolean foundInSingleQueue) {
// don't remove last job of batch
IndexBatch batch = existing.getBatch();
if (batch != null && batch.getCurrentCount() != batch.getTotalDocuments() - 1)
batch.remove(1);
// remove from queue
if (foundInSingleQueue) this._singleThreadQueue.remove(existing);
else this._queue.remove(existing);
}
}