All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.bigdata.service.master.AbstractResourceScanner Maven / Gradle / Ivy

Go to download

Blazegraph(TM) DB Core Platform. It contains all Blazegraph DB dependencies other than Blueprints.

There is a newer version: 2.1.4
Show newest version
/*

Copyright (C) SYSTAP, LLC DBA Blazegraph 2006-2016.  All rights reserved.

Contact:
     SYSTAP, LLC DBA Blazegraph
     2501 Calvert ST NW #106
     Washington, DC 20008
     licenses@blazegraph.com

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
/*
 * Created on Jul 11, 2009
 */

package com.bigdata.service.master;

import java.util.LinkedList;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.log4j.Logger;

import com.bigdata.relation.accesspath.BlockingBuffer;

/**
 * Abstract base class for the scanner for a mapped master job. The
 * {@link Callable} should return the #of resources which were accepted for
 * processing.
 * 
 * @author Bryan Thompson
 * @version $Id$
 */
public abstract class AbstractResourceScanner implements Callable {

    protected static final Logger log = Logger
            .getLogger(AbstractResourceScanner.class);
    
    /**
     * The master buffer onto which the scanner drops chunks of resources
     * for processing.
     */
    private final BlockingBuffer buffer;

    /**
     * A queue used to combine the individual resources reported to
     * {@link #accept(Object)} into chunks before they are added to the
     * {@link #buffer}.
     */
    private final ArrayBlockingQueue queue;

    /**
     * Lock used to serialize the decision to transfer a chunk from the queue to
     * the buffer.
     */
    private final ReentrantLock queueLock = new ReentrantLock();
    
    /**
     * The #of resources accepted by the scanner.
     */
    private final AtomicLong acceptCount = new AtomicLong();

    /**
     * The #of chunks of resources which have been added to the buffer.
     */
    private final AtomicLong chunkCount = new AtomicLong();

    /**
     * Return the #of accepted resources.
     */
    final public long getAcceptCount() {

        return acceptCount.get();

    }

    /**
     * @param buffer
     *            The buffer to which the resources should be added.
     */
    protected AbstractResourceScanner(final BlockingBuffer buffer) {

        if (buffer == null)
            throw new IllegalArgumentException();

        this.buffer = buffer;

        this.queue = new ArrayBlockingQueue(2 * buffer.getMinimumChunkSize());
        
    }

    /**
     * Invokes {@link #runScanner()}, queuing and transferring chunks of
     * resources to the {@link BlockingBuffer} specified to the ctor. When
     * {@link #runScanner()} completes normally, the remaining resources
     * are transferred from the internal queue to the {@link BlockingBuffer}.
     * 
     * @return The #of resources accepted by the scanner.
     */
    final public Long call() throws Exception {

        // run the scanner.
        runScanner();

        // flush the last chunk to the blocking buffer.
        flushQueue();
        
        // #of resources accepted by the scanner.
        return acceptCount.get();
        
    }
    
    /**
     * Run the scanner.
     * 
     * @throws Exception
     */
    protected abstract void runScanner() throws Exception;
    
    /**
     * Accept a resource for processing.
     * 
     * @param resource
     *            The resource.
     */
    public void accept(final V resource) throws InterruptedException {

        if (resource == null)
            throw new IllegalArgumentException();

        if (log.isDebugEnabled())
            log.debug("accept: " + resource);
        
        this.acceptCount.incrementAndGet();

        // add the resource to the queue.
        queue.add(resource);

        /*
         * Synchronize callers. If there are multiple threads accepting
         * resources then only one thread at a time will cause the chunk to be
         * drained from the queue and placed onto the buffer.
         */
        queueLock.lockInterruptibly();
        try {

            if (queue.size() >= buffer.getMinimumChunkSize()) {

                // drain a chunk, transferring it to the buffer.
                transferChunk();

            }

        } finally {
         
            queueLock.unlock();
            
        }

    }
    
    /**
     * Drain a chunk from the queue, transferring it to the buffer (blocks if
     * the buffer is full).
     */
    @SuppressWarnings("unchecked")
    private void transferChunk() {
        
        final LinkedList c = new LinkedList();

        // drain chunk containing up to the desired chunk size.
        queue.drainTo(c, buffer.getMinimumChunkSize());

        final int chunkSize = c.size();
        
        if (chunkSize == 0)
            return;
        
        // allocate array of the appropriate component type.
        final V[] a = (V[]) java.lang.reflect.Array.newInstance(c.getFirst()
                .getClass(), chunkSize);

        // copy the chunk onto the array.
        int i = 0;
        for (V v : c) {

            assert v != null : "null @ index=" + i;
            
            a[i++] = v;

        }
        assert i == chunkSize : "i=" + i + ", chunkSize=" + chunkSize;

        if(log.isInfoEnabled()) {
            
            log.info("chunkSize=" + chunkSize + ", naccepted=" + acceptCount
                    + ", chunkCount=" + chunkCount);
            
        }
        
        /*
         * Add the chunk to the buffer.
         * 
         * Note: this will block if the queue is full.
         */
        buffer.add(a);

        chunkCount.incrementAndGet();
        
    }
    
    /**
     * Drain anything left in the queue, transferring it in chunks to the buffer
     * (blocks if the buffer is full).
     */
    private void flushQueue() {

        if (log.isInfoEnabled())
            log.info("Flushing queue to buffer.");
        
        while(!queue.isEmpty()) {

            // transfer a chunk from the queue to the buffer.
            transferChunk();
        
        }

    }

}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy