com.proofpoint.discovery.store.BatchProcessor Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of discovery-server Show documentation
Discovery server
There is a newer version: 1.36
/*
 * Copyright 2010 Proofpoint, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.proofpoint.discovery.store;

import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.proofpoint.log.Logger;
import com.proofpoint.reporting.Gauge;
import com.proofpoint.stats.CounterStat;
import org.weakref.jmx.Nested;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

public class BatchProcessor
{
    private final static Logger log = Logger.get(BatchProcessor.class);

    private final BatchHandler handler;
    private final int maxBatchSize;
    private final BlockingQueue queue;
    private final String name;

    private ExecutorService executor;
    private volatile Future future;

    private final CounterStat processedEntries = new CounterStat();
    private final CounterStat droppedEntries = new CounterStat();
    private final CounterStat errors = new CounterStat();

    public BatchProcessor(String name, BatchHandler handler, int maxBatchSize, int queueSize)
    {
        Preconditions.checkNotNull(name, "name is null");
        Preconditions.checkNotNull(handler, "handler is null");
        Preconditions.checkArgument(queueSize > 0, "queue size needs to be a positive integer");
        Preconditions.checkArgument(maxBatchSize > 0, "max batch size needs to be a positive integer");

        this.name = name;
        this.handler = handler;
        this.maxBatchSize = maxBatchSize;
        this.queue = new ArrayBlockingQueue<>(queueSize);
    }

    @PostConstruct
    public synchronized void start()
    {
        if (future == null) {
            executor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("batch-processor-" + name + "-%d").build());

            future = executor.submit(new Runnable() {
                public void run()
                {
                    while (!Thread.interrupted()) {
                        final List entries = new ArrayList<>(maxBatchSize);

                        try {
                            T first = queue.take();
                            entries.add(first);
                            queue.drainTo(entries, maxBatchSize - 1);

                            handler.processBatch(Collections.unmodifiableList(entries));

                            processedEntries.update(entries.size());
                        }
                        catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                        }
                        catch (Throwable t) {
                            errors.update(1);
                            log.warn(t, "Error handling batch");
                        }

                        // TODO: expose timestamp of last execution via jmx
                    }
                }
            });
        }
    }

    @Nested
    public CounterStat getProcessedEntries()
    {
        return processedEntries;
    }

    @Nested
    public CounterStat getDroppedEntries()
    {
        return droppedEntries;
    }

    @Nested
    public CounterStat getErrors()
    {
        return errors;
    }

    @Gauge
    public long getQueueSize()
    {
        return queue.size();
    }

    @PreDestroy
    public synchronized void stop()
    {
        if (future != null) {
            future.cancel(true);
            executor.shutdownNow();

            future = null;
        }
    }

    public void put(T entry)
    {
        Preconditions.checkState(!future.isCancelled(), "Processor is not running");
        Preconditions.checkNotNull(entry, "entry is null");

        while (!queue.offer(entry)) {
            // throw away oldest and try again
            if (queue.poll() != null) {
                droppedEntries.update(1);
            }
        }
    }

    public static interface BatchHandler
    {
        void processBatch(Collection entries)
                throws Exception;
    }
}