io.engineblock.activityimpl.marker.ReorderingContiguousOutputChunker Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of eb-api Show documentation
The driver API for engineblock; Provides the interfaces needed to build drivers that can be loaded by engineblock core
There is a newer version: 2.12.65
/*
 *
 *    Copyright 2016 jshook
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 * /
 */

package io.engineblock.activityimpl.marker;

import io.engineblock.activityapi.Activity;
import io.engineblock.activityapi.cycletracking.buffers.results.CycleResult;
import io.engineblock.activityapi.cycletracking.buffers.results.CycleResultsIntervalSegment;
import io.engineblock.activityapi.cycletracking.buffers.results.CycleResultsSegment;
import io.engineblock.activityapi.input.ContiguousInput;
import io.engineblock.activityapi.output.Output;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

/**
 * This is the default cycle marker implementation for EngineBlock.
 * 
 * This cycle marker wraps another tracking structure in order to
 * allow for flexible buffering methods. The extents are buffer segments
 * which can be managed atomically. They are chained here in two chains:
 * The marking chain and the tracking chain. When the atomic marking head
 * is non-null, then marking is possible, but marking calls block otherwise.
 * The same is true for the tracking head element.
 * 

 * The nowMarking and nowTracking conditions are meant to be locked and awaited
 * by marking and tracking calls respectively. Conversely, they are expected
 * to be signaled by tracking and marking calls.
 * 
 * This implementation needs to be adapted to handle early exit of either
 * marker or tracker threads with no deadlock.
 */
public class ReorderingContiguousOutputChunker implements Output  {

    private final static Logger logger = LoggerFactory.getLogger(ReorderingContiguousOutputChunker.class);
    private final int extentSize;
    private final int maxExtents;
    private List readers = new ArrayList<>();
    private AtomicLong min;
    private AtomicLong nextMin;
    private AtomicReference markingExtents = new AtomicReference<>();
    private ReentrantLock lock = new ReentrantLock(false);
    private Condition nowMarking = lock.newCondition();
    private Semaphore mutex = new Semaphore(1, false);

    public ReorderingContiguousOutputChunker(long min, long nextRangeMin, int extentSize, int maxExtents) {
        this.min = new AtomicLong(min);
        this.nextMin = new AtomicLong(nextRangeMin);
        this.extentSize = extentSize;
        this.maxExtents = maxExtents;
        initExtents();
    }

    public ReorderingContiguousOutputChunker(Activity activity) {

        if (!(activity.getInputDispenserDelegate().getInput(0) instanceof ContiguousInput)) {
            throw new RuntimeException("This type of marker may not be used with non-contiguous inputs yet.");
            // If you are looking at this code, it's because we count updates to extents to provide
            // efficient marker extent handling. The ability to use segmented inputs with markers will
            // come in a future append.
        }
        this.min = new AtomicLong(activity.getActivityDef().getStartCycle());
        this.nextMin = new AtomicLong(activity.getActivityDef().getEndCycle());
        long stride = activity.getParams().getOptionalLong("stride").orElse(1L);
        long cycleCount = nextMin.get() - min.get();
        if ((cycleCount % stride) != 0) {
            throw new RuntimeException("stride must evenly divide into cycles.");
            // TODO: Consider setting cycles to " ...
        }
        this.extentSize = calculateExtentSize(cycleCount, stride);
        this.maxExtents = 3;
        initExtents();
    }

    private void initExtents() {
        ByteTrackerExtent extent = new ByteTrackerExtent(min.get(), (min.get() + extentSize));
        this.markingExtents.set(extent);
        for (int i = 0; i < maxExtents; i++) {
            extent = extent.extend();
            logger.debug("added tracker extent " + extent.rangeSummary());
        }
        logger.info("using max " + maxExtents + " extents with getCount: " + extentSize);
    }


    @Override
    public synchronized void onCycleResultSegment(CycleResultsSegment segment) {
        logger.trace("on-cycle-result-segment: (" +segment +")");
        for (CycleResult cr : segment) {
            onCycleResult(cr.getCycle(),cr.getResult());
        }
    }

    @Override
    public synchronized boolean onCycleResult(long completedCycle, int result) {
        logger.trace("on-cycle-result: (" + completedCycle + "," + result +")");

        try {
        while (true) {
            ByteTrackerExtent extent = this.markingExtents.get();
            long unmarked = extent.markResult(completedCycle, result);

            if (unmarked > 0) {
                return true;
            } else if (unmarked == 0) {
                try {
                    mutex.acquire();
                    ByteTrackerExtent head = this.markingExtents.get();
                    while (head.isFullyFilled()) {
                        head.extend();
                        if (!this.markingExtents.compareAndSet(head, head.getNextExtent().get())) {
                            throw new RuntimeException("Unable to swap head extent.");
                        }
                        onFullyFilled(head);
                        head = this.markingExtents.get();
                    }
                    mutex.release();
                } catch (InterruptedException ignored) {
                } catch (Throwable t) {
                    throw t;
                }
                return true;
            } else {
                System.out.println("whoops");
            }
        } } catch (Throwable t) {
            throw t;
        }
    }

    @Override
    public synchronized void close() throws Exception {
        mutex.acquire();
        ByteTrackerExtent e = this.markingExtents.get();
        while (e != null) {
            onFullyFilled(e);
            e = e.getNextExtent().get();
        }
        mutex.release();

        for (Output reader : this.readers) {
            logger.debug("closing downstream reader: " + reader);
            reader.close();
        }

    }

    private void onFullyFilled(ByteTrackerExtent extent) {
        logger.debug("MARKER>: fully filled: " + extent);
        for (Output reader : readers) {
            CycleResultsIntervalSegment remainingSegment = extent.getRemainingSegment();
            if (remainingSegment!=null) {
                reader.onCycleResultSegment(remainingSegment);
            }
        }
    }

    private void onFullyServed(ByteTrackerExtent firstReadable) {
        logger.debug("TRACKER: fully tracked: " + firstReadable);
    }

    public void addExtentReader(Output reader) {
        this.readers.add(reader);
    }

    public synchronized void removeExtentReader(Output reader) {
        this.readers.remove(reader);
    }

    private int calculateExtentSize(long cycleCount, long stride) {
        if (cycleCount <= 2000000) {
            return (int) cycleCount;
        }
        for (int cs = 2000000; cs > 500000; cs--) {
            if ((cycleCount % cs) == 0 && (cs % stride) == 0) {
                return cs;
            }
        }
        throw new RuntimeException("no even divisor of cycleCount and Stride between 500K and 2M, with cycles=" + cycleCount + ",  and stride=" + stride);
    }


    @Override
    public String toString() {
        return "ConcurrentOutputSegmenter{" +
                "extentSize=" + extentSize +
                ", maxExtents=" + maxExtents +
                ", readers=" + readers +
                ", min=" + min +
                ", nextMin=" + nextMin +
                ", markingExtents/Chain=" + markingExtents.get().getChainSize() +
                '}';
    }

}