org.neo4j.kernel.impl.index.schema.BlockEntryStreamMerger Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of neo4j-kernel Show documentation
Neo4j kernel is a lightweight, embedded Java database designed to store data structured as graphs rather than tables. For more information, see http://neo4j.org.
There is a newer version: 5.25.1
Show newest version
/*
 * Copyright (c) "Neo4j"
 * Neo4j Sweden AB [http://neo4j.com]
 *
 * This file is part of Neo4j.
 *
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see .
 */
package org.neo4j.kernel.impl.index.schema;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

import org.neo4j.index.internal.gbptree.Layout;
import org.neo4j.io.IOUtils;
import org.neo4j.kernel.api.index.IndexSample;
import org.neo4j.util.Preconditions;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

/**
 * A merger of {@link BlockEntry} in a streaming fashion. It takes as input one or more {@link BlockEntryCursor}, merges them
 * and on the fly adds batches of merged results to its output, which is consumable by either other {@link BlockEntryStreamMerger}
 * or a final consumer.
 *
 * For convenience instances are themselves valid as input into another {@link BlockEntryStreamMerger}.
 */
class BlockEntryStreamMerger implements BlockEntryCursor, Callable
{
    static final int QUEUE_SIZE = 10;

    private final List> input;
    private final Layout layout;
    private final BlockStorage.Cancellation cancellation;
    private final ArrayBlockingQueue> mergedOutput;
    private final int batchSize;
    private final Comparator samplingComparator;
    private KEY prevKey;
    private long sampledValues;
    private long uniqueValues;
    private volatile boolean halted;
    // This cursor will be used by the single thread reading from this merged stream
    private BlockEntryCursor currentOutput;

    BlockEntryStreamMerger( List> input, Layout layout, Comparator samplingComparator,
            BlockStorage.Cancellation cancellation, int batchSize, int queueSize )
    {
        this.input = input;
        this.layout = layout;
        this.cancellation = cancellation;
        this.batchSize = batchSize;
        this.mergedOutput = new ArrayBlockingQueue<>( queueSize );
        this.samplingComparator = samplingComparator;
    }

    @Override
    public Void call() throws IOException
    {
        try
        {
            MergingBlockEntryReader mergingReader = new MergingBlockEntryReader<>( layout );
            input.forEach( mergingReader::addSource );
            List> merged = new ArrayList<>( batchSize );
            while ( alive() && mergingReader.next() )
            {
                merged.add( new BlockEntry<>( mergingReader.key(), mergingReader.value() ) );
                if ( merged.size() == batchSize )
                {
                    offer( merged );
                    merged = new ArrayList<>( batchSize );
                }
            }
            if ( !merged.isEmpty() )
            {
                offer( merged );
            }
            return null;
        }
        finally
        {
            halted = true;
        }
    }

    /**
     * Called from another entry processor, either another merger like this one or a writer of the final data stream.
     * @return {@code true} if a new entry was selected (accessed via {@link #key()} and {@link #value()}, or {@code false}
     * if the end of the stream has been reached.
     */
    @Override
    public boolean next() throws IOException
    {
        do
        {
            if ( currentOutput != null && currentOutput.next() )
            {
                return true;
            }
            currentOutput = nextOutputBatchOrNull();
        }
        while ( currentOutput != null );
        return false;
    }

    @Override
    public KEY key()
    {
        return currentOutput.key();
    }

    @Override
    public VALUE value()
    {
        return currentOutput.value();
    }

    @Override
    public void close() throws IOException
    {
        IOUtils.closeAll( input );
    }

    private boolean alive()
    {
        return !halted && !cancellation.cancelled();
    }

    private void offer( List> entries )
    {
        if ( samplingComparator != null )
        {
            includeInSample( entries );
        }

        ListBasedBlockEntryCursor batch = new ListBasedBlockEntryCursor<>( entries );
        try
        {
            while ( alive() && !mergedOutput.offer( batch, 10, MILLISECONDS ) )
            {   // Then just stay here and try
                Thread.onSpinWait();
            }
        }
        catch ( InterruptedException e )
        {
            Thread.currentThread().interrupt();
            halted = true;
        }
    }

    void halt()
    {
        halted = true;
    }

    private void includeInSample( List> entries )
    {
        for ( BlockEntry entry : entries )
        {
            KEY key = entry.key();
            if ( prevKey == null || samplingComparator.compare( key, prevKey ) != 0 )
            {
                prevKey = key;
                uniqueValues++;
            }
            sampledValues++;
        }
    }

    IndexSample buildIndexSample()
    {
        Preconditions.checkState( samplingComparator != null, "I haven't been sampling at all" );
        return new IndexSample( sampledValues, uniqueValues, sampledValues );
    }

    private BlockEntryCursor nextOutputBatchOrNull()
    {
        // Keep polling the output if:
        // - output isn't empty
        // - output is empty but this merger is still going
        while ( alive() || !mergedOutput.isEmpty() )
        {
            try
            {
                BlockEntryCursor result = mergedOutput.poll( 10, TimeUnit.MILLISECONDS );
                if ( result != null )
                {
                    return result;
                }
            }
            catch ( InterruptedException e )
            {
                Thread.currentThread().interrupt();
            }
        }
        return null;
    }
}