org.modeshape.jcr.query.BufferManager Maven / Gradle / Ivy
/*
* ModeShape (http://www.modeshape.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.modeshape.jcr.query;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.mapdb.BTreeKeySerializer;
import org.mapdb.DB;
import org.mapdb.DB.BTreeMapMaker;
import org.mapdb.DB.HTreeSetMaker;
import org.mapdb.DBMaker;
import org.mapdb.HTreeMap;
import org.mapdb.Serializer;
import org.modeshape.common.collection.SingleIterator;
import org.modeshape.common.collection.Supplier;
import org.modeshape.jcr.ExecutionContext;
import org.modeshape.jcr.index.local.MapDB;
import org.modeshape.jcr.index.local.MapDB.ComparableUniqueKeyComparator;
import org.modeshape.jcr.index.local.MapDB.Serializers;
import org.modeshape.jcr.index.local.MapDB.UniqueKey;
import org.modeshape.jcr.index.local.MapDB.UniqueKeyComparator;
import org.modeshape.jcr.index.local.MapDB.UniqueKeyBTreeSerializer;
import org.modeshape.jcr.query.Tuples.TupleFactory;
import org.modeshape.jcr.query.model.TypeSystem.TypeFactory;
import org.modeshape.jcr.value.ValueFactories;
/**
* A manager of temporary buffers used in the query system.
*
* @author Randall Hauch ([email protected])
*/
public class BufferManager implements Serializers, AutoCloseable {
/**
* A basic buffer interface.
*
* @author Randall Hauch ([email protected])
*/
public static interface Buffer extends AutoCloseable {
/**
* Determine if this buffer is empty. This may be more efficient than {@link #size()}.
*
* @return true if the buffer is empty, or false otherwise
*/
boolean isEmpty();
/**
* Get the size of the buffer. This might be more expensive if the buffer is not created with the flag to keep the size.
*
* @return the size of the buffer.
*/
long size();
/**
* Close the buffer and release all resources. The buffer is not usable after this.
*/
@Override
void close();
}
/**
* A buffer that maintains the insertion order of a series of values.
*
* @param the type of value
* @author Randall Hauch ([email protected])
*/
public static interface QueueBuffer extends Buffer, Iterable {
/**
* Add a value to the end of this buffer.
*
* @param value the value to be added; may be null
*/
void append( T value );
/**
* Get an iterator over all of the values.
*
* @return the iterator; never null
*/
@Override
Iterator iterator();
}
public static interface Predicate {
/**
* Add the value to the buffer only if the buffer does not yet contain the value.
*
* @param value the value
* @return true if the buffer has not yet seen that value, or false otherwise
*/
boolean addIfAbsent( T value );
}
/**
* A buffer used to determine distinct values.
*
* @param the type of the distinct value
* @author Randall Hauch ([email protected])
*/
public static interface DistinctBuffer extends Buffer, Iterable, Predicate {
/**
* Get an iterator over all of the records.
*
* @return the iterator; never null
*/
@Override
Iterator iterator();
}
/**
* A buffer used to sort values into ascending or descending order.
*
* @param the type of the sorting value
* @param the type of record that is to be sorted
* @author Randall Hauch ([email protected])
*/
public static interface SortingBuffer extends Buffer {
/**
* Put the supplied value into the buffer given its sortable value.
*
* @param sortable the value of the record that is to be used for sorting
* @param record the record
*/
void put( SortType sortable,
RecordType record );
/**
* Get an iterator over all of the records in ascending order.
*
* @return the ascending iterator
*/
Iterator ascending();
/**
* Get an iterator over all of the records in descending order.
*
* @return the ascending iterator
*/
Iterator descending();
/**
* Get an iterator over all of the records that have the given sortable value.
*
* @param key the sortable value
* @return the iterator; null if there is no entry, or an iterator with one or more values
*/
Iterator getAll( SortType key );
/**
* Get an iterator over all of the records that have a sortable value with in the specified range.
*
* @param lowerKey the minimum sortable value; may be null if there is no minimum
* @param includeLowerKey true if the values equal to or greater than {@code lowerKey} value is to be included, or false
* if only values greater than {@code lowerKey} should be included
* @param upperKey the maximum sortable value; may be null if there is no maximum
* @param includeUpperKey true if the values equal to or less than {@code upperKey} value are to be included, or false if
* only values less than {@code upperKey} should be included
* @return the iterator; null if there is no entry, or an iterator with one or more values
*/
Iterator getAll( SortType lowerKey,
boolean includeLowerKey,
SortType upperKey,
boolean includeUpperKey );
}
/**
* An object use to create a new {@link DistinctBuffer}.
*
* @see BufferManager#createDistinctBuffer(Serializer)
* @param the type of value to be used in the buffer
* @author Randall Hauch ([email protected])
*/
public static interface QueueBufferMaker {
/**
* Specify whether to store the bufer on the heap.
*
* @param useHeap true if the buffer's contents are to be stored on the heap, or false if off-heap storage should be used.
* @return this maker instance; never null
*/
QueueBufferMaker useHeap( boolean useHeap );
/**
* Create the {@link DistinctBuffer} instance.
*
* @return the distinct buffer; never null
*/
QueueBuffer make();
}
/**
* An object use to create a new {@link DistinctBuffer}.
*
* @see BufferManager#createDistinctBuffer(Serializer)
* @param the type of value to be used in the buffer
* @author Randall Hauch ([email protected])
*/
public static interface DistinctBufferMaker {
/**
* Specify whether to store the bufer on the heap.
*
* @param useHeap true if the buffer's contents are to be stored on the heap, or false if off-heap storage should be used.
* @return this maker instance; never null
*/
DistinctBufferMaker useHeap( boolean useHeap );
/**
* Specify whether to keep track of the buffer size when adding value. Doing so may slow adds, but it will make returning
* the size very quick. Adds may be faster if this is set to false, but asking the buffer for its size requires iterating
* over the entire buffer and thus may be slower.
*
* @param keepBufferSize true if the buffer is to efficiently track its size, or false if it can skip this and, only if
* {@link Buffer#size()} is called, compute the size in a brute force manner.
* @return this maker instance; never null
*/
DistinctBufferMaker keepSize( boolean keepBufferSize );
/**
* Create the {@link DistinctBuffer} instance.
*
* @return the distinct buffer; never null
*/
DistinctBuffer make();
}
/**
* An object use to create a new {@link SortingBuffer}.
*
* @see BufferManager#createSortingBuffer(BTreeKeySerializer, Serializer)
* @param the type of sortable value
* @param the type of record to be placed into the buffer
* @author Randall Hauch ([email protected])
*/
public static interface SortingBufferMaker {
/**
* Specify whether to store the bufer on the heap.
*
* @param useHeap true if the buffer's contents are to be stored on the heap, or false if off-heap storage should be used.
* @return this maker instance; never null
*/
SortingBufferMaker useHeap( boolean useHeap );
/**
* Specify whether to keep track of the buffer size when adding value. Doing so may slow adds, but it will make returning
* the size very quick. Adds may be faster if this is set to false, but asking the buffer for its size requires iterating
* over the entire buffer and thus may be slower.
*
* @param keepBufferSize true if the buffer is to efficiently track its size, or false if it can skip this and, only if
* {@link Buffer#size()} is called, compute the size in a brute force manner.
* @return this maker instance; never null
*/
SortingBufferMaker keepSize( boolean keepBufferSize );
/**
* Create the {@link SortingBuffer} instance.
*
* @return the distinct buffer; never null
*/
SortingBuffer make();
}
protected static final class DbHolder implements AutoCloseable {
private final AtomicReference reference = new AtomicReference<>();
private final Lock lock = new ReentrantLock();
private final Supplier supplier;
protected DbHolder( Supplier supplier ) {
this.supplier = supplier;
}
public DB get() {
DB db = reference.get();
if (db == null) {
try {
lock.lock();
// Allocate it ...
db = supplier.get();
reference.set(db);
} finally {
lock.unlock();
}
}
return db;
}
@Override
public void close() {
DB db = reference.getAndSet(null);
if (db != null) {
db.close();
}
}
}
private final static Supplier OFF_HEAP_DB_SUPPLIER = new Supplier() {
@Override
public DB get() {
return DBMaker.newMemoryDirectDB().make();
}
};
private final static Supplier ON_HEAP_DB_SUPPLIER = new Supplier() {
@Override
public DB get() {
return DBMaker.newMemoryDB().make();
}
};
private final Serializers serializers;
private final DbHolder offheap;
private final DbHolder onheap;
private final AtomicLong dbCounter = new AtomicLong();
public BufferManager( ExecutionContext context ) {
this(context, OFF_HEAP_DB_SUPPLIER, ON_HEAP_DB_SUPPLIER);
}
protected BufferManager( ExecutionContext context,
Supplier offheapDbSupplier,
Supplier onheapDbSupplier ) {
offheap = new DbHolder(offheapDbSupplier);
onheap = new DbHolder(onheapDbSupplier);
// Create the serializers ...
ValueFactories factories = context.getValueFactories();
serializers = MapDB.serializers(factories);
}
@Override
public void close() {
RuntimeException error = null;
try {
onheap.close();
} catch (RuntimeException e) {
error = e;
} finally {
try {
offheap.close();
} catch (RuntimeException e) {
if (error == null) error = e;
}
if (error != null) throw error;
}
}
/**
* Obtain a maker object that can create a new {@link QueueBuffer}.
*
* @param serializer the serializer for the value
* @return the maker; never null
*/
public QueueBufferMaker createQueueBuffer( Serializer serializer ) {
return new MakeOrderedBuffer("buffer-" + dbCounter.incrementAndGet(), serializer);
}
/**
* Obtain a maker object that can create a new {@link DistinctBuffer}.
*
* @param distinctSerializer the serializer for the distinct value
* @return the maker; never null
*/
public DistinctBufferMaker createDistinctBuffer( Serializer distinctSerializer ) {
return new MakeDistinctBuffer("buffer-" + dbCounter.incrementAndGet(), distinctSerializer);
}
/**
* Obtain a maker object that can create a new {@link SortingBuffer} that will keep a single values for any given key.
*
* @param keySerializer the serializer for the keys
* @param valueSerializer the serializer for the values
* @return the maker; never null
*/
public SortingBufferMaker createSortingBuffer( BTreeKeySerializer keySerializer,
Serializer valueSerializer ) {
return new MakeSortingBuffer("buffer-" + dbCounter.incrementAndGet(), keySerializer, valueSerializer);
}
/**
* Obtain a maker object that can create a new {@link SortingBuffer} that can store multiple values for any given key.
*
* @param keySerializer the serializer for the keys
* @param keyComparator the comparator for the keys, or null if natural ordering should be used
* @param valueSerializer the serializer for the values
* @return the maker; never null
*/
public , V> SortingBufferMaker createSortingWithDuplicatesBuffer( Serializer keySerializer,
Comparator keyComparator,
Serializer valueSerializer ) {
return new MakeSortingWithDuplicatesBuffer("buffer-" + dbCounter.incrementAndGet(), keySerializer, keyComparator,
valueSerializer);
}
@Override
public Serializer serializerFor( Class type ) {
return null;
}
@Override
public BTreeKeySerializer bTreeKeySerializerFor( Class type,
Comparator comparator,
boolean pack ) {
return null;
}
/**
* Obtain a serializer for the given value type.
*
* @param type the type; may not be null
* @return the serializer
*/
public Serializer serializerFor( TypeFactory type ) {
if (type instanceof TupleFactory) {
return ((TupleFactory)type).getSerializer(this);
}
return serializers.serializerFor(type.getType());
}
/**
* Obtain a serializer for the given key type.
*
* @param type the type; may not be null
* @param pack true if the serializer can/should pack keys together when possible, or false otherwise
* @return the serializer
*/
public BTreeKeySerializer bTreeKeySerializerFor( TypeFactory type,
boolean pack ) {
return serializers.bTreeKeySerializerFor(type.getType(), type.getComparator(), pack);
}
protected final DB db( boolean useHeap ) {
return useHeap ? onheap.get() : offheap.get();
}
protected final void delete( String name,
boolean onHeap ) {
db(onHeap).delete(name);
}
protected abstract class CloseableBuffer implements Buffer {
protected final String name;
protected final boolean onHeap;
protected CloseableBuffer( String name,
boolean onHeap ) {
this.name = name;
this.onHeap = onHeap;
}
@Override
public void close() {
BufferManager.this.delete(name, onHeap);
}
}
protected final class CloseableQueueBuffer extends CloseableBuffer implements QueueBuffer {
protected final Map buffer;
private final AtomicLong size = new AtomicLong();
protected CloseableQueueBuffer( String name,
boolean onHeap,
Map buffer ) {
super(name, onHeap);
this.buffer = buffer;
}
@Override
public boolean isEmpty() {
return buffer.isEmpty();
}
@Override
public long size() {
return size.get();
}
@Override
public void append( T value ) {
buffer.put(size.getAndIncrement(), value);
}
@Override
public Iterator iterator() {
final AtomicLong counter = new AtomicLong(0L);
return new Iterator() {
@Override
public boolean hasNext() {
return counter.get() < buffer.size();
}
@Override
public T next() {
Long key = counter.getAndIncrement();
if (key.intValue() < buffer.size()) {
return buffer.get(key);
}
throw new NoSuchElementException();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public String toString() {
return "QueueBuffer(" + name + ",size=" + size.get() + ")";
}
}
protected final class CloseableDistinctBuffer extends CloseableBuffer implements DistinctBuffer {
private final Set buffer;
protected CloseableDistinctBuffer( String name,
boolean onHeap,
Set buffer ) {
super(name, onHeap);
this.buffer = buffer;
}
@Override
public boolean isEmpty() {
return buffer.isEmpty();
}
@Override
public long size() {
return buffer.size();
}
@Override
public boolean addIfAbsent( T value ) {
return buffer.add(value);
}
@Override
public Iterator iterator() {
return buffer.iterator();
}
@Override
public String toString() {
return "DistinctBuffer(" + name + ")";
}
}
protected final class CloseableSortingBuffer extends CloseableBuffer implements SortingBuffer {
private final NavigableMap buffer;
protected CloseableSortingBuffer( String name,
boolean onHeap,
NavigableMap buffer ) {
super(name, onHeap);
this.buffer = buffer;
}
@Override
public boolean isEmpty() {
return buffer.isEmpty();
}
@Override
public long size() {
return buffer.size();
}
@Override
public void put( K sortable,
V record ) {
buffer.put(sortable, record);
}
@Override
public Iterator getAll( K key ) {
V value = buffer.get(key);
return value == null ? null : new SingleIterator(value);
}
@Override
public Iterator getAll( K lowerKey,
boolean includeLowerKey,
K upperKey,
boolean includeUpperKey ) {
if (lowerKey == null) {
if (upperKey == null) {
// It is unbounded ...
return buffer.values().iterator();
}
return buffer.headMap(upperKey, includeUpperKey).values().iterator();
}
assert lowerKey != null;
if (upperKey == null) {
return buffer.tailMap(lowerKey, includeLowerKey).values().iterator();
}
return buffer.subMap(lowerKey, includeLowerKey, upperKey, includeUpperKey).values().iterator();
}
@Override
public Iterator ascending() {
final Iterator> entryIter = buffer.entrySet().iterator();
return new Iterator() {
@Override
public boolean hasNext() {
return entryIter.hasNext();
}
@Override
public V next() {
return entryIter.next().getValue();
}
@Override
public void remove() {
entryIter.remove();
}
};
}
@Override
public Iterator descending() {
final Iterator> entryIter = buffer.descendingMap().entrySet().iterator();
return new Iterator() {
@Override
public boolean hasNext() {
return entryIter.hasNext();
}
@Override
public V next() {
return entryIter.next().getValue();
}
@Override
public void remove() {
entryIter.remove();
}
};
}
@Override
public String toString() {
return "SortingBuffer(" + name + ")";
}
}
protected final class CloseableSortingBufferWithDuplicates, V> extends CloseableBuffer
implements SortingBuffer {
private final NavigableMap, V> buffer;
private final AtomicLong counter = new AtomicLong();
protected CloseableSortingBufferWithDuplicates( String name,
boolean onHeap,
NavigableMap, V> buffer ) {
super(name, onHeap);
this.buffer = buffer;
}
@Override
public boolean isEmpty() {
return buffer.isEmpty();
}
@Override
public long size() {
return buffer.size();
}
@Override
public void put( K sortable,
V record ) {
buffer.put(new UniqueKey(sortable, counter.incrementAndGet()), record);
}
@Override
public Iterator getAll( K key ) {
UniqueKey lowest = new UniqueKey(key, 0);
UniqueKey pastHighest = new UniqueKey(key, Long.MAX_VALUE);
SortedMap, V> map = buffer.subMap(lowest, pastHighest);
if (map == null || map.isEmpty()) return null;
final Iterator, V>> entryIter = map.entrySet().iterator();
return new Iterator() {
@Override
public boolean hasNext() {
return entryIter.hasNext();
}
@Override
public V next() {
return entryIter.next().getValue();
}
@Override
public void remove() {
entryIter.remove();
}
};
}
@Override
public Iterator getAll( K lowerKey,
boolean includeLowerKey,
K upperKey,
boolean includeUpperKey ) {
UniqueKey lowest = includeLowerKey ? new UniqueKey(lowerKey, 0) : new UniqueKey(lowerKey, Long.MAX_VALUE);
UniqueKey highest = includeUpperKey ? new UniqueKey(upperKey, Long.MAX_VALUE) : new UniqueKey(upperKey, 0L);
if (upperKey == null) {
if (lowerKey == null) return Collections.emptyList().iterator();
return buffer.tailMap(lowest, includeLowerKey).values().iterator();
} else if (lowerKey == null) {
assert upperKey != null;
return buffer.headMap(highest, includeUpperKey).values().iterator();
}
assert lowerKey != null;
assert upperKey != null;
return buffer.subMap(lowest, includeLowerKey, highest, includeUpperKey).values().iterator();
}
@Override
public Iterator ascending() {
final Iterator, V>> entryIter = buffer.entrySet().iterator();
return new Iterator() {
@Override
public boolean hasNext() {
return entryIter.hasNext();
}
@Override
public V next() {
return entryIter.next().getValue();
}
@Override
public void remove() {
entryIter.remove();
}
};
}
@Override
public Iterator descending() {
final Iterator, V>> entryIter = buffer.descendingMap().entrySet().iterator();
return new Iterator() {
@Override
public boolean hasNext() {
return entryIter.hasNext();
}
@Override
public V next() {
return entryIter.next().getValue();
}
@Override
public void remove() {
entryIter.remove();
}
};
}
@Override
public String toString() {
return "SortingBufferWithDuplicateKeys(" + name + ")";
}
}
protected final class MakeOrderedBuffer implements QueueBufferMaker {
private final String name;
private boolean useHeap = true;
private final Serializer serializer;
protected MakeOrderedBuffer( String name,
Serializer serializer ) {
assert name != null;
assert serializer != null;
this.name = name;
this.serializer = serializer;
}
@Override
public MakeOrderedBuffer useHeap( boolean useHeap ) {
this.useHeap = useHeap;
return this;
}
@Override
public QueueBuffer make() {
HTreeMap values = db(useHeap).createHashMap(name).valueSerializer(serializer).counterEnable().make();
return new CloseableQueueBuffer(name, useHeap, values);
}
}
protected final class MakeDistinctBuffer implements DistinctBufferMaker {
private final String name;
private boolean useHeap = true;
private boolean keepsize = false;
private final Serializer serializer;
protected MakeDistinctBuffer( String name,
Serializer serializer ) {
assert name != null;
assert serializer != null;
this.name = name;
this.serializer = serializer;
}
@Override
public DistinctBufferMaker keepSize( boolean keepBufferSize ) {
this.keepsize = keepBufferSize;
return this;
}
@Override
public DistinctBufferMaker useHeap( boolean useHeap ) {
this.useHeap = useHeap;
return this;
}
@Override
public DistinctBuffer make() {
HTreeSetMaker maker = db(useHeap).createHashSet(name).serializer(serializer);
if (keepsize) maker = maker.counterEnable();
Set buffer = maker.make();
return new CloseableDistinctBuffer(name, useHeap, buffer);
}
}
protected final class MakeSortingBuffer implements SortingBufferMaker {
private final String name;
private boolean useHeap = true;
private boolean keepsize = false;
private final BTreeKeySerializer keySerializer;
private final Serializer valueSerializer;
protected MakeSortingBuffer( String name,
BTreeKeySerializer keySerializer,
Serializer valueSerializer ) {
this.name = name;
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
}
@Override
public SortingBufferMaker keepSize( boolean keepBufferSize ) {
this.keepsize = keepBufferSize;
return this;
}
@Override
public SortingBufferMaker useHeap( boolean useHeap ) {
this.useHeap = useHeap;
return this;
}
@Override
public SortingBuffer make() {
BTreeMapMaker maker = db(useHeap).createTreeMap(name).keySerializer(keySerializer).valueSerializer(valueSerializer);
if (keepsize) maker = maker.counterEnable();
NavigableMap buffer = maker.make();
return new CloseableSortingBuffer(name, useHeap, buffer);
}
}
protected final class MakeSortingWithDuplicatesBuffer, V> implements SortingBufferMaker {
private final String name;
private boolean useHeap = true;
private boolean keepsize = false;
private final Serializer keySerializer;
private final Serializer valueSerializer;
private final Comparator keyComparator;
@SuppressWarnings( "unchecked" )
protected MakeSortingWithDuplicatesBuffer( String name,
Serializer keySerializer,
Comparator keyComparator,
Serializer valueSerializer ) {
this.name = name;
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
this.keyComparator = (Comparator)keyComparator;
}
@Override
public SortingBufferMaker keepSize( boolean keepBufferSize ) {
this.keepsize = keepBufferSize;
return this;
}
@Override
public SortingBufferMaker useHeap( boolean useHeap ) {
this.useHeap = useHeap;
return this;
}
@Override
public SortingBuffer make() {
Comparator> comparator = this.keyComparator != null ? new UniqueKeyComparator(keyComparator) : new ComparableUniqueKeyComparator();
BTreeKeySerializer> uniqueKeySerializer = new UniqueKeyBTreeSerializer(keySerializer, comparator);
BTreeMapMaker maker = db(useHeap).createTreeMap(name).keySerializer(uniqueKeySerializer)
.valueSerializer(valueSerializer);
if (keepsize) maker = maker.counterEnable();
NavigableMap, V> buffer = maker.make();
return new CloseableSortingBufferWithDuplicates(name, useHeap, buffer);
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy