com.googlecode.cqengine.index.navigable.NavigableIndex Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cqengine Show documentation
Show all versions of cqengine Show documentation
Collection Query Engine: NoSQL indexing and query engine for Java collections with ultra-low latency
/**
* Copyright 2012-2015 Niall Gallagher
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.googlecode.cqengine.index.navigable;
import com.googlecode.concurrenttrees.common.LazyIterator;
import com.googlecode.cqengine.attribute.Attribute;
import com.googlecode.cqengine.attribute.SimpleAttribute;
import com.googlecode.cqengine.attribute.SimpleNullableAttribute;
import com.googlecode.cqengine.index.support.*;
import com.googlecode.cqengine.quantizer.Quantizer;
import com.googlecode.cqengine.query.Query;
import com.googlecode.cqengine.query.option.DeduplicationOption;
import com.googlecode.cqengine.query.option.QueryOptions;
import com.googlecode.cqengine.query.simple.*;
import com.googlecode.cqengine.resultset.connective.ResultSetUnion;
import com.googlecode.cqengine.resultset.connective.ResultSetUnionAll;
import com.googlecode.cqengine.resultset.filter.QuantizedResultSet;
import com.googlecode.cqengine.resultset.ResultSet;
import com.googlecode.cqengine.resultset.iterator.IteratorUtil;
import com.googlecode.cqengine.resultset.iterator.UnmodifiableIterator;
import com.googlecode.cqengine.resultset.stored.StoredResultSet;
import com.googlecode.cqengine.resultset.stored.StoredSetBasedResultSet;
import java.util.*;
import java.util.concurrent.*;
/**
* An index backed by a {@link ConcurrentSkipListMap}.
*
* Supports query types:
*
* -
* {@link Equal}
*
* -
* {@link LessThan}
*
* -
* {@link GreaterThan}
*
* -
* {@link Between}
*
*
*
* The constructor of this index accepts {@link Factory} objects, from which it will create the map and value sets it
* uses internally. This allows the application to "tune" the construction parameters of these maps/sets,
* by supplying custom factories.
* For default settings, supply {@link DefaultIndexMapFactory} and {@link DefaultValueSetFactory}.
*
* @author Niall Gallagher
*/
public class NavigableIndex, O> extends AbstractMapBasedAttributeIndex>> implements SortedKeyStatisticsAttributeIndex {
protected static final int INDEX_RETRIEVAL_COST = 40;
/**
* Package-private constructor, used by static factory methods. Creates a new NavigableIndex initialized to index
* the supplied attribute.
*
* @param indexMapFactory A factory used to create the main map-based data structure used by the index
* @param valueSetFactory A factory used to create sets to store values in the index
* @param attribute The attribute on which the index will be built
*/
protected NavigableIndex(Factory>> indexMapFactory, Factory> valueSetFactory, Attribute attribute) {
super(indexMapFactory, valueSetFactory, attribute, new HashSet>() {{
add(Equal.class);
add(LessThan.class);
add(GreaterThan.class);
add(Between.class);
add(Has.class);
}});
}
/**
* {@inheritDoc}
*
* This index is mutable.
* @return true
*/
@Override
public boolean isMutable() {
return true;
}
/**
* {@inheritDoc}
*/
@Override
public ResultSet retrieve(final Query query, final QueryOptions queryOptions) {
Class queryClass = query.getClass();
final boolean indexIsQuantized = isQuantized();
// Process Equal queries in the same was as HashIndex...
if (queryClass.equals(Equal.class)) {
final Equal equal = (Equal) query;
return new ResultSet() {
@Override
public Iterator iterator() {
ResultSet rs = indexMap.get(getQuantizedValue(equal.getValue()));
return rs == null ? Collections.emptySet().iterator() : filterForQuantization(rs, equal, queryOptions).iterator();
}
@Override
public boolean contains(O object) {
ResultSet rs = indexMap.get(getQuantizedValue(equal.getValue()));
return rs != null && filterForQuantization(rs, equal, queryOptions).contains(object);
}
@Override
public boolean matches(O object) {
return query.matches(object, queryOptions);
}
@Override
public int size() {
ResultSet rs = indexMap.get(getQuantizedValue(equal.getValue()));
return rs == null ? 0 : filterForQuantization(rs, equal, queryOptions).size();
}
@Override
public int getRetrievalCost() {
return INDEX_RETRIEVAL_COST;
}
@Override
public int getMergeCost() {
// Return size of entire stored set as merge cost...
ResultSet rs = indexMap.get(getQuantizedValue(equal.getValue()));
return rs == null ? 0 : rs.size();
}
@Override
public void close() {
// No op.
}
@Override
public Query getQuery() {
return query;
}
@Override
public QueryOptions getQueryOptions() {
return queryOptions;
}
};
}
else if (queryClass.equals(Has.class)) {
// If a query option specifying logical deduplication is supplied return ResultSetUnion,
// otherwise return ResultSetUnionAll.
// We can avoid deduplication if the index is built on a SimpleAttribute however,
// because the same object could not exist in more than one StoredResultSet...
if (DeduplicationOption.isLogicalElimination(queryOptions)
&& !(getAttribute() instanceof SimpleAttribute || getAttribute() instanceof SimpleNullableAttribute)) {
return new ResultSetUnion(indexMap.values(), query, queryOptions) {
@Override
public int getRetrievalCost() {
return INDEX_RETRIEVAL_COST;
}
};
}
else {
return new ResultSetUnionAll(indexMap.values(), query, queryOptions) {
@Override
public int getRetrievalCost() {
return INDEX_RETRIEVAL_COST;
}
};
}
}
// Process LessThan, GreaterThan and Between queries as follows...
final IndexRangeLookupFunction lookupFunction;
if (queryClass.equals(LessThan.class)) {
final LessThan lessThan = (LessThan) query;
lookupFunction = new IndexRangeLookupFunction(query, false, true) {
@Override
public Iterable> perform() {
return indexMap.headMap(
getQuantizedValue(lessThan.getValue()),
lessThan.isValueInclusive() || indexIsQuantized
).values();
}
};
}
else if (queryClass.equals(GreaterThan.class)) {
final GreaterThan greaterThan = (GreaterThan) query;
lookupFunction = new IndexRangeLookupFunction(query, true, false) {
@Override
public Iterable> perform() {
return indexMap.tailMap(
getQuantizedValue(greaterThan.getValue()),
greaterThan.isValueInclusive() || indexIsQuantized
).values();
}
};
}
else if (queryClass.equals(Between.class)) {
final Between between = (Between) query;
lookupFunction = new IndexRangeLookupFunction(query, true, true) {
@Override
public Iterable> perform() {
return indexMap.subMap(
getQuantizedValue(between.getLowerValue()),
between.isLowerInclusive() || indexIsQuantized,
getQuantizedValue(between.getUpperValue()),
between.isUpperInclusive() || indexIsQuantized
).values();
}
};
}
else {
throw new IllegalStateException("Unsupported query: " + query);
}
// Fetch results using the supplied function.
// This should return a Collection which is actually just a view onto the index
// which presents a collection of sets containing values selected by the function...
@SuppressWarnings({"unchecked"})
Iterable> results = (Iterable>) lookupFunction.perform();
// Add filtering for quantization (implemented by subclass supporting a Quantizer, a no-op in this class)...
results = addFilteringForQuantization(results, lookupFunction, queryOptions);
// If a query option specifying logical deduplication is supplied return ResultSetUnion,
// otherwise return ResultSetUnionAll.
// We can avoid deduplication if the index is built on a SimpleAttribute however,
// because the same object could not exist in more than one StoredResultSet...
if (DeduplicationOption.isLogicalElimination(queryOptions)
&& !(getAttribute() instanceof SimpleAttribute || getAttribute() instanceof SimpleNullableAttribute)) {
return new ResultSetUnion(results, query, queryOptions) {
@Override
public int getRetrievalCost() {
return INDEX_RETRIEVAL_COST;
}
};
}
else {
return new ResultSetUnionAll(results, query, queryOptions) {
@Override
public int getRetrievalCost() {
return INDEX_RETRIEVAL_COST;
}
};
}
}
/**
* An interface which when implemented encapsulates the logic to retrieve zero or more {@link ResultSet}s from an
* index, where the logic for selecting these {@link ResultSet}s is implemented by the {@link #perform()} method.
*
* @param The type of object stored in the result sets
*/
protected abstract class IndexRangeLookupFunction {
protected final boolean filterFirstResultSet;
protected final boolean filterLastResultSet;
protected final Query query;
/**
* The following arguments are useful when the index uses a {@link com.googlecode.cqengine.quantizer.Quantizer},
* and so the stored sets of objects might need to be filtered on retrieval.
*
* @param query The query against which objects should be filtered
* @param filterFirstResultSet True if the first {@link StoredResultSet} returned by the function should be
* filtered to return only objects which actually match the query - typically true for {@link GreaterThan} and
* {@link Between} queries and false for {@link LessThan} queries
* @param filterLastResultSet True if the last {@link StoredResultSet} returned by the function should be
* filtered to return only objects which actually match the query - typically true for {@link LessThan} and
* {@link Between} queries and false for {@link GreaterThan} queries
*/
protected IndexRangeLookupFunction(Query query, boolean filterFirstResultSet, boolean filterLastResultSet) {
this.query = query;
this.filterFirstResultSet = filterFirstResultSet;
this.filterLastResultSet = filterLastResultSet;
}
protected abstract Iterable> perform();
}
@Override
public CloseableIterable getDistinctKeys(QueryOptions queryOptions) {
return wrapNonCloseable(getDistinctKeysInRange(null, true, null, true));
}
@Override
public CloseableIterable getDistinctKeys(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive, QueryOptions queryOptions) {
return wrapNonCloseable(getDistinctKeysInRange(lowerBound, lowerInclusive, upperBound, upperInclusive));
}
@Override
public CloseableIterable getDistinctKeysDescending(QueryOptions queryOptions) {
return wrapNonCloseable(getDistinctKeysInRange(null, true, null, true).descendingSet());
}
@Override
public CloseableIterable getDistinctKeysDescending(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive, QueryOptions queryOptions) {
return wrapNonCloseable(getDistinctKeysInRange(lowerBound, lowerInclusive, upperBound, upperInclusive).descendingSet());
}
NavigableSet getDistinctKeysInRange(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive) {
NavigableSet results;
if (lowerBound != null && upperBound != null) {
results = indexMap.keySet().subSet(lowerBound, lowerInclusive, upperBound, upperInclusive);
}
else if (lowerBound != null) {
results = indexMap.keySet().tailSet(lowerBound, lowerInclusive);
}
else if (upperBound != null) {
results = indexMap.keySet().headSet(upperBound, upperInclusive);
}
else {
results = indexMap.keySet();
}
return results;
}
@Override
public Integer getCountForKey(A key, QueryOptions queryOptions) {
return super.getCountForKey(key);
}
@Override
public Integer getCountOfDistinctKeys(QueryOptions queryOptions) {
return super.getCountOfDistinctKeys(queryOptions);
}
@Override
public CloseableIterable> getStatisticsForDistinctKeys(QueryOptions queryOptions) {
return super.getStatisticsForDistinctKeys(queryOptions);
}
@Override
public CloseableIterable> getStatisticsForDistinctKeysDescending(final QueryOptions queryOptions) {
final CloseableIterator distinctKeysDescending = getDistinctKeysDescending(queryOptions).iterator();
return wrapNonCloseable(new Iterable>() {
@Override
public Iterator> iterator() {
return new LazyIterator>() {
@Override
protected KeyStatistics computeNext() {
if (distinctKeysDescending.hasNext()){
A key = distinctKeysDescending.next();
return new KeyStatistics(key, getCountForKey(key, queryOptions));
}else{
return endOfData();
}
}
};
}
});
}
@Override
public CloseableIterable> getKeysAndValues(QueryOptions queryOptions) {
return wrapNonCloseable(IteratorUtil.flatten(getKeysAndValuesInRange(null, true, null, true)));
}
@Override
public CloseableIterable> getKeysAndValues(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive, QueryOptions queryOptions) {
return wrapNonCloseable(IteratorUtil.flatten(getKeysAndValuesInRange(lowerBound, lowerInclusive, upperBound, upperInclusive)));
}
@Override
public CloseableIterable> getKeysAndValuesDescending(QueryOptions queryOptions) {
return wrapNonCloseable(IteratorUtil.flatten(getKeysAndValuesInRange(null, true, null, true).descendingMap()));
}
@Override
public CloseableIterable> getKeysAndValuesDescending(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive, QueryOptions queryOptions) {
return wrapNonCloseable(IteratorUtil.flatten(getKeysAndValuesInRange(lowerBound, lowerInclusive, upperBound, upperInclusive).descendingMap()));
}
NavigableMap> getKeysAndValuesInRange(A lowerBound, boolean lowerInclusive, A upperBound, boolean upperInclusive) {
NavigableMap> results;
if (lowerBound != null && upperBound != null) {
results = indexMap.subMap(lowerBound, lowerInclusive, upperBound, upperInclusive);
}
else if (lowerBound != null) {
results = indexMap.tailMap(lowerBound, lowerInclusive);
}
else if (upperBound != null) {
results = indexMap.headMap(upperBound, upperInclusive);
}
else {
results = indexMap;
}
return results;
}
// ---------- Hook methods which can be overridden by subclasses using a Quantizer ----------
/**
* A no-op method which may be overridden by subclasses which use a
* {@link com.googlecode.cqengine.quantizer.Quantizer}.
*
* This default implementation simply returns the given attribute value unmodified.
*
* Returns an {@link Iterable} which is similar to the one supplied, but which transparently wraps the first
* {@link ResultSet} and/or the last {@link ResultSet} returned by the supplied {@link Iterable} in a
* {@link QuantizedResultSet}.
*
* A {@link QuantizedResultSet} transparently filters objects in the wrapped {@link ResultSet} to ensure that
* they match the given {@link Query}. This is necessary when the index uses a
* {@link com.googlecode.cqengine.quantizer.Quantizer}, where objects having several adjacent attribute values
* will be stored together in the same {@link StoredResultSet} in the index.
*
* For range queries ({@link LessThan}, {@link GreaterThan}, {@link Between}), the {@link StoredResultSet}s
* in an index using a quantizer at the keys in the index referenced by the query, are not guaranteed to only
* contain objects matching the values in the query, due to objects being mixed with their adjacent counterparts.
* Therefore it is necessary to filter the {@link ResultSet} at either end of the range.
*
* @param resultSets {@link com.googlecode.cqengine.resultset.ResultSet}s stored in the index which were found to match the range query, in ascending
* order of their associated keys
* @param lookupFunction Contains parameters specifying whether the first or last {@link com.googlecode.cqengine.resultset.ResultSet}s should be
* wrapped in a {@link com.googlecode.cqengine.resultset.filter.QuantizedResultSet}, and also encapsulates the {@link com.googlecode.cqengine.query.Query} against which objects should
* be filtered
* @param queryOptions Optional parameters for the query
*
* @return An {@link Iterable} optionally with the first and/or last {@link ResultSet}s wrapped in
* {@link QuantizedResultSet}s
*/
protected Iterable> addFilteringForQuantization(final Iterable> resultSets, final IndexRangeLookupFunction lookupFunction, QueryOptions queryOptions) {
return resultSets;
}
/**
* A no-op method which can be overridden by a subclass to return a {@link ResultSet} which filters objects from the
* given {@link ResultSet}, to return only those objects matching the query, for the case that the index is using a
* {@link com.googlecode.cqengine.quantizer.Quantizer}.
*
* This default implementation simply returns the given {@link ResultSet} unmodified.
*
* @param storedResultSet A {@link com.googlecode.cqengine.resultset.ResultSet} stored against a quantized key in the index
* @param query The query against which results should be matched
* @param queryOptions Optional parameters for the query
*
* @return A {@link ResultSet} which filters objects from the given {@link ResultSet},
* to return only those objects matching the query
*/
protected ResultSet filterForQuantization(ResultSet storedResultSet, Query query, QueryOptions queryOptions) {
return storedResultSet;
}
// ---------- Static factory methods to create NavigableIndexes ----------
/**
* Creates a new {@code NavigableIndex} on the given attribute. The attribute can be a {@link SimpleAttribute} or a
* {@link com.googlecode.cqengine.attribute.MultiValueAttribute}, as long as the type of the attribute referenced
* implements {@link Comparable}.
*
* @param attribute The attribute on which the index will be built, a {@link SimpleAttribute} or a
* {@link com.googlecode.cqengine.attribute.MultiValueAttribute} where the type of the attribute referenced
* implements {@link Comparable}
* @param The type of the attribute
* @param The type of the object containing the attribute
* @return A new HashIndex which will build an index on this attribute
*/
public static , O> NavigableIndex onAttribute(Attribute attribute) {
return onAttribute(new DefaultIndexMapFactory(), new DefaultValueSetFactory(), attribute);
}
/**
* Creates a new {@code NavigableIndex} on the given attribute. The attribute can be a {@link SimpleAttribute} or a
* {@link com.googlecode.cqengine.attribute.MultiValueAttribute}, as long as the type of the attribute referenced
* implements {@link Comparable}.
*
* @param indexMapFactory A factory used to create the main map-based data structure used by the index
* @param valueSetFactory A factory used to create sets to store values in the index
* @param attribute The attribute on which the index will be built, a {@link SimpleAttribute} or a
* {@link com.googlecode.cqengine.attribute.MultiValueAttribute} where the type of the attribute referenced
* implements {@link Comparable}
* @param The type of the attribute
* @param The type of the object containing the attribute
* @return A new HashIndex which will build an index on this attribute
*/
public static , O> NavigableIndex onAttribute(Factory>> indexMapFactory, Factory> valueSetFactory, Attribute attribute) {
return new NavigableIndex(indexMapFactory, valueSetFactory, attribute);
}
/**
* Creates a {@link NavigableIndex} on the given attribute using the given {@link Quantizer}.
*
* @param quantizer A {@link Quantizer} to use in this index
* @param attribute The attribute on which the index will be built
* @param The type of the object containing the attribute
* @return A {@link NavigableIndex} on the given attribute using the given {@link Quantizer}
*/
public static , O> NavigableIndex withQuantizerOnAttribute(final Quantizer quantizer, Attribute attribute) {
return withQuantizerOnAttribute(new DefaultIndexMapFactory(), new DefaultValueSetFactory(), quantizer, attribute);
}
/**
* Creates a {@link NavigableIndex} on the given attribute using the given {@link Quantizer}.
*
* @param indexMapFactory A factory used to create the main map-based data structure used by the index
* @param valueSetFactory A factory used to create sets to store values in the index
* @param quantizer A {@link Quantizer} to use in this index
* @param attribute The attribute on which the index will be built
* @param The type of the object containing the attribute
* @return A {@link NavigableIndex} on the given attribute using the given {@link Quantizer}
*/
public static , O> NavigableIndex withQuantizerOnAttribute(Factory>> indexMapFactory, Factory> valueSetFactory, final Quantizer quantizer, Attribute attribute) {
return new NavigableIndex(indexMapFactory, valueSetFactory, attribute) {
// ---------- Override the hook methods related to Quantizer ----------
@Override
protected Iterable> addFilteringForQuantization(final Iterable> resultSets, final IndexRangeLookupFunction lookupFunction, final QueryOptions queryOptions) {
if (!lookupFunction.filterFirstResultSet && !lookupFunction.filterLastResultSet) {
// No filtering required, return the same iterable...
return resultSets;
}
return new Iterable>() {
@Override
public Iterator> iterator() {
return new UnmodifiableIterator>() {
Iterator> resultSetsIterator = resultSets.iterator();
boolean firstResultSet = true;
@Override
public boolean hasNext() {
return resultSetsIterator.hasNext();
}
@Override
public ResultSet next() {
ResultSet rs = resultSetsIterator.next();
if (lookupFunction.filterFirstResultSet && firstResultSet) {
// Filtering is enabled for first ResultSet, and we are processing first ResultSet.
// Wrap it in QuantizedResultSet...
firstResultSet = false;
return new QuantizedResultSet(rs, lookupFunction.query, queryOptions);
}
else if (!lookupFunction.filterLastResultSet || resultSetsIterator.hasNext()) {
// Filtering is disabled for last ResultSet,
// or we are processing a ResultSet which is neither first nor last.
// Don't wrap it...
return rs;
}
else {
// Filtering is enabled for last ResultSet and we are processing last ResultSet.
// Wrap it in QuantizedResultSet...
return new QuantizedResultSet(rs, lookupFunction.query, queryOptions);
}
}
};
}
};
}
@Override
protected A getQuantizedValue(A attributeValue) {
return quantizer.getQuantizedValue(attributeValue);
}
@Override
protected ResultSet filterForQuantization(ResultSet storedResultSet, Query query, QueryOptions queryOptions) {
return new QuantizedResultSet(storedResultSet, query, queryOptions);
}
@Override
public boolean isQuantized() {
return true;
}
};
}
/**
* Creates an index map using default settings.
*/
public static class DefaultIndexMapFactory implements Factory>> {
@Override
public ConcurrentNavigableMap> create() {
return new ConcurrentSkipListMap>();
}
}
/**
* Creates a value set using default settings.
*/
public static class DefaultValueSetFactory implements Factory> {
@Override
public StoredResultSet create() {
return new StoredSetBasedResultSet(Collections.newSetFromMap(new ConcurrentHashMap()));
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy