org.apache.lucene.util.FieldCacheSanityChecker Maven / Gradle / Ivy
Show all versions of infinispan-embedded-query
package org.apache.lucene.util;
/**
* Copyright 2009 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
import org.apache.lucene.store.AlreadyClosedException;
/**
* Provides methods for sanity checking that entries in the FieldCache
* are not wasteful or inconsistent.
*
*
* Lucene 2.9 Introduced numerous enhancements into how the FieldCache
* is used by the low levels of Lucene searching (for Sorting and
* ValueSourceQueries) to improve both the speed for Sorting, as well
* as reopening of IndexReaders. But these changes have shifted the
* usage of FieldCache from "top level" IndexReaders (frequently a
* MultiReader or DirectoryReader) down to the leaf level SegmentReaders.
* As a result, existing applications that directly access the FieldCache
* may find RAM usage increase significantly when upgrading to 2.9 or
* Later. This class provides an API for these applications (or their
* Unit tests) to check at run time if the FieldCache contains "insane"
* usages of the FieldCache.
*
* @lucene.experimental
* @see FieldCache
* @see FieldCacheSanityChecker.Insanity
* @see FieldCacheSanityChecker.InsanityType
*/
public final class FieldCacheSanityChecker {
private boolean estimateRam;
public FieldCacheSanityChecker() {
/* NOOP */
}
/**
* If set, estimate size for all CacheEntry objects will be calculateed.
*/
public void setRamUsageEstimator(boolean flag) {
estimateRam = flag;
}
/**
* Quick and dirty convenience method
* @see #check
*/
public static Insanity[] checkSanity(FieldCache cache) {
return checkSanity(cache.getCacheEntries());
}
/**
* Quick and dirty convenience method that instantiates an instance with
* "good defaults" and uses it to test the CacheEntrys
* @see #check
*/
public static Insanity[] checkSanity(CacheEntry... cacheEntries) {
FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
sanityChecker.setRamUsageEstimator(true);
return sanityChecker.check(cacheEntries);
}
/**
* Tests a CacheEntry[] for indication of "insane" cache usage.
*
* NOTE:FieldCache CreationPlaceholder objects are ignored.
* (:TODO: is this a bad idea? are we masking a real problem?)
*
*/
public Insanity[] check(CacheEntry... cacheEntries) {
if (null == cacheEntries || 0 == cacheEntries.length)
return new Insanity[0];
if (estimateRam) {
for (int i = 0; i < cacheEntries.length; i++) {
cacheEntries[i].estimateSize();
}
}
// the indirect mapping lets MapOfSet dedup identical valIds for us
//
// maps the (valId) identityhashCode of cache values to
// sets of CacheEntry instances
final MapOfSets valIdToItems = new MapOfSets<>(new HashMap>(17));
// maps ReaderField keys to Sets of ValueIds
final MapOfSets readerFieldToValIds = new MapOfSets<>(new HashMap>(17));
//
// any keys that we know result in more then one valId
final Set valMismatchKeys = new HashSet<>();
// iterate over all the cacheEntries to get the mappings we'll need
for (int i = 0; i < cacheEntries.length; i++) {
final CacheEntry item = cacheEntries[i];
final Object val = item.getValue();
// It's OK to have dup entries, where one is eg
// float[] and the other is the Bits (from
// getDocWithField())
if (val instanceof Bits) {
continue;
}
if (val instanceof FieldCache.CreationPlaceholder)
continue;
final ReaderField rf = new ReaderField(item.getReaderKey(),
item.getFieldName());
final Integer valId = Integer.valueOf(System.identityHashCode(val));
// indirect mapping, so the MapOfSet will dedup identical valIds for us
valIdToItems.put(valId, item);
if (1 < readerFieldToValIds.put(rf, valId)) {
valMismatchKeys.add(rf);
}
}
final List insanity = new ArrayList<>(valMismatchKeys.size() * 3);
insanity.addAll(checkValueMismatch(valIdToItems,
readerFieldToValIds,
valMismatchKeys));
insanity.addAll(checkSubreaders(valIdToItems,
readerFieldToValIds));
return insanity.toArray(new Insanity[insanity.size()]);
}
/**
* Internal helper method used by check that iterates over
* valMismatchKeys and generates a Collection of Insanity
* instances accordingly. The MapOfSets are used to populate
* the Insanity objects.
* @see InsanityType#VALUEMISMATCH
*/
private Collection checkValueMismatch(MapOfSets valIdToItems,
MapOfSets readerFieldToValIds,
Set valMismatchKeys) {
final List insanity = new ArrayList<>(valMismatchKeys.size() * 3);
if (! valMismatchKeys.isEmpty() ) {
// we have multiple values for some ReaderFields
final Map> rfMap = readerFieldToValIds.getMap();
final Map> valMap = valIdToItems.getMap();
for (final ReaderField rf : valMismatchKeys) {
final List badEntries = new ArrayList<>(valMismatchKeys.size() * 2);
for(final Integer value: rfMap.get(rf)) {
for (final CacheEntry cacheEntry : valMap.get(value)) {
badEntries.add(cacheEntry);
}
}
CacheEntry[] badness = new CacheEntry[badEntries.size()];
badness = badEntries.toArray(badness);
insanity.add(new Insanity(InsanityType.VALUEMISMATCH,
"Multiple distinct value objects for " +
rf.toString(), badness));
}
}
return insanity;
}
/**
* Internal helper method used by check that iterates over
* the keys of readerFieldToValIds and generates a Collection
* of Insanity instances whenever two (or more) ReaderField instances are
* found that have an ancestry relationships.
*
* @see InsanityType#SUBREADER
*/
private Collection checkSubreaders( MapOfSets valIdToItems,
MapOfSets readerFieldToValIds) {
final List insanity = new ArrayList<>(23);
Map> badChildren = new HashMap<>(17);
MapOfSets badKids = new MapOfSets<>(badChildren); // wrapper
Map> viToItemSets = valIdToItems.getMap();
Map> rfToValIdSets = readerFieldToValIds.getMap();
Set seen = new HashSet<>(17);
Set readerFields = rfToValIdSets.keySet();
for (final ReaderField rf : readerFields) {
if (seen.contains(rf)) continue;
List