Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import static org.apache.solr.common.params.CommonParams.SORT;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.IntArrayList;
import com.carrotsearch.hppc.IntIntHashMap;
import com.carrotsearch.hppc.IntLongHashMap;
import com.carrotsearch.hppc.cursors.IntIntCursor;
import com.carrotsearch.hppc.cursors.IntLongCursor;
import com.carrotsearch.hppc.procedures.IntProcedure;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.EmptyDocValuesProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Pruning;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.GroupParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.QueryElevationComponent;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.NumberType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.util.IntFloatDynamicMap;
import org.apache.solr.util.IntIntDynamicMap;
import org.apache.solr.util.IntLongDynamicMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The CollapsingQParserPlugin is a PostFilter that performs field collapsing. This is a high
* performance alternative to standard Solr field collapsing (with ngroups) when the number of
* distinct groups in the result set is high.
*
*
Sample syntax:
*
*
Collapse based on the highest scoring document:
*
*
fq=(!collapse field=field_name}
*
*
Collapse based on the min value of a numeric field:
*
*
There are three null policies:
* ignore : removes docs with a null value in the collapse field (default).
* expand : treats each doc with a null value in the collapse field as a separate group.
* collapse : collapses all docs with a null value into a single group using either highest score,
* or min/max.
*
*
The CollapsingQParserPlugin fully supports the QueryElevationComponent
*/
public class CollapsingQParserPlugin extends QParserPlugin {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String NAME = "collapse";
public static final String HINT_TOP_FC = "top_fc";
/**
* Indicates that values in the collapse field are unique per contiguous block, and a single pass
* "block based" collapse algorithm can be used. This behavior is the default for collapsing on
* the _root_ field, but may also be enabled for other fields that have the same
* characteristics. This hint will be ignored if other options prevent the use of this single pass
* approach (notable: nullPolicy=collapse)
*
*
Do NOT use this hint if the index is not laid out such that each unique
* value in the collapse field is garuntteed to only exist in one contiguous block, otherwise the
* results of the collapse filter will include more then one document per collapse value.
*/
public static final String HINT_BLOCK = "block";
/**
* If elevation is used in combination with the collapse query parser, we can define that we only
* want to return the representative and not all elevated docs by setting this parameter to false
* (true by default).
*/
public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing";
/**
* @deprecated use {@link NullPolicy} instead.
*/
@Deprecated public static final String NULL_COLLAPSE = "collapse";
@Deprecated public static final String NULL_IGNORE = "ignore";
@Deprecated public static final String NULL_EXPAND = "expand";
@Deprecated public static final String HINT_MULTI_DOCVALUES = "multi_docvalues";
public enum NullPolicy {
IGNORE("ignore", 0),
COLLAPSE("collapse", 1),
EXPAND("expand", 2);
private final String name;
private final int code;
NullPolicy(String name, int code) {
this.name = name;
this.code = code;
}
public String getName() {
return name;
}
public int getCode() {
return code;
}
public static NullPolicy fromString(String nullPolicy) {
if (StrUtils.isNullOrEmpty(nullPolicy)) {
return DEFAULT_POLICY;
}
switch (nullPolicy) {
case "ignore":
return IGNORE;
case "collapse":
return COLLAPSE;
case "expand":
return EXPAND;
default:
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Invalid nullPolicy: " + nullPolicy);
}
}
static final NullPolicy DEFAULT_POLICY = IGNORE;
}
@Override
public QParser createParser(
String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) {
return new CollapsingQParser(qstr, localParams, params, request);
}
private static class CollapsingQParser extends QParser {
public CollapsingQParser(
String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) {
super(qstr, localParams, params, request);
}
@Override
public Query parse() throws SyntaxError {
try {
return new CollapsingPostFilter(localParams, params, req);
} catch (Exception e) {
throw new SyntaxError(e.getMessage(), e);
}
}
}
public enum GroupHeadSelectorType {
MIN,
MAX,
SORT,
SCORE;
public static final EnumSet MIN_MAX = EnumSet.of(MIN, MAX);
}
/** Models all the information about how group head documents should be selected */
public static final class GroupHeadSelector {
/**
* The param value for this selector whose meaning depends on type. (ie: a field or valuesource
* for MIN/MAX, a sort string for SORT, "score" for SCORE). Will never be null.
*/
public final String selectorText;
/** The type for this selector, will never be null */
public final GroupHeadSelectorType type;
private GroupHeadSelector(String s, GroupHeadSelectorType type) {
assert null != s;
assert null != type;
this.selectorText = s;
this.type = type;
}
@Override
public boolean equals(final Object other) {
if (other instanceof GroupHeadSelector) {
final GroupHeadSelector that = (GroupHeadSelector) other;
return (this.type == that.type) && this.selectorText.equals(that.selectorText);
}
return false;
}
@Override
public int hashCode() {
return 17 * (31 + selectorText.hashCode()) * (31 + type.hashCode());
}
@Override
public String toString() {
return "GroupHeadSelector(selectorText=" + this.selectorText + ", type=" + this.type + ")";
}
/** returns a new GroupHeadSelector based on the specified local params */
public static GroupHeadSelector build(final SolrParams localParams) {
final String sortString =
StrUtils.isBlank(localParams.get(SORT)) ? null : localParams.get(SORT);
final String max = StrUtils.isBlank(localParams.get("max")) ? null : localParams.get("max");
final String min = StrUtils.isBlank(localParams.get("min")) ? null : localParams.get("min");
if (1 < numNotNull(min, max, sortString)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"At most one localparam for selecting documents (min, max, sort) may be specified: "
+ localParams.toString());
}
if (null != sortString) {
return new GroupHeadSelector(sortString, GroupHeadSelectorType.SORT);
} else if (null != min) {
return new GroupHeadSelector(min, GroupHeadSelectorType.MIN);
} else if (null != max) {
return new GroupHeadSelector(max, GroupHeadSelectorType.MAX);
}
// default
return new GroupHeadSelector("score", GroupHeadSelectorType.SCORE);
}
}
public static class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter {
private String collapseField;
private final GroupHeadSelector groupHeadSelector;
private final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector
public String hint;
private boolean needsScores = true;
private boolean needsScores4Collapsing = false;
private NullPolicy nullPolicy;
private Set boosted; // ordered by "priority"
private int size;
public String getField() {
return this.collapseField;
}
@Override
public void setCache(boolean cache) {}
@Override
public boolean getCache() {
return false;
}
// Only a subset of fields in hashCode/equals?
@Override
public int hashCode() {
int hashCode = classHash();
hashCode = 31 * hashCode + collapseField.hashCode();
hashCode = 31 * hashCode + groupHeadSelector.hashCode();
hashCode = 31 * hashCode + nullPolicy.hashCode();
return hashCode;
}
@Override
public boolean equals(Object other) {
return sameClassAs(other) && equalsTo(getClass().cast(other));
}
private boolean equalsTo(CollapsingPostFilter other) {
return collapseField.equals(other.collapseField)
&& groupHeadSelector.equals(other.groupHeadSelector)
&& nullPolicy == other.nullPolicy;
}
@Override
public void visit(QueryVisitor visitor) {
visitor.visitLeaf(this);
}
@Override
public int getCost() {
return Math.max(super.getCost(), 100);
}
@Override
public String toString(String s) {
return "CollapsingPostFilter(field="
+ this.collapseField
+ ", nullPolicy="
+ this.nullPolicy.getName()
+ ", "
+ this.groupHeadSelector
+ (hint == null ? "" : ", hint=" + this.hint)
+ ", size="
+ this.size
+ ")";
}
public CollapsingPostFilter(
SolrParams localParams, SolrParams params, SolrQueryRequest request) {
// Don't allow collapsing if grouping is being used.
if (request.getParams().getBool(GroupParams.GROUP, false)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Can not use collapse with Grouping enabled");
}
this.collapseField = localParams.get("field");
if (this.collapseField == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Required 'field' param is missing.");
}
// if unknown field, this would fail fast
SchemaField collapseFieldSf = request.getSchema().getField(this.collapseField);
if (!(collapseFieldSf.isUninvertible() || collapseFieldSf.hasDocValues())) {
// uninvertible=false and docvalues=false
// field can't be indexed=false and uninvertible=true
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field '"
+ collapseField
+ "' should be either docValues enabled or indexed with uninvertible enabled");
} else if (collapseFieldSf.multiValued()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Collapsing not supported on multivalued fields");
}
this.groupHeadSelector = GroupHeadSelector.build(localParams);
if (groupHeadSelector.type.equals(GroupHeadSelectorType.SORT)
&& CollapseScore.wantsCScore(groupHeadSelector.selectorText)) {
// we can't support Sorts that wrap functions that include "cscore()" because
// the abstraction layer for Sort/SortField rewriting gives each clause it's own
// context Map which we don't have access to -- so for now, give a useful error
// (as early as possible) if attempted
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Using cscore() as a function in the 'sort' local "
+ "param of the collapse parser is not supported");
}
this.sortSpec =
GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
? SortSpecParsing.parseSortSpec(groupHeadSelector.selectorText, request)
: null;
this.hint = localParams.get("hint");
this.size = localParams.getInt("size", 100000); // Only used for collapsing on int fields.
{
final SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
assert null != info;
// may be null in some esoteric corner usages
final ResponseBuilder rb = info.getResponseBuilder();
final SortSpec topSort = null == rb ? null : rb.getSortSpec();
this.needsScores4Collapsing =
GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)
|| (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
&& this.sortSpec.includesScore())
|| (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)
&& CollapseScore.wantsCScore(groupHeadSelector.selectorText));
this.needsScores =
needsScores4Collapsing
|| (info.getRsp().getReturnFields().wantsScore()
|| (null != topSort && topSort.includesScore())
|| (this.boosted != null));
if (this.needsScores && null != rb) {
// regardless of why we need scores ensure the IndexSearcher will compute them
// for the "real" docs. (ie: maybe we need them because we were
// asked to compute them for the collapsed docs, maybe we need them because in
// order to find the groupHead we need them computed for us.
rb.setFieldFlags(rb.getFieldFlags() | SolrIndexSearcher.GET_SCORES);
}
}
this.nullPolicy = NullPolicy.fromString(localParams.get("nullPolicy"));
}
@Override
@SuppressWarnings({"unchecked"})
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
try {
SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher;
CollectorFactory collectorFactory = new CollectorFactory();
// Deal with boosted docs.
// We have to deal with it here rather then the constructor because
// because the QueryElevationComponent runs after the Queries are constructed.
IntIntHashMap boostDocsMap = null;
Map
needsScores
*
* @lucene.internal
*/
abstract static class AbstractBlockSortSpecCollector extends AbstractBlockCollector {
/**
* Helper method for extracting a {@link Sort} out of a {@link SortSpec} or creating
* one synthetically for "min/max" {@link GroupHeadSelector} against a {@link FunctionQuery}
* or simple field name.
*
* @return appropriate (already re-written) Sort to use with a AbstractBlockSortSpecCollector
*/
public static Sort getSort(
final GroupHeadSelector groupHeadSelector,
final SortSpec sortSpec,
final FunctionQuery funcQuery,
final SolrIndexSearcher searcher)
throws IOException {
if (null != sortSpec) {
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
// a "feature" of SortSpec is that getSort() is null if we're just using 'score desc'
if (null == sortSpec.getSort()) {
return Sort.RELEVANCE.rewrite(searcher);
}
return sortSpec.getSort().rewrite(searcher);
} // else: min/max on field or value source...
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
assert !CollapseScore.wantsCScore(groupHeadSelector.selectorText);
final boolean reverse = GroupHeadSelectorType.MAX.equals(groupHeadSelector.type);
final SortField sf =
(null != funcQuery)
? funcQuery.getValueSource().getSortField(reverse)
: searcher.getSchema().getField(groupHeadSelector.selectorText).getSortField(reverse);
return (new Sort(sf)).rewrite(searcher);
}
private final BlockBasedSortFieldsCompare sortsCompare;
public AbstractBlockSortSpecCollector(
final String collapseField,
final int nullPolicy,
final IntIntHashMap boostDocsMap,
final Sort sort,
final boolean needsScores) {
super(collapseField, nullPolicy, boostDocsMap, needsScores);
this.sortsCompare = new BlockBasedSortFieldsCompare(sort.getSort());
}
@Override
public void setScorer(Scorable scorer) throws IOException {
sortsCompare.setScorer(scorer);
super.setScorer(scorer);
}
private void setCurrentGroupBestMatch(final int contextDocId, final boolean isBoosted)
throws IOException {
currentGroupState.setBestDocForCurrentGroup(contextDocId, isBoosted);
if (needsScores) {
currentGroupState.score = scorer.score();
}
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
super.doSetNextReader(context);
this.sortsCompare.setNextReader(context);
}
/**
* This method should be called by subclasses for each doc + group encountered
*
* @param contextDoc a valid doc id relative to the current reader context
* @param docGroup some uique identifier for the group - the base class makes no assumptions
* about it's meaning
* @see #collectDocWithNullGroup
*/
protected void collectDocWithGroup(int contextDoc, int docGroup) throws IOException {
assert 0 <= contextDoc;
final boolean isBoosted = isBoostedAdvanceExact(contextDoc);
if (-1 < currentGroupState.docID() && docGroup == currentGroupState.getCurrentGroup()) {
// we have an existing group, and contextDoc is in that group.
if (isBoosted) {
// this doc is the best and should be immediately collected regardless of sort values
setCurrentGroupBestMatch(contextDoc, isBoosted);
delegateCollect();
} else if (currentGroupState.hasBoostedDocs()) {
// No-Op: nothing about this doc matters since we've already collected boosted docs in
// this group
// No-Op
} else {
// check if it's the new 'best' doc in this group...
if (sortsCompare.testAndSetGroupValues(contextDoc)) {
setCurrentGroupBestMatch(contextDoc, isBoosted);
}
}
} else {
// We have a document that starts a new group (or may be the first doc+group we've collected
// this segmen)
// first collect the prior group if needed...
maybeDelegateCollect();
// then setup the new group and current best match
currentGroupState.resetForNewGroup();
currentGroupState.setCurrentGroup(docGroup);
sortsCompare.setGroupValues(contextDoc);
setCurrentGroupBestMatch(contextDoc, isBoosted);
if (isBoosted) { // collect immediately
delegateCollect();
}
}
}
/**
* This method should be called by subclasses for each doc encountered that is not in a group
* (ie: null group)
*
* @param contextDoc a valid doc id relative to the current reader context
* @see #collectDocWithGroup
*/
protected void collectDocWithNullGroup(int contextDoc) throws IOException {
assert 0 <= contextDoc;
// NOTE: with 'null group' docs, it doesn't matter if they are boosted since we don't suppor
// collapsing nulls
// this doc is definitely not part of any prior group, so collect if needed...
maybeDelegateCollect();
if (expandNulls) {
// set & immediately collect our current doc...
setCurrentGroupBestMatch(contextDoc, false);
// NOTE: sort values don't matter
delegateCollect();
} else {
// we're ignoring nulls, so: No-Op.
}
// either way re-set for the next doc / group
currentGroupState.resetForNewGroup();
}
}
/**
* A block based score collector that uses a field's "ord" as the group ids
*
* @lucene.internal
*/
static class BlockOrdSortSpecCollector extends AbstractBlockSortSpecCollector {
private SortedDocValues segmentValues;
public BlockOrdSortSpecCollector(
final String collapseField,
final int nullPolicy,
final IntIntHashMap boostDocsMap,
final Sort sort,
final boolean needsScores)
throws IOException {
super(collapseField, nullPolicy, boostDocsMap, sort, needsScores);
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
super.doSetNextReader(context);
this.segmentValues = DocValues.getSorted(context.reader(), collapseField);
}
@Override
public void collect(int contextDoc) throws IOException {
if (segmentValues.advanceExact(contextDoc)) {
int ord = segmentValues.ordValue();
collectDocWithGroup(contextDoc, ord);
} else {
collectDocWithNullGroup(contextDoc);
}
}
}
/**
* A block based score collector that uses a field's numeric value as the group ids
*
* @lucene.internal
*/
static class BlockIntSortSpecCollector extends AbstractBlockSortSpecCollector {
private NumericDocValues segmentValues;
public BlockIntSortSpecCollector(
final String collapseField,
final int nullPolicy,
final IntIntHashMap boostDocsMap,
final Sort sort,
final boolean needsScores)
throws IOException {
super(collapseField, nullPolicy, boostDocsMap, sort, needsScores);
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
super.doSetNextReader(context);
this.segmentValues = DocValues.getNumeric(context.reader(), collapseField);
}
@Override
public void collect(int contextDoc) throws IOException {
if (segmentValues.advanceExact(contextDoc)) {
int group = (int) segmentValues.longValue();
collectDocWithGroup(contextDoc, group);
} else {
collectDocWithNullGroup(contextDoc);
}
}
}
private static class CollectorFactory {
/**
* @see #isNumericCollapsible
*/
private static final EnumSet NUMERIC_COLLAPSIBLE_TYPES =
EnumSet.of(NumberType.INTEGER, NumberType.FLOAT);
private boolean isNumericCollapsible(FieldType collapseFieldType) {
return NUMERIC_COLLAPSIBLE_TYPES.contains(collapseFieldType.getNumberType());
}
public DelegatingCollector getCollector(
String collapseField,
GroupHeadSelector groupHeadSelector,
SortSpec sortSpec,
int nullPolicy,
String hint,
boolean needsScores4Collapsing,
boolean needsScores,
int size,
IntIntHashMap boostDocs,
SolrIndexSearcher searcher)
throws IOException {
DocValuesProducer docValuesProducer = null;
FunctionQuery funcQuery = null;
// block collapsing logic is much simpler and uses less memory, but is only viable in specific
// situations
final boolean blockCollapse =
(("_root_".equals(collapseField) || HINT_BLOCK.equals(hint))
// because we currently handle all min/max cases using
// AbstractBlockSortSpecCollector, we can't handle functions wrapping cscore()
// (for the same reason cscore() isn't supported in 'sort' local param)
&& (!CollapseScore.wantsCScore(groupHeadSelector.selectorText))
//
&& NullPolicy.COLLAPSE.getCode() != nullPolicy);
if (HINT_BLOCK.equals(hint) && !blockCollapse) {
log.debug(
"Query specifies hint={} but other local params prevent the use block based collapse",
HINT_BLOCK);
}
FieldType collapseFieldType = searcher.getSchema().getField(collapseField).getType();
if (collapseFieldType instanceof StrField) {
// if we are using blockCollapse, then there is no need to bother with TOP_FC
if (HINT_TOP_FC.equals(hint) && !blockCollapse) {
@SuppressWarnings("resource")
final LeafReader uninvertingReader = getTopFieldCacheReader(searcher, collapseField);
docValuesProducer =
new EmptyDocValuesProducer() {
@Override
public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
SortedDocValues values = uninvertingReader.getSortedDocValues(collapseField);
if (values != null) {
return values;
} else {
return DocValues.emptySorted();
}
}
};
} else {
docValuesProducer =
new EmptyDocValuesProducer() {
@Override
public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField);
}
};
}
} else {
if (HINT_TOP_FC.equals(hint)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"top_fc hint is only supported when collapsing on String Fields");
}
}
FieldType minMaxFieldType = null;
if (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)) {
final String text = groupHeadSelector.selectorText;
if (!text.contains("(")) {
minMaxFieldType = searcher.getSchema().getField(text).getType();
} else {
SolrParams params = new ModifiableSolrParams();
try (SolrQueryRequest request = new LocalSolrQueryRequest(searcher.getCore(), params)) {
FunctionQParser functionQParser = new FunctionQParser(text, null, null, request);
funcQuery = (FunctionQuery) functionQParser.parse();
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
}
int maxDoc = searcher.maxDoc();
int leafCount = searcher.getTopReaderContext().leaves().size();
SolrRequestInfo req = SolrRequestInfo.getRequestInfo();
boolean collectElevatedDocsWhenCollapsing =
req != null
&& req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true);
if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {
if (collapseFieldType instanceof StrField) {
if (blockCollapse) {
return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs);
}
return new OrdScoreCollector(
maxDoc,
leafCount,
docValuesProducer,
nullPolicy,
boostDocs,
searcher,
collectElevatedDocsWhenCollapsing);
} else if (isNumericCollapsible(collapseFieldType)) {
if (blockCollapse) {
return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs);
}
return new IntScoreCollector(
maxDoc,
leafCount,
nullPolicy,
size,
collapseField,
boostDocs,
searcher,
collectElevatedDocsWhenCollapsing);
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");
}
} else { // min, max, sort, etc.. something other then just "score"
if (collapseFieldType instanceof StrField) {
if (blockCollapse) {
// NOTE: for now we don't worry about wether this is a sortSpec of min/max
// groupHeadSelector, we use a "sort spec' based block collector unless/until there is
// some (performance?) reason to specialize
return new BlockOrdSortSpecCollector(
collapseField,
nullPolicy,
boostDocs,
BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher),
needsScores || needsScores4Collapsing);
}
return new OrdFieldValueCollector(
maxDoc,
leafCount,
docValuesProducer,
nullPolicy,
groupHeadSelector,
sortSpec,
needsScores4Collapsing,
needsScores,
minMaxFieldType,
boostDocs,
funcQuery,
searcher,
collectElevatedDocsWhenCollapsing);
} else if (isNumericCollapsible(collapseFieldType)) {
if (blockCollapse) {
// NOTE: for now we don't worry about wether this is a sortSpec of min/max
// groupHeadSelector, we use a "sort spec' based block collector unless/until there is
// some (performance?) reason to specialize
return new BlockIntSortSpecCollector(
collapseField,
nullPolicy,
boostDocs,
BlockOrdSortSpecCollector.getSort(groupHeadSelector, sortSpec, funcQuery, searcher),
needsScores || needsScores4Collapsing);
}
return new IntFieldValueCollector(
maxDoc,
size,
leafCount,
nullPolicy,
collapseField,
groupHeadSelector,
sortSpec,
needsScores4Collapsing,
needsScores,
minMaxFieldType,
boostDocs,
funcQuery,
searcher,
collectElevatedDocsWhenCollapsing);
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");
}
}
}
}
public static final class CollapseScore {
/**
* Inspects the GroupHeadSelector to determine if this CollapseScore is needed. If it is, then
* "this" will be added to the readerContext using the "CSCORE" key, and true will be returned.
* If not returns false.
*/
public boolean setupIfNeeded(
final GroupHeadSelector groupHeadSelector,
final Map readerContext) {
// HACK, but not really any better options until/unless we can recursively
// ask value sources if they depend on score
if (wantsCScore(groupHeadSelector.selectorText)) {
readerContext.put("CSCORE", this);
return true;
}
return false;
}
/**
* Huge HACK, but not really any better options until/unless we can recursively ask value
* sources if they depend on score
*/
public static boolean wantsCScore(final String text) {
return (text.contains("cscore()"));
}
private CollapseScore() {
// No-Op
}
public float score;
}
/*
* Collapse Strategies
*/
/**
* The abstract base Strategy for collapse strategies that collapse on an ordinal using min/max
* field value to select the group head.
*/
private abstract static class OrdFieldValueStrategy {
protected int nullPolicy;
protected IntIntDynamicMap ords;
protected Scorable scorer;
protected FloatArrayList nullScores;
protected float nullScore;
protected IntFloatDynamicMap scores;
protected FixedBitSet collapsedSet;
protected int nullDoc = -1;
protected boolean needsScores;
private final BoostedDocsCollector boostedDocsCollector;
public abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException;
public abstract void setNextReader(LeafReaderContext context) throws IOException;
public OrdFieldValueStrategy(
int maxDoc,
int valueCount,
int nullPolicy,
boolean needsScores,
BoostedDocsCollector boostedDocsCollector,
SortedDocValues values) {
this.ords = new IntIntDynamicMap(valueCount, -1);
this.nullPolicy = nullPolicy;
this.needsScores = needsScores;
this.collapsedSet = new FixedBitSet(maxDoc);
this.boostedDocsCollector = boostedDocsCollector;
if (this.needsScores) {
this.scores = new IntFloatDynamicMap(valueCount, 0.0f);
if (nullPolicy == NullPolicy.EXPAND.getCode()) {
nullScores = new FloatArrayList();
}
}
}
public FixedBitSet getCollapsedSet() {
// Handle the boosted docs.
boostedDocsCollector.purgeGroupsThatHaveBoostedDocs(
collapsedSet,
(ord) -> {
ords.remove(ord);
},
() -> {
nullDoc = -1;
});
// Build the sorted DocSet of group heads.
if (nullDoc > -1) {
this.collapsedSet.set(nullDoc);
}
ords.forEachValue(doc -> collapsedSet.set(doc));
return collapsedSet;
}
public void setScorer(Scorable scorer) throws IOException {
this.scorer = scorer;
}
public FloatArrayList getNullScores() {
return nullScores;
}
public float getNullScore() {
return this.nullScore;
}
public IntFloatDynamicMap getScores() {
return scores;
}
}
/*
* Strategy for collapsing on ordinal using min/max of an int field to select the group head.
*/
private static class OrdIntStrategy extends OrdFieldValueStrategy {
private final String field;
private NumericDocValues minMaxValues;
private IntCompare comp;
private int nullVal;
private IntIntDynamicMap ordVals;
public OrdIntStrategy(
int maxDoc,
int nullPolicy,
int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
BoostedDocsCollector boostedDocsCollector,
SortedDocValues values)
throws IOException {
super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values);
this.field = groupHeadSelector.selectorText;
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxIntComp();
this.ordVals = new IntIntDynamicMap(valueCount, Integer.MIN_VALUE);
} else {
comp = new MinIntComp();
this.ordVals = new IntIntDynamicMap(valueCount, Integer.MAX_VALUE);
this.nullVal = Integer.MAX_VALUE;
}
}
@Override
public void setNextReader(LeafReaderContext context) throws IOException {
this.minMaxValues = DocValues.getNumeric(context.reader(), this.field);
}
@Override
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
int currentVal;
if (minMaxValues.advanceExact(contextDoc)) {
currentVal = (int) minMaxValues.longValue();
} else {
currentVal = 0;
}
if (ord > -1) {
if (comp.test(currentVal, ordVals.get(ord))) {
ords.put(ord, globalDoc);
ordVals.put(ord, currentVal);
if (needsScores) {
scores.put(ord, scorer.score());
}
}
} else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) {
if (comp.test(currentVal, nullVal)) {
nullVal = currentVal;
nullDoc = globalDoc;
if (needsScores) {
nullScore = scorer.score();
}
}
} else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
nullScores.add(scorer.score());
}
}
}
}
/**
* Strategy for collapsing on ordinal and using the min/max value of a float field to select the
* group head
*/
private static class OrdFloatStrategy extends OrdFieldValueStrategy {
private final String field;
private NumericDocValues minMaxValues;
private FloatCompare comp;
private float nullVal;
private IntFloatDynamicMap ordVals;
public OrdFloatStrategy(
int maxDoc,
int nullPolicy,
int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
BoostedDocsCollector boostedDocsCollector,
SortedDocValues values)
throws IOException {
super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values);
this.field = groupHeadSelector.selectorText;
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxFloatComp();
this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
this.nullVal = -Float.MAX_VALUE;
} else {
comp = new MinFloatComp();
this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE);
this.nullVal = Float.MAX_VALUE;
}
}
@Override
public void setNextReader(LeafReaderContext context) throws IOException {
this.minMaxValues = DocValues.getNumeric(context.reader(), this.field);
}
@Override
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
int currentMinMax;
if (minMaxValues.advanceExact(contextDoc)) {
currentMinMax = (int) minMaxValues.longValue();
} else {
currentMinMax = 0;
}
float currentVal = Float.intBitsToFloat(currentMinMax);
if (ord > -1) {
if (comp.test(currentVal, ordVals.get(ord))) {
ords.put(ord, globalDoc);
ordVals.put(ord, currentVal);
if (needsScores) {
scores.put(ord, scorer.score());
}
}
} else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) {
if (comp.test(currentVal, nullVal)) {
nullVal = currentVal;
nullDoc = globalDoc;
if (needsScores) {
nullScore = scorer.score();
}
}
} else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
nullScores.add(scorer.score());
}
}
}
}
/*
* Strategy for collapsing on ordinal and using the min/max value of a long
* field to select the group head
*/
private static class OrdLongStrategy extends OrdFieldValueStrategy {
private final String field;
private NumericDocValues minMaxVals;
private LongCompare comp;
private long nullVal;
private IntLongDynamicMap ordVals;
public OrdLongStrategy(
int maxDoc,
int nullPolicy,
int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
BoostedDocsCollector boostedDocsCollector,
SortedDocValues values)
throws IOException {
super(maxDoc, valueCount, nullPolicy, needsScores, boostedDocsCollector, values);
this.field = groupHeadSelector.selectorText;
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxLongComp();
this.ordVals = new IntLongDynamicMap(valueCount, Long.MIN_VALUE);
} else {
this.nullVal = Long.MAX_VALUE;
comp = new MinLongComp();
this.ordVals = new IntLongDynamicMap(valueCount, Long.MAX_VALUE);
}
}
@Override
public void setNextReader(LeafReaderContext context) throws IOException {
this.minMaxVals = DocValues.getNumeric(context.reader(), this.field);
}
@Override
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
long currentVal;
if (minMaxVals.advanceExact(contextDoc)) {
currentVal = minMaxVals.longValue();
} else {
currentVal = 0;
}
if (ord > -1) {
if (comp.test(currentVal, ordVals.get(ord))) {
ords.put(ord, globalDoc);
ordVals.put(ord, currentVal);
if (needsScores) {
scores.put(ord, scorer.score());
}
}
} else if (this.nullPolicy == NullPolicy.COLLAPSE.getCode()) {
if (comp.test(currentVal, nullVal)) {
nullVal = currentVal;
nullDoc = globalDoc;
if (needsScores) {
nullScore = scorer.score();
}
}
} else if (this.nullPolicy == NullPolicy.EXPAND.getCode()) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
nullScores.add(scorer.score());
}
}
}
}
/*
* Strategy for collapsing on ordinal and using the min/max value of a value source function
* to select the group head
*/
private static class OrdValueSourceStrategy extends OrdFieldValueStrategy {
private FloatCompare comp;
private float nullVal;
private ValueSource valueSource;
private FunctionValues functionValues;
private IntFloatDynamicMap ordVals;
private Map