All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.solr.handler.component.PivotFacetField Maven / Gradle / Ivy

Go to download

Apache Solr (module: core)

There is a newer version: 9.7.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.handler.component;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;

/**
 * Models a single field somewhere in a hierarchy of fields as part of a pivot facet. This pivot
 * field contains {@link PivotFacetValue}s which may each contain a nested {@link PivotFacetField}
 * child. This PivotFacetField may itself be a child of a {@link PivotFacetValue}
 * parent.
 *
 * @see PivotFacetValue
 * @see PivotFacetFieldValueCollection
 */
public class PivotFacetField {

  public final String field;

  // null if this is a top level pivot,
  // otherwise the value of the parent pivot we are nested under
  public final PivotFacetValue parentValue;

  public final PivotFacetFieldValueCollection valueCollection;

  // Facet parameters relating to this field
  private final int facetFieldLimit;
  private final int facetFieldMinimumCount;
  private final int facetFieldOffset;
  private final String facetFieldSort;

  private final Map numberOfValuesContributedByShard = new HashMap<>();
  private final Map shardLowestCount = new HashMap<>();

  private boolean needRefinementAtThisLevel = true;

  private PivotFacetField(ResponseBuilder rb, PivotFacetValue parent, String fieldName) {

    field = fieldName;
    parentValue = parent;

    // facet params
    SolrParams parameters = rb.req.getParams();
    facetFieldMinimumCount = parameters.getFieldInt(field, FacetParams.FACET_PIVOT_MINCOUNT, 1);
    facetFieldOffset = parameters.getFieldInt(field, FacetParams.FACET_OFFSET, 0);
    facetFieldLimit = parameters.getFieldInt(field, FacetParams.FACET_LIMIT, 100);
    String defaultSort =
        (facetFieldLimit > 0) ? FacetParams.FACET_SORT_COUNT : FacetParams.FACET_SORT_INDEX;
    facetFieldSort = parameters.getFieldParam(field, FacetParams.FACET_SORT, defaultSort);

    valueCollection =
        new PivotFacetFieldValueCollection(
            facetFieldMinimumCount, facetFieldOffset, facetFieldLimit, facetFieldSort);

    if ((facetFieldLimit < 0)
        ||
        // TODO: possible refinement issue if limit=0 & mincount=0 & missing=true
        // (ie: we only want the missing count for this field)
        (facetFieldLimit <= 0 && facetFieldMinimumCount == 0)
        || (facetFieldSort.equals(FacetParams.FACET_SORT_INDEX) && facetFieldMinimumCount <= 0)) {
      // in any of these cases, there's no need to refine this level of the pivot
      needRefinementAtThisLevel = false;
    }
  }

  /**
   * A recursive method that walks up the tree of pivot fields/values to build a list of String
   * representations of the values that lead down to this PivotFacetField.
   *
   * @return A mutable List of the pivot values leading down to this pivot field, will never be null
   *     but may contain nulls and may be empty if this is a top level pivot field
   * @see PivotFacetValue#getValuePath
   */
  public List getValuePath() {
    if (null != parentValue) {
      return parentValue.getValuePath();
    }
    return new ArrayList(3);
  }

  /**
   * A recursive method to construct a new PivotFacetField object from the contents of
   * the {@link NamedList}s provided by the specified shard, relative to a parent value (if this is
   * not the top field in the pivot hierarchy)
   *
   * 
The associated child {@link PivotFacetValue}s will be recursively built as well.
   *
   * @see PivotFacetValue#createFromNamedList
   * @param shardNumber the id of the shard that provided this data
   * @param rb The response builder of the current request
   * @param owner the parent value in the current pivot (may be null)
   * @param pivotValues the data from the specified shard for this pivot field, may be null or empty
   * @return the new PivotFacetField, null if pivotValues is null or empty.
   */
  public static PivotFacetField createFromListOfNamedLists(
      int shardNumber,
      ResponseBuilder rb,
      PivotFacetValue owner,
      List> pivotValues) {

    if (null == pivotValues || pivotValues.size() <= 0) return null;

    NamedList firstValue = pivotValues.get(0);
    PivotFacetField createdPivotFacetField =
        new PivotFacetField(rb, owner, PivotFacetHelper.getField(firstValue));

    int lowestCount = Integer.MAX_VALUE;

    for (NamedList pivotValue : pivotValues) {

      lowestCount = Math.min(lowestCount, PivotFacetHelper.getCount(pivotValue));

      PivotFacetValue newValue =
          PivotFacetValue.createFromNamedList(shardNumber, rb, createdPivotFacetField, pivotValue);
      createdPivotFacetField.valueCollection.add(newValue);
    }

    createdPivotFacetField.shardLowestCount.put(shardNumber, lowestCount);
    createdPivotFacetField.numberOfValuesContributedByShard.put(shardNumber, pivotValues.size());

    return createdPivotFacetField;
  }

  /**
   * Destructive method that recursively prunes values from the data structure based on the counts
   * for those values and the effective sort, mincount, limit, and offset being used for each field.
   *
   * 
This method should only be called after all refinement is completed just prior calling
   * {@link #convertToListOfNamedLists}
   *
   * @see PivotFacet#getTrimmedPivotsAsListOfNamedLists
   * @see PivotFacetFieldValueCollection#trim
   */
  public void trim() {
    // SOLR-6331...
    //
    // we can probably optimize the memory usage by trimming each level of the pivot once
    // we know we've fully refined the values at that level
    // (ie: fold this logic into refineNextLevelOfFacets)
    this.valueCollection.trim();
  }

  /**
   * Recursively sorts the collection of values associated with this field, and any sub-pivots those
   * values have.
   *
   * @see FacetParams#FACET_SORT
   * @see PivotFacetFieldValueCollection#sort
   */
  public void sort() {
    this.valueCollection.sort();
  }

  /**
   * A recursive method for generating NamedLists from this field suitable for
   * including in a pivot facet response to the original distributed request.
   */
  public List> convertToListOfNamedLists() {

    List> convertedPivotList = null;

    if (valueCollection.size() > 0) {
      convertedPivotList = new ArrayList<>();
      for (PivotFacetValue pivot : valueCollection)
        convertedPivotList.add(pivot.convertToNamedList());
    }

    return convertedPivotList;
  }

  /**
   * A recursive method for determining which {@link PivotFacetValue}s need to be refined for this
   * pivot.
   *
   * @see PivotFacet#queuePivotRefinementRequests
   */
  public void queuePivotRefinementRequests(PivotFacet pf) {

    if (needRefinementAtThisLevel) {

      if (0 < facetFieldMinimumCount) {
        // missing is always a candidate for refinement if at least one shard met the minimum
        PivotFacetValue missing = valueCollection.getMissingValue();
        if (null != missing) {
          processDefiniteCandidateElement(pf, valueCollection.getMissingValue());
        }
      }

      if (!valueCollection.getExplicitValuesList().isEmpty()) {

        if (FacetParams.FACET_SORT_COUNT.equals(facetFieldSort)) {
          // we only need to things that are currently in our limit,
          // or might be in our limit if we get increased counts from shards that
          // didn't include this value the first time
          final int indexOfCountThreshold =
              Math.min(
                      valueCollection.getExplicitValuesListSize(),
                      facetFieldOffset + facetFieldLimit)
                  - 1;
          final int countThreshold = valueCollection.getAt(indexOfCountThreshold).getCount();

          int positionInResults = 0;

          for (PivotFacetValue value : valueCollection.getExplicitValuesList()) {
            if (positionInResults <= indexOfCountThreshold) {
              // This element is within the top results, so we need to get information
              // from all of the shards.
              processDefiniteCandidateElement(pf, value);
            } else {
              // This element is not within the top results, but may still need to be refined.
              processPossibleCandidateElement(pf, value, countThreshold);
            }

            positionInResults++;
          }
        } else { // FACET_SORT_INDEX
          // everything needs refined to see what the per-shard mincount excluded
          for (PivotFacetValue value : valueCollection.getExplicitValuesList()) {
            processDefiniteCandidateElement(pf, value);
          }
        }
      }

      needRefinementAtThisLevel = false;
    }

    if (pf.isRefinementsRequired()) {
      // if any refinements are needed, then we need to stop and wait to
      // see how the picture may change before drilling down to child pivot fields
      return;
    } else {
      // Since outstanding requests have been filled, then we can drill down
      // to the next deeper level and check it.
      refineNextLevelOfFacets(pf);
    }
  }

  /**
   * Adds refinement requests for the value for each shard that has not already contributed a count
   * for this value.
   */
  private void processDefiniteCandidateElement(PivotFacet pf, PivotFacetValue value) {

    for (int shard = pf.knownShards.nextSetBit(0);
        0 <= shard;
        shard = pf.knownShards.nextSetBit(shard + 1)) {
      if (!value.shardHasContributed(shard)) {
        if ( // if we're doing index order, we need to refine anything
        // (mincount may have excluded from a shard)
        FacetParams.FACET_SORT_INDEX.equals(facetFieldSort)
            || ( // 'missing' value isn't affected by limit, needs refined if shard didn't provide
            null == value.getValue()
                ||
                // if we are doing count order, we need to refine if the limit was hit
                // (if not, the shard doesn't have the value or it would have returned already)
                numberOfValuesContributedByShardWasLimitedByFacetFieldLimit(shard))) {
          pf.addRefinement(shard, value);
        }
      }
    }
  }

  private boolean numberOfValuesContributedByShardWasLimitedByFacetFieldLimit(int shardNumber) {
    return facetFieldLimit <= numberOfValuesContributedByShard(shardNumber);
  }

  private int numberOfValuesContributedByShard(final int shardNumber) {
    return numberOfValuesContributedByShard.containsKey(shardNumber)
        ? numberOfValuesContributedByShard.get(shardNumber)
        : 0;
  }

  /**
   * Checks the {@link #lowestCountContributedbyShard} for each shard, combined with the counts we
   * already know, to see if this value is a viable candidate -- Does not make sense when using
   * {@link FacetParams#FACET_SORT_INDEX}
   *
   * @see #processDefiniteCandidateElement
   */
  private void processPossibleCandidateElement(
      PivotFacet pf, PivotFacetValue value, final int refinementThreshold) {

    assert FacetParams.FACET_SORT_COUNT.equals(facetFieldSort)
        : "Method only makes sense when sorting by count";

    int maxPossibleCountAfterRefinement = value.getCount();

    for (int shard = pf.knownShards.nextSetBit(0);
        0 <= shard;
        shard = pf.knownShards.nextSetBit(shard + 1)) {
      if (!value.shardHasContributed(shard)) {
        maxPossibleCountAfterRefinement += lowestCountContributedbyShard(shard);
      }
    }

    if (refinementThreshold <= maxPossibleCountAfterRefinement) {
      processDefiniteCandidateElement(pf, value);
    }
  }

  private int lowestCountContributedbyShard(int shardNumber) {
    return (shardLowestCount.containsKey(shardNumber)) ? shardLowestCount.get(shardNumber) : 0;
  }

  private void refineNextLevelOfFacets(PivotFacet pf) {

    List explicitValsToRefine = valueCollection.getNextLevelValuesToRefine();

    for (PivotFacetValue value : explicitValsToRefine) {
      if (null != value.getChildPivot()) {
        value.getChildPivot().queuePivotRefinementRequests(pf);
      }
    }

    PivotFacetValue missing = this.valueCollection.getMissingValue();
    if (null != missing && null != missing.getChildPivot()) {
      missing.getChildPivot().queuePivotRefinementRequests(pf);
    }
  }

  private void incrementShardValueCount(int shardNumber) {
    if (!numberOfValuesContributedByShard.containsKey(shardNumber)) {
      numberOfValuesContributedByShard.put(shardNumber, 1);
    } else {
      numberOfValuesContributedByShard.put(
          shardNumber, numberOfValuesContributedByShard.get(shardNumber) + 1);
    }
  }

  private void contributeValueFromShard(
      int shardNumber, ResponseBuilder rb, NamedList shardValue) {

    incrementShardValueCount(shardNumber);

    @SuppressWarnings("rawtypes")
    Comparable value = PivotFacetHelper.getValue(shardValue);
    int count = PivotFacetHelper.getCount(shardValue);

    // We're changing values so we most mark the collection as dirty
    valueCollection.markDirty();

    if ((!shardLowestCount.containsKey(shardNumber)) || shardLowestCount.get(shardNumber) > count) {
      shardLowestCount.put(shardNumber, count);
    }

    PivotFacetValue facetValue = valueCollection.get(value);
    if (null == facetValue) {
      // never seen before, we need to create it from scratch
      facetValue = PivotFacetValue.createFromNamedList(shardNumber, rb, this, shardValue);
      this.valueCollection.add(facetValue);
    } else {
      facetValue.mergeContributionFromShard(shardNumber, rb, shardValue);
    }
  }

  /**
   * Recursively merges the contributions from the specified shard for each {@link PivotFacetValue}
   * represended in the response.
   *
   * @see PivotFacetValue#mergeContributionFromShard
   * @param shardNumber the id of the shard that provided this data
   * @param rb The response builder of the current request
   * @param response the data from the specified shard for this pivot field, may be null
   */
  public void contributeFromShard(
      int shardNumber, ResponseBuilder rb, List> response) {
    if (null == response) return;

    for (NamedList responseValue : response) {
      contributeValueFromShard(shardNumber, rb, responseValue);
    }
  }

  @Override
  public String toString() {
    return String.format(Locale.ROOT, "P:%s F:%s V:%s", parentValue, field, valueCollection);
  }
}
 















© 2015 - 2024 Weber Informatics LLC | Privacy Policy