org.apache.solr.highlight.LuceneRegexFragmenter Maven / Gradle / Ivy
Show all versions of solr-core Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.highlight;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.highlight.Fragmenter;
/**
* Fragmenter that tries to produce snippets that "look" like a regular expression.
*
* NOTE: the default for maxAnalyzedChars is much lower for this fragmenter. After
* this limit is exhausted, fragments are produced in the same way as GapFragmenter
*/
class LuceneRegexFragmenter implements Fragmenter {
// ** defaults
public static final int DEFAULT_FRAGMENT_SIZE = 70;
public static final int DEFAULT_INCREMENT_GAP = 50;
public static final float DEFAULT_SLOP = 0.6f;
public static final int DEFAULT_MAX_ANALYZED_CHARS = 10000;
// ** settings
// desired length of fragments, in characters
protected int targetFragChars;
// increment gap which indicates a new fragment should occur
// (often due to multi-valued fields)
protected int incrementGapThreshold;
// factor by which we are allowed to bend the frag size (larger or smaller)
protected float slop;
// analysis limit (ensures we don't waste too much time on long fields)
protected int maxAnalyzedChars;
// default desirable pattern for text fragments.
protected Pattern textRE;
// ** state
protected int currentNumFrags;
protected int currentOffset;
protected int targetOffset;
protected int[] hotspots;
private PositionIncrementAttribute posIncAtt;
private OffsetAttribute offsetAtt;
// ** other
// note: could dynamically change size of sentences extracted to match
// target frag size
public static final String DEFAULT_PATTERN_RAW = "[-\\w ,\\n\"']{20,200}";
public static final Pattern DEFAULT_PATTERN = Pattern.compile(DEFAULT_PATTERN_RAW);
public LuceneRegexFragmenter() {
this(DEFAULT_FRAGMENT_SIZE, DEFAULT_INCREMENT_GAP, DEFAULT_SLOP, DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(int targetFragChars) {
this(targetFragChars, DEFAULT_INCREMENT_GAP, DEFAULT_SLOP, DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(
int targetFragChars, int incrementGapThreshold, float slop, int maxAnalyzedChars) {
this(targetFragChars, incrementGapThreshold, slop, maxAnalyzedChars, DEFAULT_PATTERN);
}
public LuceneRegexFragmenter(
int targetFragChars,
int incrementGapThreshold,
float slop,
int maxAnalyzedChars,
Pattern targetPattern) {
this.targetFragChars = targetFragChars;
this.incrementGapThreshold = incrementGapThreshold;
this.slop = slop;
this.maxAnalyzedChars = maxAnalyzedChars;
this.textRE = targetPattern;
}
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
*/
@Override
public void start(String originalText, TokenStream tokenStream) {
currentNumFrags = 1;
currentOffset = 0;
addHotSpots(originalText);
posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
}
////////////////////////////////////
// pre-analysis
////////////////////////////////////
protected void addHotSpots(String text) {
// System.out.println("hot spotting");
ArrayList temphs = new ArrayList<>(text.length() / targetFragChars);
Matcher match = textRE.matcher(text);
int cur = 0;
while (match.find() && cur < maxAnalyzedChars) {
int start = match.start(), end = match.end();
temphs.add(start);
temphs.add(end);
cur = end;
// System.out.println("Matched " + match.group());
}
hotspots = new int[temphs.size()];
for (int i = 0; i < temphs.size(); i++) {
hotspots[i] = temphs.get(i);
}
// perhaps not necessary--I don't know if re matches are non-overlapping
Arrays.sort(hotspots);
}
////////////////////////////////////
// fragmenting
////////////////////////////////////
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
*/
@Override
public boolean isNewFragment() {
boolean isNewFrag = false;
int minFragLen = (int) ((1.0f - slop) * targetFragChars);
int endOffset = offsetAtt.endOffset();
// ** determin isNewFrag
if (posIncAtt.getPositionIncrement() > incrementGapThreshold) {
// large position gaps always imply new fragments
isNewFrag = true;
} else if (endOffset - currentOffset < minFragLen) {
// we're not in our range of flexibility
isNewFrag = false;
} else if (targetOffset > 0) {
// we've already decided on a target
isNewFrag = endOffset > targetOffset;
} else {
// we might be able to do something
int minOffset = currentOffset + minFragLen;
int maxOffset = (int) (currentOffset + (1.0f + slop) * targetFragChars);
int hotIndex;
// look for a close hotspot
hotIndex = Arrays.binarySearch(hotspots, endOffset);
if (hotIndex < 0) hotIndex = -hotIndex;
if (hotIndex >= hotspots.length) {
// no more hotspots in this input stream
targetOffset = currentOffset + targetFragChars;
} else if (hotspots[hotIndex] > maxOffset) {
// no hotspots within slop
targetOffset = currentOffset + targetFragChars;
} else {
// try to find hotspot in slop
int goal = hotspots[hotIndex];
while (goal < minOffset && hotIndex < hotspots.length) {
hotIndex++;
goal = hotspots[hotIndex];
}
targetOffset = goal <= maxOffset ? goal : currentOffset + targetFragChars;
}
isNewFrag = endOffset > targetOffset;
}
// ** operate on isNewFrag
if (isNewFrag) {
currentNumFrags++;
currentOffset = endOffset;
targetOffset = -1;
}
return isNewFrag;
}
}