org.apache.solr.legacy.LegacyNumericTokenStream Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.legacy;
import java.util.Objects;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils;
/**
* Expert: This class provides a {@link TokenStream}
* for indexing numeric values that can be used by {@link
* org.apache.solr.legacy.LegacyNumericRangeQuery}.
*
* Note that for simple usage, {@link org.apache.solr.legacy.LegacyIntField}, {@link
* org.apache.solr.legacy.LegacyLongField}, {@link org.apache.solr.legacy.LegacyFloatField} or {@link org.apache.solr.legacy.LegacyDoubleField} is
* recommended. These fields disable norms and
* term freqs, as they are not usually needed during
* searching. If you need to change these settings, you
* should use this class.
*
*
Here's an example usage, for an int field:
*
*
* FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
* fieldType.setOmitNorms(true);
* fieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
* Field field = new Field(name, new LegacyNumericTokenStream(precisionStep).setIntValue(value), fieldType);
* document.add(field);
*
*
* For optimal performance, re-use the TokenStream and Field instance
* for more than one document:
*
*
* LegacyNumericTokenStream stream = new LegacyNumericTokenStream(precisionStep);
* FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
* fieldType.setOmitNorms(true);
* fieldType.setIndexOptions(IndexOptions.DOCS_ONLY);
* Field field = new Field(name, stream, fieldType);
* Document document = new Document();
* document.add(field);
*
* for(all documents) {
* stream.setIntValue(value)
* writer.addDocument(document);
* }
*
*
* This stream is not intended to be used in analyzers;
* it's more for iterating the different precisions during
* indexing a specific numeric value.
* NOTE: as token streams are only consumed once
* the document is added to the index, if you index more
* than one numeric field, use a separate LegacyNumericTokenStream
* instance for each.
*
* See {@link org.apache.solr.legacy.LegacyNumericRangeQuery} for more details on the
* precisionStep
* parameter as well as how numeric fields work under the hood.
*
* @deprecated Please switch to {@link org.apache.lucene.index.PointValues} instead
*
* @since 2.9
*/
@Deprecated
public final class LegacyNumericTokenStream extends TokenStream {
/** The full precision token gets this token type assigned. */
public static final String TOKEN_TYPE_FULL_PREC = "fullPrecNumeric";
/** The lower precision tokens gets this token type assigned. */
public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
/** Expert: Use this attribute to get the details of the currently generated token.
* @lucene.experimental
* @since 4.0
*/
public interface LegacyNumericTermAttribute extends Attribute {
/** Returns current shift value, undefined before first token */
int getShift();
/** Returns current token's raw value as {@code long} with all {@link #getShift} applied, undefined before first token */
long getRawValue();
/** Returns value size in bits (32 for {@code float}, {@code int}; 64 for {@code double}, {@code long}) */
int getValueSize();
/** Don't call this method!
* @lucene.internal */
void init(long value, int valSize, int precisionStep, int shift);
/** Don't call this method!
* @lucene.internal */
void setShift(int shift);
/** Don't call this method!
* @lucene.internal */
int incShift();
}
// just a wrapper to prevent adding CTA
private static final class NumericAttributeFactory extends AttributeFactory {
private final AttributeFactory delegate;
NumericAttributeFactory(AttributeFactory delegate) {
this.delegate = delegate;
}
@Override
public AttributeImpl createAttributeInstance(Class attClass) {
if (CharTermAttribute.class.isAssignableFrom(attClass))
throw new IllegalArgumentException("LegacyNumericTokenStream does not support CharTermAttribute.");
return delegate.createAttributeInstance(attClass);
}
}
/** Implementation of {@link org.apache.solr.legacy.LegacyNumericTokenStream.LegacyNumericTermAttribute}.
* @lucene.internal
* @since 4.0
*/
public static final class LegacyNumericTermAttributeImpl extends AttributeImpl implements LegacyNumericTermAttribute,TermToBytesRefAttribute {
private long value = 0L;
private int valueSize = 0, shift = 0, precisionStep = 0;
private BytesRefBuilder bytes = new BytesRefBuilder();
/**
* Creates, but does not yet initialize this attribute instance
* @see #init(long, int, int, int)
*/
public LegacyNumericTermAttributeImpl() {}
@Override
public BytesRef getBytesRef() {
assert valueSize == 64 || valueSize == 32;
if (shift >= valueSize) {
bytes.clear();
} else if (valueSize == 64) {
LegacyNumericUtils.longToPrefixCoded(value, shift, bytes);
} else {
LegacyNumericUtils.intToPrefixCoded((int) value, shift, bytes);
}
return bytes.get();
}
@Override
public int getShift() { return shift; }
@Override
public void setShift(int shift) { this.shift = shift; }
@Override
public int incShift() {
return (shift += precisionStep);
}
@Override
public long getRawValue() { return value & ~((1L << shift) - 1L); }
@Override
public int getValueSize() { return valueSize; }
@Override
public void init(long value, int valueSize, int precisionStep, int shift) {
this.value = value;
this.valueSize = valueSize;
this.precisionStep = precisionStep;
this.shift = shift;
}
@Override
public void clear() {
// this attribute has no contents to clear!
// we keep it untouched as it's fully controlled by outer class.
}
@Override
public void reflectWith(AttributeReflector reflector) {
reflector.reflect(TermToBytesRefAttribute.class, "bytes", getBytesRef());
reflector.reflect(LegacyNumericTermAttribute.class, "shift", shift);
reflector.reflect(LegacyNumericTermAttribute.class, "rawValue", getRawValue());
reflector.reflect(LegacyNumericTermAttribute.class, "valueSize", valueSize);
}
@Override
public void copyTo(AttributeImpl target) {
final LegacyNumericTermAttribute a = (LegacyNumericTermAttribute) target;
a.init(value, valueSize, precisionStep, shift);
}
@Override
public LegacyNumericTermAttributeImpl clone() {
LegacyNumericTermAttributeImpl t = (LegacyNumericTermAttributeImpl)super.clone();
// Do a deep clone
t.bytes = new BytesRefBuilder();
t.bytes.copyBytes(getBytesRef());
return t;
}
@Override
public int hashCode() {
return Objects.hash(precisionStep, shift, value, valueSize);
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
LegacyNumericTermAttributeImpl other = (LegacyNumericTermAttributeImpl) obj;
if (precisionStep != other.precisionStep) return false;
if (shift != other.shift) return false;
if (value != other.value) return false;
if (valueSize != other.valueSize) return false;
return true;
}
}
/**
* Creates a token stream for numeric values using the default precisionStep
* {@link org.apache.solr.legacy.LegacyNumericUtils#PRECISION_STEP_DEFAULT} (16). The stream is not yet initialized,
* before using set a value using the various set???Value() methods.
*/
public LegacyNumericTokenStream() {
this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, LegacyNumericUtils.PRECISION_STEP_DEFAULT);
}
/**
* Creates a token stream for numeric values with the specified
* precisionStep. The stream is not yet initialized,
* before using set a value using the various set???Value() methods.
*/
public LegacyNumericTokenStream(final int precisionStep) {
this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, precisionStep);
}
/**
* Expert: Creates a token stream for numeric values with the specified
* precisionStep using the given
* {@link org.apache.lucene.util.AttributeFactory}.
* The stream is not yet initialized,
* before using set a value using the various set???Value() methods.
*/
public LegacyNumericTokenStream(AttributeFactory factory, final int precisionStep) {
super(new NumericAttributeFactory(factory));
if (precisionStep < 1)
throw new IllegalArgumentException("precisionStep must be >=1");
this.precisionStep = precisionStep;
numericAtt.setShift(-precisionStep);
}
/**
* Initializes the token stream with the supplied long value.
* @param value the value, for which this TokenStream should enumerate tokens.
* @return this instance, because of this you can use it the following way:
* new Field(name, new LegacyNumericTokenStream(precisionStep).setLongValue(value))
*/
public LegacyNumericTokenStream setLongValue(final long value) {
numericAtt.init(value, valSize = 64, precisionStep, -precisionStep);
return this;
}
/**
* Initializes the token stream with the supplied int value.
* @param value the value, for which this TokenStream should enumerate tokens.
* @return this instance, because of this you can use it the following way:
* new Field(name, new LegacyNumericTokenStream(precisionStep).setIntValue(value))
*/
public LegacyNumericTokenStream setIntValue(final int value) {
numericAtt.init(value, valSize = 32, precisionStep, -precisionStep);
return this;
}
/**
* Initializes the token stream with the supplied double value.
* @param value the value, for which this TokenStream should enumerate tokens.
* @return this instance, because of this you can use it the following way:
* new Field(name, new LegacyNumericTokenStream(precisionStep).setDoubleValue(value))
*/
public LegacyNumericTokenStream setDoubleValue(final double value) {
numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep, -precisionStep);
return this;
}
/**
* Initializes the token stream with the supplied float value.
* @param value the value, for which this TokenStream should enumerate tokens.
* @return this instance, because of this you can use it the following way:
* new Field(name, new LegacyNumericTokenStream(precisionStep).setFloatValue(value))
*/
public LegacyNumericTokenStream setFloatValue(final float value) {
numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep, -precisionStep);
return this;
}
@Override
public void reset() {
if (valSize == 0)
throw new IllegalStateException("call set???Value() before usage");
numericAtt.setShift(-precisionStep);
}
@Override
public boolean incrementToken() {
if (valSize == 0)
throw new IllegalStateException("call set???Value() before usage");
// this will only clear all other attributes in this TokenStream
clearAttributes();
final int shift = numericAtt.incShift();
typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
return (shift < valSize);
}
/** Returns the precision step. */
public int getPrecisionStep() {
return precisionStep;
}
@Override
public String toString() {
// We override default because it can throw cryptic "illegal shift value":
return getClass().getSimpleName() + "(precisionStep=" + precisionStep + " valueSize=" + numericAtt.getValueSize() + " shift=" + numericAtt.getShift() + ")";
}
// members
private final LegacyNumericTermAttribute numericAtt = addAttribute(LegacyNumericTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private int valSize = 0; // valSize==0 means not initialized
private final int precisionStep;
}