org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesProducer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of lucene-backward-codecs Show documentation
Show all versions of lucene-backward-codecs Show documentation
Apache Lucene (module: backward-codecs)
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.backward_codecs.lucene80;
import java.io.IOException;
import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
import org.apache.lucene.backward_codecs.packed.LegacyDirectReader;
import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.compress.LZ4;
/** reader for {@link Lucene80DocValuesFormat} */
final class Lucene80DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap numerics = new IntObjectHashMap<>();
private final IntObjectHashMap binaries = new IntObjectHashMap<>();
private final IntObjectHashMap sorted = new IntObjectHashMap<>();
private final IntObjectHashMap sortedSets = new IntObjectHashMap<>();
private final IntObjectHashMap sortedNumerics = new IntObjectHashMap<>();
private final IndexInput data;
private final int maxDoc;
private int version = -1;
/** expert: instantiates a new reader */
Lucene80DocValuesProducer(
SegmentReadState state,
String dataCodec,
String dataExtension,
String metaCodec,
String metaExtension)
throws IOException {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
this.maxDoc = state.segmentInfo.maxDoc();
// read in the entries from the metadata file.
try (ChecksumIndexInput in =
EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)) {
Throwable priorE = null;
try {
version =
CodecUtil.checkIndexHeader(
in,
metaCodec,
Lucene80DocValuesFormat.VERSION_START,
Lucene80DocValuesFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
readFields(state.segmentInfo.name, in, state.fieldInfos);
} catch (Throwable exception) {
priorE = exception;
} finally {
CodecUtil.checkFooter(in, priorE);
}
}
String dataName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
this.data = EndiannessReverserUtil.openInput(state.directory, dataName, state.context);
boolean success = false;
try {
final int version2 =
CodecUtil.checkIndexHeader(
data,
dataCodec,
Lucene80DocValuesFormat.VERSION_START,
Lucene80DocValuesFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException(
"Format versions mismatch: meta=" + version + ", data=" + version2, data);
}
// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
// for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
// such as file truncation.
CodecUtil.retrieveChecksum(data);
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(this.data);
}
}
}
private void readFields(String segmentName, IndexInput meta, FieldInfos infos)
throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
byte type = meta.readByte();
if (type == Lucene80DocValuesFormat.NUMERIC) {
numerics.put(info.number, readNumeric(meta));
} else if (type == Lucene80DocValuesFormat.BINARY) {
final boolean compressed;
if (version >= Lucene80DocValuesFormat.VERSION_CONFIGURABLE_COMPRESSION) {
String value = info.getAttribute(Lucene80DocValuesFormat.MODE_KEY);
if (value == null) {
throw new IllegalStateException(
"missing value for "
+ Lucene80DocValuesFormat.MODE_KEY
+ " for field: "
+ info.name
+ " in segment: "
+ segmentName);
}
Lucene80DocValuesFormat.Mode mode = Lucene80DocValuesFormat.Mode.valueOf(value);
compressed = mode == Lucene80DocValuesFormat.Mode.BEST_COMPRESSION;
} else {
compressed = version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED;
}
binaries.put(info.number, readBinary(meta, compressed));
} else if (type == Lucene80DocValuesFormat.SORTED) {
sorted.put(info.number, readSorted(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_SET) {
sortedSets.put(info.number, readSortedSet(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.number, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
}
}
private NumericEntry readNumeric(IndexInput meta) throws IOException {
NumericEntry entry = new NumericEntry();
readNumeric(meta, entry);
return entry;
}
private void readNumeric(IndexInput meta, NumericEntry entry) throws IOException {
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
entry.numValues = meta.readLong();
int tableSize = meta.readInt();
if (tableSize > 256) {
throw new CorruptIndexException("invalid table size: " + tableSize, meta);
}
if (tableSize >= 0) {
entry.table = new long[tableSize];
for (int i = 0; i < tableSize; ++i) {
entry.table[i] = meta.readLong();
}
}
if (tableSize < -1) {
entry.blockShift = -2 - tableSize;
} else {
entry.blockShift = -1;
}
entry.bitsPerValue = meta.readByte();
entry.minValue = meta.readLong();
entry.gcd = meta.readLong();
entry.valuesOffset = meta.readLong();
entry.valuesLength = meta.readLong();
entry.valueJumpTableOffset = meta.readLong();
}
private BinaryEntry readBinary(IndexInput meta, boolean compressed) throws IOException {
final BinaryEntry entry = new BinaryEntry();
entry.compressed = compressed;
entry.dataOffset = meta.readLong();
entry.dataLength = meta.readLong();
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
entry.numDocsWithField = meta.readInt();
entry.minLength = meta.readInt();
entry.maxLength = meta.readInt();
if ((entry.compressed && entry.numDocsWithField > 0) || entry.minLength < entry.maxLength) {
entry.addressesOffset = meta.readLong();
// Old count of uncompressed addresses
long numAddresses = entry.numDocsWithField + 1L;
// New count of compressed addresses - the number of compresseed blocks
if (entry.compressed) {
entry.numCompressedChunks = meta.readVInt();
entry.docsPerChunkShift = meta.readVInt();
entry.maxUncompressedChunkSize = meta.readVInt();
numAddresses = entry.numCompressedChunks;
}
final int blockShift = meta.readVInt();
entry.addressesMeta = LegacyDirectMonotonicReader.loadMeta(meta, numAddresses, blockShift);
entry.addressesLength = meta.readLong();
}
return entry;
}
private SortedEntry readSorted(IndexInput meta) throws IOException {
SortedEntry entry = new SortedEntry();
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
entry.numDocsWithField = meta.readInt();
entry.bitsPerValue = meta.readByte();
entry.ordsOffset = meta.readLong();
entry.ordsLength = meta.readLong();
readTermDict(meta, entry);
return entry;
}
private SortedSetEntry readSortedSet(IndexInput meta) throws IOException {
SortedSetEntry entry = new SortedSetEntry();
byte multiValued = meta.readByte();
switch (multiValued) {
case 0: // singlevalued
entry.singleValueEntry = readSorted(meta);
return entry;
case 1: // multivalued
break;
default:
throw new CorruptIndexException("Invalid multiValued flag: " + multiValued, meta);
}
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
entry.bitsPerValue = meta.readByte();
entry.ordsOffset = meta.readLong();
entry.ordsLength = meta.readLong();
entry.numDocsWithField = meta.readInt();
entry.addressesOffset = meta.readLong();
final int blockShift = meta.readVInt();
entry.addressesMeta =
LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift);
entry.addressesLength = meta.readLong();
readTermDict(meta, entry);
return entry;
}
private static void readTermDict(IndexInput meta, TermsDictEntry entry) throws IOException {
entry.termsDictSize = meta.readVLong();
int termsDictBlockCode = meta.readInt();
if (Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_CODE == termsDictBlockCode) {
// This is a LZ4 compressed block.
entry.compressed = true;
entry.termsDictBlockShift = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
} else {
entry.termsDictBlockShift = termsDictBlockCode;
}
final int blockShift = meta.readInt();
final long addressesSize =
(entry.termsDictSize + (1L << entry.termsDictBlockShift) - 1) >>> entry.termsDictBlockShift;
entry.termsAddressesMeta =
LegacyDirectMonotonicReader.loadMeta(meta, addressesSize, blockShift);
entry.maxTermLength = meta.readInt();
// Read one more int for compressed term dict.
if (entry.compressed) {
entry.maxBlockLength = meta.readInt();
}
entry.termsDataOffset = meta.readLong();
entry.termsDataLength = meta.readLong();
entry.termsAddressesOffset = meta.readLong();
entry.termsAddressesLength = meta.readLong();
entry.termsDictIndexShift = meta.readInt();
final long indexSize =
(entry.termsDictSize + (1L << entry.termsDictIndexShift) - 1) >>> entry.termsDictIndexShift;
entry.termsIndexAddressesMeta =
LegacyDirectMonotonicReader.loadMeta(meta, 1 + indexSize, blockShift);
entry.termsIndexOffset = meta.readLong();
entry.termsIndexLength = meta.readLong();
entry.termsIndexAddressesOffset = meta.readLong();
entry.termsIndexAddressesLength = meta.readLong();
}
private SortedNumericEntry readSortedNumeric(IndexInput meta) throws IOException {
SortedNumericEntry entry = new SortedNumericEntry();
readNumeric(meta, entry);
entry.numDocsWithField = meta.readInt();
if (entry.numDocsWithField != entry.numValues) {
entry.addressesOffset = meta.readLong();
final int blockShift = meta.readVInt();
entry.addressesMeta =
LegacyDirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift);
entry.addressesLength = meta.readLong();
}
return entry;
}
@Override
public void close() throws IOException {
data.close();
}
private static class NumericEntry {
long[] table;
int blockShift;
byte bitsPerValue;
long docsWithFieldOffset;
long docsWithFieldLength;
short jumpTableEntryCount;
byte denseRankPower;
long numValues;
long minValue;
long gcd;
long valuesOffset;
long valuesLength;
long valueJumpTableOffset; // -1 if no jump-table
}
private static class BinaryEntry {
boolean compressed;
long dataOffset;
long dataLength;
long docsWithFieldOffset;
long docsWithFieldLength;
short jumpTableEntryCount;
byte denseRankPower;
int numDocsWithField;
int minLength;
int maxLength;
long addressesOffset;
long addressesLength;
LegacyDirectMonotonicReader.Meta addressesMeta;
int numCompressedChunks;
int docsPerChunkShift;
int maxUncompressedChunkSize;
}
private static class TermsDictEntry {
long termsDictSize;
int termsDictBlockShift;
LegacyDirectMonotonicReader.Meta termsAddressesMeta;
int maxTermLength;
long termsDataOffset;
long termsDataLength;
long termsAddressesOffset;
long termsAddressesLength;
int termsDictIndexShift;
LegacyDirectMonotonicReader.Meta termsIndexAddressesMeta;
long termsIndexOffset;
long termsIndexLength;
long termsIndexAddressesOffset;
long termsIndexAddressesLength;
boolean compressed;
int maxBlockLength;
}
private static class SortedEntry extends TermsDictEntry {
long docsWithFieldOffset;
long docsWithFieldLength;
short jumpTableEntryCount;
byte denseRankPower;
int numDocsWithField;
byte bitsPerValue;
long ordsOffset;
long ordsLength;
}
private static class SortedSetEntry extends TermsDictEntry {
SortedEntry singleValueEntry;
long docsWithFieldOffset;
long docsWithFieldLength;
short jumpTableEntryCount;
byte denseRankPower;
int numDocsWithField;
byte bitsPerValue;
long ordsOffset;
long ordsLength;
LegacyDirectMonotonicReader.Meta addressesMeta;
long addressesOffset;
long addressesLength;
}
private static class SortedNumericEntry extends NumericEntry {
int numDocsWithField;
LegacyDirectMonotonicReader.Meta addressesMeta;
long addressesOffset;
long addressesLength;
}
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.number);
return getNumeric(entry);
}
private abstract static class DenseNumericDocValues extends NumericDocValues {
final int maxDoc;
int doc = -1;
DenseNumericDocValues(int maxDoc) {
this.maxDoc = maxDoc;
}
@Override
public int docID() {
return doc;
}
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int advance(int target) throws IOException {
if (target >= maxDoc) {
return doc = NO_MORE_DOCS;
}
return doc = target;
}
@Override
public boolean advanceExact(int target) {
doc = target;
return true;
}
@Override
public long cost() {
return maxDoc;
}
}
private abstract static class SparseNumericDocValues extends NumericDocValues {
final IndexedDISI disi;
SparseNumericDocValues(IndexedDISI disi) {
this.disi = disi;
}
@Override
public int advance(int target) throws IOException {
return disi.advance(target);
}
@Override
public boolean advanceExact(int target) throws IOException {
return disi.advanceExact(target);
}
@Override
public int nextDoc() throws IOException {
return disi.nextDoc();
}
@Override
public int docID() {
return disi.docID();
}
@Override
public long cost() {
return disi.cost();
}
}
private NumericDocValues getNumeric(NumericEntry entry) throws IOException {
if (entry.docsWithFieldOffset == -2) {
// empty
return DocValues.emptyNumeric();
} else if (entry.docsWithFieldOffset == -1) {
// dense
if (entry.bitsPerValue == 0) {
return new DenseNumericDocValues(maxDoc) {
@Override
public long longValue() throws IOException {
return entry.minValue;
}
};
} else {
final RandomAccessInput slice =
data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
if (entry.blockShift >= 0) {
// dense but split into blocks of different bits per value
return new DenseNumericDocValues(maxDoc) {
final VaryingBPVReader vBPVReader = new VaryingBPVReader(entry, slice);
@Override
public long longValue() throws IOException {
return vBPVReader.getLongValue(doc);
}
};
} else {
final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
if (entry.table != null) {
final long[] table = entry.table;
return new DenseNumericDocValues(maxDoc) {
@Override
public long longValue() throws IOException {
return table[(int) values.get(doc)];
}
};
} else {
final long mul = entry.gcd;
final long delta = entry.minValue;
return new DenseNumericDocValues(maxDoc) {
@Override
public long longValue() throws IOException {
return mul * values.get(doc) + delta;
}
};
}
}
}
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numValues);
if (entry.bitsPerValue == 0) {
return new SparseNumericDocValues(disi) {
@Override
public long longValue() throws IOException {
return entry.minValue;
}
};
} else {
final RandomAccessInput slice =
data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
if (entry.blockShift >= 0) {
// sparse and split into blocks of different bits per value
return new SparseNumericDocValues(disi) {
final VaryingBPVReader vBPVReader = new VaryingBPVReader(entry, slice);
@Override
public long longValue() throws IOException {
final int index = disi.index();
return vBPVReader.getLongValue(index);
}
};
} else {
final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
if (entry.table != null) {
final long[] table = entry.table;
return new SparseNumericDocValues(disi) {
@Override
public long longValue() throws IOException {
return table[(int) values.get(disi.index())];
}
};
} else {
final long mul = entry.gcd;
final long delta = entry.minValue;
return new SparseNumericDocValues(disi) {
@Override
public long longValue() throws IOException {
return mul * values.get(disi.index()) + delta;
}
};
}
}
}
}
}
private LongValues getNumericValues(NumericEntry entry) throws IOException {
if (entry.bitsPerValue == 0) {
return new LongValues() {
@Override
public long get(long index) {
return entry.minValue;
}
};
} else {
final RandomAccessInput slice =
data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
if (entry.blockShift >= 0) {
return new LongValues() {
final VaryingBPVReader vBPVReader = new VaryingBPVReader(entry, slice);
@Override
public long get(long index) {
try {
return vBPVReader.getLongValue(index);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
};
} else {
final LongValues values = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
if (entry.table != null) {
final long[] table = entry.table;
return new LongValues() {
@Override
public long get(long index) {
return table[(int) values.get(index)];
}
};
} else if (entry.gcd != 1) {
final long gcd = entry.gcd;
final long minValue = entry.minValue;
return new LongValues() {
@Override
public long get(long index) {
return values.get(index) * gcd + minValue;
}
};
} else if (entry.minValue != 0) {
final long minValue = entry.minValue;
return new LongValues() {
@Override
public long get(long index) {
return values.get(index) + minValue;
}
};
} else {
return values;
}
}
}
}
private abstract static class DenseBinaryDocValues extends BinaryDocValues {
final int maxDoc;
int doc = -1;
DenseBinaryDocValues(int maxDoc) {
this.maxDoc = maxDoc;
}
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int docID() {
return doc;
}
@Override
public long cost() {
return maxDoc;
}
@Override
public int advance(int target) throws IOException {
if (target >= maxDoc) {
return doc = NO_MORE_DOCS;
}
return doc = target;
}
@Override
public boolean advanceExact(int target) throws IOException {
doc = target;
return true;
}
}
private abstract static class SparseBinaryDocValues extends BinaryDocValues {
final IndexedDISI disi;
SparseBinaryDocValues(IndexedDISI disi) {
this.disi = disi;
}
@Override
public int nextDoc() throws IOException {
return disi.nextDoc();
}
@Override
public int docID() {
return disi.docID();
}
@Override
public long cost() {
return disi.cost();
}
@Override
public int advance(int target) throws IOException {
return disi.advance(target);
}
@Override
public boolean advanceExact(int target) throws IOException {
return disi.advanceExact(target);
}
}
private BinaryDocValues getUncompressedBinary(BinaryEntry entry) throws IOException {
if (entry.docsWithFieldOffset == -2) {
return DocValues.emptyBinary();
}
final IndexInput bytesSlice = data.slice("fixed-binary", entry.dataOffset, entry.dataLength);
if (entry.docsWithFieldOffset == -1) {
// dense
if (entry.minLength == entry.maxLength) {
// fixed length
final int length = entry.maxLength;
return new DenseBinaryDocValues(maxDoc) {
final BytesRef bytes = new BytesRef(new byte[length], 0, length);
@Override
public BytesRef binaryValue() throws IOException {
bytesSlice.seek((long) doc * length);
bytesSlice.readBytes(bytes.bytes, 0, length);
return bytes;
}
};
} else {
// variable length
final RandomAccessInput addressesData =
this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
return new DenseBinaryDocValues(maxDoc) {
final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength);
@Override
public BytesRef binaryValue() throws IOException {
long startOffset = addresses.get(doc);
bytes.length = (int) (addresses.get(doc + 1L) - startOffset);
bytesSlice.seek(startOffset);
bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
return bytes;
}
};
}
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numDocsWithField);
if (entry.minLength == entry.maxLength) {
// fixed length
final int length = entry.maxLength;
return new SparseBinaryDocValues(disi) {
final BytesRef bytes = new BytesRef(new byte[length], 0, length);
@Override
public BytesRef binaryValue() throws IOException {
bytesSlice.seek((long) disi.index() * length);
bytesSlice.readBytes(bytes.bytes, 0, length);
return bytes;
}
};
} else {
// variable length
final RandomAccessInput addressesData =
this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
return new SparseBinaryDocValues(disi) {
final BytesRef bytes = new BytesRef(new byte[entry.maxLength], 0, entry.maxLength);
@Override
public BytesRef binaryValue() throws IOException {
final int index = disi.index();
long startOffset = addresses.get(index);
bytes.length = (int) (addresses.get(index + 1L) - startOffset);
bytesSlice.seek(startOffset);
bytesSlice.readBytes(bytes.bytes, 0, bytes.length);
return bytes;
}
};
}
}
}
// Decompresses blocks of binary values to retrieve content
static class BinaryDecoder {
private final LongValues addresses;
private final IndexInput compressedData;
// Cache of last uncompressed block
private long lastBlockId = -1;
private final int[] uncompressedDocStarts;
private int uncompressedBlockLength = 0;
private final byte[] uncompressedBlock;
private final BytesRef uncompressedBytesRef;
private final int docsPerChunk;
private final int docsPerChunkShift;
public BinaryDecoder(
LongValues addresses,
IndexInput compressedData,
int biggestUncompressedBlockSize,
int docsPerChunkShift) {
super();
this.addresses = addresses;
this.compressedData = compressedData;
// pre-allocate a byte array large enough for the biggest uncompressed block needed.
this.uncompressedBlock = new byte[biggestUncompressedBlockSize];
uncompressedBytesRef = new BytesRef(uncompressedBlock);
this.docsPerChunk = 1 << docsPerChunkShift;
this.docsPerChunkShift = docsPerChunkShift;
uncompressedDocStarts = new int[docsPerChunk + 1];
}
BytesRef decode(int docNumber) throws IOException {
int blockId = docNumber >> docsPerChunkShift;
int docInBlockId = docNumber % docsPerChunk;
assert docInBlockId < docsPerChunk;
// already read and uncompressed?
if (blockId != lastBlockId) {
lastBlockId = blockId;
long blockStartOffset = addresses.get(blockId);
compressedData.seek(blockStartOffset);
uncompressedBlockLength = 0;
int onlyLength = -1;
for (int i = 0; i < docsPerChunk; i++) {
if (i == 0) {
// The first length value is special. It is shifted and has a bit to denote if
// all other values are the same length
int lengthPlusSameInd = compressedData.readVInt();
int sameIndicator = lengthPlusSameInd & 1;
int firstValLength = lengthPlusSameInd >>> 1;
if (sameIndicator == 1) {
onlyLength = firstValLength;
}
uncompressedBlockLength += firstValLength;
} else {
if (onlyLength == -1) {
// Various lengths are stored - read each from disk
uncompressedBlockLength += compressedData.readVInt();
} else {
// Only one length
uncompressedBlockLength += onlyLength;
}
}
uncompressedDocStarts[i + 1] = uncompressedBlockLength;
}
if (uncompressedBlockLength == 0) {
uncompressedBytesRef.offset = 0;
uncompressedBytesRef.length = 0;
return uncompressedBytesRef;
}
assert uncompressedBlockLength <= uncompressedBlock.length;
LZ4.decompress(
EndiannessReverserUtil.wrapDataInput(compressedData),
uncompressedBlockLength,
uncompressedBlock,
0);
}
uncompressedBytesRef.offset = uncompressedDocStarts[docInBlockId];
uncompressedBytesRef.length =
uncompressedDocStarts[docInBlockId + 1] - uncompressedBytesRef.offset;
return uncompressedBytesRef;
}
}
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.number);
if (entry.compressed) {
return getCompressedBinary(entry);
} else {
return getUncompressedBinary(entry);
}
}
private BinaryDocValues getCompressedBinary(BinaryEntry entry) throws IOException {
if (entry.docsWithFieldOffset == -2) {
return DocValues.emptyBinary();
}
if (entry.docsWithFieldOffset == -1) {
// dense
final RandomAccessInput addressesData =
this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
return new DenseBinaryDocValues(maxDoc) {
BinaryDecoder decoder =
new BinaryDecoder(
addresses, data.clone(), entry.maxUncompressedChunkSize, entry.docsPerChunkShift);
@Override
public BytesRef binaryValue() throws IOException {
return decoder.decode(doc);
}
};
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numDocsWithField);
final RandomAccessInput addressesData =
this.data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesData);
return new SparseBinaryDocValues(disi) {
BinaryDecoder decoder =
new BinaryDecoder(
addresses, data.clone(), entry.maxUncompressedChunkSize, entry.docsPerChunkShift);
@Override
public BytesRef binaryValue() throws IOException {
return decoder.decode(disi.index());
}
};
}
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedEntry entry = sorted.get(field.number);
return getSorted(entry);
}
private SortedDocValues getSorted(SortedEntry entry) throws IOException {
if (entry.docsWithFieldOffset == -2) {
return DocValues.emptySorted();
}
final LongValues ords;
if (entry.bitsPerValue == 0) {
ords =
new LongValues() {
@Override
public long get(long index) {
return 0L;
}
};
} else {
final RandomAccessInput slice = data.randomAccessSlice(entry.ordsOffset, entry.ordsLength);
ords = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
}
if (entry.docsWithFieldOffset == -1) {
// dense
return new BaseSortedDocValues(entry, data) {
int doc = -1;
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int docID() {
return doc;
}
@Override
public long cost() {
return maxDoc;
}
@Override
public int advance(int target) throws IOException {
if (target >= maxDoc) {
return doc = NO_MORE_DOCS;
}
return doc = target;
}
@Override
public boolean advanceExact(int target) {
doc = target;
return true;
}
@Override
public int ordValue() {
return (int) ords.get(doc);
}
};
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numDocsWithField);
return new BaseSortedDocValues(entry, data) {
@Override
public int nextDoc() throws IOException {
return disi.nextDoc();
}
@Override
public int docID() {
return disi.docID();
}
@Override
public long cost() {
return disi.cost();
}
@Override
public int advance(int target) throws IOException {
return disi.advance(target);
}
@Override
public boolean advanceExact(int target) throws IOException {
return disi.advanceExact(target);
}
@Override
public int ordValue() {
return (int) ords.get(disi.index());
}
};
}
}
private abstract static class BaseSortedDocValues extends SortedDocValues {
final SortedEntry entry;
final IndexInput data;
final TermsEnum termsEnum;
BaseSortedDocValues(SortedEntry entry, IndexInput data) throws IOException {
this.entry = entry;
this.data = data;
this.termsEnum = termsEnum();
}
@Override
public int getValueCount() {
return Math.toIntExact(entry.termsDictSize);
}
@Override
public BytesRef lookupOrd(int ord) throws IOException {
termsEnum.seekExact(ord);
return termsEnum.term();
}
@Override
public int lookupTerm(BytesRef key) throws IOException {
SeekStatus status = termsEnum.seekCeil(key);
switch (status) {
case FOUND:
return Math.toIntExact(termsEnum.ord());
case NOT_FOUND:
case END:
default:
return Math.toIntExact(-1L - termsEnum.ord());
}
}
@Override
public TermsEnum termsEnum() throws IOException {
return new TermsDict(entry, data);
}
}
private abstract static class BaseSortedSetDocValues extends SortedSetDocValues {
final SortedSetEntry entry;
final IndexInput data;
final TermsEnum termsEnum;
BaseSortedSetDocValues(SortedSetEntry entry, IndexInput data) throws IOException {
this.entry = entry;
this.data = data;
this.termsEnum = termsEnum();
}
@Override
public long getValueCount() {
return entry.termsDictSize;
}
@Override
public BytesRef lookupOrd(long ord) throws IOException {
termsEnum.seekExact(ord);
return termsEnum.term();
}
@Override
public long lookupTerm(BytesRef key) throws IOException {
SeekStatus status = termsEnum.seekCeil(key);
switch (status) {
case FOUND:
return termsEnum.ord();
case NOT_FOUND:
case END:
default:
return -1L - termsEnum.ord();
}
}
@Override
public TermsEnum termsEnum() throws IOException {
return new TermsDict(entry, data);
}
}
private static class TermsDict extends BaseTermsEnum {
static final int LZ4_DECOMPRESSOR_PADDING = 7;
final TermsDictEntry entry;
final LongValues blockAddresses;
final IndexInput bytes;
final long blockMask;
final LongValues indexAddresses;
final IndexInput indexBytes;
final BytesRef term;
long ord = -1;
BytesRef blockBuffer = null;
ByteArrayDataInput blockInput = null;
long currentCompressedBlockStart = -1;
long currentCompressedBlockEnd = -1;
TermsDict(TermsDictEntry entry, IndexInput data) throws IOException {
this.entry = entry;
RandomAccessInput addressesSlice =
data.randomAccessSlice(entry.termsAddressesOffset, entry.termsAddressesLength);
blockAddresses =
LegacyDirectMonotonicReader.getInstance(entry.termsAddressesMeta, addressesSlice);
bytes = data.slice("terms", entry.termsDataOffset, entry.termsDataLength);
blockMask = (1L << entry.termsDictBlockShift) - 1;
RandomAccessInput indexAddressesSlice =
data.randomAccessSlice(entry.termsIndexAddressesOffset, entry.termsIndexAddressesLength);
indexAddresses =
LegacyDirectMonotonicReader.getInstance(
entry.termsIndexAddressesMeta, indexAddressesSlice);
indexBytes = data.slice("terms-index", entry.termsIndexOffset, entry.termsIndexLength);
term = new BytesRef(entry.maxTermLength);
if (entry.compressed) {
// add 7 padding bytes can help decompression run faster.
int bufferSize = entry.maxBlockLength + LZ4_DECOMPRESSOR_PADDING;
blockBuffer = new BytesRef(new byte[bufferSize], 0, bufferSize);
}
}
@Override
public BytesRef next() throws IOException {
if (++ord >= entry.termsDictSize) {
return null;
}
if ((ord & blockMask) == 0L) {
if (this.entry.compressed) {
decompressBlock();
} else {
term.length = bytes.readVInt();
bytes.readBytes(term.bytes, 0, term.length);
}
} else {
DataInput input = this.entry.compressed ? blockInput : bytes;
final int token = Byte.toUnsignedInt(input.readByte());
int prefixLength = token & 0x0F;
int suffixLength = 1 + (token >>> 4);
if (prefixLength == 15) {
prefixLength += input.readVInt();
}
if (suffixLength == 16) {
suffixLength += input.readVInt();
}
term.length = prefixLength + suffixLength;
input.readBytes(term.bytes, prefixLength, suffixLength);
}
return term;
}
@Override
public void seekExact(long ord) throws IOException {
if (ord < 0 || ord >= entry.termsDictSize) {
throw new IndexOutOfBoundsException();
}
final long blockIndex = ord >>> entry.termsDictBlockShift;
final long blockAddress = blockAddresses.get(blockIndex);
bytes.seek(blockAddress);
this.ord = (blockIndex << entry.termsDictBlockShift) - 1;
do {
next();
} while (this.ord < ord);
}
private BytesRef getTermFromIndex(long index) throws IOException {
assert index >= 0 && index <= (entry.termsDictSize - 1) >>> entry.termsDictIndexShift;
final long start = indexAddresses.get(index);
term.length = (int) (indexAddresses.get(index + 1) - start);
indexBytes.seek(start);
indexBytes.readBytes(term.bytes, 0, term.length);
return term;
}
private long seekTermsIndex(BytesRef text) throws IOException {
long lo = 0L;
long hi = (entry.termsDictSize - 1) >>> entry.termsDictIndexShift;
while (lo <= hi) {
final long mid = (lo + hi) >>> 1;
getTermFromIndex(mid);
final int cmp = term.compareTo(text);
if (cmp <= 0) {
lo = mid + 1;
} else {
hi = mid - 1;
}
}
assert hi < 0 || getTermFromIndex(hi).compareTo(text) <= 0;
assert hi == ((entry.termsDictSize - 1) >>> entry.termsDictIndexShift)
|| getTermFromIndex(hi + 1).compareTo(text) > 0;
return hi;
}
private BytesRef getFirstTermFromBlock(long block) throws IOException {
assert block >= 0 && block <= (entry.termsDictSize - 1) >>> entry.termsDictBlockShift;
final long blockAddress = blockAddresses.get(block);
bytes.seek(blockAddress);
term.length = bytes.readVInt();
bytes.readBytes(term.bytes, 0, term.length);
return term;
}
private long seekBlock(BytesRef text) throws IOException {
long index = seekTermsIndex(text);
if (index == -1L) {
return -1L;
}
long ordLo = index << entry.termsDictIndexShift;
long ordHi = Math.min(entry.termsDictSize, ordLo + (1L << entry.termsDictIndexShift)) - 1L;
long blockLo = ordLo >>> entry.termsDictBlockShift;
long blockHi = ordHi >>> entry.termsDictBlockShift;
while (blockLo <= blockHi) {
final long blockMid = (blockLo + blockHi) >>> 1;
getFirstTermFromBlock(blockMid);
final int cmp = term.compareTo(text);
if (cmp <= 0) {
blockLo = blockMid + 1;
} else {
blockHi = blockMid - 1;
}
}
assert blockHi < 0 || getFirstTermFromBlock(blockHi).compareTo(text) <= 0;
assert blockHi == ((entry.termsDictSize - 1) >>> entry.termsDictBlockShift)
|| getFirstTermFromBlock(blockHi + 1).compareTo(text) > 0;
return blockHi;
}
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
final long block = seekBlock(text);
if (block == -1) {
// before the first term
seekExact(0L);
return SeekStatus.NOT_FOUND;
}
final long blockAddress = blockAddresses.get(block);
this.ord = block << entry.termsDictBlockShift;
bytes.seek(blockAddress);
if (this.entry.compressed) {
decompressBlock();
} else {
term.length = bytes.readVInt();
bytes.readBytes(term.bytes, 0, term.length);
}
while (true) {
int cmp = term.compareTo(text);
if (cmp == 0) {
return SeekStatus.FOUND;
} else if (cmp > 0) {
return SeekStatus.NOT_FOUND;
}
if (next() == null) {
return SeekStatus.END;
}
}
}
private void decompressBlock() throws IOException {
// The first term is kept uncompressed, so no need to decompress block if only
// look up the first term when doing seek block.
term.length = bytes.readVInt();
bytes.readBytes(term.bytes, 0, term.length);
long offset = bytes.getFilePointer();
if (offset < entry.termsDataLength - 1) {
// Avoid decompress again if we are reading a same block.
if (currentCompressedBlockStart != offset) {
int decompressLength = bytes.readVInt();
// Decompress the remaining of current block
LZ4.decompress(
EndiannessReverserUtil.wrapDataInput(bytes), decompressLength, blockBuffer.bytes, 0);
currentCompressedBlockStart = offset;
currentCompressedBlockEnd = bytes.getFilePointer();
} else {
// Skip decompression but need to re-seek to block end.
bytes.seek(currentCompressedBlockEnd);
}
// Reset the buffer.
blockInput = new ByteArrayDataInput(blockBuffer.bytes, 0, blockBuffer.length);
}
}
@Override
public BytesRef term() throws IOException {
return term;
}
@Override
public long ord() throws IOException {
return ord;
}
@Override
public long totalTermFreq() throws IOException {
return -1L;
}
@Override
public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int docFreq() throws IOException {
throw new UnsupportedOperationException();
}
}
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.number);
if (entry.numValues == entry.numDocsWithField) {
return DocValues.singleton(getNumeric(entry));
}
final RandomAccessInput addressesInput =
data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
final LongValues values = getNumericValues(entry);
if (entry.docsWithFieldOffset == -1) {
// dense
return new SortedNumericDocValues() {
int doc = -1;
long start, end;
int count;
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int docID() {
return doc;
}
@Override
public long cost() {
return maxDoc;
}
@Override
public int advance(int target) throws IOException {
if (target >= maxDoc) {
return doc = NO_MORE_DOCS;
}
start = addresses.get(target);
end = addresses.get(target + 1L);
count = (int) (end - start);
return doc = target;
}
@Override
public boolean advanceExact(int target) throws IOException {
start = addresses.get(target);
end = addresses.get(target + 1L);
count = (int) (end - start);
doc = target;
return true;
}
@Override
public long nextValue() throws IOException {
return values.get(start++);
}
@Override
public int docValueCount() {
return count;
}
};
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numDocsWithField);
return new SortedNumericDocValues() {
boolean set;
long start, end;
int count;
@Override
public int nextDoc() throws IOException {
set = false;
return disi.nextDoc();
}
@Override
public int docID() {
return disi.docID();
}
@Override
public long cost() {
return disi.cost();
}
@Override
public int advance(int target) throws IOException {
set = false;
return disi.advance(target);
}
@Override
public boolean advanceExact(int target) throws IOException {
set = false;
return disi.advanceExact(target);
}
@Override
public long nextValue() throws IOException {
set();
return values.get(start++);
}
@Override
public int docValueCount() {
set();
return count;
}
private void set() {
if (set == false) {
final int index = disi.index();
start = addresses.get(index);
end = addresses.get(index + 1L);
count = (int) (end - start);
set = true;
}
}
};
}
}
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry entry = sortedSets.get(field.number);
if (entry.singleValueEntry != null) {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}
final RandomAccessInput slice = data.randomAccessSlice(entry.ordsOffset, entry.ordsLength);
final LongValues ords = LegacyDirectReader.getInstance(slice, entry.bitsPerValue);
final RandomAccessInput addressesInput =
data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
final LongValues addresses =
LegacyDirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
if (entry.docsWithFieldOffset == -1) {
// dense
return new BaseSortedSetDocValues(entry, data) {
int doc = -1;
long curr;
int count;
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int docID() {
return doc;
}
@Override
public long cost() {
return maxDoc;
}
@Override
public int advance(int target) throws IOException {
if (target >= maxDoc) {
return doc = NO_MORE_DOCS;
}
curr = addresses.get(target);
long end = addresses.get(target + 1L);
count = (int) (end - curr);
return doc = target;
}
@Override
public boolean advanceExact(int target) throws IOException {
curr = addresses.get(target);
long end = addresses.get(target + 1L);
count = (int) (end - curr);
doc = target;
return true;
}
@Override
public long nextOrd() throws IOException {
return ords.get(curr++);
}
@Override
public int docValueCount() {
return count;
}
};
} else {
// sparse
final IndexedDISI disi =
new IndexedDISI(
data,
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
entry.denseRankPower,
entry.numDocsWithField);
return new BaseSortedSetDocValues(entry, data) {
boolean set;
long curr;
int count;
@Override
public int nextDoc() throws IOException {
set = false;
return disi.nextDoc();
}
@Override
public int docID() {
return disi.docID();
}
@Override
public long cost() {
return disi.cost();
}
@Override
public int advance(int target) throws IOException {
set = false;
return disi.advance(target);
}
@Override
public boolean advanceExact(int target) throws IOException {
set = false;
return disi.advanceExact(target);
}
private void set() {
if (set == false) {
final int index = disi.index();
curr = addresses.get(index);
long end = addresses.get(index + 1L);
count = (int) (end - curr);
set = true;
}
}
@Override
public long nextOrd() throws IOException {
set();
return ords.get(curr++);
}
@Override
public int docValueCount() {
set();
return count;
}
};
}
}
@Override
public DocValuesSkipper getSkipper(FieldInfo field) {
return null;
}
@Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
}
/**
* Reader for longs split into blocks of different bits per values. The longs are requested by
* index and must be accessed in monotonically increasing order.
*/
// Note: The order requirement could be removed as the jump-tables allow for backwards iteration
// Note 2: The rankSlice is only used if an advance of > 1 block is called. Its construction could
// be lazy
private class VaryingBPVReader {
final RandomAccessInput slice; // 2 slices to avoid cache thrashing when using rank
final RandomAccessInput rankSlice;
final NumericEntry entry;
final int shift;
final long mul;
final int mask;
long block = -1;
long delta;
long offset;
long blockEndOffset;
LongValues values;
VaryingBPVReader(NumericEntry entry, RandomAccessInput slice) throws IOException {
this.entry = entry;
this.slice = slice;
this.rankSlice =
entry.valueJumpTableOffset == -1
? null
: data.randomAccessSlice(
entry.valueJumpTableOffset, data.length() - entry.valueJumpTableOffset);
shift = entry.blockShift;
mul = entry.gcd;
mask = (1 << shift) - 1;
}
long getLongValue(long index) throws IOException {
final long block = index >>> shift;
if (this.block != block) {
int bitsPerValue;
do {
// If the needed block is the one directly following the current block, it is cheaper to
// avoid the cache
if (rankSlice != null && block != this.block + 1) {
blockEndOffset = rankSlice.readLong(block * Long.BYTES) - entry.valuesOffset;
this.block = block - 1;
}
offset = blockEndOffset;
bitsPerValue = slice.readByte(offset++);
delta = slice.readLong(offset);
offset += Long.BYTES;
if (bitsPerValue == 0) {
blockEndOffset = offset;
} else {
final int length = slice.readInt(offset);
offset += Integer.BYTES;
blockEndOffset = offset + length;
}
this.block++;
} while (this.block != block);
values =
bitsPerValue == 0
? LongValues.ZEROES
: LegacyDirectReader.getInstance(slice, bitsPerValue, offset);
}
return mul * values.get(index & mask) + delta;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy