• Home
• org.apache.lucene
• lucene-test-framework
• 10.1.0
• source code
• RandomPostingsTester.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.apache.lucene.tests.index.RandomPostingsTester Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.tests.index;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.function.IntToLongFunction;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.Impact;
import org.apache.lucene.index.Impacts;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.internal.tests.IndexPackageAccess;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
import org.apache.lucene.tests.util.automaton.AutomatonTestUtil.RandomAcceptedStrings;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.Operations;

/** Helper class extracted from BasePostingsFormatTestCase to exercise a postings format. */
public class RandomPostingsTester {

  private static final IndexPackageAccess INDEX_PACKAGE_ACCESS =
      TestSecrets.getIndexPackageAccess();
  private static final IntToLongFunction DOC_TO_NORM = doc -> 1 + (doc & 0x0f);

  /** Which features to test. */
  public enum Option {
    // Sometimes use .advance():
    SKIPPING,

    // Sometimes reuse the PostingsEnum across terms:
    REUSE_ENUMS,

    // Sometimes pass non-null live docs:
    LIVE_DOCS,

    // Sometimes seek to term using previously saved TermState:
    TERM_STATE,

    // Sometimes don't fully consume docs from the enum
    PARTIAL_DOC_CONSUME,

    // Sometimes don't fully consume positions at each doc
    PARTIAL_POS_CONSUME,

    // Sometimes check payloads
    PAYLOADS,

    // Test w/ multiple threads
    THREADS
  }

  private long totalPostings;
  private long totalPayloadBytes;

  // Holds all postings:
  private final Map> fields;

  private final FieldInfos fieldInfos;

  List allTerms;
  private int maxDoc;

  final Random random;

  public RandomPostingsTester(Random random) throws IOException {
    fields = new TreeMap<>();

    this.random = random;

    final int numFields = TestUtil.nextInt(random, 1, 5);
    if (LuceneTestCase.VERBOSE) {
      System.out.println("TEST: " + numFields + " fields");
    }
    maxDoc = 0;

    FieldInfo[] fieldInfoArray = new FieldInfo[numFields];
    int fieldUpto = 0;
    while (fieldUpto < numFields) {
      String field = TestUtil.randomSimpleString(random);
      if (fields.containsKey(field)) {
        continue;
      }

      fieldInfoArray[fieldUpto] =
          new FieldInfo(
              field,
              fieldUpto,
              false,
              false,
              true,
              IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
              DocValuesType.NONE,
              DocValuesSkipIndexType.NONE,
              -1,
              new HashMap<>(),
              0,
              0,
              0,
              0,
              VectorEncoding.FLOAT32,
              VectorSimilarityFunction.EUCLIDEAN,
              false,
              false);
      fieldUpto++;

      SortedMap postings = new TreeMap<>();
      fields.put(field, postings);
      Set seenTerms = new HashSet<>();

      int numTerms;
      if (random.nextInt(10) == 7) {
        numTerms = LuceneTestCase.atLeast(random, 50);
      } else {
        numTerms = TestUtil.nextInt(random, 2, 20);
      }

      while (postings.size() < numTerms) {
        int termUpto = postings.size();
        // Cannot contain surrogates else default Java string sort order (by UTF16 code unit) is
        // different from Lucene:
        String term = TestUtil.randomSimpleString(random);
        if (seenTerms.contains(term)) {
          continue;
        }
        seenTerms.add(term);

        if (LuceneTestCase.TEST_NIGHTLY && termUpto == 0 && fieldUpto == 1) {
          // Make 1 big term:
          term = "big_" + term;
        } else if (termUpto == 1 && fieldUpto == 1) {
          // Make 1 medium term:
          term = "medium_" + term;
        } else if (random.nextBoolean()) {
          // Low freq term:
          term = "low_" + term;
        } else {
          // Very low freq term (don't multiply by RANDOM_MULTIPLIER):
          term = "verylow_" + term;
        }

        long termSeed = random.nextLong();
        postings.put(new BytesRef(term), new SeedAndOrd(termSeed));

        // NOTE: sort of silly: we enum all the docs just to
        // get the maxDoc
        PostingsEnum postingsEnum = getSeedPostings(term, termSeed, IndexOptions.DOCS, true);
        int doc;
        int lastDoc = 0;
        while ((doc = postingsEnum.nextDoc()) != PostingsEnum.NO_MORE_DOCS) {
          lastDoc = doc;
        }
        maxDoc = Math.max(lastDoc, maxDoc);
      }

      // assign ords
      long ord = 0;
      for (SeedAndOrd ent : postings.values()) {
        ent.ord = ord++;
      }
    }

    fieldInfos = new FieldInfos(fieldInfoArray);

    // It's the count, not the last docID:
    maxDoc++;

    allTerms = new ArrayList<>();
    for (Map.Entry> fieldEnt : fields.entrySet()) {
      String field = fieldEnt.getKey();
      long ord = 0;
      for (Map.Entry termEnt : fieldEnt.getValue().entrySet()) {
        allTerms.add(new FieldAndTerm(field, termEnt.getKey(), ord++));
      }
    }

    if (LuceneTestCase.VERBOSE) {
      System.out.println(
          "TEST: done init postings; "
              + allTerms.size()
              + " total terms, across "
              + fieldInfos.size()
              + " fields");
    }
  }

  public static SeedPostings getSeedPostings(
      String term, long seed, IndexOptions options, boolean allowPayloads) {
    int minDocFreq, maxDocFreq;
    if (term.startsWith("big_")) {
      minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 50000;
      maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 70000;
    } else if (term.startsWith("medium_")) {
      minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 3000;
      maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 6000;
    } else if (term.startsWith("low_")) {
      minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER;
      maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 40;
    } else {
      minDocFreq = 1;
      maxDocFreq = 3;
    }

    return new SeedPostings(seed, minDocFreq, maxDocFreq, options, allowPayloads);
  }

  /** Given the same random seed this always enumerates the same random postings */
  public static class SeedPostings extends PostingsEnum {
    // Used only to generate docIDs; this way if you pull w/
    // or w/o positions you get the same docID sequence:
    private final Random docRandom;
    private final Random random;
    public int docFreq;
    private final int maxDocSpacing;
    private final int payloadSize;
    private final boolean fixedPayloads;
    private final BytesRef payload;
    private final boolean doPositions;
    private final boolean allowPayloads;

    private int docID = -1;
    private int freq;
    public int upto;

    private int pos;
    private int offset;
    private int startOffset;
    private int endOffset;
    private int posSpacing;
    private int posUpto;

    public SeedPostings(
        long seed, int minDocFreq, int maxDocFreq, IndexOptions options, boolean allowPayloads) {
      random = new Random(seed);
      docRandom = new Random(random.nextLong());
      docFreq = TestUtil.nextInt(random, minDocFreq, maxDocFreq);
      this.allowPayloads = allowPayloads;

      // TODO: more realistic to inversely tie this to numDocs:
      maxDocSpacing = TestUtil.nextInt(random, 1, 100);

      if (random.nextInt(10) == 7) {
        // 10% of the time create big payloads:
        payloadSize = 1 + random.nextInt(3);
      } else {
        payloadSize = 1 + random.nextInt(1);
      }

      fixedPayloads = random.nextBoolean();
      byte[] payloadBytes = new byte[payloadSize];
      payload = new BytesRef(payloadBytes);
      doPositions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS.compareTo(options) <= 0;
    }

    @Override
    public int nextDoc() {
      while (true) {
        _nextDoc();
        return docID;
      }
    }

    private int _nextDoc() {
      if (docID == -1) {
        docID = 0;
      }
      // Must consume random:
      while (posUpto < freq) {
        nextPosition();
      }

      if (upto < docFreq) {
        if (upto == 0 && docRandom.nextBoolean()) {
          // Sometimes index docID = 0
        } else if (maxDocSpacing == 1) {
          docID++;
        } else {
          // TODO: sometimes have a biggish gap here!
          docID += TestUtil.nextInt(docRandom, 1, maxDocSpacing);
        }

        if (random.nextInt(200) == 17) {
          freq = TestUtil.nextInt(random, 1, 1000);
        } else if (random.nextInt(10) == 17) {
          freq = TestUtil.nextInt(random, 1, 20);
        } else {
          freq = TestUtil.nextInt(random, 1, 4);
        }

        pos = 0;
        offset = 0;
        posUpto = 0;
        posSpacing = TestUtil.nextInt(random, 1, 100);

        upto++;
        return docID;
      } else {
        return docID = NO_MORE_DOCS;
      }
    }

    @Override
    public int docID() {
      return docID;
    }

    @Override
    public int freq() {
      return freq;
    }

    @Override
    public int nextPosition() {
      if (!doPositions) {
        posUpto = freq;
        return -1;
      }
      assert posUpto < freq;

      if (posUpto == 0 && random.nextBoolean()) {
        // Sometimes index pos = 0
      } else if (posSpacing == 1) {
        pos++;
      } else {
        pos += TestUtil.nextInt(random, 1, posSpacing);
      }

      if (payloadSize != 0) {
        if (fixedPayloads) {
          payload.length = payloadSize;
          random.nextBytes(payload.bytes);
        } else {
          int thisPayloadSize = random.nextInt(payloadSize);
          if (thisPayloadSize != 0) {
            payload.length = payloadSize;
            random.nextBytes(payload.bytes);
          } else {
            payload.length = 0;
          }
        }
      } else {
        payload.length = 0;
      }
      if (!allowPayloads) {
        payload.length = 0;
      }

      startOffset = offset + random.nextInt(5);
      endOffset = startOffset + random.nextInt(10);
      offset = endOffset;

      posUpto++;
      return pos;
    }

    @Override
    public int startOffset() {
      return startOffset;
    }

    @Override
    public int endOffset() {
      return endOffset;
    }

    @Override
    public BytesRef getPayload() {
      return payload.length == 0 ? null : payload;
    }

    @Override
    public int advance(int target) throws IOException {
      return slowAdvance(target);
    }

    @Override
    public long cost() {
      return docFreq;
    }
  }

  /** Holds one field, term and ord. */
  public static class FieldAndTerm {
    final String field;
    final BytesRef term;
    final long ord;

    public FieldAndTerm(String field, BytesRef term, long ord) {
      this.field = field;
      this.term = BytesRef.deepCopyOf(term);
      this.ord = ord;
    }
  }

  private static class SeedAndOrd {
    final long seed;
    long ord;

    public SeedAndOrd(long seed) {
      this.seed = seed;
    }
  }

  private static class SeedFields extends Fields {
    final Map> fields;
    final FieldInfos fieldInfos;
    final IndexOptions maxAllowed;
    final boolean allowPayloads;

    public SeedFields(
        Map> fields,
        FieldInfos fieldInfos,
        IndexOptions maxAllowed,
        boolean allowPayloads) {
      this.fields = fields;
      this.fieldInfos = fieldInfos;
      this.maxAllowed = maxAllowed;
      this.allowPayloads = allowPayloads;
    }

    @Override
    public Iterator iterator() {
      return fields.keySet().iterator();
    }

    @Override
    public Terms terms(String field) {
      SortedMap terms = fields.get(field);
      if (terms == null) {
        return null;
      } else {
        return new SeedTerms(terms, fieldInfos.fieldInfo(field), maxAllowed, allowPayloads);
      }
    }

    @Override
    public int size() {
      return fields.size();
    }
  }

  private static class SeedTerms extends Terms {
    final SortedMap terms;
    final FieldInfo fieldInfo;
    final IndexOptions maxAllowed;
    final boolean allowPayloads;

    public SeedTerms(
        SortedMap terms,
        FieldInfo fieldInfo,
        IndexOptions maxAllowed,
        boolean allowPayloads) {
      this.terms = terms;
      this.fieldInfo = fieldInfo;
      this.maxAllowed = maxAllowed;
      this.allowPayloads = allowPayloads;
    }

    @Override
    public TermsEnum iterator() {
      SeedTermsEnum termsEnum = new SeedTermsEnum(terms, maxAllowed, allowPayloads);
      termsEnum.reset();

      return termsEnum;
    }

    @Override
    public long size() {
      return terms.size();
    }

    @Override
    public long getSumTotalTermFreq() {
      throw new UnsupportedOperationException();
    }

    @Override
    public long getSumDocFreq() {
      throw new UnsupportedOperationException();
    }

    @Override
    public int getDocCount() {
      throw new UnsupportedOperationException();
    }

    @Override
    public boolean hasFreqs() {
      return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
    }

    @Override
    public boolean hasOffsets() {
      return fieldInfo
              .getIndexOptions()
              .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
          >= 0;
    }

    @Override
    public boolean hasPositions() {
      return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
    }

    @Override
    public boolean hasPayloads() {
      return allowPayloads && fieldInfo.hasPayloads();
    }
  }

  private static class SeedTermsEnum extends BaseTermsEnum {
    final SortedMap terms;
    final IndexOptions maxAllowed;
    final boolean allowPayloads;

    private Iterator> iterator;

    private Map.Entry current;

    public SeedTermsEnum(
        SortedMap terms, IndexOptions maxAllowed, boolean allowPayloads) {
      this.terms = terms;
      this.maxAllowed = maxAllowed;
      this.allowPayloads = allowPayloads;
    }

    void reset() {
      iterator = terms.entrySet().iterator();
    }

    @Override
    public SeekStatus seekCeil(BytesRef text) {
      SortedMap tailMap = terms.tailMap(text);
      if (tailMap.isEmpty()) {
        return SeekStatus.END;
      } else {
        iterator = tailMap.entrySet().iterator();
        current = iterator.next();
        if (tailMap.firstKey().equals(text)) {
          return SeekStatus.FOUND;
        } else {
          return SeekStatus.NOT_FOUND;
        }
      }
    }

    @Override
    public BytesRef next() {
      if (iterator.hasNext()) {
        current = iterator.next();
        return term();
      } else {
        return null;
      }
    }

    @Override
    public void seekExact(long ord) {
      throw new UnsupportedOperationException();
    }

    @Override
    public BytesRef term() {
      return current.getKey();
    }

    @Override
    public long ord() {
      return current.getValue().ord;
    }

    @Override
    public int docFreq() {
      throw new UnsupportedOperationException();
    }

    @Override
    public long totalTermFreq() {
      throw new UnsupportedOperationException();
    }

    @Override
    public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
      if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) {
        if (maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
          return null;
        }
        if (PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS)
            && maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) {
          return null;
        }
        if (PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) && allowPayloads == false) {
          return null;
        }
      }
      if (PostingsEnum.featureRequested(flags, PostingsEnum.FREQS)
          && maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS) < 0) {
        return null;
      }
      return getSeedPostings(
          current.getKey().utf8ToString(), current.getValue().seed, maxAllowed, allowPayloads);
    }

    @Override
    public ImpactsEnum impacts(int flags) throws IOException {
      throw new UnsupportedOperationException();
    }
  }

  private static class ThreadState {
    // Only used with REUSE option:
    public PostingsEnum reusePostingsEnum;
  }

  private FieldInfos currentFieldInfos;

  // maxAllowed = the "highest" we can index, but we will still
  // randomly index at lower IndexOption
  public FieldsProducer buildIndex(
      Codec codec,
      Directory dir,
      IndexOptions maxAllowed,
      boolean allowPayloads,
      boolean alwaysTestMax)
      throws IOException {
    SegmentInfo segmentInfo =
        new SegmentInfo(
            dir,
            Version.LATEST,
            Version.LATEST,
            "_0",
            maxDoc,
            false,
            false,
            codec,
            Collections.emptyMap(),
            StringHelper.randomId(),
            new HashMap<>(),
            null);

    int maxIndexOption = Arrays.asList(IndexOptions.values()).indexOf(maxAllowed);
    if (LuceneTestCase.VERBOSE) {
      System.out.println("\nTEST: now build index");
    }

    // TODO use allowPayloads

    FieldInfo[] newFieldInfoArray = new FieldInfo[fields.size()];
    for (int fieldUpto = 0; fieldUpto < fields.size(); fieldUpto++) {
      FieldInfo oldFieldInfo = fieldInfos.fieldInfo(fieldUpto);

      // Randomly picked the IndexOptions to index this
      // field with:
      IndexOptions indexOptions =
          IndexOptions.values()[
              alwaysTestMax ? maxIndexOption : TestUtil.nextInt(random, 1, maxIndexOption)];
      boolean doPayloads =
          indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;

      newFieldInfoArray[fieldUpto] =
          new FieldInfo(
              oldFieldInfo.name,
              fieldUpto,
              false,
              false,
              doPayloads,
              indexOptions,
              DocValuesType.NONE,
              DocValuesSkipIndexType.NONE,
              -1,
              new HashMap<>(),
              0,
              0,
              0,
              0,
              VectorEncoding.FLOAT32,
              VectorSimilarityFunction.EUCLIDEAN,
              false,
              false);
    }

    FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);

    // Estimate that flushed segment size will be 25% of
    // what we use in RAM:
    long bytes = totalPostings * 8 + totalPayloadBytes;

    SegmentWriteState writeState =
        new SegmentWriteState(
            null,
            dir,
            segmentInfo,
            newFieldInfos,
            null,
            new IOContext(new FlushInfo(maxDoc, bytes)));

    Fields seedFields = new SeedFields(fields, newFieldInfos, maxAllowed, allowPayloads);

    NormsProducer fakeNorms =
        new NormsProducer() {

          @Override
          public void close() throws IOException {}

          @Override
          public NumericDocValues getNorms(FieldInfo field) throws IOException {
            if (newFieldInfos.fieldInfo(field.number).hasNorms()) {
              return new NumericDocValues() {

                int doc = -1;

                @Override
                public int nextDoc() throws IOException {
                  if (++doc == segmentInfo.maxDoc()) {
                    return doc = NO_MORE_DOCS;
                  }
                  return doc;
                }

                @Override
                public int docID() {
                  return doc;
                }

                @Override
                public long cost() {
                  return segmentInfo.maxDoc();
                }

                @Override
                public int advance(int target) throws IOException {
                  return doc =
                      target >= segmentInfo.maxDoc() ? DocIdSetIterator.NO_MORE_DOCS : target;
                }

                @Override
                public boolean advanceExact(int target) throws IOException {
                  doc = target;
                  return true;
                }

                @Override
                public long longValue() throws IOException {
                  return DOC_TO_NORM.applyAsLong(doc);
                }
              };
            } else {
              return null;
            }
          }

          @Override
          public void checkIntegrity() throws IOException {}
        };
    FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState);
    boolean success = false;
    try {
      consumer.write(seedFields, fakeNorms);
      success = true;
    } finally {
      if (success) {
        IOUtils.close(consumer);
      } else {
        IOUtils.closeWhileHandlingException(consumer);
      }
    }

    if (LuceneTestCase.VERBOSE) {
      System.out.println("TEST: after indexing: files=");
      for (String file : dir.listAll()) {
        System.out.println("  " + file + ": " + dir.fileLength(file) + " bytes");
      }
    }

    currentFieldInfos = newFieldInfos;

    SegmentReadState readState =
        new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.DEFAULT);

    return codec.postingsFormat().fieldsProducer(readState);
  }

  private void verifyEnum(
      Random random,
      ThreadState threadState,
      String field,
      BytesRef term,
      TermsEnum termsEnum,

      // Maximum options (docs/freqs/positions/offsets) to test:
      IndexOptions maxTestOptions,
      IndexOptions maxIndexOptions,
      EnumSet options,
      boolean alwaysTestMax)
      throws IOException {

    if (LuceneTestCase.VERBOSE) {
      System.out.println("  verifyEnum: options=" + options + " maxTestOptions=" + maxTestOptions);
    }

    // Make sure TermsEnum really is positioned on the
    // expected term:
    assertEquals(term, termsEnum.term());

    FieldInfo fieldInfo = currentFieldInfos.fieldInfo(field);

    // NOTE: can be empty list if we are using liveDocs:
    SeedPostings expected =
        getSeedPostings(
            term.utf8ToString(), fields.get(field).get(term).seed, maxIndexOptions, true);
    assertEquals(expected.docFreq, termsEnum.docFreq());

    boolean allowFreqs =
        fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0
            && maxTestOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
    boolean doCheckFreqs = allowFreqs && (alwaysTestMax || random.nextInt(3) <= 2);

    boolean allowPositions =
        fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
            && maxTestOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
    boolean doCheckPositions = allowPositions && (alwaysTestMax || random.nextInt(3) <= 2);

    boolean allowOffsets =
        fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
                >= 0
            && maxTestOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    boolean doCheckOffsets = allowOffsets && (alwaysTestMax || random.nextInt(3) <= 2);

    boolean doCheckPayloads =
        options.contains(Option.PAYLOADS)
            && allowPositions
            && fieldInfo.hasPayloads()
            && (alwaysTestMax || random.nextInt(3) <= 2);

    PostingsEnum prevPostingsEnum = null;

    PostingsEnum postingsEnum;

    if (!doCheckPositions) {
      if (allowPositions && random.nextInt(10) == 7) {
        // 10% of the time, even though we will not check positions, pull a DocsAndPositions enum

        if (options.contains(Option.REUSE_ENUMS) && random.nextInt(10) < 9) {
          prevPostingsEnum = threadState.reusePostingsEnum;
        }

        int flags = PostingsEnum.POSITIONS;
        if (alwaysTestMax || random.nextBoolean()) {
          flags |= PostingsEnum.OFFSETS;
        }
        if (alwaysTestMax || random.nextBoolean()) {
          flags |= PostingsEnum.PAYLOADS;
        }

        if (LuceneTestCase.VERBOSE) {
          System.out.println("  get DocsEnum (but we won't check positions) flags=" + flags);
        }

        threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, flags);
        postingsEnum = threadState.reusePostingsEnum;
      } else {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("  get DocsEnum");
        }
        if (options.contains(Option.REUSE_ENUMS) && random.nextInt(10) < 9) {
          prevPostingsEnum = threadState.reusePostingsEnum;
        }
        threadState.reusePostingsEnum =
            termsEnum.postings(
                prevPostingsEnum, doCheckFreqs ? PostingsEnum.FREQS : PostingsEnum.NONE);
        postingsEnum = threadState.reusePostingsEnum;
      }
    } else {
      if (options.contains(Option.REUSE_ENUMS) && random.nextInt(10) < 9) {
        prevPostingsEnum = threadState.reusePostingsEnum;
      }

      int flags = PostingsEnum.POSITIONS;
      if (alwaysTestMax || doCheckOffsets || random.nextInt(3) == 1) {
        flags |= PostingsEnum.OFFSETS;
      }
      if (alwaysTestMax || doCheckPayloads || random.nextInt(3) == 1) {
        flags |= PostingsEnum.PAYLOADS;
      }

      if (LuceneTestCase.VERBOSE) {
        System.out.println("  get DocsEnum flags=" + flags);
      }

      threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, flags);
      postingsEnum = threadState.reusePostingsEnum;
    }

    assertNotNull("null DocsEnum", postingsEnum);
    int initialDocID = postingsEnum.docID();
    assertEquals("initial docID should be -1: " + postingsEnum, -1, initialDocID);

    if (LuceneTestCase.VERBOSE) {
      if (prevPostingsEnum == null) {
        System.out.println("  got enum=" + postingsEnum);
      } else if (prevPostingsEnum == postingsEnum) {
        System.out.println("  got reuse enum=" + postingsEnum);
      } else {
        System.out.println(
            "  got enum=" + postingsEnum + " (reuse of " + prevPostingsEnum + " failed)");
      }
    }

    // 10% of the time don't consume all docs:
    int stopAt;
    if (!alwaysTestMax
        && options.contains(Option.PARTIAL_DOC_CONSUME)
        && expected.docFreq > 1
        && random.nextInt(10) == 7) {
      stopAt = random.nextInt(expected.docFreq - 1);
      if (LuceneTestCase.VERBOSE) {
        System.out.println(
            "  will not consume all docs (" + stopAt + " vs " + expected.docFreq + ")");
      }
    } else {
      stopAt = expected.docFreq;
      if (LuceneTestCase.VERBOSE) {
        System.out.println("  consume all docs");
      }
    }

    double skipChance = alwaysTestMax ? 0.5 : random.nextDouble();
    int numSkips =
        expected.docFreq < 3 ? 1 : TestUtil.nextInt(random, 1, Math.min(20, expected.docFreq / 3));
    int skipInc = expected.docFreq / numSkips;
    int skipDocInc = maxDoc / numSkips;

    // Sometimes do 100% skipping:
    boolean doAllSkipping = options.contains(Option.SKIPPING) && random.nextInt(7) == 1;

    double freqAskChance = alwaysTestMax ? 1.0 : random.nextDouble();
    double payloadCheckChance = alwaysTestMax ? 1.0 : random.nextDouble();
    double offsetCheckChance = alwaysTestMax ? 1.0 : random.nextDouble();

    if (LuceneTestCase.VERBOSE) {
      if (options.contains(Option.SKIPPING)) {
        System.out.println("  skipChance=" + skipChance + " numSkips=" + numSkips);
      } else {
        System.out.println("  no skipping");
      }
      if (doCheckFreqs) {
        System.out.println("  freqAskChance=" + freqAskChance);
      }
      if (doCheckPayloads) {
        System.out.println("  payloadCheckChance=" + payloadCheckChance);
      }
      if (doCheckOffsets) {
        System.out.println("  offsetCheckChance=" + offsetCheckChance);
      }
    }

    while (expected.upto <= stopAt) {
      if (expected.upto == stopAt) {
        if (stopAt == expected.docFreq) {
          assertEquals(
              "DocsEnum should have ended but didn't",
              PostingsEnum.NO_MORE_DOCS,
              postingsEnum.nextDoc());

          // Common bug is to forget to set this.doc=NO_MORE_DOCS in the enum!:
          assertEquals(
              "DocsEnum should have ended but didn't",
              PostingsEnum.NO_MORE_DOCS,
              postingsEnum.docID());
        }
        break;
      }

      if (options.contains(Option.SKIPPING)
          && (doAllSkipping || random.nextDouble() <= skipChance)) {
        int targetDocID = -1;
        if (expected.upto < stopAt && random.nextBoolean()) {
          // Pick target we know exists:
          final int skipCount = TestUtil.nextInt(random, 1, skipInc);
          for (int skip = 0; skip < skipCount; skip++) {
            if (expected.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
              break;
            }
          }
        } else {
          // Pick random target (might not exist):
          final int skipDocIDs = TestUtil.nextInt(random, 1, skipDocInc);
          if (skipDocIDs > 0) {
            targetDocID = expected.docID() + skipDocIDs;
            expected.advance(targetDocID);
          }
        }

        if (expected.upto >= stopAt) {
          int target = random.nextBoolean() ? maxDoc : PostingsEnum.NO_MORE_DOCS;
          if (LuceneTestCase.VERBOSE) {
            System.out.println("  now advance to end (target=" + target + ")");
          }
          assertEquals(
              "DocsEnum should have ended but didn't",
              PostingsEnum.NO_MORE_DOCS,
              postingsEnum.advance(target));
          break;
        } else {
          if (LuceneTestCase.VERBOSE) {
            if (targetDocID != -1) {
              System.out.println(
                  "  now advance to random target="
                      + targetDocID
                      + " ("
                      + expected.upto
                      + " of "
                      + stopAt
                      + ") current="
                      + postingsEnum.docID());
            } else {
              System.out.println(
                  "  now advance to known-exists target="
                      + expected.docID()
                      + " ("
                      + expected.upto
                      + " of "
                      + stopAt
                      + ") current="
                      + postingsEnum.docID());
            }
          }
          int docID = postingsEnum.advance(targetDocID != -1 ? targetDocID : expected.docID());
          assertEquals("docID is wrong", expected.docID(), docID);
        }
      } else {
        expected.nextDoc();
        if (LuceneTestCase.VERBOSE) {
          System.out.println(
              "  now nextDoc to "
                  + expected.docID()
                  + " ("
                  + expected.upto
                  + " of "
                  + stopAt
                  + ")");
        }
        int docID = postingsEnum.nextDoc();
        assertEquals("docID is wrong", expected.docID(), docID);
        if (docID == PostingsEnum.NO_MORE_DOCS) {
          break;
        }
      }

      if (doCheckFreqs && random.nextDouble() <= freqAskChance) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("    now freq()=" + expected.freq());
        }
        int freq = postingsEnum.freq();
        assertEquals("freq is wrong", expected.freq(), freq);
      }

      if (doCheckPositions) {
        int freq = postingsEnum.freq();
        int numPosToConsume;
        if (!alwaysTestMax
            && options.contains(Option.PARTIAL_POS_CONSUME)
            && random.nextInt(5) == 1) {
          numPosToConsume = random.nextInt(freq);
        } else {
          numPosToConsume = freq;
        }

        for (int i = 0; i < numPosToConsume; i++) {
          int pos = expected.nextPosition();
          if (LuceneTestCase.VERBOSE) {
            System.out.println("    now nextPosition to " + pos);
          }
          assertEquals("position is wrong", pos, postingsEnum.nextPosition());

          if (doCheckPayloads) {
            BytesRef expectedPayload = expected.getPayload();
            if (random.nextDouble() <= payloadCheckChance) {
              if (LuceneTestCase.VERBOSE) {
                System.out.println(
                    "      now check expectedPayload length="
                        + (expectedPayload == null ? 0 : expectedPayload.length));
              }
              if (expectedPayload == null || expectedPayload.length == 0) {
                assertNull("should not have payload", postingsEnum.getPayload());
              } else {
                BytesRef payload = postingsEnum.getPayload();
                assertNotNull("should have payload but doesn't", payload);

                assertEquals("payload length is wrong", expectedPayload.length, payload.length);
                for (int byteUpto = 0; byteUpto < expectedPayload.length; byteUpto++) {
                  assertEquals(
                      "payload bytes are wrong",
                      expectedPayload.bytes[expectedPayload.offset + byteUpto],
                      payload.bytes[payload.offset + byteUpto]);
                }

                // make a deep copy
                payload = BytesRef.deepCopyOf(payload);
                assertEquals(
                    "2nd call to getPayload returns something different!",
                    payload,
                    postingsEnum.getPayload());
              }
            } else {
              if (LuceneTestCase.VERBOSE) {
                System.out.println(
                    "      skip check payload length="
                        + (expectedPayload == null ? 0 : expectedPayload.length));
              }
            }
          }

          if (doCheckOffsets) {
            if (random.nextDouble() <= offsetCheckChance) {
              if (LuceneTestCase.VERBOSE) {
                System.out.println(
                    "      now check offsets: startOff="
                        + expected.startOffset()
                        + " endOffset="
                        + expected.endOffset());
              }
              assertEquals(
                  "startOffset is wrong", expected.startOffset(), postingsEnum.startOffset());
              assertEquals("endOffset is wrong", expected.endOffset(), postingsEnum.endOffset());
            } else {
              if (LuceneTestCase.VERBOSE) {
                System.out.println("      skip check offsets");
              }
            }
          } else if (fieldInfo
                  .getIndexOptions()
                  .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
              < 0) {
            if (LuceneTestCase.VERBOSE) {
              System.out.println("      now check offsets are -1");
            }
            assertEquals("startOffset isn't -1", -1, postingsEnum.startOffset());
            assertEquals("endOffset isn't -1", -1, postingsEnum.endOffset());
          }
        }
      }
    }

    if (options.contains(Option.SKIPPING)) {
      final IntToLongFunction docToNorm;
      if (fieldInfo.hasNorms()) {
        docToNorm = DOC_TO_NORM;
      } else {
        docToNorm = doc -> 1L;
      }

      // First check impacts and block uptos
      int max = -1;
      List impactsCopy = null;
      int flags = PostingsEnum.FREQS;
      if (doCheckPositions) {
        flags |= PostingsEnum.POSITIONS;
        if (doCheckOffsets) {
          flags |= PostingsEnum.OFFSETS;
        }
        if (doCheckPayloads) {
          flags |= PostingsEnum.PAYLOADS;
        }
      }

      ImpactsEnum impactsEnum = termsEnum.impacts(flags);
      PostingsEnum postings = termsEnum.postings(null, flags);
      for (int doc = impactsEnum.nextDoc(); ; doc = impactsEnum.nextDoc()) {
        assertEquals(postings.nextDoc(), doc);
        if (doc == DocIdSetIterator.NO_MORE_DOCS) {
          break;
        }
        int freq = postings.freq();
        assertEquals("freq is wrong", freq, impactsEnum.freq());
        for (int i = 0; i < freq; ++i) {
          int pos = postings.nextPosition();
          assertEquals("position is wrong", pos, impactsEnum.nextPosition());
          if (doCheckOffsets) {
            assertEquals("startOffset is wrong", postings.startOffset(), impactsEnum.startOffset());
            assertEquals("endOffset is wrong", postings.endOffset(), impactsEnum.endOffset());
          }
          if (doCheckPayloads) {
            assertEquals("payload is wrong", postings.getPayload(), impactsEnum.getPayload());
          }
        }
        if (doc > max) {
          impactsEnum.advanceShallow(doc);
          Impacts impacts = impactsEnum.getImpacts();
          INDEX_PACKAGE_ACCESS.checkImpacts(impacts, doc);
          impactsCopy =
              impacts.getImpacts(0).stream().map(i -> new Impact(i.freq, i.norm)).toList();
        }
        freq = impactsEnum.freq();
        long norm = docToNorm.applyAsLong(doc);
        int idx =
            Collections.binarySearch(
                impactsCopy, new Impact(freq, norm), Comparator.comparing(i -> i.freq));
        if (idx < 0) {
          idx = -1 - idx;
        }
        assertTrue(
            "Got "
                + new Impact(freq, norm)
                + " in postings, but no impact triggers equal or better scores in "
                + impactsCopy,
            idx <= impactsCopy.size() && impactsCopy.get(idx).norm <= norm);
      }

      // Now check advancing
      impactsEnum = termsEnum.impacts(flags);
      postings = termsEnum.postings(postings, flags);

      max = -1;
      while (true) {
        int doc = impactsEnum.docID();
        boolean advance;
        int target;
        if (random.nextBoolean()) {
          advance = false;
          target = doc + 1;
        } else {
          advance = true;
          int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
          target = impactsEnum.docID() + delta;
        }

        if (target > max && random.nextBoolean()) {
          int delta = Math.min(random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - target);
          max = target + delta;

          impactsEnum.advanceShallow(target);
          Impacts impacts = impactsEnum.getImpacts();
          INDEX_PACKAGE_ACCESS.checkImpacts(impacts, target);
          impactsCopy = Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
          for (int level = 0; level < impacts.numLevels(); ++level) {
            if (impacts.getDocIdUpTo(level) >= max) {
              impactsCopy =
                  impacts.getImpacts(level).stream().map(i -> new Impact(i.freq, i.norm)).toList();
              break;
            }
          }
        }

        if (advance) {
          doc = impactsEnum.advance(target);
        } else {
          doc = impactsEnum.nextDoc();
        }

        assertEquals(postings.advance(target), doc);
        if (doc == DocIdSetIterator.NO_MORE_DOCS) {
          break;
        }
        int freq = postings.freq();
        assertEquals("freq is wrong", freq, impactsEnum.freq());
        for (int i = 0; i < postings.freq(); ++i) {
          int pos = postings.nextPosition();
          assertEquals("position is wrong", pos, impactsEnum.nextPosition());
          if (doCheckOffsets) {
            assertEquals("startOffset is wrong", postings.startOffset(), impactsEnum.startOffset());
            assertEquals("endOffset is wrong", postings.endOffset(), impactsEnum.endOffset());
          }
          if (doCheckPayloads) {
            assertEquals("payload is wrong", postings.getPayload(), impactsEnum.getPayload());
          }
        }

        if (doc > max) {
          int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
          max = doc + delta;
          Impacts impacts = impactsEnum.getImpacts();
          INDEX_PACKAGE_ACCESS.checkImpacts(impacts, doc);
          impactsCopy = Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
          for (int level = 0; level < impacts.numLevels(); ++level) {
            if (impacts.getDocIdUpTo(level) >= max) {
              impactsCopy =
                  impacts.getImpacts(level).stream().map(i -> new Impact(i.freq, i.norm)).toList();
              break;
            }
          }
        }

        freq = impactsEnum.freq();
        long norm = docToNorm.applyAsLong(doc);
        int idx =
            Collections.binarySearch(
                impactsCopy, new Impact(freq, norm), Comparator.comparing(i -> i.freq));
        if (idx < 0) {
          idx = -1 - idx;
        }
        assertTrue(
            "Got "
                + new Impact(freq, norm)
                + " in postings, but no impact triggers equal or better scores in "
                + impactsCopy,
            idx <= impactsCopy.size() && impactsCopy.get(idx).norm <= norm);
      }
    }
  }

  private static class TestThread extends Thread {
    private Fields fieldsSource;
    private final EnumSet options;
    private final IndexOptions maxIndexOptions;
    private final IndexOptions maxTestOptions;
    private final boolean alwaysTestMax;
    private RandomPostingsTester postingsTester;
    private final Random random;

    public TestThread(
        Random random,
        RandomPostingsTester postingsTester,
        Fields fieldsSource,
        EnumSet options,
        IndexOptions maxTestOptions,
        IndexOptions maxIndexOptions,
        boolean alwaysTestMax) {
      this.random = random;
      this.fieldsSource = fieldsSource;
      this.options = options;
      this.maxTestOptions = maxTestOptions;
      this.maxIndexOptions = maxIndexOptions;
      this.alwaysTestMax = alwaysTestMax;
      this.postingsTester = postingsTester;
    }

    @Override
    public void run() {
      try {
        try {
          postingsTester.testTermsOneThread(
              random, fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
        } catch (Throwable t) {
          throw new RuntimeException(t);
        }
      } finally {
        fieldsSource = null;
        postingsTester = null;
      }
    }
  }

  public void testTerms(
      final Fields fieldsSource,
      final EnumSet options,
      final IndexOptions maxTestOptions,
      final IndexOptions maxIndexOptions,
      final boolean alwaysTestMax)
      throws Exception {

    if (options.contains(Option.THREADS)) {
      int numThreads = LuceneTestCase.TEST_NIGHTLY ? TestUtil.nextInt(random, 2, 5) : 2;
      Thread[] threads = new Thread[numThreads];
      for (int threadUpto = 0; threadUpto < numThreads; threadUpto++) {
        threads[threadUpto] =
            new TestThread(
                new Random(random.nextLong()),
                this,
                fieldsSource,
                options,
                maxTestOptions,
                maxIndexOptions,
                alwaysTestMax);
        threads[threadUpto].start();
      }
      for (int threadUpto = 0; threadUpto < numThreads; threadUpto++) {
        threads[threadUpto].join();
      }
    } else {
      testTermsOneThread(
          random, fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
    }
  }

  private void testTermsOneThread(
      Random random,
      Fields fieldsSource,
      EnumSet options,
      IndexOptions maxTestOptions,
      IndexOptions maxIndexOptions,
      boolean alwaysTestMax)
      throws IOException {

    ThreadState threadState = new ThreadState();

    // Test random terms/fields:
    List termStates = new ArrayList<>();
    List termStateTerms = new ArrayList<>();

    boolean supportsOrds = true;

    Collections.shuffle(allTerms, random);
    int upto = 0;
    while (upto < allTerms.size()) {

      boolean useTermState = termStates.size() != 0 && random.nextInt(5) == 1;
      boolean useTermOrd = supportsOrds && useTermState == false && random.nextInt(5) == 1;

      FieldAndTerm fieldAndTerm;
      TermsEnum termsEnum;

      TermState termState = null;

      if (!useTermState) {
        // Seek by random field+term:
        fieldAndTerm = allTerms.get(upto++);
        if (LuceneTestCase.VERBOSE) {
          if (useTermOrd) {
            System.out.println(
                "\nTEST: seek to term="
                    + fieldAndTerm.field
                    + ":"
                    + fieldAndTerm.term.utf8ToString()
                    + " using ord="
                    + fieldAndTerm.ord);
          } else {
            System.out.println(
                "\nTEST: seek to term="
                    + fieldAndTerm.field
                    + ":"
                    + fieldAndTerm.term.utf8ToString());
          }
        }
      } else {
        // Seek by previous saved TermState
        int idx = random.nextInt(termStates.size());
        fieldAndTerm = termStateTerms.get(idx);
        if (LuceneTestCase.VERBOSE) {
          System.out.println(
              "\nTEST: seek using TermState to term="
                  + fieldAndTerm.field
                  + ":"
                  + fieldAndTerm.term.utf8ToString());
        }
        termState = termStates.get(idx);
      }

      Terms terms = fieldsSource.terms(fieldAndTerm.field);
      assertNotNull(terms);
      termsEnum = terms.iterator();

      if (!useTermState) {
        if (useTermOrd) {
          // Try seek by ord sometimes:
          try {
            termsEnum.seekExact(fieldAndTerm.ord);
          } catch (
              @SuppressWarnings("unused")
              UnsupportedOperationException uoe) {
            supportsOrds = false;
            assertTrue(termsEnum.seekExact(fieldAndTerm.term));
          }
        } else {
          assertTrue(termsEnum.seekExact(fieldAndTerm.term));
        }
      } else {
        termsEnum.seekExact(fieldAndTerm.term, termState);
      }

      // check we really seeked to the right place
      assertEquals(fieldAndTerm.term, termsEnum.term());

      long termOrd;
      if (supportsOrds) {
        try {
          termOrd = termsEnum.ord();
        } catch (
            @SuppressWarnings("unused")
            UnsupportedOperationException uoe) {
          supportsOrds = false;
          termOrd = -1;
        }
      } else {
        termOrd = -1;
      }

      if (termOrd != -1) {
        // PostingsFormat supports ords
        assertEquals(fieldAndTerm.ord, termsEnum.ord());
      }

      boolean savedTermState = false;

      if (options.contains(Option.TERM_STATE) && !useTermState && random.nextInt(5) == 1) {
        // Save away this TermState:
        termStates.add(termsEnum.termState());
        termStateTerms.add(fieldAndTerm);
        savedTermState = true;
      }

      verifyEnum(
          random,
          threadState,
          fieldAndTerm.field,
          fieldAndTerm.term,
          termsEnum,
          maxTestOptions,
          maxIndexOptions,
          options,
          alwaysTestMax);

      // Sometimes save term state after pulling the enum:
      if (options.contains(Option.TERM_STATE)
          && !useTermState
          && !savedTermState
          && random.nextInt(5) == 1) {
        // Save away this TermState:
        termStates.add(termsEnum.termState());
        termStateTerms.add(fieldAndTerm);
        useTermState = true;
      }

      // 10% of the time make sure you can pull another enum
      // from the same term:
      if (alwaysTestMax || random.nextInt(10) == 7) {
        // Try same term again
        if (LuceneTestCase.VERBOSE) {
          System.out.println("TEST: try enum again on same term");
        }

        verifyEnum(
            random,
            threadState,
            fieldAndTerm.field,
            fieldAndTerm.term,
            termsEnum,
            maxTestOptions,
            maxIndexOptions,
            options,
            alwaysTestMax);
      }
    }

    // Test Terms.intersect:
    for (String field : fields.keySet()) {
      while (true) {
        Automaton a = AutomatonTestUtil.randomAutomaton(random);
        a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
        CompiledAutomaton ca = new CompiledAutomaton(a, false, true, false);
        if (ca.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
          // Keep retrying until we get an A that will really "use" the PF's intersect code:
          continue;
        }
        // System.out.println("A:\n" + a.toDot());

        BytesRef startTerm = null;
        if (random.nextBoolean()) {
          RandomAcceptedStrings ras = new RandomAcceptedStrings(a);
          for (int iter = 0; iter < 100; iter++) {
            int[] codePoints = ras.getRandomAcceptedString(random);
            if (codePoints.length == 0) {
              continue;
            }
            startTerm = new BytesRef(UnicodeUtil.newString(codePoints, 0, codePoints.length));
            break;
          }
          // Don't allow empty string startTerm:
          if (startTerm == null) {
            continue;
          }
        }
        TermsEnum intersected = fieldsSource.terms(field).intersect(ca, startTerm);

        Set intersectedTerms = new HashSet<>();
        BytesRef term;
        while ((term = intersected.next()) != null) {
          if (startTerm != null) {
            // NOTE: not <=
            assertTrue(startTerm.compareTo(term) < 0);
          }
          intersectedTerms.add(BytesRef.deepCopyOf(term));
          verifyEnum(
              random,
              threadState,
              field,
              term,
              intersected,
              maxTestOptions,
              maxIndexOptions,
              options,
              alwaysTestMax);
        }

        if (ca.runAutomaton == null) {
          assertTrue(intersectedTerms.isEmpty());
        } else {
          for (BytesRef term2 : fields.get(field).keySet()) {
            boolean expected;
            if (startTerm != null && startTerm.compareTo(term2) >= 0) {
              expected = false;
            } else {
              expected = ca.runAutomaton.run(term2.bytes, term2.offset, term2.length);
            }
            assertEquals("term=" + term2, expected, intersectedTerms.contains(term2));
          }
        }

        break;
      }
    }
  }

  public void testFields(Fields fields) throws Exception {
    Iterator iterator = fields.iterator();
    while (iterator.hasNext()) {
      iterator.next();
      try {
        iterator.remove();
        throw new AssertionError("Fields.iterator() allows for removal");
      } catch (
          @SuppressWarnings("unused")
          UnsupportedOperationException expected) {
        // expected;
      }
    }
    assertFalse(iterator.hasNext());
    LuceneTestCase.expectThrows(NoSuchElementException.class, iterator::next);
  }

  /**
   * Indexes all fields/terms at the specified IndexOptions, and fully tests at that IndexOptions.
   */
  public void testFull(Codec codec, Path path, IndexOptions options, boolean withPayloads)
      throws Exception {
    Directory dir = LuceneTestCase.newFSDirectory(path);

    // TODO test thread safety of buildIndex too
    FieldsProducer fieldsProducer = buildIndex(codec, dir, options, withPayloads, true);

    testFields(fieldsProducer);

    IndexOptions[] allOptions = IndexOptions.values();
    int maxIndexOption = Arrays.asList(allOptions).indexOf(options);

    for (int i = 0; i <= maxIndexOption; i++) {
      testTerms(fieldsProducer, EnumSet.allOf(Option.class), allOptions[i], options, true);
      if (withPayloads) {
        // If we indexed w/ payloads, also test enums w/o accessing payloads:
        testTerms(
            fieldsProducer,
            EnumSet.complementOf(EnumSet.of(Option.PAYLOADS)),
            allOptions[i],
            options,
            true);
      }
    }

    fieldsProducer.close();
    dir.close();
  }
}