• Home
• org.apache.lucene
• lucene-test-framework
• 6.0.1
• source code
• FSTTester.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.apache.lucene.util.fst.FSTTester Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.util.fst;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.packed.PackedInts;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

/** Helper class to test FSTs. */
public class FSTTester {

  final Random random;
  final List> pairs;
  final int inputMode;
  final Outputs outputs;
  final Directory dir;
  final boolean doReverseLookup;
  long nodeCount;
  long arcCount;

  public FSTTester(Random random, Directory dir, int inputMode, List> pairs, Outputs outputs, boolean doReverseLookup) {
    this.random = random;
    this.dir = dir;
    this.inputMode = inputMode;
    this.pairs = pairs;
    this.outputs = outputs;
    this.doReverseLookup = doReverseLookup;
  }

  static String inputToString(int inputMode, IntsRef term) {
    return inputToString(inputMode, term, true);
  }

  static String inputToString(int inputMode, IntsRef term, boolean isValidUnicode) {
    if (!isValidUnicode) {
      return term.toString();
    } else if (inputMode == 0) {
      // utf8
      return toBytesRef(term).utf8ToString() + " " + term;
    } else {
      // utf32
      return UnicodeUtil.newString(term.ints, term.offset, term.length) + " " + term;
    }
  }

  private static BytesRef toBytesRef(IntsRef ir) {
    BytesRef br = new BytesRef(ir.length);
    for(int i=0;i= 0 && x <= 255;
      br.bytes[i] = (byte) x;
    }
    br.length = ir.length;
    return br;
  }

  static String getRandomString(Random random) {
    final String term;
    if (random.nextBoolean()) {
      term = TestUtil.randomRealisticUnicodeString(random);
    } else {
      // we want to mix in limited-alphabet symbols so
      // we get more sharing of the nodes given how few
      // terms we are testing...
      term = simpleRandomString(random);
    }
    return term;
  }

  static String simpleRandomString(Random r) {
    final int end = r.nextInt(10);
    if (end == 0) {
      // allow 0 length
      return "";
    }
    final char[] buffer = new char[end];
    for (int i = 0; i < end; i++) {
      buffer[i] = (char) TestUtil.nextInt(r, 97, 102);
    }
    return new String(buffer, 0, end);
  }

  static IntsRef toIntsRef(String s, int inputMode) {
    return toIntsRef(s, inputMode, new IntsRefBuilder());
  }

  static IntsRef toIntsRef(String s, int inputMode, IntsRefBuilder ir) {
    if (inputMode == 0) {
      // utf8
      return toIntsRef(new BytesRef(s), ir);
    } else {
      // utf32
      return toIntsRefUTF32(s, ir);
    }
  }

  static IntsRef toIntsRefUTF32(String s, IntsRefBuilder ir) {
    final int charLength = s.length();
    int charIdx = 0;
    int intIdx = 0;
    ir.clear();
    while(charIdx < charLength) {
      ir.grow(intIdx+1);
      final int utf32 = s.codePointAt(charIdx);
      ir.append(utf32);
      charIdx += Character.charCount(utf32);
      intIdx++;
    }
    return ir.get();
  }

  static IntsRef toIntsRef(BytesRef br, IntsRefBuilder ir) {
    ir.grow(br.length);
    ir.clear();
    for(int i=0;i implements Comparable> {
    public final IntsRef input;
    public final T output;

    public InputOutput(IntsRef input, T output) {
      this.input = input;
      this.output = output;
    }

    @Override
    public int compareTo(InputOutput other) {
      if (other instanceof InputOutput) {
        return input.compareTo((other).input);
      } else {
        throw new IllegalArgumentException();
      }
    }
  }

  public void doTest(boolean testPruning) throws IOException {
    // no pruning
    doTest(0, 0, true);

    if (testPruning) {
      // simple pruning
      doTest(TestUtil.nextInt(random, 1, 1 + pairs.size()), 0, true);
        
      // leafy pruning
      doTest(0, TestUtil.nextInt(random, 1, 1 + pairs.size()), true);
    }
  }

  // runs the term, returning the output, or null if term
  // isn't accepted.  if prefixLength is non-null it must be
  // length 1 int array; prefixLength[0] is set to the length
  // of the term prefix that matches
  private T run(FST fst, IntsRef term, int[] prefixLength) throws IOException {
    assert prefixLength == null || prefixLength.length == 1;
    final FST.Arc arc = fst.getFirstArc(new FST.Arc());
    final T NO_OUTPUT = fst.outputs.getNoOutput();
    T output = NO_OUTPUT;
    final FST.BytesReader fstReader = fst.getBytesReader();

    for(int i=0;i<=term.length;i++) {
      final int label;
      if (i == term.length) {
        label = FST.END_LABEL;
      } else {
        label = term.ints[term.offset+i];
      }
      // System.out.println("   loop i=" + i + " label=" + label + " output=" + fst.outputs.outputToString(output) + " curArc: target=" + arc.target + " isFinal?=" + arc.isFinal());
      if (fst.findTargetArc(label, arc, arc, fstReader) == null) {
        // System.out.println("    not found");
        if (prefixLength != null) {
          prefixLength[0] = i;
          return output;
        } else {
          return null;
        }
      }
      output = fst.outputs.add(output, arc.output);
    }

    if (prefixLength != null) {
      prefixLength[0] = term.length;
    }

    return output;
  }

  private T randomAcceptedWord(FST fst, IntsRefBuilder in) throws IOException {
    FST.Arc arc = fst.getFirstArc(new FST.Arc());

    final List> arcs = new ArrayList<>();
    in.clear();
    final T NO_OUTPUT = fst.outputs.getNoOutput();
    T output = NO_OUTPUT;
    final FST.BytesReader fstReader = fst.getBytesReader();

    while(true) {
      // read all arcs:
      fst.readFirstTargetArc(arc, arc, fstReader);
      arcs.add(new FST.Arc().copyFrom(arc));
      while(!arc.isLast()) {
        fst.readNextArc(arc, fstReader);
        arcs.add(new FST.Arc().copyFrom(arc));
      }
      
      // pick one
      arc = arcs.get(random.nextInt(arcs.size()));
      arcs.clear();

      // accumulate output
      output = fst.outputs.add(output, arc.output);

      // append label
      if (arc.label == FST.END_LABEL) {
        break;
      }

      in.append(arc.label);
    }

    return output;
  }


  FST doTest(int prune1, int prune2, boolean allowRandomSuffixSharing) throws IOException {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("\nTEST: prune1=" + prune1 + " prune2=" + prune2);
    }

    final boolean willRewrite = random.nextBoolean();

    final Builder builder = new Builder<>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4,
                                              prune1, prune2,
                                              prune1==0 && prune2==0,
                                              allowRandomSuffixSharing ? random.nextBoolean() : true,
                                              allowRandomSuffixSharing ? TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE,
                                              outputs,
                                              willRewrite,
                                              PackedInts.DEFAULT,
                                              true,
                                              15);
    if (LuceneTestCase.VERBOSE) {
      if (willRewrite) {
        System.out.println("TEST: packed FST");
      } else {
        System.out.println("TEST: non-packed FST");
      }
    }

    for(InputOutput pair : pairs) {
      if (pair.output instanceof List) {
        @SuppressWarnings("unchecked") List longValues = (List) pair.output;
        @SuppressWarnings("unchecked") final Builder