test.it.unimi.dsi.big.mg4j.tool.IndexTest Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of mg4j-big Show documentation
MG4J (Managing Gigabytes for Java) is a free full-text search engine for large document collections written in Java. The big version is a fork of the original MG4J that can handle more than 2^31 terms and documents.
The newest version!
package it.unimi.dsi.big.mg4j.tool;

import static it.unimi.dsi.big.mg4j.index.CompressionFlags.DEFAULT_PAYLOAD_INDEX;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.FREQUENCIES_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.GLOBCOUNTS_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.INDEX_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.OFFSETS_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.PROPERTIES_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.SIZES_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.TERMMAP_EXTENSION;
import static it.unimi.dsi.big.mg4j.index.DiskBasedIndex.TERMS_EXTENSION;
import static it.unimi.dsi.logging.ProgressLogger.DEFAULT_LOG_INTERVAL;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import it.unimi.dsi.Util;
import it.unimi.dsi.big.io.FileLinesCollection;
import it.unimi.dsi.big.io.FileLinesCollection.FileLinesIterator;
import it.unimi.dsi.big.mg4j.document.AbstractDocumentSequence;
import it.unimi.dsi.big.mg4j.document.CompositeDocumentSequence;
import it.unimi.dsi.big.mg4j.document.DateArrayDocumentCollection;
import it.unimi.dsi.big.mg4j.document.Document;
import it.unimi.dsi.big.mg4j.document.DocumentCollection;
import it.unimi.dsi.big.mg4j.document.DocumentFactory;
import it.unimi.dsi.big.mg4j.document.DocumentFactory.FieldType;
import it.unimi.dsi.big.mg4j.document.DocumentIterator;
import it.unimi.dsi.big.mg4j.document.DocumentSequence;
import it.unimi.dsi.big.mg4j.document.IdentityDocumentFactory;
import it.unimi.dsi.big.mg4j.document.InputStreamDocumentSequence;
import it.unimi.dsi.big.mg4j.document.IntArrayDocumentCollection;
import it.unimi.dsi.big.mg4j.document.MapVirtualDocumentCollection;
import it.unimi.dsi.big.mg4j.document.SimpleCompressedDocumentCollectionBuilder;
import it.unimi.dsi.big.mg4j.document.StringArrayDocumentCollection;
import it.unimi.dsi.big.mg4j.document.ZipDocumentCollectionBuilder;
import it.unimi.dsi.big.mg4j.index.CompressionFlags;
import it.unimi.dsi.big.mg4j.index.CompressionFlags.Coding;
import it.unimi.dsi.big.mg4j.index.CompressionFlags.Component;
import it.unimi.dsi.big.mg4j.index.DiskBasedIndex;
import it.unimi.dsi.big.mg4j.index.DowncaseTermProcessor;
import it.unimi.dsi.big.mg4j.index.Index;
import it.unimi.dsi.big.mg4j.index.IndexIterator;
import it.unimi.dsi.big.mg4j.index.IndexReader;
import it.unimi.dsi.big.mg4j.index.TermProcessor;
import it.unimi.dsi.big.mg4j.index.cluster.DocumentalPartitioningStrategy;
import it.unimi.dsi.big.mg4j.index.cluster.DocumentalStrategies;
import it.unimi.dsi.big.mg4j.index.cluster.IndexCluster;
import it.unimi.dsi.big.mg4j.index.cluster.LexicalPartitioningStrategy;
import it.unimi.dsi.big.mg4j.index.cluster.LexicalStrategies;
import it.unimi.dsi.big.mg4j.index.remote.IndexServer;
import it.unimi.dsi.big.mg4j.tool.Scan.VirtualDocumentFragment;
import it.unimi.dsi.big.util.ShiftAddXorSignedStringMap;
import it.unimi.dsi.big.util.StringMap;
import it.unimi.dsi.bits.TransformationStrategies;
import it.unimi.dsi.fastutil.Arrays;
import it.unimi.dsi.fastutil.Swapper;
import it.unimi.dsi.fastutil.ints.AbstractIntComparator;
import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntArrays;
import it.unimi.dsi.fastutil.ints.IntIterators;
import it.unimi.dsi.fastutil.io.BinIO;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.ObjectRBTreeSet;
import it.unimi.dsi.fastutil.objects.Reference2ObjectOpenHashMap;
import it.unimi.dsi.io.FastBufferedReader;
import it.unimi.dsi.io.WordReader;
import it.unimi.dsi.lang.MutableString;
import it.unimi.dsi.big.mg4j.index.BitStreamIndex;
import it.unimi.dsi.sux4j.mph.MWHCFunction;
import it.unimi.dsi.util.Properties;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.InvocationTargetException;
import java.net.ConnectException;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.Map;

import junit.framework.Assert;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class IndexTest {
	static {
		Util.ensureLog4JIsConfigured( Level.INFO );
	}

	private static StringMap createMap( String basename ) throws IOException {
		FileLinesCollection flc = new FileLinesCollection( basename, "UTF-8" );
		return new ShiftAddXorSignedStringMap( flc.iterator(), new MWHCFunction( flc, TransformationStrategies.utf16() ) );
	}
	
	
	private String basename;

	private final int NUMBER_OF_DOCUMENTS = 100;

	private final int[] INTEGER_DOCUMENT = new int[ NUMBER_OF_DOCUMENTS ];

	private final Date[] DATE_DOCUMENT = new Date[ NUMBER_OF_DOCUMENTS ];

	@SuppressWarnings("unchecked")
	private final Int2ObjectMap[] VIRTUAL_DOCUMENT = new Int2ObjectMap[ NUMBER_OF_DOCUMENTS ];
	{
		for ( int i = INTEGER_DOCUMENT.length; i-- != 0; )
			INTEGER_DOCUMENT[ i ] = i;
		for ( int i = DATE_DOCUMENT.length; i-- != 0; )
			DATE_DOCUMENT[ i ] = new Date( i * 86400000L );
		for ( int i = VIRTUAL_DOCUMENT.length; i-- != 0; ) {
			VIRTUAL_DOCUMENT[ i ] = new Int2ObjectArrayMap();
			VIRTUAL_DOCUMENT[ i ].put( i - 1, "link _ to previous document link" );
			VIRTUAL_DOCUMENT[ i ].put( i, "link to this document link" );
			VIRTUAL_DOCUMENT[ i ].put( i + 1, "link to next document link" );
		}
	}

	private final VirtualDocumentResolver RESOLVER = new MapVirtualDocumentCollection.TrivialVirtualDocumentResolver( NUMBER_OF_DOCUMENTS );

	public static Reference2ObjectOpenHashMap defaultStandardIndex() {
		return new Reference2ObjectOpenHashMap( CompressionFlags.DEFAULT_STANDARD_INDEX );
	}
	
	public final static TermProcessor KILL_A_PROCESSOR = KillATermProcessor.getInstance();

	public final static class KillATermProcessor implements TermProcessor {
		private static final long serialVersionUID = 1L;

		private static final KillATermProcessor INSTANCE = new KillATermProcessor();

		public TermProcessor copy() {
			return this;
		}

		public static TermProcessor getInstance() {
			return INSTANCE;
		}

		public boolean processPrefix( MutableString prefix ) {
			return true;
		}

		public boolean processTerm( MutableString term ) {
			return term.indexOf( 'a' ) == -1;
		}
	};

	final static int[] INDEXED_FIELD = { 0, 1, 2 };

	/**
	 * Checks that the two provided indices are byte-by-byte the same, and that property files
	 * coincide except for the provided property keys.
	 * 
	 * @param basename0 the basename of an index.
	 * @param basename1 the basename of an index.
	 * @param excludedProperty a list of property keys that will not be considered when evaluating
	 * the equality of property fields.
	 */
	private void sameIndex( final String basename0, final String basename1, final String... excludedProperty ) throws IOException, ConfigurationException {
		// The two indices must be byte-by-byte identical in all components
		for ( String ext : new String[] { INDEX_EXTENSION, OFFSETS_EXTENSION, TERMS_EXTENSION, SIZES_EXTENSION, FREQUENCIES_EXTENSION, GLOBCOUNTS_EXTENSION } ) {
			File f0 = new File( basename0 + ext );
			File f1 = new File( basename1 + ext );
			assertEquals( ext, Boolean.valueOf( f0.exists() ), Boolean.valueOf( f1.exists() ) );
			if ( ext != SIZES_EXTENSION && f0.exists() ) assertTrue( ext, IOUtils.contentEquals( new FileInputStream( f0 ), new FileInputStream( f1 ) ) );
		}

		Properties properties0 = new Properties( basename0 + PROPERTIES_EXTENSION );
		Properties properties1 = new Properties( basename1 + PROPERTIES_EXTENSION );
		for ( String p : excludedProperty ) {
			properties0.setProperty( p, null );
			properties1.setProperty( p, null );
		}

		assertEquals( properties0, properties1 );
	}

	public static void sameContent( CharSequence basename0, CharSequence basename1, it.unimi.dsi.big.io.FileLinesCollection.FileLinesIterator fileLinesIterator ) throws ConfigurationException, SecurityException, IOException, URISyntaxException,
			ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
		sameContent( it.unimi.dsi.big.mg4j.index.Index.getInstance( basename0 ), it.unimi.dsi.big.mg4j.index.Index.getInstance( basename1 ), fileLinesIterator );
	}

	public static void sameContent( CharSequence basename0, CharSequence basename1 ) throws ConfigurationException, SecurityException, IOException, URISyntaxException, ClassNotFoundException,
			InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
		sameContent( basename0, basename1, null );
	}

	public static void sameContent( it.unimi.dsi.big.mg4j.index.Index index0, it.unimi.dsi.big.mg4j.index.Index index1 ) throws IOException {
		sameContent( index0, index1, null );
	}

	public static void sameContent( it.unimi.dsi.big.mg4j.index.Index index0, it.unimi.dsi.big.mg4j.index.Index index1, final FileLinesIterator terms ) throws IOException {
		assertEquals( Boolean.valueOf( index0.hasCounts ), Boolean.valueOf( index1.hasCounts ) );
		assertEquals( Boolean.valueOf( index0.hasPositions ), Boolean.valueOf( index1.hasPositions ) );
		assertEquals( Boolean.valueOf( index0.hasPayloads ), Boolean.valueOf( index1.hasPayloads ) );
		assertEquals( index0.numberOfTerms, index1.numberOfTerms );
		assertEquals( index0.numberOfDocuments, index1.numberOfDocuments );

		final long numTerms = index0.numberOfTerms;
		long document;
		int[] p0 = IntArrays.EMPTY_ARRAY, p1 = IntArrays.EMPTY_ARRAY;
		boolean hasCounts = index0.hasCounts, hasPositions = index0.hasPositions;
		final IndexReader reader0 = index0.getReader(), reader1 = index1.getReader();
		IndexIterator i0, i1;
		for ( int i = 0; i < numTerms; i++ ) {
			if ( terms != null ) {
				final CharSequence term = terms.next();
				i0 = reader0.documents( term );
				i1 = reader1.documents( term );
			}
			else {
				i0 = reader0.documents( i );
				i1 = reader1.documents( i );
			}

			while ( i0.mayHaveNext() && i1.mayHaveNext() ) {
				assertEquals( "term " + i, document = i0.nextDocument(), i1.nextDocument() );
				if ( hasCounts ) {
					assertEquals( "term " + i + ", document " + document, i0.count(), i1.count() );
					if ( hasPositions ) {
						if ( i0.count() > p0.length ) p0 = new int[ i0.count() ];
						if ( i1.count() > p1.length ) p1 = new int[ i1.count() ];
						i0.positions( p0 );
						i1.positions( p1 );
						for ( int p = 0; p < i0.count(); p++ ) assertEquals( "term " + i + ", document " + document + ", position " + p, p0[ p ], p1[ p ] );
					}
				}
			}
		}
		reader0.close();
		reader1.close();
	}


	public int processDocument( WordReader wordReader, int documentIndex, int startPos, Object2ObjectOpenHashMap> termMap, TermProcessor termProcessor )
			throws IOException {
		assertTrue( documentIndex >= 0 );
		Object2ObjectOpenHashMap terms = new Object2ObjectOpenHashMap();
		MutableString word = new MutableString(), nonWord = new MutableString();

		int pos = startPos;
		while ( wordReader.next( word, nonWord ) ) {
			if ( word.length() == 0 ) continue;
			if ( !termProcessor.processTerm( word ) ) {
				pos++;
				continue;
			}
			IntArrayList positions = terms.get( word );
			if ( positions == null ) terms.put( word.copy(), positions = new IntArrayList() );
			positions.add( pos++ );
		}

		for ( MutableString term : terms.keySet() ) {
			ObjectArrayList list = termMap.get( term );
			IntArrayList positions = terms.get( term );
			if ( list == null ) termMap.put( term, list = new ObjectArrayList() );

			int[] t = new int[ positions.size() + 1 ];
			t[ 0 ] = documentIndex;
			System.arraycopy( positions.elements(), 0, t, 1, positions.size() );
			list.add( t );
		}

		return pos;
	}

	/**
	 * Checks that the fields indexed by the given indices have been indexed correctly by performing
	 * a mock index construction over the given sequence.
	 * 
	 * @param sequence a document sequence.
	 * @param resolver the virtual document resolver used to index the collection (we assume the
	 * same for all virtual fields), or null.
	 * @param gap the virtual document gap (we assume the same for all virtual fields; it is
	 * immaterial if no field is virtual).
	 * @param index a list of indices that have indexed one or more fields of sequence.
	 */
	@SuppressWarnings("unchecked")
	public void checkAgainstContent( DocumentSequence sequence, int[] map, VirtualDocumentResolver resolver, int gap, Index... index ) throws IOException {
		DocumentIterator iterator = sequence.iterator();
		DocumentFactory factory = sequence.factory();
		Document document;
		final int n = index.length;
		final int[] field = new int[ n ];
		final int[][] currMaxPos = new int[ n ][];
		final int[] maxDoc = new int[ n ];
		IntArrays.fill( maxDoc, -1 );
		final Object2ObjectOpenHashMap>[] termMap = new Object2ObjectOpenHashMap[ n ];
		final IntArrayList[] payloadPointers = new IntArrayList[ n ];
		final ObjectArrayList