src.it.unimi.dsi.big.mg4j.search.DifferenceDocumentIterator Maven / Gradle / Ivy
Show all versions of mg4j-big Show documentation
package it.unimi.dsi.big.mg4j.search;
/*
* MG4J: Managing Gigabytes for Java (big)
*
* Copyright (C) 2007-2011 Sebastiano Vigna
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the Free
* Software Foundation; either version 3 of the License, or (at your option)
* any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, see In the lattice of interval antichains, the Brouwerian difference is obtained by deleting from
* the first operand (the minuend) all intervals that contain some interval of the second operand (the subtrahend). Thus,
* Brouwerian difference can be fruitfully employed to kill intervals containing a term or, even
* more fruitfully, to change at query time the granularity of an index by subtracting from the
* results of a query those length-two intervals
* that cross the cutpoints between the desired parts of the index.
*
*
Additionally, this class provides interval enlargement—by using a suitable
* {@linkplain #getInstance(DocumentIterator, DocumentIterator, int, int) factory method} each interval
* returned by the subtrahend will be enlarged to the left and to the right by the given amount (e.g.,
* if the left margin is 1 and the right margin is 2 the interval [2..3] will turn into [1..5]).
*
*
Note that while {@link #accept(DocumentIteratorVisitor)} will recursively visit both
* the minuend and the subtrahend, {@link #acceptOnTruePaths(DocumentIteratorVisitor)} will
* visit only the minuend.
*
* @author Sebastiano Vigna
* @since 1.2
*/
public class DifferenceDocumentIterator extends AbstractDocumentIterator implements DocumentIterator {
private static final boolean DEBUG = false;
private final static boolean ASSERTS = false;
/** The first operand. */
final private DocumentIterator minuendIterator;
/** The second operand. */
final private DocumentIterator subtrahendIterator;
/** If not null, the sole index involved in this iterator. */
final private Index soleIndex;
/** A map from indices to interval iterators. */
final private Reference2ReferenceArrayMap intervalIterators;
/** A map from indices to the iterators returned for the current document. The key set may
* not contain an index because the related iterator has never been requested. Moreover,
* the iterator in this map for a given index may differ from the one in {@link #intervalIterators}
* because it could be {@link IntervalIterators#TRUE} (in fact, in that case it may even
* happen that {@link #intervalIterators} does not contain the index). */
final private Reference2ReferenceArrayMap currentIterators;
/** An unmodifiable wrapper around {@link #currentIterators}. */
final private Reference2ReferenceMap unmodifiableCurrentIterators;
/** A margin that will be added to the left of each interval. */
private final int leftMargin;
/** A margin that will be added to the right of each interval. */
private final int rightMargin;
/** If true, for the current document we have intervals for the minuend but not for the subtrahend. */
private boolean noSubtrahend;
/** If true, the subtrahend may be nonempty. */
private boolean maybeNonEmptySubtrahend;
/** Creates a new difference document iterator given a minuend and a subtrahend iterator.
* @param minuendIterator the minuend.
* @param subtrahendIterator the subtrahend.
*/
protected DifferenceDocumentIterator( final DocumentIterator minuendIterator, final DocumentIterator subtrahendIterator, final int leftMargin, final int rightMargin ) {
if ( leftMargin < 0 || rightMargin < 0 ) throw new IllegalArgumentException( "Illegal margins: " + leftMargin + ", " + rightMargin );
this.minuendIterator = minuendIterator;
this.subtrahendIterator = subtrahendIterator;
this.leftMargin = leftMargin;
this.rightMargin = rightMargin;
final int n = minuendIterator.indices().size();
soleIndex = n == 1 ? indices().iterator().next() : null;
// If the subtrahend is empty, the result is equal to the minuend.
maybeNonEmptySubtrahend = subtrahendIterator.mayHaveNext();
intervalIterators = new Reference2ReferenceArrayMap( n );
currentIterators = new Reference2ReferenceArrayMap( n );
unmodifiableCurrentIterators = Reference2ReferenceMaps.unmodifiable( currentIterators );
}
/** Returns new difference document iterator given a minuend and a subtrahend iterator.
* @param minuendIterator the minuend.
* @param subtrahendIterator the subtrahend.
*/
public static DocumentIterator getInstance( final DocumentIterator minuendIterator, final DocumentIterator subtrahendIterator ) {
return getInstance( minuendIterator, subtrahendIterator, 0, 0 );
}
/** Returns new difference document iterator given a minuend and a subtrahend iterator.
* @param minuendIterator the minuend.
* @param subtrahendIterator the subtrahend.
* @param leftMargin a margin that will be added to the left of each interval.
* @param rightMargin a margin that will be added to the right of each interval.
*/
public static DocumentIterator getInstance( final DocumentIterator minuendIterator, final DocumentIterator subtrahendIterator, final int leftMargin, final int rightMargin ) {
return new DifferenceDocumentIterator( minuendIterator, subtrahendIterator, leftMargin, rightMargin );
}
public ReferenceSet indices() {
return minuendIterator.indices();
}
public long nextDocument() throws IOException {
do currentIterators.clear(); while( ( curr = minuendIterator.nextDocument() ) != -1 && maybeNonEmptySubtrahend && ! isValid() );
final long d = curr;
curr = fromNextDocument( d );
return d;
}
public boolean mayHaveNext() {
return minuendIterator.mayHaveNext();
}
public long skipTo( final long n ) throws IOException {
if ( curr >= n ) return curr;
currentIterators.clear();
// We first try to get a candidate document.
if ( ( curr = minuendIterator.skipTo( n ) ) != END_OF_LIST ) {
// We must manually check that we are on a valid document
if ( isValid() ) return curr;
// If not, we try to find the next valid document.
nextDocument();
}
return curr;
}
private boolean isValid() throws IOException {
// An easy optimisation for the case in which the subtrahend does not include the current document.
if ( noSubtrahend = ( subtrahendIterator.skipTo( curr ) != curr ) ) return true;
/* The policy here is that a difference iterator is valid is at least one of the underlying
* interval iterators would return at least one interval. */
if ( soleIndex != null ) return intervalIterator( soleIndex ).hasNext();
for( Index index: indices() ) if ( intervalIterator( index ).hasNext() ) return true;
return false;
}
public Reference2ReferenceMap intervalIterators() throws IOException {
for( Index index : indices() ) intervalIterator( index );
return unmodifiableCurrentIterators;
}
public IntervalIterator intervalIterator() throws IOException {
if ( soleIndex == null ) throw new IllegalStateException();
return intervalIterator( soleIndex );
}
public IntervalIterator intervalIterator( final Index index ) throws IOException {
if ( DEBUG ) System.err.println( this + ".intervalIterator(" + index + ")" );
ensureOnADocument();
IntervalIterator intervalIterator;
// If the iterator has been created and it's ready, we just return it.
if ( ( intervalIterator = currentIterators.get( index ) ) != null ) return intervalIterator;
intervalIterator = minuendIterator.intervalIterator( index );
if ( intervalIterator == IntervalIterators.FALSE ) return IntervalIterators.FALSE;
IntervalIterator subtrahendIntervalIterator;
if ( maybeNonEmptySubtrahend && ! noSubtrahend ) {
subtrahendIntervalIterator = subtrahendIterator.intervalIterator( index );
if ( subtrahendIntervalIterator == IntervalIterators.TRUE ) intervalIterator = IntervalIterators.FALSE;
else if ( intervalIterator != IntervalIterators.TRUE && subtrahendIntervalIterator != IntervalIterators.FALSE ) {
intervalIterator = intervalIterators.get( index );
if ( intervalIterator == null ) intervalIterators.put( index, intervalIterator = new DifferenceIntervalIterator( index ) );
intervalIterator.reset();
if ( ! intervalIterator.hasNext() ) intervalIterator = IntervalIterators.FALSE;
}
// Otherwise, we can just use the minuend iterator.
}
currentIterators.put( index, intervalIterator );
if ( DEBUG ) System.err.println( "Returning interval iterator " + intervalIterator );
return intervalIterator;
}
public void dispose() throws IOException {
minuendIterator.dispose();
subtrahendIterator.dispose();
}
public T accept( final DocumentIteratorVisitor visitor ) throws IOException {
if ( ! visitor.visitPre( this ) ) return null;
final T[] a = visitor.newArray( 2 );
if ( a == null ) {
if ( minuendIterator.accept( visitor ) == null ) return null;
if ( subtrahendIterator.accept( visitor ) == null ) return null;
}
else {
if ( ( a[ 0 ] = minuendIterator.accept( visitor ) ) == null ) return null;
if ( ( a[ 1 ] = subtrahendIterator.accept( visitor ) ) == null ) return null;
}
return visitor.visitPost( this, a );
}
public T acceptOnTruePaths( final DocumentIteratorVisitor visitor ) throws IOException {
if ( ! visitor.visitPre( this ) ) return null;
final T[] a = visitor.newArray( 1 );
if ( a == null ) {
if ( minuendIterator.acceptOnTruePaths( visitor ) == null ) return null;
}
else {
if ( ( a[ 0 ] = minuendIterator.acceptOnTruePaths( visitor ) ) == null ) return null;
}
return visitor.visitPost( this, a );
}
public String toString() {
return getClass().getSimpleName() + "(" + minuendIterator + ( leftMargin == 0 && rightMargin == 0 ? " - " : " -[" + leftMargin + "," + rightMargin + "] " ) + subtrahendIterator + ")";
}
/** An interval iterator returning just the interval shorter than {@link #threshold}. */
private class DifferenceIntervalIterator extends AbstractIntervalIterator implements IntervalIterator {
/** The index of this iterator. */
final Index index;
/** The underlying minuend interal iterator. */
private IntervalIterator minuendIntervalIterator;
/** The underlying minuend interal iterator. */
private IntervalIterator subtrahendIntervalIterator;
/** The last interval returned by {@link #subtrahendIntervalIterator}. */
private Interval subtrahendInterval;
public DifferenceIntervalIterator( final Index index ) {
this.index = index;
}
public void reset( ) throws IOException {
next = null;
subtrahendInterval = Intervals.MINUS_INFINITY;
minuendIntervalIterator = minuendIterator.intervalIterator( index );
subtrahendIntervalIterator = subtrahendIterator.intervalIterator( index );
if ( ASSERTS ) assert minuendIntervalIterator != IntervalIterators.TRUE;
if ( ASSERTS ) assert minuendIntervalIterator != IntervalIterators.FALSE;
if ( ASSERTS ) assert minuendIntervalIterator.hasNext();
if ( ASSERTS ) assert subtrahendIntervalIterator != IntervalIterators.TRUE;
if ( ASSERTS ) assert subtrahendIntervalIterator != IntervalIterators.FALSE;
if ( ASSERTS ) assert subtrahendIntervalIterator.hasNext();
}
public void intervalTerms( final LongSet terms ) {
// Just delegate to minuend
minuendIntervalIterator.intervalTerms( terms );
}
public Interval nextInterval() throws IOException {
if ( next != null ) {
final Interval result = next;
next = null;
return result;
}
if ( subtrahendInterval == Intervals.MINUS_INFINITY ) subtrahendInterval = subtrahendIntervalIterator.nextInterval();
Interval minuendInterval;
while( ( minuendInterval = minuendIntervalIterator.nextInterval() ) != null ) {
while( subtrahendInterval != null &&
subtrahendInterval.left - leftMargin < minuendInterval.left &&
subtrahendInterval.right + rightMargin < minuendInterval.right )
subtrahendInterval = subtrahendIntervalIterator.nextInterval();
if ( subtrahendInterval == null ||
subtrahendInterval.left - leftMargin < minuendInterval.left ||
subtrahendInterval.right + rightMargin > minuendInterval.right ) return minuendInterval;
}
return null;
}
public int extent() {
return minuendIntervalIterator.extent(); // TODO: check this
}
public String toString() {
return getClass().getSimpleName() + "(" + minuendIntervalIterator + ( leftMargin == 0 && rightMargin == 0 ? " - " : " -[" + leftMargin + "," + rightMargin + "] " ) + subtrahendIntervalIterator + ")";
}
}
}