org.hibernate.search.indexes.serialization.impl.CopyTokenStream Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of hibernate-search-engine Show documentation

Core of the Object/Lucene mapper, query engine and index management

There is a newer version: 5.11.12.Final

Show newest version

/*
 * Hibernate Search, full-text search for your domain model
 *
 * License: GNU Lesser General Public License (LGPL), version 2.1 or later
 * See the lgpl.txt file in the root directory or .
 */
package org.hibernate.search.indexes.serialization.impl;

import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.AttributeImpl;

import org.hibernate.search.indexes.serialization.spi.SerializableTokenStream;
import org.hibernate.search.util.logging.impl.Log;
import org.hibernate.search.util.logging.impl.LoggerFactory;
import java.lang.invoke.MethodHandles;

/**
 * Copy all AttributeImpl for each incrementToken
 *
 * Inspired by org.apache.lucene.analysis.CachingTokenFilter
 * Original file released under the ASL 2.0 license
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * @author Emmanuel Bernard
 */
public class CopyTokenStream extends TokenStream implements Serializable {
	private static final Log log = LoggerFactory.make( MethodHandles.lookup() );

	private List> cache;
	private int index;

	public static SerializableTokenStream buildSerializableTokenStream(TokenStream tokenStream) {
		try {
			List> stream = createAttributeLists( tokenStream );
			return new SerializableTokenStream( stream );
		}
		catch (IOException e) {
			throw log.unableToReadTokenStream();
		}
	}

	public CopyTokenStream(List> stream) {
		this.index = 0;
		this.cache = stream;
	}

	@Override
	public final boolean incrementToken() throws IOException {
		if ( index >= cache.size() ) {
			// the cache is exhausted, return false
			return false;
		}

		// Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
		setState( index );
		index++;
		return true;
	}

	private void setState(int localIndex) {
		for ( AttributeImpl attr : cache.get( localIndex ) ) {
			addAttributeImpl( attr );
		}
	}

	@Override
	public final void end() throws IOException {
		if ( cache.size() > 0 ) {
			setState( cache.size() - 1 );
		}
	}

	@Override
	public void reset() throws IOException {
		index = 0;
	}

	private static List> createAttributeLists(TokenStream input) throws IOException {
		List> results = new ArrayList<>();
		input.reset();
		while ( input.incrementToken() ) {
			List attrs = new ArrayList<>();
			results.add( attrs );
			Iterator iter = input.getAttributeImplsIterator();
			while ( iter.hasNext() ) {
				//we need to clone as AttributeImpl instances can be reused across incrementToken() calls
				attrs.add( iter.next().clone() );
			}
		}
		input.end();
		return results;
	}
}