proj.zoie.api.ZoieIndexReader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of zoie-core Show documentation
zoie core library
There is a newer version: 3.3.0
package proj.zoie.api;
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import it.unimi.dsi.fastutil.longs.LongSet;

import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.log4j.Logger;
import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermPositions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.ReaderUtil;

import proj.zoie.api.impl.DefaultIndexReaderMerger;
import proj.zoie.api.indexing.IndexReaderDecorator;

public abstract class ZoieIndexReader extends FilterIndexReader
{
  private static final Logger log = Logger.getLogger(ZoieIndexReader.class);
	public static final long DELETED_UID = Long.MIN_VALUE;
	private AtomicLong zoieRefCounter = new AtomicLong(1);
	public void incZoieRef()
	{
	  zoieRefCounter.incrementAndGet();
	}
	public void decZoieRef()
	{
	  long refCount = zoieRefCounter.decrementAndGet();
	  if (refCount < 0)
	  {
	    log.warn("refCount should never be lower than 0");
	  }
	  if (refCount == 0)
	  {
	    try
      {
        in.decRef();
      } catch (IOException e)
      {
        log.error("decZoieRef", e);
      }
	  }
	}
	protected int[] _delDocIds;
	protected long _minUID;
	protected long _maxUID;
	protected boolean _noDedup = false;
	protected final IndexReaderDecorator _decorator;
	protected DocIDMapper _docIDMapper;

	public static  List extractDecoratedReaders(List> readerList) throws IOException
	{
	  LinkedList retList = new LinkedList();
	  for (ZoieIndexReader reader : readerList)
	  {
	    retList.addAll(reader.getDecoratedReaders());
	  }
	  return retList;
	}

	public static class SubReaderInfo
	{
	  public final R subreader;
	  public final int subdocid;

	  SubReaderInfo(R subreader,int subdocid)
	  {
	    this.subreader = subreader;
	    this.subdocid = subdocid;
	  }
	}

	public interface SubReaderAccessor
	{
	  SubReaderInfo getSubReaderInfo(int docid);
	}

	public interface SubZoieReaderAccessor
	{
	  SubReaderInfo> getSubReaderInfo(int docid);
	  SubReaderInfo> geSubReaderInfoFromUID(long uid);
	}

	public static  SubReaderAccessor getSubReaderAccessor(List readerList)
	{
	  int size = readerList.size();
	  final IndexReader[] subR = new IndexReader[size];
	  final int[] starts = new int[size+1];

	  int maxDoc = 0;

	  int i = 0;
	  for (R r : readerList)
	  {
	    starts[i] = maxDoc;
	    subR[i] = r;
	    maxDoc += r.maxDoc();          // compute maxDocs
	    i++;
	  }
	  starts[size] = maxDoc;

	  return new SubReaderAccessor()
	  {
	    public SubReaderInfo getSubReaderInfo(int docid)
	    {
	      int subIdx = ReaderUtil.subIndex(docid, starts);
	      int subdocid = docid - starts[subIdx];
	      R subreader = (R)(subR[subIdx]);
	      return new SubReaderInfo(subreader, subdocid);
	    }
	  };
	}
	
	public static  SubZoieReaderAccessor getSubZoieReaderAccessor(List> readerList)
	{
	  int size = readerList.size();
	  final ZoieIndexReader[] subR = new ZoieIndexReader[size];
	  final int[] starts = new int[size+1];

	  int maxDoc = 0;

	  int i = 0;
	  for (ZoieIndexReader r : readerList)
	  {
	    starts[i] = maxDoc;
	    subR[i] = r;
	    maxDoc += r.maxDoc();          // compute maxDocs
	    i++;
	  }
	  starts[size] = maxDoc;

    	  return new SubZoieReaderAccessor()
    	  {
  		
	  		public SubReaderInfo> getSubReaderInfo(int docid) {
	  			int subIdx = ReaderUtil.subIndex(docid, starts);
	  		  	int subdocid = docid - starts[subIdx];
	  		  	return new SubReaderInfo>((ZoieIndexReader)(subR[subIdx]), subdocid);
	  		}

			public SubReaderInfo> geSubReaderInfoFromUID(long uid) {
				SubReaderInfo> info = null;
				for (int i=0;i subReader = (ZoieIndexReader)subR[i];
					DocIDMapper mapper = subReader.getDocIDMaper();
					int docid = mapper.getDocID(uid);
					if (docid!=DocIDMapper.NOT_FOUND){
						info = new SubReaderInfo>(subReader,docid);
						break;
					}
				}
				return info;
			}
	  	  };
    	}
	
	public static  ZoieIndexReader open(IndexReader r) throws IOException{
		return open(r,null);
	}
	
	public static  ZoieIndexReader open(IndexReader r,IndexReaderDecorator decorator) throws IOException{
		return new ZoieMultiReader(r,decorator);
	}
	
	public static  ZoieIndexReader open(Directory dir,IndexReaderDecorator decorator) throws IOException{
		IndexReader r = IndexReader.open(dir, true);
		// load zoie reader
		try{
			return open(r,decorator);
		}
		catch(IOException ioe){
			if (r!=null){
				r.close();
			}
			throw ioe;
		}
	}
	
	public static  R mergeIndexReaders(List> readerList,IndexReaderMerger merger){
		return merger.mergeIndexReaders(readerList);
	}
	
	public static  MultiReader mergeIndexReaders(List> readerList){
		return mergeIndexReaders(readerList,new DefaultIndexReaderMerger());
	}
		
	protected ZoieIndexReader(IndexReader in,IndexReaderDecorator decorator) throws IOException
	{
		super(in);
		_decorator = decorator;
		_delDocIds = null;//new ContextAccessor(this, "delset");// new InheritableThreadLocal();
		_minUID=Long.MAX_VALUE;
		_maxUID=Long.MIN_VALUE;
	}
	
	abstract public List getDecoratedReaders() throws IOException;
    abstract public void setDelDocIds();
    abstract public void markDeletes(LongSet delDocs, LongSet deltedUIDs);
    abstract public void commitDeletes();
	     
	public IndexReader getInnerReader(){
		return in;
	}
	
	@Override
	public boolean hasDeletions()
	{
	  if(!_noDedup)
	  {
		int[] delSet = _delDocIds;
	    if(delSet != null && delSet.length > 0) return true;
	  }
	  return in.hasDeletions();
	}
	
	protected abstract boolean hasIndexDeletions();
	
	public boolean hasDuplicates()
	{
		int[] delSet = _delDocIds;//.get();
		return (delSet!=null && delSet.length > 0);
	}

	@Override
	abstract public boolean isDeleted(int docid);
	
	abstract public byte[] getStoredValue(long uid) throws IOException;
	
	public boolean isDuplicate(int docid)
	{
	  int[] delSet = _delDocIds;//.get();
	  return delSet!=null && Arrays.binarySearch(delSet, docid) >= 0;
	}
	
	public boolean isDuplicateUID(long uid){
	  int docid = _docIDMapper.getDocID(uid);
	  return isDuplicate(docid);
	}
	
	public int[] getDelDocIds()
	{
	  return _delDocIds;//.get();
	}
	
	public long getMinUID()
	{
		return _minUID;
	}
	
	public long getMaxUID()
	{
		return _maxUID;
	}

	abstract public long getUID(int docid);
	
	public DocIDMapper getDocIDMaper(){
		return _docIDMapper;
	}
	
	public void setDocIDMapper(DocIDMapper docIDMapper){
		_docIDMapper = docIDMapper;
	}
	
	public void setNoDedup(boolean noDedup)
	{
	  _noDedup = noDedup;
	}

	@Override
	abstract public ZoieIndexReader[] getSequentialSubReaders();
	
	@Override
	abstract public TermDocs termDocs() throws IOException;
	
	@Override
	abstract public TermPositions termPositions() throws IOException;
	
  /**
   * makes exact shallow copy of a given ZoieMultiReader
   * @return a shallow copy of a given ZoieMultiReader
   * @throws IOException
   */
  public abstract ZoieIndexReader copy() throws IOException;
}