org.netbeans.modules.parsing.lucene.LuceneIndex Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.netbeans.modules.parsing.lucene;
import org.apache.lucene.store.LockObtainFailedException;
import org.netbeans.modules.parsing.lucene.support.LowMemoryWatcher;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.ref.SoftReference;
import java.net.URI;
import java.nio.channels.ClosedByInterruptException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Version;
import org.netbeans.api.annotations.common.CheckForNull;
import org.netbeans.api.annotations.common.NonNull;
import org.netbeans.api.annotations.common.NullAllowed;
import org.netbeans.modules.parsing.lucene.support.Convertor;
import org.netbeans.modules.parsing.lucene.support.Index;
import org.netbeans.modules.parsing.lucene.support.IndexReaderInjection;
import org.netbeans.modules.parsing.lucene.support.StoppableConvertor;
import org.openide.util.BaseUtilities;
import org.openide.util.Exceptions;
import org.openide.util.Pair;
import org.openide.util.Parameters;
import org.openide.util.RequestProcessor;
/**
* Note - there can be only a single IndexWriter at a time for the dir index. For consistency, the Writer is
* kept in a thread-local variable until it is committed. Lucene will throw an exception if another Writer creation
* attempt is done (by another thread, presumably).
*
* It should be thread-safe (according to Lucene docs) to use an IndexWriter while Readers are opened.
*
* As Reader and Writers can be used in parallel, all query+store operations use readLock so they can run in parallel.
* Operations which affect the whole index (close, clear) use write lock. RefreshReader called internally from writer's commit (close)
* is incompatible with parallel reads, as it closes the old reader - uses writeLock.
*
* Locks must be acquired in the order [rwLock, LuceneIndex]. The do* method synchronize on the DirCache instance and must be called
* if the code already holds rwLock.
*
* @author Tomas Zezula
*/
//@NotTreadSafe
public class LuceneIndex implements Index.Transactional, Index.WithTermFrequencies, Runnable {
private static final String PROP_INDEX_POLICY = "java.index.useMemCache"; //NOI18N
private static final String PROP_DIR_TYPE = "java.index.dir"; //NOI18N
private static final String DIR_TYPE_MMAP = "mmap"; //NOI18N
private static final String DIR_TYPE_NIO = "nio"; //NOI18N
private static final String DIR_TYPE_IO = "io"; //NOI18N
private static final CachePolicy DEFAULT_CACHE_POLICY = CachePolicy.DYNAMIC;
private static final CachePolicy cachePolicy = getCachePolicy();
private static final Logger LOGGER = Logger.getLogger(LuceneIndex.class.getName());
private final DirCache dirCache;
public static LuceneIndex create (final File cacheRoot, final Analyzer analyzer) throws IOException {
return new LuceneIndex (cacheRoot, analyzer);
}
static boolean awaitPendingEvictors() throws InterruptedException {
try {
return DirCache.EVICTOR_RP.submit(new Runnable() {
@Override
public void run() {
}
}, Boolean.TRUE).get();
} catch (ExecutionException e) {
return false;
}
}
/** Creates a new instance of LuceneIndex */
private LuceneIndex (final File refCacheRoot, final Analyzer analyzer) throws IOException {
assert refCacheRoot != null;
assert analyzer != null;
this.dirCache = new DirCache(
refCacheRoot,
cachePolicy,
analyzer);
}
@Override
public void query (
final @NonNull Collection result,
final @NonNull Convertor convertor,
@NullAllowed FieldSelector selector,
final @NullAllowed AtomicBoolean cancel,
final @NonNull Query... queries
) throws IOException, InterruptedException {
Parameters.notNull("queries", queries); //NOI18N
Parameters.notNull("convertor", convertor); //NOI18N
Parameters.notNull("result", result); //NOI18N
if (selector == null) {
selector = AllFieldsSelector.INSTANCE;
}
IndexReader in = null;
try {
in = dirCache.acquireReader();
if (in == null) {
LOGGER.log(Level.FINE, "{0} is invalid!", this);
return;
}
final BitSet bs = new BitSet(in.maxDoc());
final Collector c = new BitSetCollector(bs);
final IndexSearcher searcher = new IndexSearcher(in);
try {
for (Query q : queries) {
if (cancel != null && cancel.get()) {
throw new InterruptedException ();
}
searcher.search(q, c);
}
} finally {
searcher.close();
}
if (convertor instanceof IndexReaderInjection) {
((IndexReaderInjection)convertor).setIndexReader(in);
}
try {
for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) {
if (cancel != null && cancel.get()) {
throw new InterruptedException ();
}
final Document doc = in.document(docNum, selector);
final T value = convertor.convert(doc);
if (value != null) {
result.add (value);
}
}
} finally {
if (convertor instanceof IndexReaderInjection) {
((IndexReaderInjection)convertor).setIndexReader(null);
}
}
} finally {
dirCache.releaseReader(in);
}
}
@Override
public void queryTerms(
final @NonNull Collection result,
final @NullAllowed Term seekTo,
final @NonNull StoppableConvertor filter,
final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException {
queryTermsImpl(result, seekTo, Convertors.newTermEnumToTermConvertor(filter), cancel);
}
@Override
public void queryTermFrequencies(
final @NonNull Collection result,
final @NullAllowed Term seekTo,
final @NonNull StoppableConvertor filter,
final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException {
queryTermsImpl(result, seekTo, Convertors.newTermEnumToFreqConvertor(filter), cancel);
}
//where
private void queryTermsImpl(
final @NonNull Collection result,
final @NullAllowed Term seekTo,
final @NonNull StoppableConvertor adapter,
final @NullAllowed AtomicBoolean cancel) throws IOException, InterruptedException {
IndexReader in = null;
try {
in = dirCache.acquireReader();
if (in == null) {
LOGGER.log(Level.FINE, "{0} is invalid!", this);
return;
}
final TermEnum terms = seekTo == null ? in.terms () : in.terms (seekTo);
try {
if (adapter instanceof IndexReaderInjection) {
((IndexReaderInjection)adapter).setIndexReader(in);
}
try {
do {
if (cancel != null && cancel.get()) {
throw new InterruptedException ();
}
final T vote = adapter.convert(terms);
if (vote != null) {
result.add(vote);
}
} while (terms.next());
} catch (StoppableConvertor.Stop stop) {
//Stop iteration of TermEnum finally {
} finally {
if (adapter instanceof IndexReaderInjection) {
((IndexReaderInjection)adapter).setIndexReader(null);
}
}
} finally {
terms.close();
}
} finally {
dirCache.releaseReader(in);
}
}
@Override
public void queryDocTerms(
final @NonNull Map> result,
final @NonNull Convertor convertor,
final @NonNull Convertor termConvertor,
@NullAllowed FieldSelector selector,
final @NullAllowed AtomicBoolean cancel,
final @NonNull Query... queries) throws IOException, InterruptedException {
Parameters.notNull("queries", queries); //NOI18N
Parameters.notNull("slector", selector); //NOI18N
Parameters.notNull("convertor", convertor); //NOI18N
Parameters.notNull("termConvertor", termConvertor); //NOI18N
Parameters.notNull("result", result); //NOI18N
if (selector == null) {
selector = AllFieldsSelector.INSTANCE;
}
IndexReader in = null;
try {
in = dirCache.acquireReader();
if (in == null) {
LOGGER.log(Level.FINE, "{0} is invalid!", this);
return;
}
final BitSet bs = new BitSet(in.maxDoc());
final Collector c = new BitSetCollector(bs);
final IndexSearcher searcher = new IndexSearcher(in);
final TermCollector termCollector = new TermCollector(c);
try {
for (Query q : queries) {
if (cancel != null && cancel.get()) {
throw new InterruptedException ();
}
if (q instanceof TermCollector.TermCollecting) {
((TermCollector.TermCollecting)q).attach(termCollector);
} else {
throw new IllegalArgumentException (
String.format("Query: %s does not implement TermCollecting", //NOI18N
q.getClass().getName()));
}
searcher.search(q, termCollector);
}
} finally {
searcher.close();
}
boolean logged = false;
if (convertor instanceof IndexReaderInjection) {
((IndexReaderInjection)convertor).setIndexReader(in);
}
try {
if (termConvertor instanceof IndexReaderInjection) {
((IndexReaderInjection)termConvertor).setIndexReader(in);
}
try {
for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) {
if (cancel != null && cancel.get()) {
throw new InterruptedException ();
}
final Document doc = in.document(docNum, selector);
final T value = convertor.convert(doc);
if (value != null) {
final Set terms = termCollector.get(docNum);
if (terms != null) {
result.put (value, convertTerms(termConvertor, terms));
} else {
if (!logged) {
LOGGER.log(Level.WARNING, "Index info [maxDoc: {0} numDoc: {1} docs: {2}]",
new Object[] {
in.maxDoc(),
in.numDocs(),
termCollector.docs()
});
logged = true;
}
LOGGER.log(Level.WARNING, "No terms found for doc: {0}", docNum);
}
}
}
} finally {
if (termConvertor instanceof IndexReaderInjection) {
((IndexReaderInjection)termConvertor).setIndexReader(null);
}
}
} finally {
if (convertor instanceof IndexReaderInjection) {
((IndexReaderInjection)convertor).setIndexReader(null);
}
}
} finally {
dirCache.releaseReader(in);
}
}
private static Set convertTerms(final Convertor convertor, final Set terms) {
final Set result = new HashSet(terms.size());
for (Term term : terms) {
result.add(convertor.convert(term));
}
return result;
}
@Override
public void run() {
dirCache.beginTrans();
}
@Override
public void commit() throws IOException {
dirCache.closeTxWriter();
}
@Override
public void rollback() throws IOException {
dirCache.rollbackTxWriter();
}
@Override
public void txStore(
final Collection toAdd,
final Collection toDelete, final Convertor docConvertor,
final Convertor queryConvertor) throws IOException {
final IndexWriter wr = dirCache.acquireWriter();
try {
try {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log(Level.FINE, "Storing in TX {0}: {1} added, {2} deleted",
new Object[] { this, toAdd.size(), toDelete.size() }
);
}
_doStore(toAdd, toDelete, docConvertor, queryConvertor, wr);
} finally {
// nothing committed upon failure - readers not affected
boolean ok = false;
try {
((FlushIndexWriter)wr).callFlush(false, true);
ok = true;
} finally {
if (!ok) {
dirCache.rollbackTxWriter();
}
}
}
} finally {
dirCache.releaseWriter(wr);
}
}
private void _doStore(
@NonNull final Collection data,
@NonNull final Collection toDelete,
@NonNull final Convertor docConvertor,
@NonNull final Convertor queryConvertor,
@NonNull final IndexWriter out) throws IOException {
try {
if (dirCache.exists()) {
for (S td : toDelete) {
out.deleteDocuments(queryConvertor.convert(td));
}
}
if (data.isEmpty()) {
return;
}
final LowMemoryWatcher lmListener = LowMemoryWatcher.getInstance();
Directory memDir = null;
IndexWriter activeOut = null;
if (lmListener.isLowMemory()) {
activeOut = out;
} else {
memDir = new RAMDirectory ();
activeOut = new IndexWriter (
memDir,
new IndexWriterConfig(
Version.LUCENE_35,
dirCache.getAnalyzer()));
}
for (Iterator it = fastRemoveIterable(data).iterator(); it.hasNext();) {
T entry = it.next();
it.remove();
final Document doc = docConvertor.convert(entry);
activeOut.addDocument(doc);
if (memDir != null && lmListener.isLowMemory()) {
activeOut.close();
out.addIndexes(memDir);
memDir = new RAMDirectory ();
activeOut = new IndexWriter (
memDir,
new IndexWriterConfig(
Version.LUCENE_35,
dirCache.getAnalyzer()));
}
}
data.clear();
if (memDir != null) {
activeOut.close();
out.addIndexes(memDir);
activeOut = null;
memDir = null;
}
} catch (RuntimeException e) {
throw Exceptions.attachMessage(e, "Lucene Index Folder: " + dirCache.folder.getAbsolutePath());
} catch (IOException e) {
throw Exceptions.attachMessage(e, "Lucene Index Folder: " + dirCache.folder.getAbsolutePath());
}
}
@Override
public void store (
final @NonNull Collection data,
final @NonNull Collection toDelete,
final @NonNull Convertor docConvertor,
final @NonNull Convertor queryConvertor,
final boolean optimize) throws IOException {
final IndexWriter wr = dirCache.acquireWriter();
dirCache.storeCloseSynchronizer.enter();
try {
try {
try {
_doStore(data, toDelete, docConvertor, queryConvertor, wr);
} finally {
LOGGER.log(Level.FINE, "Committing {0}", this);
dirCache.releaseWriter(wr);
}
} finally {
dirCache.close(wr);
}
} finally {
dirCache.storeCloseSynchronizer.exit();
}
}
@Override
public Status getStatus (boolean force) throws IOException {
return dirCache.getStatus(force);
}
@Override
public void clear () throws IOException {
dirCache.clear();
}
@Override
public void close () throws IOException {
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.log(Level.FINEST, "Closing index: {0} {1}", //NOI18N
new Object[]{
this.dirCache.toString(),
Thread.currentThread().getStackTrace()});
}
dirCache.close(true);
}
@Override
public String toString () {
return getClass().getSimpleName()+"["+this.dirCache.toString()+"]"; //NOI18N
}
private static CachePolicy getCachePolicy() {
final String value = System.getProperty(PROP_INDEX_POLICY); //NOI18N
if (Boolean.TRUE.toString().equals(value) ||
CachePolicy.ALL.getSystemName().equals(value)) {
return CachePolicy.ALL;
}
if (Boolean.FALSE.toString().equals(value) ||
CachePolicy.NONE.getSystemName().equals(value)) {
return CachePolicy.NONE;
}
if (CachePolicy.DYNAMIC.getSystemName().equals(value)) {
return CachePolicy.DYNAMIC;
}
return DEFAULT_CACHE_POLICY;
}
private static Iterable fastRemoveIterable(final Collection c) {
return c instanceof ArrayList ?
new Iterable() {
@Override
public Iterator iterator() {
return new Iterator() {
private final ListIterator delegate = ((List)c).listIterator();
@Override
public boolean hasNext() {
return delegate.hasNext();
}
@Override
public T next() {
return delegate.next();
}
@Override
public void remove() {
delegate.set(null);
}
};
}
} :
c;
}
//
private enum CachePolicy {
NONE("none", false), //NOI18N
DYNAMIC("dynamic", true), //NOI18N
ALL("all", true); //NOI18N
private final String sysName;
private final boolean hasMemCache;
CachePolicy(final String sysName, final boolean hasMemCache) {
assert sysName != null;
this.sysName = sysName;
this.hasMemCache = hasMemCache;
}
String getSystemName() {
return sysName;
}
boolean hasMemCache() {
return hasMemCache;
}
}
private static final class DirCache implements Evictable {
private static final RequestProcessor EVICTOR_RP =
new RequestProcessor(LuceneIndex.DirCache.class.getName(), 1);
private final File folder;
private final RecordOwnerLockFactory lockFactory;
private final CachePolicy cachePolicy;
private final Analyzer analyzer;
private final StoreCloseSynchronizer storeCloseSynchronizer;
private volatile FSDirectory fsDir;
//@GuardedBy("this")
private RAMDirectory memDir;
private CleanReference ref;
private IndexReader reader;
private volatile boolean closed;
private volatile Throwable closeStackTrace;
private volatile Status validCache;
private final IndexWriterReference indexWriterRef = new IndexWriterReference();
private final ReadWriteLock rwLock = new java.util.concurrent.locks.ReentrantReadWriteLock();
private DirCache(
final @NonNull File folder,
final @NonNull CachePolicy cachePolicy,
final @NonNull Analyzer analyzer) throws IOException {
assert folder != null;
assert cachePolicy != null;
assert analyzer != null;
this.folder = new Folder(folder);
this.lockFactory = new RecordOwnerLockFactory();
this.fsDir = createFSDirectory(this.folder, lockFactory);
this.cachePolicy = cachePolicy;
this.analyzer = analyzer;
this.storeCloseSynchronizer = new StoreCloseSynchronizer();
}
Analyzer getAnalyzer() {
return this.analyzer;
}
void clear() throws IOException {
Future sync;
while (true) {
rwLock.writeLock().lock();
try {
sync = storeCloseSynchronizer.getSync();
if (sync == null) {
doClear();
break;
}
} finally {
rwLock.writeLock().unlock();
}
try {
sync.get();
} catch (InterruptedException ex) {
break;
} catch (ExecutionException ex) {
Exceptions.printStackTrace(ex);
}
}
}
private synchronized void doClear() throws IOException {
checkPreconditions();
// already write locked
doClose(false, true);
try {
lockFactory.forceClearLocks();
final String[] content = fsDir.listAll();
boolean dirty = false;
if (content != null) {
for (String file : content) {
try {
fsDir.deleteFile(file);
} catch (IOException e) {
//Some temporary files
if (fsDir.fileExists(file)) {
dirty = true;
}
}
}
}
if (dirty) {
//Try to delete dirty files and log what's wrong
final File cacheDir = fsDir.getDirectory();
final File[] children = cacheDir.listFiles();
if (children != null) {
for (final File child : children) {
if (!child.delete()) {
throw RecordOwnerLockFactory.annotateException(
new IOException("Cannot delete: " + child.getAbsolutePath()),
folder,
Thread.getAllStackTraces()); //NOI18N
}
}
}
}
} finally {
//Need to recreate directory, see issue: #148374
this.fsDir.close();
this.fsDir = createFSDirectory(this.folder, this.lockFactory);
}
}
void close(IndexWriter writer) throws IOException {
if (writer == null) {
return;
}
boolean success = false;
try {
writer.close();
success = true;
} finally {
LOGGER.log(Level.FINE, "TX writer cleared for {0}", this);
indexWriterRef.release();
try {
if (!success) {
if (IndexWriter.isLocked(fsDir)) {
IndexWriter.unlock(fsDir);
}
}
} catch (IOException ioe) {
LOGGER.log(
Level.WARNING,
"Cannot unlock index {0} while recovering, {1}.", //NOI18N
new Object[] {
folder.getAbsolutePath(),
ioe.getMessage()
});
} finally {
refreshReader();
}
}
}
void close (final boolean closeFSDir) throws IOException {
Future sync;
while (true) {
rwLock.writeLock().lock();
try {
sync = storeCloseSynchronizer.getSync();
if (sync == null) {
doClose(closeFSDir, false);
break;
}
} finally {
rwLock.writeLock().unlock();
}
try {
sync.get();
} catch (InterruptedException ex) {
break;
} catch (ExecutionException ex) {
Exceptions.printStackTrace(ex);
}
}
}
synchronized void doClose (
final boolean closeFSDir,
final boolean ensureClosed) throws IOException {
try {
rollbackTxWriter();
if (this.reader != null) {
this.reader.close();
if (ensureClosed) {
try {
while (this.reader.getRefCount() > 0) {
this.reader.decRef();
}
} catch (IllegalStateException e) {
//pass closed
}
}
this.reader = null;
}
} finally {
if (memDir != null) {
assert cachePolicy.hasMemCache();
if (this.ref != null) {
this.ref.clear();
}
final Directory tmpDir = this.memDir;
memDir = null;
tmpDir.close();
}
if (closeFSDir) {
this.closeStackTrace = new Throwable();
this.closed = true;
this.fsDir.close();
}
}
}
boolean exists() {
try {
return IndexReader.indexExists(this.fsDir);
} catch (IOException e) {
return false;
} catch (RuntimeException e) {
LOGGER.log(Level.INFO, "Broken index: " + folder.getAbsolutePath(), e);
return false;
}
}
Status getStatus (boolean force) throws IOException {
checkPreconditions();
Status valid = validCache;
if (force || valid == null) {
rwLock.writeLock().lock();
try {
Status res = Status.INVALID;
if (lockFactory.hasLocks()) {
if (indexWriterRef.get() != null) {
res = Status.WRITING;
} else {
LOGGER.log(Level.WARNING, "Locked index folder: {0}", folder.getAbsolutePath()); //NOI18N
if (force) {
clear();
}
}
} else {
if (!exists()) {
res = Status.EMPTY;
} else if (force) {
try {
getReader();
res = Status.VALID;
} catch (java.io.IOException e) {
clear();
} catch (RuntimeException e) {
clear();
}
} else {
res = Status.VALID;
}
}
valid = res;
validCache = valid;
} finally {
rwLock.writeLock().unlock();
}
}
return valid;
}
boolean closeTxWriter() throws IOException {
IndexWriter writer = indexWriterRef.get();
if (writer != null) {
LOGGER.log(Level.FINE, "Committing {0}", this);
close(writer);
return true;
} else {
return false;
}
}
boolean rollbackTxWriter() throws IOException {
final IndexWriter writer = indexWriterRef.get();
if (writer != null) {
try {
writer.rollback();
return true;
} finally {
indexWriterRef.release();
}
} else {
return false;
}
}
void beginTrans() {
indexWriterRef.beginTrans();
}
/**
* The writer operates under readLock(!) since we do not want to lock out readers,
* but just close, clear and commit operations.
*
* @return
* @throws IOException
*/
IndexWriter acquireWriter () throws IOException {
checkPreconditions();
hit();
boolean ok = false;
rwLock.readLock().lock();
try {
try {
final IndexWriter writer = indexWriterRef.acquire(new Callable() {
@NonNull
public IndexWriter call() throws IOException {
final IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_35, analyzer);
//Linux: The posix::fsync(int) is very slow on Linux ext3,
//minimize number of files sync is done on.
//http://netbeans.org/bugzilla/show_bug.cgi?id=208224
//All OS: The CFS is better for SSD disks.
final TieredMergePolicy mergePolicy = new TieredMergePolicy();
mergePolicy.setNoCFSRatio(1.0);
iwc.setMergePolicy(mergePolicy);
return new FlushIndexWriter (fsDir, iwc);
}
});
ok = true;
return writer;
} catch (LockObtainFailedException lf) {
//Already annotated
throw lf;
} catch (IOException ioe) {
//Issue #149757 - logging
throw RecordOwnerLockFactory.annotateException (
ioe,
folder,
null);
}
} finally {
if (!ok) {
rwLock.readLock().unlock();
}
}
}
void releaseWriter(@NonNull final IndexWriter w) {
assert indexWriterRef.get() == w || indexWriterRef.get() == null;
rwLock.readLock().unlock();
}
IndexReader acquireReader() throws IOException {
rwLock.readLock().lock();
IndexReader r = null;
try {
r = getReader();
return r;
} finally {
if (r == null) {
rwLock.readLock().unlock();
}
}
}
void releaseReader(IndexReader r) {
if (r == null) {
return;
}
assert r == this.reader;
rwLock.readLock().unlock();
}
private synchronized IndexReader getReader () throws IOException {
checkPreconditions();
hit();
if (this.reader == null) {
if (validCache != Status.VALID &&
validCache != Status.WRITING &&
validCache != null) {
return null;
}
//Issue #149757 - logging
try {
Directory source;
if (cachePolicy.hasMemCache() && fitsIntoMem(fsDir)) {
memDir = new RAMDirectory(fsDir);
if (cachePolicy == CachePolicy.DYNAMIC) {
ref = new CleanReference (new RAMDirectory[] {this.memDir});
}
source = memDir;
} else {
source = fsDir;
}
assert source != null;
this.reader = IndexReader.open(source,true);
} catch (final FileNotFoundException | ClosedByInterruptException | InterruptedIOException e) {
//Either the index dir does not exist or the thread is interrupted
//pass - returns null
} catch (IOException ioe) {
if (validCache == null) {
return null;
} else {
throw RecordOwnerLockFactory.annotateException(
ioe,
folder,
Thread.getAllStackTraces());
}
}
}
hit();
return this.reader;
}
void refreshReader() throws IOException {
try {
if (cachePolicy.hasMemCache()) {
close(false);
} else {
rwLock.writeLock().lock();
try {
synchronized (this) {
if (reader != null) {
final IndexReader newReader = IndexReader.openIfChanged(reader);
if (newReader != null) {
reader.close();
reader = newReader;
}
}
}
} finally {
rwLock.writeLock().unlock();
}
}
} finally {
validCache = Status.VALID;
}
}
@Override
public String toString() {
return this.folder.getAbsolutePath();
}
@Override
public void evicted() {
EVICTOR_RP.execute(new Runnable() {
@Override
public void run() {
boolean needsClose = true;
synchronized (this) {
if (memDir != null) {
if (ref != null) {
ref.clearHRef();
}
needsClose = false;
}
}
if (needsClose) {
try {
close(false);
LOGGER.log(Level.FINE, "Evicted index: {0}", folder.getAbsolutePath()); //NOI18N
} catch (IOException ex) {
Exceptions.printStackTrace(ex);
}
}
}
});
}
private synchronized void hit() {
if (reader != null) {
final URI uri = BaseUtilities.toURI(folder);
if (memDir != null) {
IndexCacheFactory.getDefault().getRAMCache().put(uri, this);
if (ref != null) {
ref.get();
}
} else {
IndexCacheFactory.getDefault().getNIOCache().put(uri, this);
}
}
}
private void checkPreconditions () throws IndexClosedException {
if (closed) {
throw (IndexClosedException) new IndexClosedException().initCause(closeStackTrace);
}
}
private static FSDirectory createFSDirectory (
final File indexFolder,
final LockFactory lockFactory) throws IOException {
assert indexFolder != null;
assert lockFactory != null;
final FSDirectory directory;
final String dirType = System.getProperty(PROP_DIR_TYPE);
if(DIR_TYPE_MMAP.equals(dirType)) {
directory = new MMapDirectory(indexFolder, lockFactory);
} else if (DIR_TYPE_NIO.equals(dirType)) {
directory = new NIOFSDirectory(indexFolder, lockFactory);
} else if (DIR_TYPE_IO.equals(dirType)) {
directory = new SimpleFSDirectory(indexFolder, lockFactory);
} else {
directory = FSDirectory.open(indexFolder, lockFactory);
}
return directory;
}
private static boolean fitsIntoMem(@NonNull final Directory dir) {
try {
long size = 0;
for (String path : dir.listAll()) {
size+=dir.fileLength(path);
}
return IndexCacheFactory.getDefault().getRAMController().shouldLoad(size);
} catch (IOException ioe) {
return false;
}
}
private static final class IndexWriterReference {
//@GuardedBy("this")
private Pair> openThread;
//@GuardedBy("this")
private Pair> txThread;
//@GuardedBy("this")
private IndexWriter indexWriter;
//@GuardedBy("this")
private boolean modified;
synchronized void beginTrans() {
assertNoModifiedWriter();
modified = false;
txThread = trace();
}
@NonNull
synchronized IndexWriter acquire (@NonNull final Callable indexWriterFactory) throws IOException {
if (indexWriter != null) {
assert openThread != null;
assertSingleThreadWriter();
} else {
try {
assert openThread == null;
indexWriter = indexWriterFactory.call();
openThread = trace();
modified = true;
} catch (IOException | RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return indexWriter;
}
@CheckForNull
synchronized IndexWriter get() {
return indexWriter;
}
synchronized void release() {
openThread = null;
txThread = null;
indexWriter = null;
modified = false;
}
private void assertSingleThreadWriter() {
assert Thread.holdsLock(this);
if (openThread.first() != Thread.currentThread()) {
final IllegalStateException e = new IllegalStateException(String.format(
"Other thread using opened writer, " + //NOI18N
"old owner Thread %s(%d), " + //NOI18N
"new owner Thread %s(%d).", //NOI18N
openThread.first(),
openThread.first().getId(),
Thread.currentThread(),
Thread.currentThread().getId()),
openThread.second() != null ?
openThread.second().second() :
null);
throw e;
}
}
private void assertNoModifiedWriter() {
assert Thread.holdsLock(this);
if (assertsEnabled() && txThread != null && modified) {
final Throwable t = new Throwable(
String.format(
"Using stale writer, possibly forgotten call to store, " + //NOI18N
"old owner Thread %s(%d) enter time: %d, " + //NOI18N
"new owner Thread %s(%d) enter time: %d.", //NOI18N
txThread.first(),
txThread.first().getId(),
txThread.second().first(),
Thread.currentThread(),
Thread.currentThread().getId(),
System.currentTimeMillis()),
txThread.second().second());
LOGGER.log(
Level.WARNING,
"Using stale writer", //NOI18N
t);
}
}
@NonNull
private static Pair> trace() {
return Pair.of(
Thread.currentThread(),
assertsEnabled() ?
Pair.of(System.currentTimeMillis(), new Exception("Owner stack")) : //NOI18N
null);
}
@SuppressWarnings("AssertWithSideEffects")
private static boolean assertsEnabled() {
boolean ae = false;
assert ae = true;
return ae;
}
}
private final class CleanReference extends SoftReference implements Runnable {
@SuppressWarnings("VolatileArrayField")
private volatile Directory[] hardRef; //clearHRef may be called by more concurrently (read lock).
private final AtomicLong size = new AtomicLong(); //clearHRef may be called by more concurrently (read lock).
private CleanReference(final RAMDirectory[] dir) {
super (dir, BaseUtilities.activeReferenceQueue());
final IndexCacheFactory.RAMController c = IndexCacheFactory.getDefault().getRAMController();
final boolean doHardRef = !c.isFull();
if (doHardRef) {
this.hardRef = dir;
long _size = dir[0].sizeInBytes();
size.set(_size);
c.acquire(_size);
}
LOGGER.log(Level.FINEST, "Caching index: {0} cache policy: {1}", //NOI18N
new Object[]{
folder.getAbsolutePath(),
cachePolicy.getSystemName()
});
}
@Override
public void run() {
try {
LOGGER.log(Level.FINEST, "Dropping cache index: {0} cache policy: {1}", //NOI18N
new Object[] {
folder.getAbsolutePath(),
cachePolicy.getSystemName()
});
close(false);
} catch (IOException ex) {
Exceptions.printStackTrace(ex);
}
}
@Override
public void clear() {
clearHRef();
super.clear();
}
void clearHRef() {
this.hardRef = null;
IndexCacheFactory.getDefault().getRAMController().release(
size.getAndSet(0));
}
}
}
//
private static class FlushIndexWriter extends IndexWriter {
public FlushIndexWriter(
@NonNull final Directory d,
@NonNull final IndexWriterConfig conf) throws CorruptIndexException, LockObtainFailedException, IOException {
super(d, conf);
}
/**
* Accessor to index flush for this package
* @param triggerMerges
* @param flushDeletes
* @throws IOException
*/
void callFlush(boolean triggerMerges, boolean flushDeletes) throws IOException {
// flushStores ignored in Lucene 3.5
super.flush(triggerMerges, true, flushDeletes);
}
}
private static final class StoreCloseSynchronizer {
private ThreadLocal isWriterThread = new ThreadLocal(){
@Override
protected Boolean initialValue() {
return Boolean.FALSE;
}
};
//@GuardedBy("this")
private int depth;
StoreCloseSynchronizer() {}
synchronized void enter() {
depth++;
isWriterThread.set(Boolean.TRUE);
}
synchronized void exit() {
assert depth > 0;
depth--;
isWriterThread.remove();
if (depth == 0) {
notifyAll();
}
}
synchronized Future getSync() {
if (depth == 0 || isWriterThread.get() == Boolean.TRUE) {
return null;
} else {
return new Future() {
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
throw new UnsupportedOperationException();
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean isDone() {
synchronized(StoreCloseSynchronizer.this) {
return depth == 0;
}
}
@Override
public Void get() throws InterruptedException, ExecutionException {
synchronized (StoreCloseSynchronizer.this) {
while (depth > 0) {
StoreCloseSynchronizer.this.wait();
}
}
return null;
}
@Override
public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
if (unit != TimeUnit.MILLISECONDS) {
throw new UnsupportedOperationException();
}
synchronized (StoreCloseSynchronizer.this) {
while (depth > 0) {
StoreCloseSynchronizer.this.wait(timeout);
}
}
return null;
}
};
}
}
}
private static final class Folder extends File {
private static final java.nio.file.InvalidPathException IPE = new java.nio.file.InvalidPathException("", "") { //NOI18N
@Override
public Throwable fillInStackTrace() {
return this;
}
};
Folder(@NonNull final File folder) {
super(folder.getAbsolutePath());
}
@Override
public File getAbsoluteFile() {
assert isAbsolute();
return this;
}
@Override
public boolean isDirectory() {
return true;
}
@Override
public boolean isFile() {
return !isDirectory();
}
@Override
public Path toPath() {
throw IPE;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy