net.intelie.live.plugins.messenger.search.SearchServiceImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of plugin-messenger Show documentation
Intelie Live Messenger Plugin
The newest version!
package net.intelie.live.plugins.messenger.search;

import com.google.common.collect.Sets;
import net.intelie.live.Live;
import net.intelie.live.plugins.messenger.search.document.SearchableDoc;
import net.intelie.live.plugins.messenger.search.document.SearchableDocImpl;
import net.intelie.pipes.types.Type;
import net.intelie.pipes.util.Preconditions;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;

import static net.intelie.live.plugins.messenger.search.SearchableFields.*;

public class SearchServiceImpl implements SearchService, Closeable {

    private static final Logger LOGGER = LoggerFactory.getLogger(SearchServiceImpl.class);
    private final List searchableEvents = new CopyOnWriteArrayList<>();

    private final FSDirectory fsDirectory;
    private final IndexWriter indexWriter;
    private final SearcherManager searcherManager;
    private final ControlledRealTimeReopenThread reopenThread;
    private final SearchableIndexWriter searchableIndexWriter;
    private final Live live;

    public SearchServiceImpl(@NotNull Live live) throws Exception {
        this(live, 30.0, 0.7);
    }

    public SearchServiceImpl(@NotNull Live live, Double targetMaxStaleSec, Double targetMinStaleSec) throws Exception {

        this.live = live;
        Path path = live.system().getDataDirectory().resolve("search");
        Files.createDirectories(path);
        LOGGER.info("Lucene Directory -> {}", path.toString());

        fsDirectory = FSDirectory.open(path);
        live.describeAction("Opened index directory " + fsDirectory, fsDirectory);

        indexWriter = new IndexWriter(fsDirectory, new IndexWriterConfig());
        live.describeAction("Opened index writer", indexWriter);

        searchableIndexWriter = new SearchableIndexWriterBase();

        searcherManager = new SearcherManager(indexWriter, null);
        live.describeAction("Opened searcher manager", searcherManager);

        reopenThread = new ControlledRealTimeReopenThread<>(indexWriter, searcherManager, targetMaxStaleSec, targetMinStaleSec);
        reopenThread.setName("fulltext index reopen thread");
        reopenThread.setDaemon(true);
        live.describeAction("Started fulltext index reopen thread", reopenThread);
        live.system().requestScheduledExecutor(1, "search update last indexed info").scheduleAtFixedRate(this::commitSettingInfo, 1, 1, TimeUnit.MINUTES);
        reopenThread.start();
    }

    @Override
    public void close() throws IOException {

        try (IndexWriter ignored1 = this.indexWriter;
             SearcherManager ignored2 = searcherManager;
             ControlledRealTimeReopenThread ignored3 = this.reopenThread) {
        }


    }

    @Override
    public synchronized AutoCloseable registerSearchableEventHandler(@NotNull SearchableEventHandler searchableEventHandler) throws Exception {
        Preconditions.checkState(searchableEvents.stream().noneMatch(handlerBase -> handlerBase.searchableName().equals(searchableEventHandler.searchableName())), "already registered");
        SearchableEventBase handlerBase = new SearchableEventBase(live, searchableIndexWriter, searchableEventHandler);
        searchableEvents.add(handlerBase);
        handlerBase.startIndexing();
        LOGGER.error("registering searchable handler for -> {}", handlerBase.searchableName());
        return () -> {
            searchableEvents.remove(handlerBase);
            handlerBase.close();
        };
    }

    @Override
    public void restartIndexing() throws Exception {
        indexWriter.flush();
        for (SearchableEventBase searchableEvent : searchableEvents) {
            searchableEvent.restartIndexing();
        }
    }

    @Override
    public Set searchableFields() {
        Set retSet = new HashSet<>();
        for (SearchableEventBase searchableEvent : searchableEvents) {
            retSet.addAll(searchableEvent.searchableFields());
        }
        return retSet;
    }

    @Override
    public int searchCount(String query) throws Exception {
        IndexSearcher searcher = searcherManager.acquire();
        try {
            return searcher.count(query(query, null, null));
        } finally {
            searcherManager.release(searcher);
        }
    }

    @Override
    public Collection> search(String query, int limit, Long lowerCursor, Long upperCursor) throws Exception {
        limit = Math.max(1, limit);
        IndexSearcher searcher = searcherManager.acquire();
        List> events = new ArrayList<>();
        try {
            TopDocs docs = searcher.search(query(query, lowerCursor, upperCursor), limit, sort());
            LOGGER.info("found docs -> {} for query = {}", docs.scoreDocs.length, query);
            List foundDocs = new ArrayList<>();
            for (ScoreDoc doc : docs.scoreDocs) {
                Document curDoc = searcher.doc(doc.doc, Sets.newHashSet(EVENT_UID, EVENT_TYPE, EVENT_SRC, CREATED_AT));
                foundDocs.add(new SearchableDocImpl(curDoc));
            }

            for (SearchableEventBase searchableEvent : searchableEvents) {
                events.addAll(searchableEvent.loadEventsFrom(foundDocs));
            }

            events.sort((map1, map2) -> {
                Double createdAt1 = Type.NUMBER.cast(map1.get(CREATED_AT));
                Double createdAt2 = Type.NUMBER.cast(map2.get(CREATED_AT));
                return Double.compare(createdAt2 != null ? createdAt2 : 0, createdAt1 != null ? createdAt1 : 0);
            });

            return events;
        } finally {
            searcherManager.release(searcher);
        }
    }

    public SearchableIndexWriter getIndexWriter() {
        return searchableIndexWriter;
    }

    private void commitSettingInfo() {
        for (SearchableEventBase searchableEvent : searchableEvents) {
            searchableEvent.commitSettingsInfo();
        }
    }

    private org.apache.lucene.search.Query query(String query, Long lowerCursor, Long upperCursor) throws ParseException {
        Set searchFields = searchableFields();
        searchFields.add(DEFAULT_FIELD);
        String[] searchFieldsArray = searchFields.toArray(new String[0]);
        MultiFieldQueryParser parser = new MultiFieldQueryParser(searchFieldsArray, new StandardAnalyzer());
        parser.setDefaultOperator(QueryParser.Operator.AND);
        org.apache.lucene.search.Query q = parser.parse(query);
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(q, BooleanClause.Occur.MUST);
        if (lowerCursor == null) {
            lowerCursor = Long.MIN_VALUE;
        }
        if (upperCursor != null) {
            org.apache.lucene.search.Query pointQuery = LongPoint.newRangeQuery(CREATED_AT, lowerCursor, upperCursor);
            org.apache.lucene.search.Query dvQuery = NumericDocValuesField.newSlowRangeQuery(CREATED_AT, lowerCursor, upperCursor);
            builder.add(new IndexOrDocValuesQuery(pointQuery, dvQuery), BooleanClause.Occur.FILTER);
        }
        return builder.build();
    }

    private static Sort sort() {
        return new Sort(new SortField(CREATED_AT, SortField.Type.LONG, true));
    }

    // this class is exposed by search index holder to allow plugins insert/delete custom event to the index
    private class SearchableIndexWriterBase implements SearchableIndexWriter {

        @Override
        public void updateDocument(Term term, Iterable doc) throws Exception {
            indexWriter.updateDocument(term, doc);
        }

        @Override
        public boolean deleteDocument(Term term) throws Exception {
            return indexWriter.deleteDocuments(term) > 0;
        }

        @Override
        public void flushBuffer() throws Exception {
            while (indexWriter.hasUncommittedChanges()) {
                indexWriter.commit();
            }
        }
    }
}