net.intelie.live.plugins.messenger.search.SearchableEventBase Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of plugin-messenger Show documentation
Intelie Live Messenger Plugin
The newest version!
package net.intelie.live.plugins.messenger.search;

import net.intelie.live.*;
import net.intelie.live.plugins.messenger.search.document.*;
import net.intelie.pipes.types.Type;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.Term;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import static net.intelie.live.plugins.messenger.search.SearchableFields.UID;

class SearchableEventBase implements AutoCloseable {
    private static final Logger LOGGER = LoggerFactory.getLogger(SearchableEventBase.class);
    private final Live live;
    private final SettingsNode lastIndexedNodeInfo;
    private volatile AtomicLong lastIndexedTimestamp = new AtomicLong();
    private final AtomicLong nIndexedEvents = new AtomicLong();
    private Live.Action runningQueryAction;
    private final Live.Action providerListenerAction;
    private final SearchableIndexWriter indexWriter;
    private final SearchableEventHandler searchableHandler;
    private final LoggedUser loggedUser;

    SearchableEventBase(Live live, SearchableIndexWriter indexWriter, SearchableEventHandler searchableHandler) throws Exception {
        this.live = live;
        this.indexWriter = indexWriter;
        this.searchableHandler = searchableHandler;
        this.loggedUser = live.auth().getLoggedUser();
        SettingsNode settings = live.settings().home().cd("search");
        lastIndexedNodeInfo = settings.cd("lastIndexedTimestamp").cd(searchableHandler.searchableName());
        String actionDescription = "Provider Listener for " +  searchableHandler.searchableName() + " searchable";
        providerListenerAction = this.live.describeAction(actionDescription,
                this.live.engine().getMainStorage().onProvidersChange(new ProviderListenerBase()));
    }

    String searchableName() {
        return searchableHandler.searchableName();
    }

    Set searchableFields(){
        return searchableHandler.searchableFields();
    }

    @Override
    public void close() throws Exception {
        try {
            providerListenerAction.close();
        } finally {
            stopIndexing();
        }
    }

    synchronized void restartIndexing() throws Exception {
        stopIndexing();
        lastIndexedTimestamp = new AtomicLong();
        nIndexedEvents.set(0L);
        lastIndexedNodeInfo.set(0L);
        startIndexing();
    }

    synchronized void stopIndexing() throws Exception{
        if (this.runningQueryAction != null) {
            this.runningQueryAction.close();
        }
    }

    synchronized void startIndexing() throws Exception {
        Long lastTimestamp = getLastIndexedTimestamp();
        Query query = searchableHandler.createQuery(lastTimestamp);
        runningQueryAction = live.engine().runQueries(query.listenWith(new QueryListener.Empty() {
            @Override
            public void onEvent(QueryEvent event, boolean history) throws Exception {
                for (Map evt : event)
                    processEventToSearchable(evt);
            }
        }));
    }

    List> loadEventsFrom(List foundDocs) throws Exception {
        UserDef user = loggedUser.getUser();
        return searchableHandler.loadFoundEvents(user, foundDocs);
    }

    private Long getLastIndexedTimestamp() {
        @Nullable Long oldLastTimestamp = lastIndexedNodeInfo.get(Long.class);
        if (oldLastTimestamp != null) {
            long now = live.time().clock().now();
            if (oldLastTimestamp > now) {
                LOGGER.warn("Last timestamp for type {} -> {} in the future (now {}), ignoring", searchableHandler.searchableName(), oldLastTimestamp, now);
                oldLastTimestamp = now;
            }
        }
        return oldLastTimestamp;
    }

    public void commitSettingsInfo() {
        try {
            long lastCommittedTimestamp = lastIndexedTimestamp.get();
            long numIndexedEvents = nIndexedEvents.getAndSet(0L);
            if (numIndexedEvents > 0){
                lastIndexedNodeInfo.withOptions(new SettingsOptions().enableLog(false), () -> {
                    lastIndexedNodeInfo.set(lastCommittedTimestamp);
                    return null;
                });
                LOGGER.info("{} {} events added to search index. Last timestamp {}", numIndexedEvents, searchableName(), lastCommittedTimestamp);
            }
        } catch (Exception e) {
            LOGGER.error("Error updating settings info for " + searchableName(), e);
        }
    }

    private void processEventToSearchable(Map event) throws Exception {
        String eventType = Type.STRING.cast(event.get(Event.TYPE));
        Double eventTimestamp = Type.NUMBER.cast(event.get(Event.TIMESTAMP));
        String eventUid = Type.STRING.cast(event.get(UID));
        if (eventType == null || eventTimestamp == null || eventUid == null)
            return;

        String uid = eventType + ':' + eventUid;
        if (event.containsKey("__delete")){
            boolean ret = indexWriter.deleteDocument(new Term(UID, uid));
            LOGGER.info(" deleting event from index {} uid-> {}", ret, eventUid);
            return;
        }

        List document = searchableHandler.eventToSearchable(uid, event);
        if(document == null || document.isEmpty()){
            return;
        }

        indexWriter.updateDocument(new Term(UID, uid), translateToLuceneDoc(document));
        lastIndexedTimestamp.accumulateAndGet(eventTimestamp.longValue(), Math::max);
        nIndexedEvents.incrementAndGet();
    }

    private List translateToLuceneDoc(List searchFields){
        List luceneFields = new ArrayList<>();
        for(SearchField sf : searchFields){
            if (sf instanceof StringSearchField){
                luceneFields.add(new StringField(sf.name(),sf.stringValue(), sf.isStore() ? Field.Store.YES : Field.Store.NO));
            }
            else if(sf instanceof TextSearchField){
                luceneFields.add(new TextField(sf.name(), sf.stringValue(), sf.isStore() ? Field.Store.YES : Field.Store.NO));
            }
            else if(sf instanceof LongPointSearchField){
                luceneFields.add(new LongPoint(sf.name(),((LongPointSearchField) sf).getValue()));
            }
            else if(sf instanceof NumericDocValuesField){
                luceneFields.add(new org.apache.lucene.document.NumericDocValuesField(sf.name(),((NumericDocValuesField) sf).getValue()));
            }
        }

        return luceneFields;
    }

    private synchronized void restartIndexingQuery() {
        try {
            stopIndexing();
            startIndexing();
        } catch (Exception e) {
            LOGGER.error("Error running {} indexing query", searchableName());
        }
    }

    private class ProviderListenerBase implements MainStorageProvider.ProviderListener {
        @Override
        public void onProviderAdded(String name, StorageProvider provider, StorageOptions options) {
            restartIndexingQuery();
        }
    }
}