com.bazoud.elasticsearch.river.git.GitRiver Maven / Gradle / Ivy

Go to download
package com.bazoud.elasticsearch.river.git;

import java.io.File;
import java.util.Map;
import java.util.regex.Pattern;

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.river.AbstractRiverComponent;
import org.elasticsearch.river.River;
import org.elasticsearch.river.RiverIndexName;
import org.elasticsearch.river.RiverName;
import org.elasticsearch.river.RiverSettings;
import org.elasticsearch.threadpool.ThreadPool;

import com.bazoud.elasticsearch.river.git.beans.Context;
import com.bazoud.elasticsearch.river.git.guava.FunctionFlow;
import com.bazoud.elasticsearch.river.git.guava.PredicateFunction;
import com.bazoud.elasticsearch.river.git.guava.functions.CloneRepositoryFunction;
import com.bazoud.elasticsearch.river.git.guava.functions.CommitIndexFunction;
import com.bazoud.elasticsearch.river.git.guava.functions.FetchRepositoryFunction;
import com.bazoud.elasticsearch.river.git.guava.functions.InitializeFunction;
import com.bazoud.elasticsearch.river.git.guava.predicates.FetchOrCloneRepositoryPredicate;
import com.google.common.base.Optional;

/**
 * @author Olivier Bazoud
 */
public class GitRiver extends AbstractRiverComponent implements River {
    private ThreadPool threadPool;
    private Client client;

    private volatile Thread indexerThread;
    private volatile boolean closed;

    private String name;
    private String uri;
    private int updateRate;
    private String indexName;
    private String typeName;
    private String workingDir;
    private String issueRegex;
    private boolean indexingDiff;

    public final static String INDEX_NAME = "git";
    public final static String TYPE_NAME = "git";

    @Inject
    protected GitRiver(RiverName riverName, RiverSettings settings, @RiverIndexName String riverIndexName, Client client, ThreadPool threadPool) {
        super(riverName, settings);
        this.client = client;
        this.threadPool = threadPool;

        logger.info("Creating Git river");

        if (settings.settings().containsKey("git")) {
            Map gitSettings = (Map) settings.settings().get("git");
            this.name = XContentMapValues.nodeStringValue(gitSettings.get("name"), null);
            this.uri = XContentMapValues.nodeStringValue(gitSettings.get("uri"), null);
            this.updateRate = XContentMapValues.nodeIntegerValue(gitSettings.get("update_rate"), 15 * 60 * 1000);
            this.typeName = XContentMapValues.nodeStringValue(gitSettings.get("type"), TYPE_NAME);
            this.workingDir = XContentMapValues.nodeStringValue(gitSettings.get("working_dir"), System.getProperty("user.home") + File.separator + ".elasticsearch-river-git");
            this.issueRegex = XContentMapValues.nodeStringValue(gitSettings.get("issue_regex"), null);
            this.indexingDiff = XContentMapValues.nodeBooleanValue(gitSettings.get("indexing_diff"), false);
            this.indexName = riverName.name();
        }
    }

    @Override
    public void start() {
        logger.info("Starting Git River: name '{}', uri '{}', updateRate '{}', indexing to '{}'/'{}'",
            this.name, this.uri, this.updateRate, this.indexName, typeName);

        try {
            client.admin().cluster().prepareHealth()
                .setWaitForYellowStatus()
                .execute().actionGet();

            client.admin().indices().prepareCreate(indexName).execute().actionGet();
            logger.info("Index '{}'/'{}' created", this.indexName, typeName);

        } catch (Exception e) {
            if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                // that's fine
            } else if (ExceptionsHelper.unwrapCause(e) instanceof ClusterBlockException) {
                // ok, not recovered yet..., lets start indexing and hope we recover by the first bulk
            } else {
                logger.warn("failed to create index '{}', disabling river...", e, indexName);
                return;
            }
        }

        indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "git_river_indexer").newThread(new GitIndexer());
        indexerThread.start();
    }

    @Override
    public void close() {
        if (closed) {
            return;
        }
        logger.info("Stopping Git River");
        indexerThread.interrupt();
        closed = true;
    }

    /**
     * @author Olivier Bazoud
     */
    public class GitIndexer implements Runnable {

        @SuppressWarnings("unchecked")
        @Override
        public void run() {
            while (true) {
                if (closed) {
                    return;
                }

                try {
                    Context context = Context.context()
                        .name(name)
                        .uri(uri)
                        .workingDir(workingDir)
                        .client(client)
                        .indexName(indexName)
                        .issuePattern(compilePattern(issueRegex))
                        .indexingDiff(indexingDiff)
                        .build();

                    FunctionFlow.flow()
                        .add(new InitializeFunction())
                        .add(
                            new PredicateFunction(
                                new FetchRepositoryFunction(),
                                new CloneRepositoryFunction(),
                                new FetchOrCloneRepositoryPredicate()
                            )
                        )
                        .add(new CommitIndexFunction())
                        .build()
                        .apply(context);

                } catch (Exception e) {
                    logger.warn("Git river exception occurred", e);
                }

                try {
                    logger.debug("Git river is going to sleep for {} ms", updateRate);
                    Thread.sleep(updateRate);
                } catch (InterruptedException e) {
                    // we shamefully swallow the interrupted exception
                    logger.warn("Git river interrupted");
                }
            }
        }

        private Optional compilePattern(String regex) {
            Optional issuePattern = Optional.absent();
            try {
                issuePattern = Optional.of(Pattern.compile(regex));
            } catch (Throwable e) {
                logger.warn("Git river exception occurred when parsing regex |{}|", issuePattern, e);
            }
            return issuePattern;
        }

    }
}