• Home
• jp.co.bizreach
• elastic-scala-httpclient_2.11
• 1.0.8
• source code
• AsyncESClient.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

jp.co.bizreach.elasticsearch4s.AsyncESClient.scala Maven / Gradle / Ivy

package jp.co.bizreach.elasticsearch4s

import ESUtils._
import org.elasticsearch.action.search.SearchRequestBuilder
import org.slf4j.LoggerFactory
import org.elasticsearch.client.support.AbstractClient
import scala.reflect.ClassTag
import com.ning.http.client.AsyncHttpClient
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global

object AsyncESClient {
  private var httpClient: AsyncHttpClient = null

  def using[T](url: String)(f: AsyncESClient => Future[T]): Future[T] = {
    val httpClient = HttpUtils.createHttpClient()
    val client = new AsyncESClient(new QueryBuilderClient(), httpClient, url)
    val future = f(client)
    future.onComplete { case t =>
      httpClient.close()
    }
    future
  }

  def apply(url: String): AsyncESClient = {
    if(httpClient == null){
      throw new IllegalStateException("AsyncHttpClient has not been initialized. Call AsyncESClient.init() at first.")
    }
    new AsyncESClient(new QueryBuilderClient(), httpClient, url)
  }

  def init() = {
    httpClient = HttpUtils.createHttpClient()
  }

  def shutdown() = {
    httpClient.close()
  }
}

class AsyncESClient(queryClient: AbstractClient, httpClient: AsyncHttpClient, url: String) {

  val logger = LoggerFactory.getLogger(classOf[AsyncESClient])

  def searchAsync(config: ESConfig)(f: SearchRequestBuilder => Unit): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug("******** ESConfig:" + config.toString)
    val searcher = queryClient.prepareSearch(config.indexName)
    config.typeName.foreach(x => searcher.setTypes(x))
    f(searcher)
    logger.debug(s"searchRequest:${searcher.toString}")

    val future = HttpUtils.postAsync(httpClient, config.preferenceUrl(url, "/_search"), searcher.toString)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def searchAllAsync(config: ESConfig)(f: SearchRequestBuilder => Unit): Future[Either[Map[String, Any], Map[String, Any]]] = {
    countAsync(config)(f).flatMap { result =>
      result match {
        case Left(x)  => Future(Left(x))
        case Right(x) => {
          val total = x("count").asInstanceOf[Int]
          searchAsync(config) { searcher =>
            f(searcher)
            searcher.setFrom(0)
            searcher.setSize(total)
          }
        }
      }
    }
  }

  def searchByTemplateAsync(config: ESConfig)(lang: String, template: String, params: AnyRef, options: Option[String] = None): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug("******** ESConfig:" + config.toString)
    val json = JsonUtils.serialize(
      Map(
        "lang" -> lang,
        "template" -> Map(
          "file" -> template
        ),
        "params" -> params
      )
    )
    logger.debug(s"searchRequest:${json}")

    val future = HttpUtils.postAsync(httpClient, config.urlWithParameters(url, "_search/template" + options.getOrElse("")), json)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def findAsync[T](config: ESConfig)(f: SearchRequestBuilder => Unit)(implicit c: ClassTag[T]): Future[Option[(String, T)]] = {
    searchAsync(config)(f).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => {
          val hits = x("hits").asInstanceOf[Map[String, Any]]("hits").asInstanceOf[Seq[Map[String, Any]]]
          if(hits.length == 0){
            None
          } else {
            Some((hits.head("_id").toString, JsonUtils.deserialize[T](JsonUtils.serialize(getDocumentMap(hits.head)))))
          }
        }
      }
    }
  }

  def findAsListAsync[T](config: ESConfig)(f: SearchRequestBuilder => Unit)(implicit c: ClassTag[T]): Future[List[(String, T)]] = {
    searchAsync(config)(f).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => createESSearchResult(x).list.map { x => (x.id, x.doc) }
      }
    }
  }

  def findAllAsListAsync[T](config: ESConfig)(f: SearchRequestBuilder => Unit)(implicit c: ClassTag[T]): Future[List[(String, T)]] = {
    findAsListAsync(config){ searcher =>
      countAsIntAsync(config)(f).map { count =>
        f(searcher)
        searcher.setFrom(0)
        searcher.setSize(count)
      }
    }
  }

  def listAsync[T](config: ESConfig)(f: SearchRequestBuilder => Unit)(implicit c: ClassTag[T]): Future[ESSearchResult[T]] = {
    searchAsync(config)(f).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => createESSearchResult(x)
      }
    }
  }

  def listAllAsync[T](config: ESConfig)(f: SearchRequestBuilder => Unit)(implicit c: ClassTag[T]): Future[ESSearchResult[T]] = {
    listAsync(config){ searcher =>
      countAsIntAsync(config)(f).map { count =>
        f(searcher)
        searcher.setFrom(0)
        searcher.setSize(count)
      }
    }
  }

  def listByTemplateAsync[T](config: ESConfig)(lang: String, template: String, params: AnyRef)(implicit c: ClassTag[T]): Future[ESSearchResult[T]] = {
    searchByTemplateAsync(config)(lang, template, params).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => createESSearchResult(x)
      }
    }
  }

  def insertJsonAsync(config: ESConfig, json: String): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug(s"insertJson:\n${json}")
    logger.debug(s"insertRequest:\n${json}")

    val future = HttpUtils.postAsync(httpClient, config.url(url), json)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def insertAsync(config: ESConfig, entity: AnyRef):  Future[Either[Map[String, Any], Map[String, Any]]] = {
    insertJsonAsync(config, JsonUtils.serialize(entity))
  }

  def updateJsonAsync(config: ESConfig, id: String, json: String): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug(s"updateJson:\n${json}")
    logger.debug(s"updateRequest:\n${json}")

    val future = HttpUtils.putAsync(httpClient, config.url(url) + "/" + id, json)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def updateAsync(config: ESConfig, id: String, entity: AnyRef): Future[Either[Map[String, Any], Map[String, Any]]] = {
    updateJsonAsync(config, id, JsonUtils.serialize(entity))
  }

  def deleteAsync(config: ESConfig, id: String): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug(s"delete id:\n${id}")

    val future = HttpUtils.deleteAsync(httpClient, config.url(url) + "/" + id)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def deleteByQueryAsync(config: ESConfig)(f: SearchRequestBuilder => Unit): Future[Either[Map[String, Any], Map[String, Any]]] = ???

  def countAsync(config: ESConfig)(f: SearchRequestBuilder => Unit): Future[Either[Map[String, Any], Map[String, Any]]] = {
    logger.debug("******** ESConfig:" + config.toString)
    val searcher = queryClient.prepareSearch(config.indexName)
    config.typeName.foreach(x => searcher.setTypes(x))
    f(searcher)
    logger.debug(s"countRequest:${searcher.toString}")

    val future = HttpUtils.postAsync(httpClient, config.preferenceUrl(url, "_count"), searcher.toString)
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def countAsIntAsync(config: ESConfig)(f: SearchRequestBuilder => Unit): Future[Int] = {
    countAsync(config)(f).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => x("count").asInstanceOf[Int]
      }
    }
  }

  def countByTemplateAsync(config: ESConfig)(lang: String, template: String, params: AnyRef): Future[Either[Map[String, Any], Map[String, Any]]] = {
    searchByTemplateAsync(config)(lang, template, params, Some("?search_type=count"))
  }

  def countByTemplateAsIntAsync(config: ESConfig)(lang: String, template: String, params: AnyRef): Future[Int] = {
    countByTemplateAsync(config)(lang: String, template: String, params: AnyRef).map { result =>
      result match {
        case Left(x)  => throw new RuntimeException(x("error").toString)
        case Right(x) => x("hits").asInstanceOf[Map[String, Any]]("total").asInstanceOf[Int]
      }
    }
  }

  def refreshAsync(config: ESConfig)(): Future[Either[Map[String, Any], Map[String, Any]]] = {
    val future = HttpUtils.postAsync(httpClient, s"${url}/${config.indexName}/_refresh", "")
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

  def scrollAsync[T, R](config: ESConfig)(f: SearchRequestBuilder => Unit)(p: (String, T) => R)(implicit c1: ClassTag[T], c2: ClassTag[R]): Future[Stream[R]] = {
    def scroll0[R](init: Boolean, searchUrl: String, body: String, stream: Stream[R], invoker: (String, Map[String, Any]) => R): Future[Stream[R]] = {
      val future = HttpUtils.postAsync(httpClient, searchUrl + "?scroll=5m" + (if(init) "&search_type=scan" else ""), body)
      future.flatMap { resultJson =>
        val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
        if(map.get("error").isDefined){
          throw new RuntimeException(map("error").toString)
        } else {
          val scrollId = map("_scroll_id").toString
          val list = map("hits").asInstanceOf[Map[String, Any]]("hits").asInstanceOf[List[Map[String, Any]]]
          list match {
            case Nil if init == false => Future(stream)
            case Nil  => scroll0(false, s"${url}/_search/scroll", scrollId, stream, invoker)
            case list => scroll0(false, s"${url}/_search/scroll", scrollId, list.map { map => invoker(map("_id").toString, getDocumentMap(map)) }.toStream #::: stream, invoker)
          }
        }
      }
    }

    logger.debug("******** ESConfig:" + config.toString)
    val searcher = queryClient.prepareSearch(config.indexName)
    config.typeName.foreach(x => searcher.setTypes(x))
    f(searcher)
    logger.debug(s"searchRequest:${searcher.toString}")

    scroll0(true, config.url(url) + "/_search", searcher.toString, Stream.empty,
      (_id: String, map: Map[String, Any]) => p(_id, JsonUtils.deserialize[T](JsonUtils.serialize(map))))
  }

  def scrollChunkAsync[T, R](config: ESConfig)(f: SearchRequestBuilder => Unit)(p: (Seq[(String, T)]) => R)(implicit c1: ClassTag[T], c2: ClassTag[R]): Future[Stream[R]] = {
    def scroll0[R](init: Boolean, searchUrl: String, body: String, stream: Stream[R], invoker: (Seq[(String, Map[String, Any])]) => R): Future[Stream[R]] = {
      val future = HttpUtils.postAsync(httpClient, searchUrl + "?scroll=5m" + (if(init) "&search_type=scan" else ""), body)
      future.flatMap { resultJson =>
        val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
        if(map.get("error").isDefined){
          throw new RuntimeException(map("error").toString)
        } else {
          val scrollId = map("_scroll_id").toString
          val list = map("hits").asInstanceOf[Map[String, Any]]("hits").asInstanceOf[List[Map[String, Any]]]
          list match {
            case Nil if init == false => Future(stream)
            case Nil  => scroll0(false, s"${url}/_search/scroll", scrollId, stream, invoker)
            case list => scroll0(false, s"${url}/_search/scroll", scrollId, Seq(invoker(list.map { map => (map("_id").toString, getDocumentMap(map)) })).toStream #::: stream, invoker)
          }
        }
      }
    }

    logger.debug("******** ESConfig:" + config.toString)
    val searcher = queryClient.prepareSearch(config.indexName)
    config.typeName.foreach(x => searcher.setTypes(x))
    f(searcher)
    logger.debug(s"searchRequest:${searcher.toString}")

    scroll0(true, config.url(url) + "/_search", searcher.toString, Stream.empty,
      (maps: Seq[(String, Map[String, Any])]) => p(maps.map { case (id, map) =>
        (id, JsonUtils.deserialize[T](JsonUtils.serialize(map)))
      })
    )
  }

  def bulkAsync[T](actions: Seq[BulkAction]): Future[Either[Map[String, Any], Map[String, Any]]] = {
    val future = HttpUtils.postAsync(httpClient, s"${url}/_bulk", actions.map(_.jsonString).mkString("", "\n", "\n"))
    future.map { resultJson =>
      val map = JsonUtils.deserialize[Map[String, Any]](resultJson)
      map.get("error").map { case message: String => Left(map) }.getOrElse(Right(map))
    }
  }

}