org.apache.flume.sink.elasticsearch.client.ElasticSearchRestClient Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flume-ng-elasticsearch-sink Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.flume.sink.elasticsearch.client;

import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.sink.elasticsearch.ElasticSearchEventSerializer;
import org.apache.flume.sink.elasticsearch.IndexNameBuilder;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.common.bytes.BytesReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

/**
 * Rest ElasticSearch client which is responsible for sending bulks of events to
 * ElasticSearch using ElasticSearch HTTP API. This is configurable, so any
 * config params required should be taken through this.
 */
public class  ElasticSearchRestClient implements ElasticSearchClient {

  private static final String INDEX_OPERATION_NAME = "index";
  private static final String INDEX_PARAM = "_index";
  private static final String TYPE_PARAM = "_type";
  private static final String TTL_PARAM = "_ttl";
  private static final String BULK_ENDPOINT = "_bulk";

  private static final Logger logger = LoggerFactory.getLogger(ElasticSearchRestClient.class);

  private final ElasticSearchEventSerializer serializer;
  private final RoundRobinList serversList;
  
  private StringBuilder bulkBuilder;
  private HttpClient httpClient;
  
  public ElasticSearchRestClient(String[] hostNames,
      ElasticSearchEventSerializer serializer) {

    for (int i = 0; i < hostNames.length; ++i) {
      if (!hostNames[i].contains("http://") && !hostNames[i].contains("https://")) {
        hostNames[i] = "http://" + hostNames[i];
      }
    }
    this.serializer = serializer;

    serversList = new RoundRobinList(Arrays.asList(hostNames));
    httpClient = new DefaultHttpClient();
    bulkBuilder = new StringBuilder();
  }

  @VisibleForTesting
  public ElasticSearchRestClient(String[] hostNames,
          ElasticSearchEventSerializer serializer, HttpClient client) {
    this(hostNames, serializer);
    httpClient = client;
  }

  @Override
  public void configure(Context context) {
  }

  @Override
  public void close() {
  }

  @Override
  public void addEvent(Event event, IndexNameBuilder indexNameBuilder, String indexType,
                       long ttlMs) throws Exception {
    BytesReference content = serializer.getContentBuilder(event).bytes();
    Map> parameters = new HashMap>();
    Map indexParameters = new HashMap();
    indexParameters.put(INDEX_PARAM, indexNameBuilder.getIndexName(event));
    indexParameters.put(TYPE_PARAM, indexType);
    if (ttlMs > 0) {
      indexParameters.put(TTL_PARAM, Long.toString(ttlMs));
    }
    parameters.put(INDEX_OPERATION_NAME, indexParameters);

    Gson gson = new Gson();
    synchronized (bulkBuilder) {
      bulkBuilder.append(gson.toJson(parameters));
      bulkBuilder.append("\n");
      bulkBuilder.append(content.toBytesArray().toUtf8());
      bulkBuilder.append("\n");
    }
  }

  @Override
  public void execute() throws Exception {
    int statusCode = 0, triesCount = 0;
    HttpResponse response = null;
    String entity;
    synchronized (bulkBuilder) {
      entity = bulkBuilder.toString();
      bulkBuilder = new StringBuilder();
    }

    while (statusCode != HttpStatus.SC_OK && triesCount < serversList.size()) {
      triesCount++;
      String host = serversList.get();
      String url = host + "/" + BULK_ENDPOINT;
      HttpPost httpRequest = new HttpPost(url);
      httpRequest.setEntity(new StringEntity(entity));
      response = httpClient.execute(httpRequest);
      statusCode = response.getStatusLine().getStatusCode();
      logger.info("Status code from elasticsearch: " + statusCode);
      if (response.getEntity() != null) {
        logger.debug("Status message from elasticsearch: " +
                     EntityUtils.toString(response.getEntity(), "UTF-8"));
      }
    }

    if (statusCode != HttpStatus.SC_OK) {
      if (response.getEntity() != null) {
        throw new EventDeliveryException(EntityUtils.toString(response.getEntity(), "UTF-8"));
      } else {
        throw new EventDeliveryException("Elasticsearch status code was: " + statusCode);
      }
    }
  }
}