org.apache.gobblin.http.ApacheHttpAsyncClient Maven / Gradle / Ivy

Go to download
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.gobblin.http;

import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.nio.conn.NHttpClientConnectionManager;
import org.apache.http.nio.reactor.ConnectingIOReactor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.ImmutableMap;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

import org.apache.gobblin.async.Callback;
import org.apache.gobblin.broker.gobblin_scopes.GobblinScopeTypes;
import org.apache.gobblin.broker.iface.SharedResourcesBroker;
import org.apache.gobblin.utils.HttpUtils;

/**
 * An asynchronous {@link HttpClient} which sends {@link HttpUriRequest} and registers a callback.
 * It encapsulates a {@link CloseableHttpClient} instance to send the {@link HttpUriRequest}
 *
 * {@link CloseableHttpAsyncClient} is used
 */
@Slf4j
public class ApacheHttpAsyncClient extends ThrottledHttpClient  {
  private static final Logger LOG = LoggerFactory.getLogger(ApacheHttpClient.class);

  public static final String HTTP_CONN_MANAGER = "connMgrType";
  public static final String POOLING_CONN_MANAGER_MAX_TOTAL_CONN = "connMgr.pooling.maxTotalConn";
  public static final String POOLING_CONN_MANAGER_MAX_PER_CONN = "connMgr.pooling.maxPerConn";
  public static final String REQUEST_TIME_OUT_MS_KEY = "reqTimeout";
  public static final String CONNECTION_TIME_OUT_MS_KEY = "connTimeout";


  private static final Config FALLBACK =
      ConfigFactory.parseMap(ImmutableMap.builder()
          .put(REQUEST_TIME_OUT_MS_KEY, TimeUnit.SECONDS.toMillis(10L))
          .put(CONNECTION_TIME_OUT_MS_KEY, TimeUnit.SECONDS.toMillis(10L))
          .put(HTTP_CONN_MANAGER, ApacheHttpClient.ConnManager.POOLING.name())
          .put(POOLING_CONN_MANAGER_MAX_TOTAL_CONN, 20)
          .put(POOLING_CONN_MANAGER_MAX_PER_CONN, 2)
          .build());

  private final CloseableHttpAsyncClient client;

  public ApacheHttpAsyncClient(HttpAsyncClientBuilder builder, Config config, SharedResourcesBroker broker) {
    super (broker, HttpUtils.createApacheHttpClientLimiterKey(config));
    config = config.withFallback(FALLBACK);

    RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT)
        .setSocketTimeout(config.getInt(REQUEST_TIME_OUT_MS_KEY))
        .setConnectTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
        .setConnectionRequestTimeout(config.getInt(CONNECTION_TIME_OUT_MS_KEY))
        .build();

    try {
      builder.disableCookieManagement().useSystemProperties().setDefaultRequestConfig(requestConfig);
      builder.setConnectionManager(getNHttpConnManager(config));
      client = builder.build();
      client.start();
    } catch (IOException e) {
      throw new RuntimeException("ApacheHttpAsyncClient cannot be initialized");
    }
  }

  private NHttpClientConnectionManager getNHttpConnManager(Config config) throws IOException {
    NHttpClientConnectionManager httpConnManager;

    String connMgrStr = config.getString(HTTP_CONN_MANAGER);
    switch (ApacheHttpClient.ConnManager.valueOf(connMgrStr.toUpperCase())) {
      case POOLING:
        ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor();
        PoolingNHttpClientConnectionManager poolingConnMgr = new PoolingNHttpClientConnectionManager(ioReactor);
        poolingConnMgr.setMaxTotal(config.getInt(POOLING_CONN_MANAGER_MAX_TOTAL_CONN));
        poolingConnMgr.setDefaultMaxPerRoute(config.getInt(POOLING_CONN_MANAGER_MAX_PER_CONN));
        httpConnManager = poolingConnMgr;
        break;
      default:
        throw new IllegalArgumentException(connMgrStr + " is not supported");
    }

    LOG.info("Using " + httpConnManager.getClass().getSimpleName());
    return httpConnManager;
  }

  /**
   * A helper class which contains a latch so that we can achieve blocking calls even using
   * http async client APIs. Same can be achieved by invoking {@link Future#get()} returned by
   * {@link org.apache.http.nio.client.HttpAsyncClient#execute(HttpUriRequest, FutureCallback)}.
   * However this method seems to have a synchronization problem. It seems like {@link Future#get()}
   * is not fully blocked before callback is triggered.
   */
  @Getter
  private static class SyncHttpResponseCallback implements FutureCallback {
    private HttpUriRequest request = null;
    private HttpResponse response = null;
    private Exception exception = null;

    private final CountDownLatch latch = new CountDownLatch(1);

    public SyncHttpResponseCallback(HttpUriRequest request) {
      this.request = request;
    }

    @Override
    public void completed(HttpResponse result) {
      log.info ("Sync apache version request: {}, statusCode: {}", request, result.getStatusLine().getStatusCode());
      response = result;
      latch.countDown();
    }

    @Override
    public void failed(Exception ex) {
      exception = ex;
      latch.countDown();
    }

    @Override
    public void cancelled() {
      throw new UnsupportedOperationException("Should not be cancelled");
    }

    public void await() throws InterruptedException {
      latch.await();
    }
  }

  @Override
  public HttpResponse sendRequestImpl(HttpUriRequest request) throws IOException {
    SyncHttpResponseCallback callback = new SyncHttpResponseCallback(request);
    this.client.execute(request, callback);

    try {
      callback.await();
      if (callback.getException() != null) {
        throw new IOException(callback.getException());
      }
      return callback.getResponse();
    } catch (InterruptedException e) {
      throw new IOException(e);
    }
  }

  @Override
  public void sendAsyncRequestImpl(HttpUriRequest request, Callback callback) throws IOException {
    this.client.execute(request, new FutureCallback() {
      @Override
      public void completed(HttpResponse result) {
        callback.onSuccess(result);
      }

      @Override
      public void failed(Exception ex) {
        callback.onFailure(ex);
      }

      @Override
      public void cancelled() {
        throw new UnsupportedOperationException();
      }
    });
  }

  @Override
  public void close() throws IOException {
    client.close();
  }
}