All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.twitter.hbc.httpclient.ClientBase Maven / Gradle / Ivy

The newest version!
/**
 * Copyright 2013 Twitter, Inc.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **/

package com.twitter.hbc.httpclient;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.twitter.hbc.RateTracker;
import com.twitter.hbc.ReconnectionManager;
import com.twitter.hbc.core.Hosts;
import com.twitter.hbc.core.HttpConstants;
import com.twitter.hbc.core.StatsReporter;
import com.twitter.hbc.core.endpoint.StreamingEndpoint;
import com.twitter.hbc.core.event.ConnectionEvent;
import com.twitter.hbc.core.event.Event;
import com.twitter.hbc.core.event.EventType;
import com.twitter.hbc.core.event.HttpResponseEvent;
import com.twitter.hbc.core.processor.HosebirdMessageProcessor;
import com.twitter.hbc.httpclient.auth.Authentication;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;

/**
 * Thread-safe.
 * TODO: better name?!?
 */
class ClientBase implements Runnable {

  private final static Logger logger = LoggerFactory.getLogger(ClientBase.class);

  private final String name;
  private final HttpClient client;

  private final StreamingEndpoint endpoint;
  private final Hosts hosts;
  private final Authentication auth;
  private final HosebirdMessageProcessor processor;
  private final ReconnectionManager reconnectionManager;

  private final AtomicReference exitEvent;

  private final CountDownLatch isRunning;

  private final RateTracker rateTracker;
  private final BlockingQueue eventsQueue;
  private final StatsReporter statsReporter;

  private final AtomicBoolean connectionEstablished;
  private final AtomicBoolean reconnect;

  ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth,
             HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker) {
    this(name, client, hosts, endpoint, auth, processor, manager, rateTracker, null);
  }

  // TODO: support setting some http timeouts?
  ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth,
             HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker,
             @Nullable BlockingQueue eventsQueue) {
    this.client = Preconditions.checkNotNull(client);
    this.name = Preconditions.checkNotNull(name);

    this.endpoint = Preconditions.checkNotNull(endpoint);
    this.hosts = Preconditions.checkNotNull(hosts);
    this.auth = Preconditions.checkNotNull(auth);

    this.processor = Preconditions.checkNotNull(processor);
    this.reconnectionManager = Preconditions.checkNotNull(manager);
    this.rateTracker = Preconditions.checkNotNull(rateTracker);

    this.eventsQueue = eventsQueue;

    this.exitEvent = new AtomicReference();

    this.isRunning = new CountDownLatch(1);
    this.statsReporter = new StatsReporter();

    this.connectionEstablished = new AtomicBoolean(false);
    this.reconnect = new AtomicBoolean(false);
  }

  @Override
  public void run() {
    // establish the initial connection
    //   if connection fails due to auth or some other 400, stop immediately
    //   if connection fails due to a 500, back off and retry
    //   if no response or other code, stop immediately
    // begin reading from the stream
    // while the stop signal hasn't been sent, and no IOException from processor, keep processing
    // if  IOException, time to restart the connection:
    //   handle http connection cleanup
    //   do some backoff, set backfill
    // if stop signal set, time to kill/clean the connection, and end this thread.
    try {
      if (client instanceof RestartableHttpClient) {
        ((RestartableHttpClient) client).setup();
      }
      rateTracker.start();
      while (!isDone()) {
        String host = hosts.nextHost();
        if (host == null) {
          setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "No hosts available"));
          break;
        }

        double rate = rateTracker.getCurrentRateSeconds();
        if (!Double.isNaN(rate)) {
          endpoint.setBackfillCount(reconnectionManager.estimateBackfill(rate));
        }

        HttpUriRequest request = HttpConstants.constructRequest(host, endpoint, auth);
        if (request != null) {
          String postContent = null;
          if (endpoint.getHttpMethod().equalsIgnoreCase(HttpConstants.HTTP_POST)) {
            postContent = endpoint.getPostParamString();
          }
          auth.signRequest(request, postContent);
          Connection conn = new Connection(client, processor);
          StatusLine status = establishConnection(conn, request);
          if (handleConnectionResult(status)) {
            rateTracker.resume();
            processConnectionData(conn);
            rateTracker.pause();
          }
          logger.info("{} Done processing, preparing to close connection", name);
          conn.close();
        } else {
          addEvent(
            new Event(
              EventType.CONNECTION_ERROR,
              String.format("Error creating request: %s, %s, %s", endpoint.getHttpMethod(), host, endpoint.getURI())
            )
          );
        }
      }
    } catch (Throwable e) {
      logger.warn(name + " Uncaught exception", e);
      Exception laundered = (e instanceof Exception) ? (Exception) e : new RuntimeException(e);
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, laundered));
    } finally {
      rateTracker.stop();
      logger.info("{} Shutting down httpclient connection manager", name);
      client.getConnectionManager().shutdown();
      isRunning.countDown();
    }
  }

  @Nullable()
  @VisibleForTesting
  StatusLine establishConnection(Connection conn, HttpUriRequest request) {
    logger.info("{} Establishing a connection", name);
    // establish connection
    StatusLine status = null;
    try {
      addEvent(new ConnectionEvent(EventType.CONNECTION_ATTEMPT, request));
      status = conn.connect(request);
    } catch (UnknownHostException e) {
      // banking on some httpHosts.nextHost() being legitimate, or else this connection will fail.
      logger.warn("{} Unknown host - {}", name, request.getURI().getHost());
      addEvent(new Event(EventType.CONNECTION_ERROR, e));
    } catch (IOException e) {
      logger.warn("{} IOException caught when establishing connection to {}", name, request.getURI());
      addEvent(new Event(EventType.CONNECTION_ERROR, e));
      reconnectionManager.handleLinearBackoff();
    } catch (Exception e) {
      logger.error(String.format("%s Unknown exception while establishing connection to %s", name, request.getURI()), e);
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, e));
    }
    return status;
  }

  /**
   * @return whether a successful connection has been established
   */
  @VisibleForTesting
  boolean handleConnectionResult(@Nullable StatusLine statusLine) {
    statsReporter.incrNumConnects();
    if (statusLine == null) {
      logger.warn("{} failed to establish connection properly", name);
      addEvent(new Event(EventType.CONNECTION_ERROR, "Failed to establish connection properly"));
      return false;
    }
    int statusCode = statusLine.getStatusCode();
    if (statusCode == HttpConstants.Codes.SUCCESS) {
      logger.debug("{} Connection successfully established", name);
      statsReporter.incrNum200s();
      connectionEstablished.set(true);
      addEvent(new HttpResponseEvent(EventType.CONNECTED, statusLine));
      reconnectionManager.resetCounts();
      return true;
    }

    logger.warn(name + " Error connecting w/ status code - {}, reason - {}", statusCode, statusLine.getReasonPhrase());
    statsReporter.incrNumConnectionFailures();
    addEvent(new HttpResponseEvent(EventType.HTTP_ERROR, statusLine));
    if (HttpConstants.FATAL_CODES.contains(statusCode)) {
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "Fatal error code: " + statusCode));
    } else if (statusCode < 500 && statusCode >= 400) {
      statsReporter.incrNum400s();
      // we will retry these a set number of times, then fail
      if (reconnectionManager.shouldReconnectOn400s()) {
        logger.debug("{} Reconnecting on {}", name, statusCode);
        reconnectionManager.handleExponentialBackoff();
      } else {
        logger.debug("{} Reconnecting retries exhausted for {}", name, statusCode);
        setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "Retries exhausted"));
      }
    } else if (statusCode >= 500) {
      statsReporter.incrNum500s();
      reconnectionManager.handleExponentialBackoff();
    } else {
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, statusLine.getReasonPhrase()));
    }
    return false;
  }

  private void processConnectionData(Connection conn) {
    logger.info("{} Processing connection data", name);
    try {
      addEvent(new Event(EventType.PROCESSING, "Processing messages"));
      while(!isDone() && !reconnect.getAndSet(false)) {
        if (conn.processResponse()) {
          statsReporter.incrNumMessages();
        } else {
          statsReporter.incrNumMessagesDropped();
        }
        rateTracker.eventObserved();
      }
    } catch (RuntimeException e) {
      logger.warn(name + " Unknown error processing connection: ", e);
      statsReporter.incrNumDisconnects();
      addEvent(new Event(EventType.DISCONNECTED, e));
    } catch (IOException ex) {
      // connection issue? whatever. let's try connecting again
      // we can't really diagnosis the actual disconnection reason without parsing (looking at disconnect message)
      // but we can make a good guess at when we're stalling. TODO
      logger.info("{} Disconnected during processing - will reconnect", name);
      statsReporter.incrNumDisconnects();
      addEvent(new Event(EventType.DISCONNECTED, ex));
    } catch (InterruptedException interrupt) {
      // interrupted while trying to append message to queue. exit
      logger.info("{} Thread interrupted during processing, exiting", name);
      statsReporter.incrNumDisconnects();
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, interrupt));
    } catch (Exception e) {
      // Unexpected exception thrown, killing everything
      logger.warn(name + " Unexpected exception during processing", e);
      statsReporter.incrNumDisconnects();
      setExitStatus(new Event(EventType.STOPPED_BY_ERROR, e));
    }
  }

  private void setExitStatus(Event event) {
    logger.info("{} exit event - {}", name, event.getMessage());
    addEvent(event);
    exitEvent.set(event);
  }

  private void addEvent(Event event) {
    if (eventsQueue != null) {
      if (!eventsQueue.offer(event)) {
        statsReporter.incrNumClientEventsDropped();
      }
    }
  }

  public void reconnect() {
    if (connectionEstablished.get()) {
      reconnect.set(true);
    }
  }

  /**
   * Stops the current connection. No reconnecting will occur. Kills thread + cleanup.
   * Waits for the loop to end
   **/
  public void stop(int waitMillis) throws InterruptedException {
    try {
      if (!isDone()) {
        setExitStatus(new Event(EventType.STOPPED_BY_USER, String.format("Stopped by user: waiting for %d ms", waitMillis)));
      }
      if (!waitForFinish(waitMillis)) {
        logger.warn("{} Client thread failed to finish in {} millis", name, waitMillis);
      }
    } finally {
      rateTracker.shutdown();
    }
  }

  public void shutdown(int millis) {
    try {
      stop(millis);
    } catch (InterruptedException e) {
      logger.warn("Client failed to shutdown due to interruption", e);
    }
  }

  public boolean isDone() {
    return exitEvent.get() != null;
  }

  public Event getExitEvent() {
    if (!isDone()) {
      throw new IllegalStateException(name + " Still running");
    }
    return exitEvent.get();
  }

  public boolean waitForFinish(int millis) throws InterruptedException {
    return isRunning.await(millis, TimeUnit.MILLISECONDS);
  }

  public void waitForFinish() throws InterruptedException {
    isRunning.await();
  }

  @Override
  public String toString() {
    return String.format("%s, endpoint: %s", getName(), endpoint.getURI());
  }

  public String getName() {
    return name;
  }

  public StreamingEndpoint getEndpoint() {
    return endpoint;
  }

  public StatsReporter.StatsTracker getStatsTracker() {
    return statsReporter.getStatsTracker();
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy