io.axual.connect.plugins.http.HttpSinkConnector Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of http-sink-connector Show documentation
This connector is used to call an HTTP service for each record on a topic
There is a newer version: 1.1.0-RC1
package io.axual.connect.plugins.http;

/*-
 * ========================LICENSE_START=================================
 * HTTP Sink Connector for Kafka Connect
 * %%
 * Copyright (C) 2020 Axual B.V.
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =========================LICENSE_END==================================
 */


import static io.axual.connect.plugins.http.HttpSinkConnectorConfig.ENDPOINT_CONFIG;

import io.axual.connect.plugins.http.exceptions.HttpSinkConnectorException;
import io.axual.connect.plugins.http.helpers.SslHelper;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.net.SocketFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import org.apache.kafka.common.config.Config;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigValue;
import org.apache.kafka.connect.connector.Task;
import org.apache.kafka.connect.sink.SinkConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * The HTTP Sink Connector class is used for sending records to a specific HTTP endpoint.

 * It accepts the configuration options defined in {@link HttpSinkConnectorConfig}.
 The
 * corresponding task class for this connector is {@link HttpSinkTask}.
 *
 * It will always create the maximum number of task configurations, just passing all
 * configuration properties on to the task.
 When {@link #validate(Map)} is called it will create
 * an enriched configuration definition using {@link HttpSinkConnectorConfig#getConfigurationDefinition(Map)}
 * and validate the supplied properties.
 If no validation errors are found, the connector will
 * try to open a socket on the target host to verify the endpoint and SSL configuration.
 * 
 */
public class HttpSinkConnector extends SinkConnector {

  private static final Logger LOG = LoggerFactory.getLogger(HttpSinkConnector.class);

  Map connectorConfig = new HashMap<>();
  final SslHelper sslHelper;

  /**
   * Constructor to inject sslHelper for testing
   *
   * @param sslHelper
   */
  HttpSinkConnector(SslHelper sslHelper) {
    this.sslHelper = sslHelper;
  }

  /**
   * Default constructor required for instantiation of connector
   */
  public HttpSinkConnector() {
    this.sslHelper = SslHelper.INSTANCE;
  }

  @Override
  public void start(Map config) {
    LOG.info("Starting connector");
    if (!connectorConfig.isEmpty()) {
      this.context.raiseError(
          new HttpSinkConnectorException("Trying to start a connector which is not stopped"));
      return;
    }
    connectorConfig.putAll(config);
  }

  @Override
  public Class taskClass() {
    return HttpSinkTask.class;
  }

  @Override
  public List> taskConfigs(int maxTasks) {
    LOG.debug("Generating {} task configurations", maxTasks);
    List> taskConfigs = new ArrayList<>(maxTasks);
    for (int task = 0; task < maxTasks; task++) {
      taskConfigs.add(new HashMap<>(connectorConfig));
    }
    return taskConfigs;
  }

  @Override
  public void stop() {
    LOG.debug("Stopping connector");
    connectorConfig.clear();
  }

  @Override
  public ConfigDef config() {
    LOG.debug("Creating default HttpSinkConnectorConfig");
    return HttpSinkConnectorConfig.getConfigurationDefinition(Collections.emptyMap());
  }

  @Override
  public Config validate(Map connectorConfigs) {
    LOG.debug("Validating connector configuration");
    ConfigDef configDef = HttpSinkConnectorConfig.getConfigurationDefinition(connectorConfigs);
    List configValues = configDef.validate(connectorConfigs);

    // Only perform connection test if no errors are reported
    if (configValues.stream().anyMatch(configValue -> !configValue.errorMessages().isEmpty())) {
      LOG.info("Found errors in configuration, skipping connectivity tests");
      return new Config(configValues);
    }

    // Create config object to open test connection
    HttpSinkConnectorConfig config = new HttpSinkConnectorConfig(connectorConfigs);

    validateEndpoint(config, configValues);
    return new Config(configValues);
  }

  // Helper method that will try to open a socket connection, or in case of a HTTPS connection,
  // an SSL socket connection to the specified endpoint.
  private void validateEndpoint(HttpSinkConnectorConfig config, List configValues) {
    LOG.info("Validating endpoint settings");
    try {
      final URL endpoint = new URL(config.getEndpoint());
      final int port = endpoint.getPort() < 0 ? endpoint.getDefaultPort() : endpoint.getPort();
      if ("https".contentEquals(endpoint.getProtocol())) {
        //load and test ssl socket
        SSLContext sslContext = sslHelper
            .getContext(config.getSslCertificateAuthorityFileLocation(),
                config.getSslCertificateAuthorityCertificates());

        SSLSocketFactory socketFactory = sslContext.getSocketFactory();
        try (SSLSocket socket = (SSLSocket) socketFactory.createSocket(endpoint.getHost(), port)) {
          socket.startHandshake();
        }
      } else {
        // If the connection is refused it will be caught by the IOException
        SocketFactory.getDefault().createSocket(endpoint.getHost(), port);
      }
      LOG.debug("Connected to URL {}", config.getEndpoint());
    } catch (SSLHandshakeException e) {
      addErrorMessage(configValues, "SSL Handshake failed",
          Collections.singleton(ENDPOINT_CONFIG));
      LOG.warn("SSL Handshake failed connecting to endpoint '{}'", config.getEndpoint(), e);
    } catch (MalformedURLException e) {
      addErrorMessage(configValues, "Endpoint is not a valid URL. " + e.getMessage(),
          Collections.singleton(ENDPOINT_CONFIG));
      LOG.warn("Endpoint '{}' is not a valid", config.getEndpoint(), e);
    } catch (IOException e) {
      addErrorMessage(configValues, "Could not connect to the endpoint, " + e.getMessage(),
          Collections.singleton(ENDPOINT_CONFIG));
      LOG.warn("Could not connect to endpoint '{}'", config.getEndpoint(), e);
    }
  }

  // helper method to add error messages to the proper configuration value
  private void addErrorMessage(List configValues, String errorMessage,
      Set keys) {
    configValues.stream()
        .filter(configValue -> keys.contains(configValue.name()))
        .forEach(configValue -> configValue.addErrorMessage(errorMessage));
  }

  @Override
  public String version() {
    return HttpConnectorInfo.getVersion();
  }
}