io.axual.connect.plugins.http.HttpSinkTask Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of http-sink-connector Show documentation
Show all versions of http-sink-connector Show documentation
This connector is used to call an HTTP service for each record on a topic
The newest version!
package io.axual.connect.plugins.http;
/*-
* ========================LICENSE_START=================================
* HTTP Sink Connector for Kafka Connect
* %%
* Copyright (C) 2020 Axual B.V.
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =========================LICENSE_END==================================
*/
import static io.axual.connect.plugins.http.HttpSinkConnectorConfig.AUTHENTICATION_PROVIDER_CLASS_PARAM_PREFIX;
import static io.axual.connect.plugins.http.HttpSinkConnectorConfig.HEADER_SELECTOR_CLASS_PARAM_PREFIX;
import static io.axual.connect.plugins.http.HttpSinkConnectorConfig.MESSAGE_FORMATTER_CLASS_PARAM_PREFIX;
import io.axual.connect.plugins.http.exceptions.HttpSinkConnectorException;
import io.axual.connect.plugins.http.helpers.SslHelper;
import io.axual.connect.plugins.http.sender.ContentLogger;
import io.axual.connect.plugins.http.sender.HttpSender;
import io.axual.connect.plugins.http.sender.HttpSenderConfiguration;
import io.axual.connect.plugins.http.sender.HttpSenderResult;
import io.axual.connect.plugins.http.sender.HttpSenderRetryStrategy;
import io.axual.connect.plugins.http.sender.IAuthenticationProvider;
import io.axual.connect.plugins.http.sender.IHeaderSelector;
import io.axual.connect.plugins.http.sender.IMessageFormatter;
import java.util.Collection;
import java.util.Map;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.connect.sink.SinkRecord;
import org.apache.kafka.connect.sink.SinkTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This {@code SinkTask} implementation will use the {@link HttpSender} to send the {@code
* SinkRecord} provided by Kafka Connect to an HTTP endpoint.
*
* When started the task will create a {@link HttpSinkConnectorConfig} to get the provided
* configuration more easily. It will then construct the {@link HttpSenderConfiguration} to
* configure the tasks instance of the {@link HttpSender}
*
* This task will try to send all records provided using the {@link #put(Collection)} method
* synchronously to the HTTP endpoint using {@link HttpSender#sendRecord(SinkRecord)}.
If the
* send has failed it will throw an exception to stop the task
*/
public class HttpSinkTask extends SinkTask {
private static final Logger LOG = LoggerFactory.getLogger(HttpSinkTask.class);
public static final String ERROR_MESSAGE_SEND_RECORD = "Could not send record";
public static final String ERROR_MESSAGE_SEND_RECORD_STATUS_INFO_FORMAT = "Could not send record. Status code : %d Reason '%s'";
HttpSinkConnectorConfig config;
HttpSenderConfiguration httpSenderConfiguration;
private final HttpSender httpSender;
private final SslHelper sslHelper;
/**
* Used for testing
*
* @param sender to inject a mocked sender
*/
HttpSinkTask(HttpSender sender, SslHelper sslHelper) {
httpSender = sender;
this.sslHelper = sslHelper;
}
public HttpSinkTask() {
httpSender = new HttpSender();
sslHelper = SslHelper.INSTANCE;
}
@Override
public String version() {
LOG.debug("Return version");
return HttpConnectorInfo.getVersion();
}
@Override
public void start(Map configs) {
LOG.info("Starting connector");
config = new HttpSinkConnectorConfig(configs);
final SSLContext sslContext = sslHelper
.getContext(config.getSslCertificateAuthorityFileLocation(),
config.getSslCertificateAuthorityCertificates());
final String[] supportedProtocols = config.getSslProtocols().toArray(new String[0]);
final String[] supportedCipherSuites = config.getSslCipherSuites().toArray(new String[0]);
final HostnameVerifier hostnameVerifier =
Boolean.FALSE.equals(config.getSslEnableHostnameVerification())
? NoopHostnameVerifier.INSTANCE :
new DefaultHostnameVerifier();
final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
supportedProtocols, supportedCipherSuites, hostnameVerifier);
final RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(config.allowRedirects())
.setCircularRedirectsAllowed(config.allowCircularRedirects())
.setMaxRedirects(config.getMaximumRedirects())
.setSocketTimeout(config.getSocketTimeoutMs())
.setConnectionRequestTimeout(config.getConnectionRequestTimeoutMs())
.setConnectTimeout(config.getConnectionTimeoutMs())
.setAuthenticationEnabled(
!(config.getAuthenticationProvider() instanceof NoopHostnameVerifier))
.build();
final HttpSenderRetryStrategy retryStrategy = new HttpSenderRetryStrategy(
config.getValidStatusCodes(), config.getMaximumRetries(), config.getRetryWaitMs());
final String loggerName = config.getContentLoggerName();
final ContentLogger contentLogger = new ContentLogger(
loggerName == null ? null : LoggerFactory.getLogger(loggerName));
final IAuthenticationProvider authenticationProvider = config.getAuthenticationProvider();
authenticationProvider
.configure(config.originalsWithPrefix(AUTHENTICATION_PROVIDER_CLASS_PARAM_PREFIX));
final IMessageFormatter messageFormatter = config.getMessageFormatter();
messageFormatter.configure(config.originalsWithPrefix(MESSAGE_FORMATTER_CLASS_PARAM_PREFIX));
final IHeaderSelector headerSelector = config.getHeaderSelector();
headerSelector.configure(config.originalsWithPrefix(HEADER_SELECTOR_CLASS_PARAM_PREFIX));
httpSenderConfiguration = new HttpSenderConfiguration(
authenticationProvider,
messageFormatter,
headerSelector,
config.getEndpoint(),
config.getMethod(),
sslsf,
requestConfig,
retryStrategy,
ContentType.parse(config.getContentType()),
config.getStaticHeaders(), contentLogger);
httpSender.configure(httpSenderConfiguration);
}
@Override
public void put(Collection records) {
LOG.info("Putting {} records", records.size());
for (SinkRecord record : records) {
HttpSenderResult result = httpSender.sendRecord(record);
if (result.isSuccess()) {
LOG.info("Record send");
} else {
final String logMessage;
if (result.getStatusLine() != null) {
// HTTP Status information is available, use it
StatusLine status = result.getStatusLine();
logMessage = String.format(ERROR_MESSAGE_SEND_RECORD_STATUS_INFO_FORMAT, status.getStatusCode(), status.getReasonPhrase());
} else {
// Use the standard log message
logMessage = ERROR_MESSAGE_SEND_RECORD;
}
// Log a predictable message
LOG.error(logMessage, result.getException());
throw new HttpSinkConnectorException(logMessage, result.getException());
}
}
}
@Override
public void flush(Map currentOffsets) {
LOG.info("Prepare to flush records");
super.flush(currentOffsets);
}
@Override
public void stop() {
LOG.info("Stopped");
config = null;
httpSenderConfiguration = null;
httpSender.close();
}
}