Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
io.debezium.connector.mysql.MySqlConnectorTask Maven / Gradle / Ivy
/*
* Copyright Debezium Authors.
*
* Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
*/
package io.debezium.connector.mysql;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.kafka.connect.source.SourceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.debezium.DebeziumException;
import io.debezium.bean.StandardBeanNames;
import io.debezium.config.Configuration;
import io.debezium.config.Field;
import io.debezium.connector.base.ChangeEventQueue;
import io.debezium.connector.binlog.BinlogEventMetadataProvider;
import io.debezium.connector.binlog.BinlogSourceTask;
import io.debezium.connector.binlog.jdbc.BinlogConnectorConnection;
import io.debezium.connector.mysql.jdbc.MySqlConnection;
import io.debezium.connector.mysql.jdbc.MySqlConnectionConfiguration;
import io.debezium.connector.mysql.jdbc.MySqlFieldReaderResolver;
import io.debezium.connector.mysql.jdbc.MySqlValueConverters;
import io.debezium.document.DocumentReader;
import io.debezium.jdbc.DefaultMainConnectionProvidingConnectionFactory;
import io.debezium.jdbc.MainConnectionProvidingConnectionFactory;
import io.debezium.pipeline.ChangeEventSourceCoordinator;
import io.debezium.pipeline.DataChangeEvent;
import io.debezium.pipeline.ErrorHandler;
import io.debezium.pipeline.EventDispatcher;
import io.debezium.pipeline.notification.NotificationService;
import io.debezium.pipeline.signal.SignalProcessor;
import io.debezium.pipeline.spi.Offsets;
import io.debezium.relational.TableId;
import io.debezium.schema.SchemaFactory;
import io.debezium.schema.SchemaNameAdjuster;
import io.debezium.snapshot.SnapshotterService;
import io.debezium.spi.snapshot.Snapshotter;
import io.debezium.spi.topic.TopicNamingStrategy;
import io.debezium.util.Clock;
/**
* The main task executing streaming from MySQL.
* Responsible for lifecycle management of the streaming code.
*
* @author Jiri Pechanec
*
*/
public class MySqlConnectorTask extends BinlogSourceTask {
private static final Logger LOGGER = LoggerFactory.getLogger(MySqlConnectorTask.class);
private static final String CONTEXT_NAME = "mysql-connector-task";
private volatile MySqlTaskContext taskContext;
private volatile ChangeEventQueue queue;
private volatile BinlogConnectorConnection connection;
private volatile BinlogConnectorConnection beanRegistryJdbcConnection;
private volatile ErrorHandler errorHandler;
private volatile MySqlDatabaseSchema schema;
@Override
public String version() {
return Module.version();
}
@Override
public ChangeEventSourceCoordinator start(Configuration configuration) {
final Clock clock = Clock.system();
final MySqlConnectorConfig connectorConfig = new MySqlConnectorConfig(configuration);
final TopicNamingStrategy topicNamingStrategy = connectorConfig.getTopicNamingStrategy(MySqlConnectorConfig.TOPIC_NAMING_STRATEGY);
final SchemaNameAdjuster schemaNameAdjuster = connectorConfig.schemaNameAdjuster();
final MySqlValueConverters valueConverters = getValueConverters(connectorConfig);
// DBZ-3238: automatically set "useCursorFetch" to true when a snapshot fetch size other than the default of -1 is given
// By default do not load whole result sets into memory
final Configuration config = configuration.edit()
.withDefault("database.responseBuffering", "adaptive")
.withDefault("database.fetchSize", 10_000)
.withDefault("database.useCursorFetch", connectorConfig.useCursorFetch())
.build();
MainConnectionProvidingConnectionFactory connectionFactory = new DefaultMainConnectionProvidingConnectionFactory<>(() -> {
final MySqlConnectionConfiguration connectionConfig = new MySqlConnectionConfiguration(config);
return new MySqlConnection(connectionConfig, MySqlFieldReaderResolver.resolve(connectorConfig));
});
connection = connectionFactory.mainConnection();
Offsets previousOffsets = getPreviousOffsets(
new MySqlPartition.Provider(connectorConfig, config),
new MySqlOffsetContext.Loader(connectorConfig));
final boolean tableIdCaseInsensitive = connection.isTableIdCaseSensitive();
this.schema = new MySqlDatabaseSchema(connectorConfig, valueConverters, topicNamingStrategy, schemaNameAdjuster, tableIdCaseInsensitive);
// Manual Bean Registration
beanRegistryJdbcConnection = connectionFactory.newConnection();
connectorConfig.getBeanRegistry().add(StandardBeanNames.CONFIGURATION, config);
connectorConfig.getBeanRegistry().add(StandardBeanNames.CONNECTOR_CONFIG, connectorConfig);
connectorConfig.getBeanRegistry().add(StandardBeanNames.DATABASE_SCHEMA, schema);
connectorConfig.getBeanRegistry().add(StandardBeanNames.JDBC_CONNECTION, beanRegistryJdbcConnection);
connectorConfig.getBeanRegistry().add(StandardBeanNames.VALUE_CONVERTER, valueConverters);
connectorConfig.getBeanRegistry().add(StandardBeanNames.OFFSETS, previousOffsets);
// Service providers
registerServiceProviders(connectorConfig.getServiceRegistry());
final SnapshotterService snapshotterService = connectorConfig.getServiceRegistry().tryGetService(SnapshotterService.class);
final Snapshotter snapshotter = snapshotterService.getSnapshotter();
validateBinlogConfiguration(snapshotter, connection);
// If the binlog position is not available it is necessary to re-execute snapshot
if (validateSnapshotFeasibility(snapshotter, previousOffsets.getTheOnlyOffset(), connection)) {
previousOffsets.resetOffset(previousOffsets.getTheOnlyPartition());
}
LOGGER.info("Closing connection before starting schema recovery");
try {
connection.close();
}
catch (SQLException e) {
throw new DebeziumException(e);
}
MySqlOffsetContext previousOffset = previousOffsets.getTheOnlyOffset();
validateAndLoadSchemaHistory(connectorConfig, connection::validateLogPosition, previousOffsets, schema, snapshotter);
LOGGER.info("Reconnecting after finishing schema recovery");
try {
connection.setAutoCommit(false);
}
catch (SQLException e) {
throw new DebeziumException(e);
}
// If the binlog position is not available it is necessary to re-execute snapshot
if (previousOffset == null) {
LOGGER.info("No previous offset found");
}
else {
LOGGER.info("Found previous offset {}", previousOffset);
}
taskContext = new MySqlTaskContext(connectorConfig, schema);
// Set up the task record queue ...
this.queue = new ChangeEventQueue.Builder()
.pollInterval(connectorConfig.getPollInterval())
.maxBatchSize(connectorConfig.getMaxBatchSize())
.maxQueueSize(connectorConfig.getMaxQueueSize())
.maxQueueSizeInBytes(connectorConfig.getMaxQueueSizeInBytes())
.loggingContextSupplier(() -> taskContext.configureLoggingContext(CONTEXT_NAME))
.buffering()
.build();
errorHandler = new MySqlErrorHandler(connectorConfig, queue, errorHandler);
final BinlogEventMetadataProvider metadataProvider = new BinlogEventMetadataProvider();
SignalProcessor signalProcessor = new SignalProcessor<>(
MySqlConnector.class, connectorConfig, Map.of(),
getAvailableSignalChannels(),
DocumentReader.defaultReader(),
previousOffsets);
resetOffset(connectorConfig, previousOffset, signalProcessor);
final Configuration heartbeatConfig = config;
final EventDispatcher dispatcher = new EventDispatcher<>(
connectorConfig,
topicNamingStrategy,
schema,
queue,
connectorConfig.getTableFilters().dataCollectionFilter(),
DataChangeEvent::new,
null,
metadataProvider,
connectorConfig.createHeartbeat(
topicNamingStrategy,
schemaNameAdjuster,
() -> new MySqlConnection(
new MySqlConnectionConfiguration(heartbeatConfig),
MySqlFieldReaderResolver.resolve(connectorConfig)),
new BinlogHeartbeatErrorHandler()),
schemaNameAdjuster,
signalProcessor);
final MySqlStreamingChangeEventSourceMetrics streamingMetrics = new MySqlStreamingChangeEventSourceMetrics(taskContext, queue, metadataProvider);
NotificationService notificationService = new NotificationService<>(getNotificationChannels(),
connectorConfig, SchemaFactory.get(), dispatcher::enqueueNotification);
ChangeEventSourceCoordinator coordinator = new ChangeEventSourceCoordinator<>(
previousOffsets,
errorHandler,
MySqlConnector.class,
connectorConfig,
new MySqlChangeEventSourceFactory(
connectorConfig,
connectionFactory,
errorHandler,
dispatcher,
clock,
schema,
taskContext,
streamingMetrics,
queue,
snapshotterService),
new MySqlChangeEventSourceMetricsFactory(streamingMetrics),
dispatcher,
schema,
signalProcessor,
notificationService,
snapshotterService);
coordinator.start(taskContext, this.queue, metadataProvider);
return coordinator;
}
private MySqlValueConverters getValueConverters(MySqlConnectorConfig configuration) {
return new MySqlValueConverters(
configuration.getDecimalMode(),
configuration.getTemporalPrecisionMode(),
configuration.getBigIntUnsignedHandlingMode().asBigIntUnsignedMode(),
configuration.binaryHandlingMode(),
configuration.isTimeAdjustedEnabled() ? MySqlValueConverters::adjustTemporal : x -> x,
configuration.getEventConvertingFailureHandlingMode(),
configuration.getServiceRegistry());
}
@Override
public List doPoll() throws InterruptedException {
final List records = queue.poll();
return records.stream().map(DataChangeEvent::getRecord).collect(Collectors.toList());
}
@Override
protected void doStop() {
try {
if (connection != null) {
connection.close();
}
}
catch (SQLException e) {
LOGGER.error("Exception while closing JDBC connection", e);
}
try {
if (beanRegistryJdbcConnection != null) {
beanRegistryJdbcConnection.close();
}
}
catch (SQLException e) {
LOGGER.error("Exception while closing JDBC bean registry connection", e);
}
if (schema != null) {
schema.close();
}
}
@Override
protected Iterable getAllConfigurationFields() {
return MySqlConnectorConfig.ALL_FIELDS;
}
@Override
@SuppressWarnings("unchecked")
protected Long getReadOnlyIncrementalSnapshotSignalOffset(MySqlOffsetContext previousOffset) {
return ((MySqlReadOnlyIncrementalSnapshotContext) previousOffset.getIncrementalSnapshotContext()).getSignalOffset();
}
}
