com.mongodb.kafka.connect.sink.MongoSinkConfig Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of mongo-kafka-connect Show documentation
Show all versions of mongo-kafka-connect Show documentation
The official MongoDB Apache Kafka Connect Connector.
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Original Work: Apache License, Version 2.0, Copyright 2017 Hans-Peter Grahsl.
*/
package com.mongodb.kafka.connect.sink;
import static com.mongodb.kafka.connect.sink.MongoSinkTask.LOGGER;
import static com.mongodb.kafka.connect.sink.MongoSinkTopicConfig.TOPIC_OVERRIDE_PREFIX;
import static com.mongodb.kafka.connect.sink.SinkConfigSoftValidator.logIncompatibleProperties;
import static com.mongodb.kafka.connect.sink.SinkConfigSoftValidator.logObsoleteProperties;
import static com.mongodb.kafka.connect.util.ServerApiConfig.addServerApiConfig;
import static com.mongodb.kafka.connect.util.SslConfigs.addSslConfigDef;
import static com.mongodb.kafka.connect.util.Validators.errorCheckingPasswordValueValidator;
import static com.mongodb.kafka.connect.util.custom.credentials.CustomCredentialProviderConstants.CUSTOM_AUTH_ENABLE_CONFIG;
import static com.mongodb.kafka.connect.util.custom.credentials.CustomCredentialProviderGenericInitializer.initializeCustomProvider;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
import static org.apache.kafka.common.config.ConfigDef.Width;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.kafka.common.config.AbstractConfig;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigDef.Importance;
import org.apache.kafka.common.config.ConfigDef.Type;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.config.ConfigValue;
import com.mongodb.ConnectionString;
import com.mongodb.kafka.connect.MongoSinkConnector;
import com.mongodb.kafka.connect.util.Validators;
import com.mongodb.kafka.connect.util.custom.credentials.CustomCredentialProvider;
public class MongoSinkConfig extends AbstractConfig {
private static final String EMPTY_STRING = "";
public static final String TOPICS_CONFIG = MongoSinkConnector.TOPICS_CONFIG;
private static final String TOPICS_DOC =
"A list of kafka topics for the sink connector, separated by commas";
public static final String TOPICS_DEFAULT = EMPTY_STRING;
private static final String TOPICS_DISPLAY = "The Kafka topics";
public static final String TOPICS_REGEX_CONFIG = "topics.regex";
private static final String TOPICS_REGEX_DOC =
"Regular expression giving topics to consume. "
+ "Under the hood, the regex is compiled to a java.util.regex.Pattern. "
+ "Only one of "
+ TOPICS_CONFIG
+ " or "
+ TOPICS_REGEX_CONFIG
+ " should be specified.";
private static final String TOPICS_REGEX_DEFAULT = EMPTY_STRING;
private static final String TOPICS_REGEX_DISPLAY = "Topics regex";
public static final String CONNECTION_URI_CONFIG = "connection.uri";
private static final String CONNECTION_URI_DEFAULT = "mongodb://localhost:27017";
private static final String CONNECTION_URI_DISPLAY = "MongoDB Connection URI";
private static final String CONNECTION_URI_DOC =
"The connection URI as supported by the official drivers. "
+ "eg: ``mongodb://user@pass@locahost/``.";
public static final String TOPIC_OVERRIDE_CONFIG = "topic.override.%s.%s";
private static final String TOPIC_OVERRIDE_DEFAULT = EMPTY_STRING;
private static final String TOPIC_OVERRIDE_DISPLAY = "Per topic configuration overrides.";
public static final String TOPIC_OVERRIDE_DOC =
"The overrides configuration allows for per topic customization of configuration. "
+ "The customized overrides are merged with the default configuration, to create the specific configuration for a topic.\n"
+ "For example, ``topic.override.foo.collection=bar`` will store data from the ``foo`` topic into the ``bar`` collection.\n"
+ "Note: All configuration options apart from '"
+ CONNECTION_URI_CONFIG
+ "' and '"
+ TOPICS_CONFIG
+ "' are overridable.";
static final String PROVIDER_CONFIG = "provider";
private static final List INVISIBLE_CONFIGS = singletonList(TOPIC_OVERRIDE_CONFIG);
private Map originals;
private final Optional> topics;
private final Optional topicsRegex;
private Map topicSinkConnectorConfigMap;
private ConnectionString connectionString;
private CustomCredentialProvider customCredentialProvider;
public MongoSinkConfig(final Map originals) {
super(CONFIG, originals, false);
this.originals = unmodifiableMap(originals);
topics =
getList(TOPICS_CONFIG).isEmpty()
? Optional.empty()
: Optional.of(unmodifiableList(getList(TOPICS_CONFIG)));
topicsRegex =
getString(TOPICS_REGEX_CONFIG).isEmpty()
? Optional.empty()
: Optional.of(Pattern.compile(getString(TOPICS_REGEX_CONFIG)));
if (topics.isPresent() && topicsRegex.isPresent()) {
throw new ConfigException(
format(
"%s and %s are mutually exclusive options, but both are set.",
TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
} else if (!topics.isPresent() && !topicsRegex.isPresent()) {
throw new ConfigException(
format("Must configure one of %s or %s", TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
}
connectionString = new ConnectionString(getPassword(CONNECTION_URI_CONFIG).value());
topicSinkConnectorConfigMap =
new ConcurrentHashMap<>(
topics.orElse(emptyList()).stream()
.collect(
Collectors.toMap((t) -> t, (t) -> new MongoSinkTopicConfig(t, originals))));
// Process and validate overrides of regex values.
if (topicsRegex.isPresent()) {
Pattern topicRegex = topicsRegex.get();
originals.keySet().stream()
.filter(k -> k.startsWith(TOPIC_OVERRIDE_PREFIX))
.forEach(
k -> {
String topic = k.substring(TOPIC_OVERRIDE_PREFIX.length()).split("\\.")[0];
if (!topicSinkConnectorConfigMap.containsKey(topic)
&& topicRegex.matcher(topic).matches()) {
topicSinkConnectorConfigMap.put(
topic, new MongoSinkTopicConfig(topic, originals));
}
});
}
// Initialize CustomCredentialProvider if mongo.custom.auth.mechanism.enable is set to true
if (Boolean.parseBoolean(originals.get(CUSTOM_AUTH_ENABLE_CONFIG))) {
customCredentialProvider = initializeCustomProvider(originals);
}
}
public static final ConfigDef CONFIG = createConfigDef();
static String createOverrideKey(final String topic, final String config) {
if (!CONFIG.configKeys().containsKey(config)) {
throw new ConfigException("Unknown configuration key: " + config);
}
return format(TOPIC_OVERRIDE_CONFIG, topic, config);
}
public CustomCredentialProvider getCustomCredentialProvider() {
return customCredentialProvider;
}
public ConnectionString getConnectionString() {
return connectionString;
}
public Optional> getTopics() {
return topics;
}
public Optional getTopicRegex() {
return topicsRegex;
}
public Map getOriginals() {
return originals;
}
public MongoSinkTopicConfig getMongoSinkTopicConfig(final String topic) {
if (!topicSinkConnectorConfigMap.containsKey(topic)) {
topics.ifPresent(
topicsList -> {
if (!topicsList.contains(topic)) {
throw new ConfigException(
format("Unknown topic: %s, must be one of: %s", topic, topicsList));
}
});
topicsRegex.ifPresent(
topicRegex -> {
if (!topicRegex.matcher(topic).matches()) {
throw new ConfigException(
format("Unknown topic: %s, does not match: %s", topic, topicRegex));
}
if (!topicSinkConnectorConfigMap.containsKey(topic)) {
topicSinkConnectorConfigMap.put(topic, new MongoSinkTopicConfig(topic, originals));
}
});
}
return topicSinkConnectorConfigMap.get(topic);
}
private static ConfigDef createConfigDef() {
ConfigDef configDef =
new ConfigDef() {
@Override
@SuppressWarnings("unchecked")
public Map validateAll(final Map props) {
logObsoleteProperties(props.keySet(), LOGGER::warn);
logIncompatibleProperties(props, LOGGER::warn);
Map results = super.validateAll(props);
INVISIBLE_CONFIGS.forEach(
c -> {
if (results.containsKey(c)) {
results.get(c).visible(false);
}
});
// Don't validate child configs if the top level configs are broken
if (results.values().stream().anyMatch((c) -> !c.errorMessages().isEmpty())) {
return results;
}
boolean hasTopicsConfig = !props.getOrDefault(TOPICS_CONFIG, "").trim().isEmpty();
boolean hasTopicsRegexConfig =
!props.getOrDefault(TOPICS_REGEX_CONFIG, "").trim().isEmpty();
if (hasTopicsConfig && hasTopicsRegexConfig) {
results
.get(TOPICS_CONFIG)
.addErrorMessage(
format(
"%s and %s are mutually exclusive options, but both are set.",
TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
} else if (!hasTopicsConfig && !hasTopicsRegexConfig) {
results
.get(TOPICS_CONFIG)
.addErrorMessage(
format("Must configure one of %s or %s", TOPICS_CONFIG, TOPICS_REGEX_CONFIG));
}
if (hasTopicsConfig) {
List topics = (List) results.get(TOPICS_CONFIG).value();
topics.forEach(
topic -> results.putAll(MongoSinkTopicConfig.validateAll(topic, props)));
} else if (hasTopicsRegexConfig) {
results.putAll(MongoSinkTopicConfig.validateRegexAll(props));
}
return results;
}
};
String group = "Connection";
int orderInGroup = 0;
configDef.define(
TOPICS_CONFIG,
Type.LIST,
TOPICS_DEFAULT,
Importance.HIGH,
TOPICS_DOC,
group,
++orderInGroup,
Width.MEDIUM,
TOPICS_DISPLAY);
configDef.define(
TOPICS_REGEX_CONFIG,
Type.STRING,
TOPICS_REGEX_DEFAULT,
Validators.isAValidRegex(),
Importance.HIGH,
TOPICS_REGEX_DOC,
group,
++orderInGroup,
Width.MEDIUM,
TOPICS_REGEX_DISPLAY);
configDef.define(
CONNECTION_URI_CONFIG,
Type.PASSWORD,
CONNECTION_URI_DEFAULT,
errorCheckingPasswordValueValidator("A valid connection string", ConnectionString::new),
Importance.HIGH,
CONNECTION_URI_DOC,
group,
++orderInGroup,
Width.MEDIUM,
CONNECTION_URI_DISPLAY);
addServerApiConfig(configDef);
addSslConfigDef(configDef);
group = "Overrides";
orderInGroup = 0;
configDef.define(
TOPIC_OVERRIDE_CONFIG,
Type.STRING,
TOPIC_OVERRIDE_DEFAULT,
Validators.topicOverrideValidator(),
Importance.LOW,
TOPIC_OVERRIDE_DOC,
group,
++orderInGroup,
Width.MEDIUM,
TOPIC_OVERRIDE_DISPLAY);
configDef.defineInternal(PROVIDER_CONFIG, Type.STRING, "", Importance.LOW);
MongoSinkTopicConfig.BASE_CONFIG.configKeys().values().forEach(configDef::define);
return configDef;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy