org.apache.flink.statefun.sdk.kafka.KafkaIngressBuilder Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.statefun.sdk.kafka;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
import org.apache.flink.statefun.sdk.annotations.ForRuntime;
import org.apache.flink.statefun.sdk.io.IngressIdentifier;
import org.apache.flink.statefun.sdk.io.IngressSpec;
import org.apache.kafka.clients.consumer.ConsumerConfig;
/**
* A builder for creating an {@link IngressSpec} for consuming data from Apache Kafka.
*
* @param The type consumed from Kafka.
*/
public final class KafkaIngressBuilder {
private final IngressIdentifier id;
private final List topics = new ArrayList<>();
private final Properties properties = new Properties();
private OptionalConfig consumerGroupId = OptionalConfig.withoutDefault();
private OptionalConfig> deserializer =
OptionalConfig.withoutDefault();
private OptionalConfig kafkaAddress = OptionalConfig.withoutDefault();
private OptionalConfig autoResetPosition =
OptionalConfig.withDefault(KafkaIngressAutoResetPosition.LATEST);
private OptionalConfig startupPosition =
OptionalConfig.withDefault(KafkaIngressStartupPosition.fromLatest());
private KafkaIngressBuilder(IngressIdentifier id) {
this.id = Objects.requireNonNull(id);
}
/**
* @param id A unique ingress identifier.
* @param The type consumed from Kafka.
* @return A new {@link KafkaIngressBuilder}.
*/
public static KafkaIngressBuilder forIdentifier(IngressIdentifier id) {
return new KafkaIngressBuilder<>(id);
}
/** @param consumerGroupId the consumer group id to use. */
public KafkaIngressBuilder withConsumerGroupId(String consumerGroupId) {
this.consumerGroupId.set(consumerGroupId);
return this;
}
/** @param kafkaAddress Comma separated addresses of the brokers. */
public KafkaIngressBuilder withKafkaAddress(String kafkaAddress) {
this.kafkaAddress.set(kafkaAddress);
return this;
}
/** @param topic The name of the topic that should be consumed. */
public KafkaIngressBuilder withTopic(String topic) {
topics.add(topic);
return this;
}
/** @param topics A list of topics that should be consumed. */
public KafkaIngressBuilder addTopics(List topics) {
this.topics.addAll(topics);
return this;
}
/** A configuration property for the KafkaConsumer. */
public KafkaIngressBuilder withProperties(Properties properties) {
this.properties.putAll(properties);
return this;
}
/** A configuration property for the KafkaProducer. */
public KafkaIngressBuilder withProperty(String name, String value) {
Objects.requireNonNull(name);
Objects.requireNonNull(value);
this.properties.setProperty(name, value);
return this;
}
/**
* @param deserializerClass The deserializer used to convert between Kafka's byte messages and
* java objects.
*/
public KafkaIngressBuilder withDeserializer(
Class> deserializerClass) {
Objects.requireNonNull(deserializerClass);
this.deserializer.set(instantiateDeserializer(deserializerClass));
return this;
}
/**
* @param autoResetPosition the auto offset reset position to use, in case consumed offsets are
* invalid.
*/
public KafkaIngressBuilder withAutoResetPosition(
KafkaIngressAutoResetPosition autoResetPosition) {
this.autoResetPosition.set(autoResetPosition);
return this;
}
/**
* Configures the position that the ingress should start consuming from. By default, the startup
* position is {@link KafkaIngressStartupPosition#fromLatest()}.
*
* Note that this configuration only affects the position when starting the application from a
* fresh start. When restoring the application from a savepoint, the ingress will always start
* consuming from the offsets persisted in the savepoint.
*
* @param startupPosition the position that the Kafka ingress should start consuming from.
* @see KafkaIngressStartupPosition
*/
public KafkaIngressBuilder withStartupPosition(KafkaIngressStartupPosition startupPosition) {
this.startupPosition.set(startupPosition);
return this;
}
/** @return A new {@link KafkaIngressSpec}. */
public KafkaIngressSpec build() {
Properties properties = resolveKafkaProperties();
return new KafkaIngressSpec<>(
id, properties, topics, deserializer.get(), startupPosition.get());
}
private Properties resolveKafkaProperties() {
Properties resultProps = new Properties();
resultProps.putAll(properties);
// for all configuration passed using named methods, overwrite corresponding properties
kafkaAddress.overwritePropertiesIfPresent(resultProps, ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
autoResetPosition.overwritePropertiesIfPresent(
resultProps, ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
consumerGroupId.overwritePropertiesIfPresent(resultProps, ConsumerConfig.GROUP_ID_CONFIG);
return resultProps;
}
private static > T instantiateDeserializer(
Class deserializerClass) {
try {
Constructor defaultConstructor = deserializerClass.getDeclaredConstructor();
defaultConstructor.setAccessible(true);
return defaultConstructor.newInstance();
} catch (NoSuchMethodException e) {
throw new IllegalStateException(
"Unable to create an instance of deserializer "
+ deserializerClass.getName()
+ "; has no default constructor",
e);
} catch (IllegalAccessException | InstantiationException | InvocationTargetException e) {
throw new IllegalStateException(
"Unable to create an instance of deserializer " + deserializerClass.getName(), e);
}
}
// ========================================================================================
// Methods for runtime usage
// ========================================================================================
@ForRuntime
KafkaIngressBuilder withDeserializer(KafkaIngressDeserializer deserializer) {
this.deserializer.set(deserializer);
return this;
}
}