io.streamnative.pulsar.handlers.kop.utils.MetadataUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
/**
* Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
*/
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.streamnative.pulsar.handlers.kop.utils;
import com.google.common.collect.Sets;
import io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration;
import io.streamnative.pulsar.handlers.kop.format.EntryFormatterFactory;
import io.streamnative.pulsar.handlers.kop.storage.PartitionLog;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.common.internals.Topic;
import org.apache.pulsar.client.admin.Clusters;
import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException;
import org.apache.pulsar.client.admin.Tenants;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride;
import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
import org.apache.pulsar.common.policies.data.BacklogQuota;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies;
import org.apache.pulsar.common.policies.data.DispatchRate;
import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode;
import org.apache.pulsar.common.policies.data.InactiveTopicPolicies;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.PublishRate;
import org.apache.pulsar.common.policies.data.RetentionPolicies;
import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
import org.apache.pulsar.common.policies.data.SubscribeRate;
import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.common.policies.data.TopicType;
/**
* Utils for KoP Metadata.
*/
@Slf4j
public class MetadataUtils {
// trigger compaction when the offset backlog reaches 1MB
public static final int MAX_COMPACTION_THRESHOLD = 1024 * 1024;
public static final int INFINITE_RETENTION_SIZE = -1;
public static final int INFINITE_RETENTION_TIMES = -1;
public static final int DISABLE_TTL = 0;
public static final BacklogQuota BACKLOG_QUOTA =
BacklogQuota.builder()
.limitSize(-1)
.limitTime(-1)
.retentionPolicy(BacklogQuota.RetentionPolicy.producer_request_hold)
.build();
public static final InactiveTopicPolicies INACTIVE_TOPIC_POLICIES = new InactiveTopicPolicies(
InactiveTopicDeleteMode.delete_when_no_subscriptions, 60, false);
public static final AutoTopicCreationOverride AUTO_TOPIC_CREATION_OVERRIDE =
AutoTopicCreationOverride.builder()
.allowAutoTopicCreation(true)
.topicType(TopicType.NON_PARTITIONED.toString())
.build();
public static final AutoSubscriptionCreationOverride AUTO_SUBSCRIPTION_CREATION_OVERRIDE =
AutoSubscriptionCreationOverride.builder()
.allowAutoSubscriptionCreation(true)
.build();
public static final DispatchRate DISPATCH_RATE =
DispatchRate.builder()
.dispatchThrottlingRateInByte(0)
.dispatchThrottlingRateInMsg(0)
.ratePeriodInSecond(1)
.relativeToPublishRate(false)
.build();
public static final SubscribeRate SUBSCRIBE_RATE = new SubscribeRate(0, 30);
public static final PublishRate PUBLISH_RATE = new PublishRate(0, 0);
public static final DelayedDeliveryPolicies DELAYED_DELIVERY_POLICIES =
DelayedDeliveryPolicies.builder()
.active(true)
.tickTime(1000)
.maxDeliveryDelayInMillis(0)
.build();
public static final long DISABLE_OFFLOAD_THRESHOLD = -1L;
public static final String DEFAULT_METADATA_NAMESPACE_CONF_FLAG = "__ksn_metadata_namespace_initialized";
public static String constructOffsetsTopicBaseName(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace()
+ "/" + Topic.GROUP_METADATA_TOPIC_NAME;
}
public static String constructTxnLogTopicBaseName(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace()
+ "/" + Topic.TRANSACTION_STATE_TOPIC_NAME;
}
public static String constructTxProducerStateTopicBaseName(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace()
+ "/__transaction_producer_state";
}
public static String constructTxnProducerIdTopicBaseName(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace()
+ "/__transaction_producerid_generator";
}
public static String constructSchemaRegistryTopicName(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKopSchemaRegistryNamespace()
+ "/" + conf.getKopSchemaRegistryTopicName();
}
public static String constructMetadataNamespace(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace();
}
public static String constructProducerIdTopicNamespace(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaMetadataNamespace();
}
public static String constructUserTopicsNamespace(String tenant, KafkaServiceConfiguration conf) {
return tenant + "/" + conf.getKafkaNamespace();
}
public static void createOffsetMetadataIfMissing(String tenant, PulsarAdmin pulsarAdmin,
ClusterData clusterData,
KafkaServiceConfiguration conf) {
KopTopic kopTopic = new KopTopic(constructOffsetsTopicBaseName(tenant, conf),
constructMetadataNamespace(tenant, conf));
createKafkaMetadataIfMissing(tenant, conf.getKafkaMetadataNamespace(), pulsarAdmin, clusterData, conf, kopTopic,
conf.getOffsetsTopicNumPartitions(), false);
}
public static void createTxnMetadataIfMissing(String tenant,
PulsarAdmin pulsarAdmin,
ClusterData clusterData,
KafkaServiceConfiguration conf) {
KopTopic kopTopic = new KopTopic(constructTxnLogTopicBaseName(tenant, conf),
constructMetadataNamespace(tenant, conf));
createKafkaMetadataIfMissing(tenant, conf.getKafkaMetadataNamespace(), pulsarAdmin, clusterData, conf, kopTopic,
conf.getKafkaTxnLogTopicNumPartitions(), false);
KopTopic kopTopicProducerState = new KopTopic(constructTxProducerStateTopicBaseName(tenant, conf),
constructMetadataNamespace(tenant, conf));
createKafkaMetadataIfMissing(tenant, conf.getKafkaMetadataNamespace(), pulsarAdmin, clusterData, conf,
kopTopicProducerState, conf.getKafkaTxnProducerStateTopicNumPartitions(), false);
if (conf.isKafkaTransactionProducerIdsStoredOnPulsar()) {
KopTopic producerIdKopTopic = new KopTopic(constructTxnProducerIdTopicBaseName(tenant, conf),
constructProducerIdTopicNamespace(tenant, conf));
createKafkaMetadataIfMissing(tenant, conf.getKafkaMetadataNamespace(),
pulsarAdmin, clusterData, conf, producerIdKopTopic,
conf.getKafkaTxnLogTopicNumPartitions(), false);
}
}
/**
* This method creates the Kafka metadata tenant and namespace if they are not currently present.
*
* - If the cluster does not exist this method will throw a PulsarServerException.NotFoundException
* - If the tenant does not exist it will be created
* - If the tenant exists but the allowedClusters list does not include the cluster this method will
* add the cluster to the allowedClusters list
* - If the namespace does not exist it will be created
* - If the namespace exists but the replicationClusters list does not include the cluster this method
* will add the cluster to the replicationClusters list
* - If the offset topic does not exist it will be created
* - If the offset topic exists but some partitions are missing, the missing partitions will be created
*
*/
private static void createKafkaMetadataIfMissing(String tenant,
String namespace,
PulsarAdmin pulsarAdmin,
ClusterData clusterData,
KafkaServiceConfiguration conf,
KopTopic kopTopic,
int partitionNum,
boolean infiniteRetention) {
if (!conf.isKafkaManageSystemNamespaces()) {
log.info("Skipping initialization of topic {} for tenant {}", kopTopic.getFullName(), tenant);
return;
}
log.info("Try create metadata if missing namespace: {}/{}, topic: {}, numPartitions: {}"
+ ", infiniteRetention: {}",
tenant, namespace, kopTopic.getFullName(), partitionNum, infiniteRetention);
String kafkaMetadataNamespace = tenant + "/" + namespace;
String cluster = conf.getClusterName();
boolean clusterExists = false;
boolean tenantExists = false;
boolean namespaceExists = false;
boolean offsetsTopicExists = false;
try {
Clusters clusters = pulsarAdmin.clusters();
if (!clusters.getClusters().contains(cluster)) {
try {
pulsarAdmin.clusters().createCluster(cluster, clusterData);
} catch (PulsarAdminException e) {
if (e instanceof ConflictException) {
log.info("Attempted to create cluster {} however it was created concurrently.", cluster);
} else {
// Re-throw all other exceptions
throw e;
}
}
} else {
ClusterData configuredClusterData = clusters.getCluster(cluster);
log.info("Cluster {} found: {}", cluster, configuredClusterData);
}
clusterExists = true;
// Check if the metadata tenant exists and create it if not
Tenants tenants = pulsarAdmin.tenants();
createTenantIfMissing(tenant, conf, cluster, tenants);
tenantExists = true;
// Check if the metadata namespace exists and create it if not
Namespaces namespaces = pulsarAdmin.namespaces();
createNamespaceIfMissing(tenant, conf, cluster, kafkaMetadataNamespace, namespaces,
infiniteRetention, true);
namespaceExists = true;
// Check if the offsets topic exists and create it if not
createTopicIfNotExist(pulsarAdmin, kopTopic.getFullName(), partitionNum);
offsetsTopicExists = true;
} catch (PulsarAdminException e) {
if (e instanceof ConflictException) {
log.info("Resources concurrent creating and cause e: ", e);
return;
}
log.error("Failed to initialize Kafka Metadata {}: {}", kafkaMetadataNamespace, e.getMessage());
} finally {
log.info("Current state of kafka metadata, cluster: {} exists: {}, tenant: {} exists: {},"
+ " namespace: {} exists: {}, topic: {} exists: {}",
cluster, clusterExists, tenant, tenantExists, kafkaMetadataNamespace, namespaceExists,
kopTopic.getOriginalName(), offsetsTopicExists);
}
}
private static void createTenantIfMissing(String tenant, KafkaServiceConfiguration conf,
String cluster, Tenants tenants) throws PulsarAdminException {
if (!tenants.getTenants().contains(tenant)) {
log.info("Tenant: {} does not exist, creating it ...", tenant);
tenants.createTenant(tenant,
TenantInfo.builder()
.adminRoles(conf.getSuperUserRoles())
.allowedClusters(Collections.singleton(cluster))
.build());
} else {
TenantInfo kafkaMetadataTenantInfo = tenants.getTenantInfo(tenant);
Set allowedClusters = kafkaMetadataTenantInfo.getAllowedClusters();
if (!allowedClusters.contains(cluster)) {
log.info("Tenant: {} exists but cluster: {} is not in the allowedClusters list, updating it ...",
tenant, cluster);
allowedClusters.add(cluster);
tenants.updateTenant(tenant, kafkaMetadataTenantInfo);
}
}
}
private static void createNamespaceIfMissing(String tenant, KafkaServiceConfiguration conf,
String cluster, String kafkaNamespace,
Namespaces namespaces,
boolean infiniteRetention,
boolean isMetadataNamespace)
throws PulsarAdminException {
if (!namespaces.getNamespaces(tenant).contains(kafkaNamespace)) {
log.info("Namespaces: {} does not exist in tenant: {}, creating it ...",
kafkaNamespace, tenant);
Set replicationClusters = Sets.newHashSet(cluster);
namespaces.createNamespace(kafkaNamespace, replicationClusters);
namespaces.setNamespaceReplicationClusters(kafkaNamespace, replicationClusters);
} else {
List replicationClusters = namespaces.getNamespaceReplicationClusters(kafkaNamespace);
if (!replicationClusters.contains(cluster)) {
log.info("Namespace: {} exists but cluster: {} is not in the replicationClusters list,"
+ "updating it ...", kafkaNamespace, cluster);
Set newReplicationClusters = Sets.newHashSet(replicationClusters);
newReplicationClusters.add(cluster);
namespaces.setNamespaceReplicationClusters(kafkaNamespace, newReplicationClusters);
}
}
if (isMetadataNamespace) {
checkAndSetDefaultNamespacePolicies(namespaces, kafkaNamespace, conf, infiniteRetention);
}
}
/**
* This method creates the Kafka tenant and namespace if they are not currently present.
*
* - If the cluster does not exist this method will throw a PulsarServerException.NotFoundException
* - If the tenant does not exist it will be created
* - If the tenant exists but the allowedClusters list does not include the cluster this method will
* add the cluster to the allowedClusters list
* - If the namespace does not exist it will be created
* - If the namespace exists but the replicationClusters list does not include the cluster this method
* will add the cluster to the replicationClusters list
* - If the offset topic does not exist it will be created
* - If the offset topic exists but some partitions are missing, the missing partitions will be created
*
*/
public static void createKafkaNamespaceIfMissing(PulsarAdmin pulsarAdmin,
ClusterData clusterData,
KafkaServiceConfiguration conf)
throws PulsarAdminException {
String cluster = conf.getClusterName();
String tenant = conf.getKafkaTenant();
String kafkaNamespace = constructUserTopicsNamespace(tenant, conf);
boolean clusterExists = false;
boolean tenantExists = false;
boolean namespaceExists = false;
try {
Clusters clusters = pulsarAdmin.clusters();
if (!clusters.getClusters().contains(cluster)) {
try {
pulsarAdmin.clusters().createCluster(cluster, clusterData);
} catch (PulsarAdminException e) {
if (e instanceof ConflictException) {
log.info("Attempted to create cluster {} however it was created concurrently.", cluster);
} else {
// Re-throw all other exceptions
throw e;
}
}
} else {
ClusterData configuredClusterData = clusters.getCluster(cluster);
log.info("Cluster {} found: {}", cluster, configuredClusterData);
}
clusterExists = true;
// Check if the kafka tenant exists and create it if not
Tenants tenants = pulsarAdmin.tenants();
createTenantIfMissing(tenant, conf, cluster, tenants);
tenantExists = true;
Namespaces namespaces = pulsarAdmin.namespaces();
// Check if the kafka namespace exists and create it if not
createNamespaceIfMissing(tenant, conf, cluster, kafkaNamespace, namespaces, false, false);
namespaceExists = true;
} catch (PulsarAdminException e) {
if (e instanceof ConflictException) {
log.info("Resources concurrent creating and cause e: ", e);
return;
}
log.error("Failed to successfully initialize Kafka Metadata {}",
kafkaNamespace, e);
throw e;
} finally {
log.info("Current state of kafka metadata, cluster: {} exists: {}, tenant: {} exists: {},"
+ " namespace: {} exists: {}",
cluster, clusterExists, tenant, tenantExists, kafkaNamespace, namespaceExists);
}
}
private static void createTopicIfNotExist(final PulsarAdmin admin,
final String topic,
final int numPartitions) throws PulsarAdminException {
Map properties = Map.of(
PartitionLog.KAFKA_ENTRY_FORMATTER_PROPERTY_NAME, EntryFormatterFactory.EntryFormat.PULSAR.name());
try {
admin.topics().getPartitionedTopicMetadata(topic);
} catch (PulsarAdminException.NotFoundException e) {
if (numPartitions > 0) {
log.info("Creating partitioned topic {} (with {} partitions) if it does not exist",
topic, numPartitions);
admin.topics().createPartitionedTopic(topic, numPartitions);
} else {
log.info("Creating non-partitioned topic {} if it does not exist", topic);
admin.topics().createNonPartitionedTopic(topic, properties);
}
}
}
public static void createSchemaRegistryMetadataIfMissing(String tenant,
PulsarAdmin pulsarAdmin,
ClusterData clusterData,
KafkaServiceConfiguration conf) {
KopTopic kopTopic = new KopTopic(constructSchemaRegistryTopicName(tenant, conf),
constructMetadataNamespace(tenant, conf));
createKafkaMetadataIfMissing(tenant, conf.getKopSchemaRegistryNamespace(), pulsarAdmin, clusterData,
conf, kopTopic, 1, true);
}
public static boolean isSystemTopic(final org.apache.pulsar.broker.service.Topic topic,
final KafkaServiceConfiguration kafkaConfig) {
final TopicName topicName = TopicName.get(topic.getName());
return topicName.getNamespacePortion().equals(kafkaConfig.getKafkaMetadataNamespace())
|| topicName.getNamespacePortion().equals(kafkaConfig.getKopSchemaRegistryNamespace())
|| topic.isSystemTopic();
}
/**
* check and set namespace default configurations only for metadata namespace.
*/
public static synchronized void checkAndSetDefaultNamespacePolicies(
Namespaces namespaces, String namespace, KafkaServiceConfiguration conf, boolean infiniteRetention)
throws PulsarAdminException {
Policies policies = namespaces.getPolicies(namespace);
if (policies != null && policies.properties != null
&& policies.properties.getOrDefault(
DEFAULT_METADATA_NAMESPACE_CONF_FLAG, "false").equals("true")) {
return;
}
if (infiniteRetention) {
log.info("Setting infinite retention for namespace {}", namespace);
namespaces.setRetention(namespace,
new RetentionPolicies(INFINITE_RETENTION_SIZE, INFINITE_RETENTION_TIMES));
// disable TTL
namespaces.setNamespaceMessageTTL(namespace, DISABLE_TTL);
} else {
namespaces.setRetention(namespace, new RetentionPolicies(
(int) conf.getOffsetsRetentionMinutes(), conf.getSystemTopicRetentionSizeInMB()));
namespaces.setNamespaceMessageTTL(namespace, conf.getOffsetsMessageTTL());
}
namespaces.setOffloadThreshold(namespace, DISABLE_OFFLOAD_THRESHOLD);
namespaces.setOffloadThresholdInSeconds(namespace, DISABLE_OFFLOAD_THRESHOLD);
namespaces.setCompactionThreshold(namespace, MAX_COMPACTION_THRESHOLD);
// disable backlog quota
namespaces.setBacklogQuota(namespace, BACKLOG_QUOTA);
// disable subscription expiration
namespaces.setSubscriptionExpirationTime(namespace, 0);
// disable deduplication
namespaces.setDeduplicationStatus(namespace, false);
// disable encryption check
namespaces.setEncryptionRequiredStatus(namespace, false);
// disable entry filters
namespaces.setNamespaceEntryFilters(namespace, null);
// disable inactive topic policies
namespaces.setInactiveTopicPolicies(namespace, INACTIVE_TOPIC_POLICIES);
// same with system topic
namespaces.setSchemaCompatibilityStrategy(namespace, SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE);
namespaces.setIsAllowAutoUpdateSchema(namespace, true);
namespaces.setSchemaValidationEnforcedAsync(namespace, false);
// follow with Pulsar default values
namespaces.setMaxSubscriptionsPerTopic(namespace, 0);
namespaces.setAutoTopicCreation(namespace, AUTO_TOPIC_CREATION_OVERRIDE);
namespaces.setAutoSubscriptionCreation(namespace, AUTO_SUBSCRIPTION_CREATION_OVERRIDE);
namespaces.setDispatchRate(namespace, DISPATCH_RATE);
namespaces.setSubscriptionDispatchRate(namespace, DISPATCH_RATE);
namespaces.setSubscribeRate(namespace, SUBSCRIBE_RATE);
namespaces.setPublishRate(namespace, PUBLISH_RATE);
namespaces.setReplicatorDispatchRate(namespace, DISPATCH_RATE);
namespaces.setSubscriptionAuthMode(namespace, SubscriptionAuthMode.None);
namespaces.setDelayedDeliveryMessages(namespace, DELAYED_DELIVERY_POLICIES);
namespaces.setMaxProducersPerTopic(namespace, 0);
namespaces.setMaxConsumersPerTopic(namespace, 0);
namespaces.setMaxConsumersPerSubscription(namespace, 0);
namespaces.setMaxTopicsPerNamespace(namespace, 0);
// this should be useless for system topic consumer, the system topic consumer is exclusive
namespaces.setMaxUnackedMessagesPerConsumer(namespace, 50000);
namespaces.setMaxUnackedMessagesPerSubscription(namespace, 200000);
namespaces.setProperty(namespace, DEFAULT_METADATA_NAMESPACE_CONF_FLAG, "true");
}
}