io.streamnative.pulsar.handlers.kop.topic.KopPersistentTopic Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
The newest version!
/**
 * Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 */
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop.topic;

import static io.streamnative.pulsar.handlers.kop.storage.PartitionLog.KAFKA_TOPIC_UUID_PROPERTY_NAME;

import io.streamnative.pulsar.handlers.kop.KafkaProtocolHandler;
import io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration;
import io.streamnative.pulsar.handlers.kop.storage.PartitionLog;
import io.streamnative.pulsar.handlers.kop.utils.MetadataUtils;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.ManagedLedger;
import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.compaction.Compactor;

@Slf4j
public class KopPersistentTopic extends PersistentTopic {

    private final KafkaProtocolHandler kafkaProtocolHandler;
    private final Object partitionLogLock = new Object();
    private PartitionLog partitionLog;
    private volatile String kafkaTopicUUID;

    public KopPersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) {
        super(topic, ledger, brokerService);
        kafkaProtocolHandler = (KafkaProtocolHandler) brokerService.getPulsar().getProtocolHandlers()
                .protocol(KafkaProtocolHandler.PROTOCOL_NAME);
    }

    @Override
    public CompletableFuture initialize() {
        return super.initialize().thenCompose(v -> {
            KafkaServiceConfiguration kafkaConfig = kafkaProtocolHandler.getKafkaConfig();
            if (MetadataUtils.isSystemTopic(this, kafkaConfig)) {
                return CompletableFuture.completedFuture(null);
            }
            TopicName topicName = TopicName.get(topic);
            CompletableFuture topicMetadataFuture = !topicName.isPartitioned()
                    ? CompletableFuture.completedFuture(null) :
                    this.getBrokerService().fetchPartitionedTopicMetadataAsync(
                            TopicName.getPartitionedTopicName(topic), true);
            return topicMetadataFuture.thenAccept(partitionedTopicMetadata -> {
                Map properties;
                if (partitionedTopicMetadata != null && partitionedTopicMetadata.partitions > 0) {
                    properties = partitionedTopicMetadata.properties;
                } else {
                    properties = getManagedLedger().getProperties();
                }
                properties = properties == null ? Map.of() : properties;
                this.updateKafkaTopicUUID(properties.get(KAFKA_TOPIC_UUID_PROPERTY_NAME));
            });
        });
    }

    @Override
    public CompletableFuture getLastDispatchablePosition() {
        if (notKafkaCompactedTopic()) {
            return super.getLastDispatchablePosition();
        }
        return getPartitionLog().awaitInitialisation()
            .thenCombine(super.getLastDispatchablePosition(), (partitionLog, pulsarLastPosition) -> {
                final var kafkaLastPosition = partitionLog.getProducerStateManager().getMaxReadPosition();
                return kafkaLastPosition.compareTo(pulsarLastPosition) < 0 ? kafkaLastPosition : pulsarLastPosition;
            }).exceptionally(e -> {
                log.warn("Return earliest as last dispatchable position due to the initialization failure of {}: {}",
                    getName(), e.getMessage());
                return PositionFactory.EARLIEST;
            });
    }

    @Override
    public Position getMaxReadPosition() {
        final Position pulsarMaxReadPosition = super.getMaxReadPosition();
        if (notKafkaCompactedTopic()) {
            return pulsarMaxReadPosition;
        }

        final PartitionLog partitionLog = getPartitionLog();
        if (partitionLog.isInitialised()) {
            Position kafkaMaxReadPosition = partitionLog.getProducerStateManager().getMaxReadPosition();
            if (kafkaMaxReadPosition.compareTo(pulsarMaxReadPosition) < 0) {
                return kafkaMaxReadPosition;
            } else {
                return pulsarMaxReadPosition;
            }
        } else {
            return PositionFactory.EARLIEST;
        }
    }

    private boolean notKafkaCompactedTopic() {
        final var kafkaConfig = kafkaProtocolHandler.getKafkaConfig();
        return (!kafkaConfig.isKafkaTransactionCoordinatorEnabled()
            || !isKafkaTopic()
            || !this.getSubscriptions().containsKey(Compactor.COMPACTION_SUBSCRIPTION)
            || MetadataUtils.isSystemTopic(this, kafkaConfig));
    }

    private PartitionLog getPartitionLog() {
        synchronized (partitionLogLock) {
            if (this.partitionLog == null || this.partitionLog.isInitialisationFailed()) {
                this.partitionLog = kafkaProtocolHandler.getReplicaManager().getLogManager().getLog(topic);
            }
            return this.partitionLog;
        }
    }

    @Override
    public CompletableFuture checkIfTransactionBufferRecoverCompletely() {
        KafkaServiceConfiguration kafkaConfig = kafkaProtocolHandler.getKafkaConfig();
        if (!isKafkaTopic() || !kafkaConfig.isKafkaTransactionCoordinatorEnabled()
                || MetadataUtils.isSystemTopic(this, kafkaConfig)) {
            return super.checkIfTransactionBufferRecoverCompletely();
        }

        return CompletableFuture.allOf(getPartitionLog().awaitInitialisation(),
                super.checkIfTransactionBufferRecoverCompletely());
    }

    public boolean isKafkaTopic() {
        return kafkaTopicUUID != null;
    }

    public synchronized void updateKafkaTopicUUID(String kafkaTopicUUID) {
        this.kafkaTopicUUID = kafkaTopicUUID;
    }
}