io.streamnative.pulsar.handlers.kop.KafkaTopicLookupService Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
There is a newer version: 4.0.0.4
Show newest version
/**
 * Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 */
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop;

import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.util.FutureUtil;

/**
 * KafkaTopicLookupManager manages a Map of topic to KafkaTopicConsumerManager.
 * For each topic, there is a KafkaTopicConsumerManager, which manages a topic and its related offset cursor.
 * This is mainly used to cache the produce/consume topic, not include offsetTopic.
 */
@RequiredArgsConstructor
@Slf4j
public class KafkaTopicLookupService {
    private final BrokerService brokerService;
    private final PulsarAdmin admin;

    public CompletableFuture> getTopic(String topic, Object requestor) {
        return getTopic(topic, requestor, true);
    }

    private CompletableFuture> getTopic(String topic, Object requestor, boolean retry) {
        final var topicName = TopicName.get(topic);
        return internalGetTopic(topic, requestor).thenCompose(optTopic -> {
            if (optTopic.isPresent() || topicName.getPartitionIndex() != 0) {
                return CompletableFuture.completedFuture(optTopic);
            } else {
                // The partition index is 0, which might also indicate it's a non-partitioned topic
                return internalGetTopic(topicName.getPartitionedTopicName(), requestor);
            }
        }).thenCompose(optTopic -> {
            if (optTopic.isPresent() || !retry) {
                return CompletableFuture.completedFuture(optTopic);
            } else {
                log.info("[{}] Try creating topic {} since it cannot be loaded", requestor, topic);
                final var partitionedTopic = topicName.getPartitionedTopicName();
                return admin.topics().getPartitionedTopicMetadataAsync(partitionedTopic)
                    .thenApply(metadata -> metadata.partitions > 0)
                    .exceptionally(e -> {
                        log.warn("Failed to get partition metadata of {}: {}", partitionedTopic, e.getMessage());
                        return false;
                    })
                    .thenCompose(isPartitionedTopic -> {
                        if (!isPartitionedTopic) {
                            // The partition metadata does not exist or it's a non-partitioned topic
                            return CompletableFuture.completedFuture(Optional.empty());
                        }
                        return admin.topics().createNonPartitionedTopicAsync(topic)
                            .exceptionally(e -> {
                                log.warn("[{}] Failed to create {}: {}", requestor, topic, e.getMessage());
                                return null;
                            })
                            .thenCompose(__ -> getTopic(topic, requestor, false));
                    });
            }
        });
    }

    private CompletableFuture> internalGetTopic(String topic, Object requestor) {
        return brokerService.getTopicIfExists(topic).thenApply(optTopic -> optTopic.map(__ -> (PersistentTopic) __))
            .exceptionally(e -> {
                final var unwrapped = FutureUtil.unwrapCompletionException(e);
                // The ServiceUnitNotReadyException is retryable, so we should print a warning log instead of error log
                if (unwrapped instanceof BrokerServiceException.ServiceUnitNotReadyException) {
                    log.warn("[{}] Failed to getTopic {}: {}", requestor, topic, unwrapped.getMessage());
                    return Optional.empty();
                } else {
                    log.error("[{}] Failed to getTopic {}", requestor, topic, unwrapped);
                    throw new CompletionException(unwrapped);
                }
            });
    }
}