io.streamnative.pulsar.handlers.kop.storage.PartitionLogManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
There is a newer version: 4.0.0.4
Show newest version
/**
 * Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 */
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop.storage;

import com.google.common.collect.Maps;
import io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration;
import io.streamnative.pulsar.handlers.kop.KafkaTopicLookupService;
import io.streamnative.pulsar.handlers.kop.RequestStats;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.kafka.common.errors.NotLeaderOrFollowerException;
import org.apache.kafka.common.utils.Time;
import org.apache.pulsar.broker.service.plugin.EntryFilter;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.util.FutureUtil;

/**
 * Manage {@link PartitionLog}.
 */
@AllArgsConstructor
@Slf4j
public class PartitionLogManager {

    @Getter
    private final KafkaServiceConfiguration kafkaConfig;
    private final RequestStats requestStats;
    private final Map logMap;
    private final Time time;
    private final List entryFilters;

    private final KafkaTopicLookupService kafkaTopicLookupService;

    private final ProducerStateManagerSnapshotBufferFactory producerStateManagerSnapshotBufferFactory;

    private final OrderedExecutor recoveryExecutor;

    public PartitionLogManager(KafkaServiceConfiguration kafkaConfig,
                               RequestStats requestStats,
                               final List entryFilters,
                               Time time,
                               KafkaTopicLookupService kafkaTopicLookupService,
                               ProducerStateManagerSnapshotBufferFactory producerStateManagerSnapshotBufferFactory,
                               OrderedExecutor recoveryExecutor) {
        this.kafkaConfig = kafkaConfig;
        this.requestStats = requestStats;
        this.logMap = Maps.newConcurrentMap();
        this.entryFilters = entryFilters;
        this.time = time;
        this.kafkaTopicLookupService = kafkaTopicLookupService;
        this.producerStateManagerSnapshotBufferFactory = producerStateManagerSnapshotBufferFactory;
        this.recoveryExecutor = recoveryExecutor;
    }

    public PartitionLog getLog(String kopTopic) {
        final var inserted = new MutableBoolean(false);
        PartitionLog res = logMap.computeIfAbsent(kopTopic, key -> {
            TopicName topicName = TopicName.get(kopTopic);
            String tenant = topicName.getTenant();

            PartitionLog partitionLog = new PartitionLog(kafkaConfig, requestStats,
                    time, key, entryFilters,
                    kafkaTopicLookupService,
                    producerStateManagerSnapshotBufferFactory.create(tenant), recoveryExecutor);
            partitionLog.initialise();
            inserted.setTrue();
            return partitionLog;
        });
        if (inserted.isTrue()) {
            res.awaitInitialisation().exceptionally(e -> {
                // in case of failure we have to remove the CompletableFuture from the map
                if (!(FutureUtil.unwrapCompletionException(e) instanceof NotLeaderOrFollowerException)) {
                    log.error("[{}] Failed to initialize", kopTopic, e);
                }
                res.close();
                logMap.remove(kopTopic, res);
                return null;
            });
        }
        return res;
    }

    PartitionLog removeAndCloseLog(String topicName) {
        log.info("Remove and close PartitionLog {}", topicName);
        PartitionLog exists =  logMap.remove(topicName);
        if (exists != null) {
            exists.close();
        }
        return exists;
    }

    int size() {
        return logMap.size();
    }

    public CompletableFuture updatePurgeAbortedTxnsOffsets() {
        List> handles = new ArrayList<>();
        logMap.values().forEach(log -> {
            if (log.isInitialised()) {
                handles.add(log.updatePurgeAbortedTxnsOffset());
            }
        });
        return FutureUtil
                .waitForAll(handles);
    }

    public CompletableFuture closeAsync() {
        final var start = System.currentTimeMillis();
        final var closeFutures = logMap.values().stream().map(PartitionLog::close).toList();
        log.info("Start closing {} PartitionLogs", closeFutures.size());
        final var future = FutureUtil.waitForAll(closeFutures);
        future.whenComplete((__, e) -> {
            final var elapsed = System.currentTimeMillis() - start;
            log.info("It takes {} ms to close {} PartitionLogs (exception: {})", elapsed, closeFutures.size(),
                (e == null) ? "null" : e.getMessage());
        });
        return future;
    }
}