All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.streamnative.pulsar.handlers.kop.AdminManager Maven / Gradle / Ivy

Go to download

Kafka on Pulsar implemented using Pulsar Protocol Handler

There is a newer version: 4.0.0.4
Show newest version
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop;

import static io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperationKey.TopicKey;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import io.streamnative.pulsar.handlers.kop.exceptions.KoPTopicException;
import io.streamnative.pulsar.handlers.kop.utils.KopTopic;
import io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperation;
import io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperationPurgatory;
import io.streamnative.pulsar.handlers.kop.utils.timer.SystemTimer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.config.ConfigResource;
import org.apache.kafka.common.errors.InvalidPartitionsException;
import org.apache.kafka.common.errors.InvalidRequestException;
import org.apache.kafka.common.errors.TopicExistsException;
import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
import org.apache.kafka.common.message.CreatePartitionsRequestData;
import org.apache.kafka.common.message.CreateTopicsRequestData;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.requests.ApiError;
import org.apache.kafka.common.requests.CreateTopicsRequest;
import org.apache.kafka.common.requests.DescribeConfigsResponse;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;

@Slf4j
public class AdminManager {

    private final DelayedOperationPurgatory topicPurgatory =
            DelayedOperationPurgatory.builder()
                    .purgatoryName("topic")
                    .timeoutTimer(SystemTimer.builder().executorName("topic").build())
                    .build();

    private final PulsarAdmin admin;
    private final int defaultNumPartitions;
    private final int maxMessageSize;

    private volatile Map> brokersCache = Maps.newHashMap();
    private final ReentrantReadWriteLock brokersCacheLock = new ReentrantReadWriteLock();

    private final Random random = new Random();
    private volatile Map controllerId = Maps.newHashMap();


    public AdminManager(PulsarAdmin admin, KafkaServiceConfiguration conf) {
        this.admin = admin;
        this.defaultNumPartitions = conf.getDefaultNumPartitions();
        this.maxMessageSize = conf.getMaxMessageSize();
    }

    public void shutdown() {
        topicPurgatory.shutdown();
    }

    public CompletableFuture> createTopicsAsync(
            Map createInfo,
            int timeoutMs,
            String namespacePrefix) {
        final Map> futureMap = new ConcurrentHashMap<>();
        final AtomicInteger numTopics = new AtomicInteger(createInfo.size());
        final CompletableFuture> resultFuture = new CompletableFuture<>();

        Runnable complete = () -> {
            // prevent `futureMap` from being modified by createPartitionedTopicAsync()'s callback
            numTopics.set(0);
            // complete the pending futures with timeout error
            futureMap.values().forEach(future -> {
                if (!future.isDone()) {
                    future.complete(new ApiError(Errors.REQUEST_TIMED_OUT, null));
                }
            });
            resultFuture.complete(futureMap.entrySet().stream().collect(Collectors.toMap(
                    Map.Entry::getKey,
                    entry -> entry.getValue().getNow(ApiError.NONE)
            )));
        };

        createInfo.forEach((topic, detail) -> {
            final CompletableFuture errorFuture = new CompletableFuture<>();
            futureMap.put(topic, errorFuture);

            KopTopic kopTopic;
            try {
                kopTopic = new KopTopic(topic, namespacePrefix);
            } catch (KoPTopicException e) {
                errorFuture.complete(ApiError.fromThrowable(e));
                if (numTopics.decrementAndGet() == 0) {
                    complete.run();
                }
                return;
            }
            int numPartitions = detail.numPartitions();
            if (numPartitions == CreateTopicsRequest.NO_NUM_PARTITIONS) {
                numPartitions = defaultNumPartitions;
            }
            if (numPartitions < 0) {
                errorFuture.complete(ApiError.fromThrowable(
                        new InvalidRequestException("The partition '" + numPartitions + "' is negative")));
                if (numTopics.decrementAndGet() == 0) {
                    complete.run();
                }
                return;
            }
            admin.topics().createPartitionedTopicAsync(kopTopic.getFullName(), numPartitions,
                            Map.of("kafkaTopicUUID", UUID.randomUUID().toString()))
                    .whenComplete((ignored, e) -> {
                        if (e == null) {
                            if (log.isDebugEnabled()) {
                                log.debug("Successfully create topic '{}'", topic);
                            }
                        } else {
                            log.error("Failed to create topic '{}': {}", topic, e);
                        }
                        if (e == null) {
                            errorFuture.complete(ApiError.NONE);
                        } else if (e instanceof PulsarAdminException.ConflictException) {
                            errorFuture.complete(ApiError.fromThrowable(
                                    new TopicExistsException("Topic '" + topic + "' already exists.")));
                        } else {
                            errorFuture.complete(ApiError.fromThrowable(e));
                        }
                        if (numTopics.decrementAndGet() == 0) {
                            complete.run();
                        }
                    });
        });

        if (timeoutMs <= 0) {
            complete.run();
        } else {
            List delayedCreateKeys =
                    createInfo.keySet().stream().map(TopicKey::new).collect(Collectors.toList());
            DelayedCreateTopics delayedCreate = new DelayedCreateTopics(timeoutMs, numTopics, complete);
            topicPurgatory.tryCompleteElseWatch(delayedCreate, delayedCreateKeys);
        }

        return resultFuture;
    }

    CompletableFuture> describeConfigsAsync(
            Map>> resourceToConfigNames, String namespacePrefix) {
        // Since Kafka's storage and policies are much different from Pulsar, here we just return a default config to
        // avoid some Kafka based systems need to send DescribeConfigs request, like confluent schema registry.
        final DescribeConfigsResponse.Config defaultTopicConfig = new DescribeConfigsResponse.Config(ApiError.NONE,
                KafkaLogConfig.getEntries().entrySet().stream().map(entry ->
                        new DescribeConfigsResponse.ConfigEntry(entry.getKey(), entry.getValue(),
                                DescribeConfigsResponse.ConfigSource.DEFAULT_CONFIG, false, false,
                                Collections.emptyList())
                ).collect(Collectors.toList()));

        Map> futureMap =
                resourceToConfigNames.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> {
                    ConfigResource resource = entry.getKey();
                    try {
                        CompletableFuture future = new CompletableFuture<>();
                        switch (resource.type()) {
                            case TOPIC:
                                KopTopic kopTopic = new KopTopic(resource.name(), namespacePrefix);
                                admin.topics().getPartitionedTopicMetadataAsync(kopTopic.getFullName())
                                        .whenComplete((metadata, e) -> {
                                            if (e != null) {
                                                if (e instanceof PulsarAdminException.NotFoundException) {
                                                    final ApiError error = new ApiError(
                                                            Errors.UNKNOWN_TOPIC_OR_PARTITION,
                                                            "Topic " + kopTopic.getOriginalName() + " doesn't exist");
                                                    future.complete(new DescribeConfigsResponse.Config(
                                                            error, Collections.emptyList()));
                                                } else {
                                                    future.complete(new DescribeConfigsResponse.Config(
                                                            ApiError.fromThrowable(e), Collections.emptyList()));
                                                }
                                            } else if (metadata.partitions > 0) {
                                                future.complete(defaultTopicConfig);
                                            } else {
                                                final ApiError error = new ApiError(Errors.INVALID_TOPIC_EXCEPTION,
                                                        "Topic " + kopTopic.getOriginalName()
                                                                + " is non-partitioned");
                                                future.complete(new DescribeConfigsResponse.Config(
                                                        error, Collections.emptyList()));
                                            }
                                        });
                                break;
                            case BROKER:
                                List dummyConfig = new ArrayList<>();
                                dummyConfig.add(buildDummyEntryConfig("num.partitions",
                                        this.defaultNumPartitions + ""));
                                dummyConfig.add(buildDummyEntryConfig("message.max.bytes", maxMessageSize + ""));
                                // this is useless in KOP, but some tools like KSQL need a value
                                dummyConfig.add(buildDummyEntryConfig("default.replication.factor", "1"));
                                dummyConfig.add(buildDummyEntryConfig("delete.topic.enable", "true"));
                                future.complete(new DescribeConfigsResponse.Config(ApiError.NONE, dummyConfig));
                                break;
                            default:
                                return CompletableFuture.completedFuture(new DescribeConfigsResponse.Config(
                                        ApiError.fromThrowable(
                                            new InvalidRequestException("Unsupported resource type: "
                                                    + resource.type())),
                                            Collections.emptyList()));
                        }
                        return future;
                    } catch (Exception e) {
                        return CompletableFuture.completedFuture(
                                new DescribeConfigsResponse.Config(ApiError.fromThrowable(e), Collections.emptyList()));
                    }
                }));
        CompletableFuture> resultFuture = new CompletableFuture<>();
        CompletableFuture.allOf(futureMap.values().toArray(new CompletableFuture[0])).whenComplete((ignored, e) -> {
            if (e != null) {
                resultFuture.completeExceptionally(e);
                return;
            }
            resultFuture.complete(futureMap.entrySet().stream().collect(
                    Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getNow(null))
            ));
        });
        return resultFuture;
    }

    private DescribeConfigsResponse.ConfigEntry buildDummyEntryConfig(String configName, String configValue) {
        return new DescribeConfigsResponse.ConfigEntry(
                configName, configValue,
                DescribeConfigsResponse.ConfigSource.DEFAULT_CONFIG, true, true,
                Collections.emptyList());
    }

    public void deleteTopic(String topicToDelete,
                            Consumer successConsumer,
                            Consumer errorConsumer) {
        admin.topics()
                .deletePartitionedTopicAsync(topicToDelete, true, true)
                .thenRun(() -> {
                    log.info("delete topic {} successfully.", topicToDelete);
                    successConsumer.accept(topicToDelete);
                })
                .exceptionally((e -> {
                    log.error("delete topic {} failed, exception: ", topicToDelete, e);
                    errorConsumer.accept(topicToDelete);
                    return null;
                }));
    }

    public void truncateTopic(String topicToDelete,
                              long offset,
                              Position position,
                              Consumer successConsumer,
                              Consumer errorConsumer) {
        log.info("truncateTopic {} at offset {}, pulsar position {}", topicToDelete, offset, position);
        if (position == null) {
            errorConsumer.accept("Cannot find position");
            return;
        }
        if (position.equals(PositionImpl.LATEST)) {
            admin.topics()
                .truncateAsync(topicToDelete)
                .thenRun(() -> {
                    log.info("truncated topic {} successfully.", topicToDelete);
                    successConsumer.accept(topicToDelete);
                })
                .exceptionally((e -> {
                    log.error("truncated topic {} failed, exception: ", topicToDelete, e);
                    errorConsumer.accept(topicToDelete);
                    return null;
                }));
        } else {
            errorConsumer.accept("Not implemented truncate topic at position " + position);
        }

    }

    CompletableFuture> createPartitionsAsync(
            Map createInfo,
            int timeoutMs,
            String namespacePrefix) {
        final Map> futureMap = new ConcurrentHashMap<>();
        final AtomicInteger numTopics = new AtomicInteger(createInfo.size());
        final CompletableFuture> resultFuture = new CompletableFuture<>();

        Runnable complete = () -> {
            // prevent `futureMap` from being modified by updatePartitionedTopicAsync()'s callback
            numTopics.set(0);
            // complete the pending futures with timeout error
            futureMap.values().forEach(future -> {
                if (!future.isDone()) {
                    future.complete(new ApiError(Errors.REQUEST_TIMED_OUT, null));
                }
            });
            resultFuture.complete(futureMap.entrySet().stream().collect(Collectors.toMap(
                    Map.Entry::getKey,
                    entry -> entry.getValue().getNow(ApiError.NONE)
            )));
        };

        createInfo.forEach((topic, newPartitions) -> {
            final CompletableFuture errorFuture = new CompletableFuture<>();
            futureMap.put(topic, errorFuture);

            try {
                KopTopic kopTopic = new KopTopic(topic, namespacePrefix);

                int numPartitions = newPartitions.count();
                if (numPartitions < 0) {
                    errorFuture.complete(ApiError.fromThrowable(
                            new InvalidPartitionsException("The partition '" + numPartitions + "' is negative")));

                    if (numTopics.decrementAndGet() == 0) {
                        complete.run();
                    }
                } else if (newPartitions.assignments() != null
                        && !newPartitions.assignments().isEmpty()) {
                    errorFuture.complete(ApiError.fromThrowable(
                            new InvalidRequestException(
                                    "Kop server currently doesn't support manual assignment replica sets '"
                                            + newPartitions
                                            .assignments()
                                            .stream()
                                            .map(CreatePartitionsRequestData.CreatePartitionsAssignment::brokerIds)
                                            .map(String::valueOf).collect(Collectors.joining(", ", "[", "]"))
                                            + "' the number of partitions must be specified ")
                    ));

                    if (numTopics.decrementAndGet() == 0) {
                        complete.run();
                    }
                } else {
                    handleUpdatePartitionsAsync(topic,
                            kopTopic,
                            numPartitions,
                            errorFuture,
                            numTopics,
                            complete);
                }

            } catch (KoPTopicException e) {
                errorFuture.complete(ApiError.fromThrowable(e));
                if (numTopics.decrementAndGet() == 0) {
                    complete.run();
                }
            }
        });


        if (timeoutMs <= 0) {
            complete.run();
        } else {
            List delayedCreateKeys =
                    createInfo.keySet().stream().map(TopicKey::new).collect(Collectors.toList());
            DelayedCreatePartitions delayedCreate = new DelayedCreatePartitions(timeoutMs, numTopics, complete);
            topicPurgatory.tryCompleteElseWatch(delayedCreate, delayedCreateKeys);
        }

        return resultFuture;
    }

    private void handleUpdatePartitionsAsync(String topic,
                                             KopTopic kopTopic,
                                             int newPartitions,
                                             CompletableFuture errorFuture,
                                             AtomicInteger numTopics,
                                             Runnable complete) {
        admin.topics().getPartitionedTopicMetadataAsync(kopTopic.getFullName())
                .whenComplete((metadata, t) -> {
                    if (t == null) {
                        int oldPartitions = metadata.partitions;
                        if (oldPartitions > newPartitions) {
                            errorFuture.complete(ApiError.fromThrowable(
                                    new InvalidPartitionsException(
                                            "Topic currently has '" + oldPartitions + "' partitions, "
                                                    + "which is higher than the requested '" + newPartitions + "'.")
                            ));
                            if (numTopics.decrementAndGet() == 0) {
                                complete.run();
                            }
                            return;
                        }

                        admin.topics().updatePartitionedTopicAsync(kopTopic.getFullName(), newPartitions)
                                .whenComplete((ignored, e) -> {
                                    if (e == null) {
                                        if (log.isDebugEnabled()) {
                                            log.debug("Successfully create topic '{}' new partitions '{}'",
                                                    topic, newPartitions);
                                        }

                                        errorFuture.complete(ApiError.NONE);
                                    } else {
                                        log.error("Failed to create topic '{}' new partitions '{}': {}",
                                                topic, newPartitions, e);

                                        errorFuture.complete(ApiError.fromThrowable(e));
                                    }

                                    if (numTopics.decrementAndGet() == 0) {
                                        complete.run();
                                    }
                                });
                    } else {
                        if (t instanceof PulsarAdminException.NotFoundException) {
                            errorFuture.complete(ApiError.fromThrowable(
                                    new UnknownTopicOrPartitionException("Topic '" + topic + "' doesn't exist.")));
                        } else {
                            errorFuture.complete(ApiError.fromThrowable(t));
                        }
                        if (numTopics.decrementAndGet() == 0) {
                            complete.run();
                        }
                    }
                });
    }

    public Collection getBrokers(String listenerName) {

        if (brokersCache.containsKey(listenerName)) {
            return brokersCache.get(listenerName);
        }

        return Collections.emptyList();
    }

    public Map> getAllBrokers() {
        return brokersCache;
    }

    public void setBrokers(Map> newBrokers) {
        brokersCacheLock.writeLock().lock();
        try {
            setControllerId(newBrokers);
            this.brokersCache = newBrokers;
        } finally {
            brokersCacheLock.writeLock().unlock();
        }
    }

    // only set when setBrokers
    private void setControllerId(Map> newBrokers) {
        Map newControllerId = Maps.newHashMap();
        newBrokers.forEach((listenerName, brokers) -> {
            if (brokers.size() == 0) {
                newControllerId.put(listenerName, MetadataResponse.NO_CONTROLLER_ID);
            } else {
                List nodes = Lists.newArrayList(brokers);
                newControllerId.put(listenerName,
                        nodes.size() > 1 ? nodes.get(random.nextInt(brokers.size())).id() : nodes.get(0).id());
            }
        });
        this.controllerId = newControllerId;
    }

    // always get the controllerId directly from the cache
    public int getControllerId(String listenerName) {
        return controllerId.getOrDefault(listenerName, MetadataResponse.NO_CONTROLLER_ID);
    }
}
    

    

    

            

    

            



    
    






    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy