io.apicurio.registry.utils.kafka.KafkaUtil Maven / Gradle / Ivy

Go to download
package io.apicurio.registry.utils.kafka;

import io.apicurio.registry.utils.ConcurrentUtil;
import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.config.TopicConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;

public class KafkaUtil {
    private static final Logger log = LoggerFactory.getLogger(KafkaUtil.class);

    public static  T result(KafkaFuture kf) {
        CompletionStage cs = toCompletionStage(kf);
        return ConcurrentUtil.result(cs);
    }

    public static  CompletionStage toCompletionStage(KafkaFuture kf) {
        CompletableFuture cf = new CompletableFuture<>();
        kf.whenComplete((v, t) -> {
            if (t != null) {
                cf.completeExceptionally(t);
            } else {
                cf.complete(v);
            }
        });
        return cf;
    }

    /**
     * Create topics with sensible defaults.
     *
     * @param properties the Kafka properties to create Kafka admin
     * @param topicNames topics to create, if they don't exist
     * @param topicConfig the config to use for the new topic
     */
    public static void createTopics(Properties properties, Set topicNames,
            Map topicConfig) {
        try (Admin admin = Admin.create(properties)) {
            createTopics(admin, topicNames, topicConfig);
        }
    }

    public static void createTopics(Properties properties, Set topicNames) {
        createTopics(properties, topicNames, null);
    }

    /**
     * Create topics with sensible defaults.
     *
     * @param admin the Kafka admin to use
     * @param topicNames topics to create, if they don't exist
     * @param topicConfig the config to use for the new topic
     */
    public static void createTopics(Admin admin, Set topicNames, Map topicConfig) {
        ConcurrentUtil.result(createTopicsAsync(admin, topicNames, topicConfig));
    }

    public static void createTopics(Admin admin, Set topicNames) {
        createTopics(admin, topicNames, null);
    }

    /**
     * Create topics with sensible defaults, async.
     *
     * @param admin the Kafka admin to use
     * @param topicNames topics to create, if they don't exist
     */
    public static CompletionStage createTopicsAsync(Admin admin, Set topicNames,
            Map topicConfig) {
        List> topicsToCreate = new ArrayList<>();
        return toCompletionStage(admin.listTopics().names()).thenCompose(topics -> {
            for (String topicName : topicNames) {
                createTopic(admin, topics, topicsToCreate, topicName, topicConfig);
            }
            // noinspection SuspiciousToArrayCall
            return CompletableFuture.allOf(topicsToCreate.toArray(new CompletableFuture[0]));
        }).thenCompose(v -> {
            if (topicsToCreate.size() > 0) {
                return toCompletionStage(admin.createTopics(
                        topicsToCreate.stream().map(ConcurrentUtil::result).collect(Collectors.toList()))
                        .all());
            } else {
                return CompletableFuture.completedFuture(null);
            }
        });
    }

    private static void createTopic(Admin admin, Set topics,
            List> topicsToCreate, String topicName,
            Map topicConfig) {
        if (!topics.contains(topicName)) {
            KafkaFuture newTopicKF = admin.describeCluster().nodes().thenApply(nodes -> {
                Map configs = new HashMap<>();
                if (topicConfig != null) {
                    configs.putAll(topicConfig);
                }
                log.info("Creating new Kafka topic: {}", topicName);
                int replicationFactor = Math.min(3, nodes.size());
                if (configs.containsKey("replication.factor")) {
                    replicationFactor = Integer.parseInt(configs.get("replication.factor"));
                }
                int minimumInSyncReplicas = Math.max(replicationFactor - 1, 1);
                configs.putIfAbsent(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG,
                        String.valueOf(minimumInSyncReplicas));
                return new NewTopic(topicName, 1, (short) replicationFactor).configs(configs);
            }).whenComplete((nt, t) -> log.info("Created new topic: {}", topicName, t));
            topicsToCreate.add(toCompletionStage(newTopicKF));
        }
    }
}