kafka.entity.changelog.topic.TopicManagerTopologySupplier Maven / Gradle / Ivy

Go to download
package kafka.entity.changelog.topic;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Metrics;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.function.Supplier;
import kafka.entity.changelog.schema.Topic;
import kafka.entity.changelog.serde.CommandSerde;
import kafka.entity.changelog.serde.EventSerde;
import kafka.entity.changelog.serde.EventsSerde;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.Consumed;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.kstream.Transformer;
import org.apache.kafka.streams.kstream.ValueTransformerWithKey;
import org.apache.kafka.streams.processor.Processor;
import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.PunctuationType;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.Stores;

import static kafka.entity.changelog.schema.Topic.*;
import static kafka.entity.changelog.schema.Topic.CreationEvent;
import static kafka.entity.changelog.schema.Topic.TopicCreated;
import static kafka.entity.changelog.schema.Topic.TopicDeleted;
import static kafka.entity.changelog.schema.Topic.TopicsFound;

public class TopicManagerTopologySupplier implements Supplier {
  static final String TOPIC_CURRENT_STATE_STORE = "topic-current";
  static final String TOPIC_HISTORY_STATE_STORE = "topic-history";

  final Counter topicsCreatedCounter = Metrics.counter("changelog.topics.created");
  final Counter topicsDeletedCounter = Metrics.counter("changelog.topics.deleted");

  final String commandTopicName;
  final String eventTopicName;
  final TopicMetadataSource metadataSource;
  final TopicCommandRepository commandRepository;
  final TopicNotificationService notificationService;

  final Duration syncInterval;

  final EventSerde eventSerde;
  final EventsSerde eventsSerde;
  final CommandSerde commandSerde;

  TopicManagerTopologySupplier(
      String commandTopicName, String eventTopicName,
      TopicMetadataSource metadataSource,
      TopicCommandRepository commandRepository,
      TopicNotificationService notificationService,
      Duration syncInterval) {
    this.commandTopicName = commandTopicName;
    this.eventTopicName = eventTopicName;
    this.metadataSource = metadataSource;
    this.commandRepository = commandRepository;
    this.notificationService = notificationService;
    this.syncInterval = syncInterval;

    eventSerde = new EventSerde();
    eventsSerde = new EventsSerde();
    commandSerde = new CommandSerde();
  }

  @Override public Topology get() {
    StreamsBuilder builder = new StreamsBuilder();

    builder.addStateStore(
        Stores.keyValueStoreBuilder(
            Stores.persistentKeyValueStore(TOPIC_CURRENT_STATE_STORE),
            Serdes.String(),
            Serdes.String()).withLoggingDisabled());
    builder.addStateStore(
        Stores.keyValueStoreBuilder(
            Stores.persistentKeyValueStore(TOPIC_HISTORY_STATE_STORE),
            Serdes.String(),
            eventsSerde).withLoggingDisabled());

    KStream[] commandStreams =
        builder.stream(commandTopicName, Consumed.with(Serdes.String(), commandSerde))
            .branch(
                (key, value) -> value.hasTopicsFound(),
                (key, value) -> value.hasCreateTopic());

    KStream eventStream =
        commandStreams[0].mapValues(Command::getTopicsFound)
            .transformValues(
                () -> new ValueTransformerWithKey>() {
                  KeyValueStore topics;

                  @Override public void init(ProcessorContext context) {
                    topics =
                        (KeyValueStore) context.getStateStore(
                            TOPIC_CURRENT_STATE_STORE);
                    context.schedule(
                        syncInterval,
                        PunctuationType.WALL_CLOCK_TIME,
                        timestamp -> {
                          TopicsFound topicsFound = metadataSource.findTopics();
                          if (topicsFound != null) {
                            Metrics.gauge("changelog.topics.found", topicsFound.getNameCount());
                            commandRepository.put(
                                Command.newBuilder().setTopicsFound(topicsFound).build());
                          }
                        });
                  }

                  @Override public List transform(String key, TopicsFound value) {
                    List events = new ArrayList<>();
                    // Check for topics removed
                    topics.all().forEachRemaining(keyValue -> {
                      if (!value.getNameList().contains(keyValue.key)) {
                        TopicDeleted topicDeleted =
                            TopicDeleted.newBuilder()
                                .setName(keyValue.key)
                                .build();
                        events.add(
                            Event.newBuilder()
                                .setTopicName(keyValue.value)
                                .setUsername("changelog")
                                .setSource("topics-found")
                                .setTimestamp(Instant.now().toEpochMilli())
                                .setCreationEvent(
                                    CreationEvent.newBuilder()
                                        //.setTopicName(keyValue.key)
                                        .setTopicDeleted(topicDeleted)
                                        .build())
                                .build());
                      }
                    });
                    // Check for new topics
                    for (String name : value.getNameList()) {
                      if (topics.get(name) == null) {
                        TopicCreated topicCreated = metadataSource.getTopicCreated(name);
                        events.add(
                            Event.newBuilder()
                                .setTopicName(name)
                                .setUsername("changelog")
                                .setSource("topics-found")
                                .setTimestamp(Instant.now().toEpochMilli())
                                .setCreationEvent(
                                    CreationEvent.newBuilder()
                                        .setTopicCreated(topicCreated)
                                        .build())
                                .build());
                      }
                    }
                    return events;
                  }

                  @Override public void close() { //Nothing to close
                  }
                }, TOPIC_CURRENT_STATE_STORE)
            .flatMapValues(value -> value)
            .selectKey((key, value) -> value.getTopicName())
            .through(eventTopicName, Produced.with(Serdes.String(), eventSerde))
            .peek((key, value) -> notificationService.send(value));

    commandStreams[1] // create topic requests
        .transform(() -> new Transformer>() {
          KeyValueStore topics;

          @Override public void init(ProcessorContext context) {
            topics =
                (KeyValueStore) context.getStateStore(TOPIC_CURRENT_STATE_STORE);
          }

          @Override
          public KeyValue transform(String key, Command value) {
            CreateTopic createTopic = value.getCreateTopic();
            String topicName = createTopic.getName();
            if (topics.get(topicName) == null) {
              try {
                metadataSource.createTopic(createTopic);
                return KeyValue.pair(
                    topicName,
                    Event.newBuilder()
                        .setTopicName(topicName)
                        .setUsername(value.getUsername())
                        .setSource(value.getSource())
                        .setTimestamp(value.getTimestamp())
                        .setCreationEvent(
                            CreationEvent.newBuilder().setTopicCreated(
                                TopicCreated.newBuilder()
                                    .setName(topicName)
                                    .setPartitions(createTopic.getPartitions())
                                    .setReplicationFactor(createTopic.getReplicationFactor())
                                    .build()))
                        .build());
              } catch (ExecutionException e) {
                e.printStackTrace();
              } catch (InterruptedException e) {
                e.printStackTrace();
              }
            }
            return null;
          }

          @Override public void close() { // nothing to close.
          }
        }, TOPIC_CURRENT_STATE_STORE)
        .filter((key, value) -> Objects.nonNull(value))
        .to(eventTopicName, Produced.with(Serdes.String(), eventSerde));

    KStream[] eventStreams =
        eventStream.branch((key, value) -> value.hasCreationEvent());

    eventStreams[0] // topics creation events
        .process(() -> new Processor() {
          KeyValueStore topicCreationStates;
          KeyValueStore topics;

          @Override public void init(ProcessorContext context) {
            topicCreationStates =
                (KeyValueStore)
                    context.getStateStore(TOPIC_HISTORY_STATE_STORE);
            topics =
                (KeyValueStore) context.getStateStore(TOPIC_CURRENT_STATE_STORE);
          }

          @Override public void process(String topicName, Event value) {
            CreationEvent creation = value.getCreationEvent();
            if (creation.hasTopicCreated()) {
              topics.putIfAbsent(topicName, topicName);
              topicsCreatedCounter.increment();
            }
            if (creation.hasTopicDeleted()) {
              topics.delete(topicName);
              topicsDeletedCounter.increment();
            }
            Events states = topicCreationStates.get(topicName);
            if (states == null) {
              states = Events.newBuilder().addEvent(value).build();
            } else {
              if (!states.getEventList().contains(value)) {
                Events.Builder inner = states.toBuilder();
                if (inner.getEventList().size() > 5) {
                  inner.removeEvent(0);
                }
                states = inner.addEvent(value).build();
              }
            }
            topicCreationStates.put(topicName, states);
          }

          @Override public void close() { // Nothing to close
          }
        }, TOPIC_HISTORY_STATE_STORE, TOPIC_CURRENT_STATE_STORE);

    return builder.build();
  }
}