kafka.entity.changelog.topic.TopicChangelog Maven / Gradle / Ivy

Go to download
package kafka.entity.changelog.topic;

import java.io.Closeable;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import kafka.entity.changelog.notification.ConsoleNotificationProvider;
import kafka.entity.changelog.notification.NotificationProvider;
import kafka.entity.changelog.schema.Topic;
import kafka.entity.changelog.serde.CommandSerde;
import kafka.entity.changelog.serde.CreationEventSerde;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static kafka.entity.changelog.topic.TopicManagerTopologySupplier.TOPIC_CURRENT_STATE_STORE;
import static kafka.entity.changelog.topic.TopicManagerTopologySupplier.TOPIC_HISTORY_STATE_STORE;
import static org.apache.kafka.streams.state.QueryableStoreTypes.keyValueStore;

public class TopicChangelog implements Runnable, Closeable {
  static final Logger LOG = LoggerFactory.getLogger(TopicChangelog.class);

  final String bootstrapServers;
  final String commandTopicName;
  final String eventTopicName;
  final Duration syncInterval;
  final String stateDir;
  final NotificationProvider notificationProvider;

  final TopicMetadataSource metadataSource;
  final TopicCommandRepository commandRepository;

  final Topology topicManagerStreamTopology;
  final Properties topicManagerStreamConfig;

  final AdminClient adminClient;
  final KafkaProducer commandProducer;
  final KafkaProducer creationEventProducer;

  volatile KafkaStreams topicManagerStream;

  TopicChangelog(Builder builder) {
    this.bootstrapServers = builder.bootstrapServers;
    this.commandTopicName = builder.commandTopicName;
    this.eventTopicName = builder.eventTopicName;
    this.syncInterval = builder.syncInterval;
    this.stateDir = builder.stateDir;
    this.notificationProvider = builder.notificationProvider;
    this.adminClient = AdminClient.create(builder.adminClientConfig);
    this.metadataSource =
        TopicMetadataSource.create(adminClient);
    this.creationEventProducer = new KafkaProducer<>(
        builder.producerConfig,
        new StringSerializer(),
        new CreationEventSerde().serializer());
    this.commandProducer = new KafkaProducer<>(
        builder.producerConfig,
        new StringSerializer(),
        new CommandSerde().serializer());
    this.commandRepository = TopicCommandRepository.create(
        commandProducer,
        builder.commandTopicName);
    this.topicManagerStreamConfig = builder.topicManagerStreamConfig;
    TopicNotificationService notificationService =
        new TopicNotificationService(builder.notificationProvider);
    this.topicManagerStreamTopology = new TopicManagerTopologySupplier(
        builder.commandTopicName,
        builder.eventTopicName,
        metadataSource,
        commandRepository,
        notificationService,
        builder.syncInterval).get();
  }

  public static Builder newBuilder() {
    return new Builder();
  }

  @Override public void run() {
    new TopicBootstrap(this).run();
    getTopicManagerStream().start();
  }

  @Override public void close() {
    if (topicManagerStream != null) topicManagerStream.close();
    if (creationEventProducer != null) creationEventProducer.close();
    if (commandProducer != null) commandProducer.close();
    if (adminClient != null) adminClient.close();
  }

  KafkaStreams getTopicManagerStream() {
    if (topicManagerStream == null) {
      synchronized (this) {
        if (topicManagerStream == null) {
          this.topicManagerStream =
              new KafkaStreams(topicManagerStreamTopology, topicManagerStreamConfig);
        }
      }
    }
    return topicManagerStream;
  }

  List listTopics() {
    try {
      ReadOnlyKeyValueStore store =
          getTopicManagerStream().store(TOPIC_CURRENT_STATE_STORE, keyValueStore());
      List values = new ArrayList<>();
      try (KeyValueIterator all = store.all()) {
        all.forEachRemaining(keyValue -> values.add(keyValue.value));
      }
      return values;
    } catch (Exception e) {
      LOG.error("Error querying topics", e);
      return null;
    }
  }

  List topicEvents(String topicName) {
    try {
      ReadOnlyKeyValueStore store =
          getTopicManagerStream().store(TOPIC_HISTORY_STATE_STORE, keyValueStore());
      Topic.Events events = store.get(topicName);
      return events.getEventList();
    } catch (Exception e) {
      LOG.error("Error querying topics", e);
      return null;
    }
  }

  @Override public String toString() {
    return this.toBuilder()
        .bootstrapServers(bootstrapServers)
        .commandTopicName(commandTopicName)
        .creationEventTopicName(eventTopicName)
        .stateDir(stateDir)
        .syncInterval(syncInterval)
        .notificationProvider(notificationProvider)
        .toString();
  }

  private Builder toBuilder() {
    return new Builder();
  }

  public static class Builder {
    String bootstrapServers = "localhost:19092";
    Duration syncInterval = Duration.ofMinutes(1L);
    String commandTopicName = "changelog_topic_command";
    String eventTopicName = "changelog_topic_event";
    String applicationId = "changelog_topic_manager";
    String stateDir = "/tmp/changelog_topic_manager";

    Properties adminClientConfig = new Properties();
    Properties producerConfig = new Properties();
    Properties topicManagerStreamConfig = new Properties();

    NotificationProvider notificationProvider = new ConsoleNotificationProvider();

    Builder() {
      this.adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      this.producerConfig.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      this.topicManagerStreamConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      this.topicManagerStreamConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
      this.topicManagerStreamConfig.put(StreamsConfig.STATE_DIR_CONFIG, stateDir);
    }

    public Builder bootstrapServers(String bootstrapServers) {
      if (bootstrapServers == null) throw new NullPointerException("bootstrapServers == null");
      this.adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      this.producerConfig.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      this.topicManagerStreamConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      return this;
    }

    public Builder stateDir(String stateDir) {
      if (stateDir == null) throw new NullPointerException("stateDir == null");
      this.stateDir = stateDir;
      return this;
    }

    public Builder syncInterval(Duration syncInterval) {
      if (syncInterval == null) throw new NullPointerException("syncInterval == null");
      this.syncInterval = syncInterval;
      return this;
    }

    public Builder creationEventTopicName(String creationEventTopicName) {
      if (creationEventTopicName == null) {
        throw new NullPointerException("creationEventTopicName == null");
      }
      this.eventTopicName = creationEventTopicName;
      return this;
    }

    public Builder commandTopicName(String commandTopicName) {
      if (commandTopicName == null) throw new NullPointerException("commandTopicName == null");
      this.commandTopicName = commandTopicName;
      return this;
    }

    public Builder notificationProvider(NotificationProvider notificationProvider) {
      if (notificationProvider == null) {
        throw new NullPointerException("notificationProvider == null");
      }
      this.notificationProvider = notificationProvider;
      return this;
    }

    public TopicChangelog build() {
      return new TopicChangelog(this);
    }

    @Override public String toString() {
      return "Builder{" +
          "bootstrapServers='" + bootstrapServers + '\'' +
          ", syncInterval=" + syncInterval +
          ", creationEventTopicName='" + eventTopicName + '\'' +
          ", commandTopicName='" + commandTopicName + '\'' +
          ", applicationId='" + applicationId + '\'' +
          ", stateDir='" + stateDir + '\'' +
          ", adminClientConfig=" + adminClientConfig +
          ", producerConfig=" + producerConfig +
          ", topicManagerStreamConfig=" + topicManagerStreamConfig +
          ", notificationProvider=" + notificationProvider +
          '}';
    }
  }
}