pl.allegro.tech.hermes.management.domain.detection.InactiveTopicsDetectionJob Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hermes-management Show documentation
Fast and reliable message broker built on top of Kafka.
There is a newer version: 2.10.4
package pl.allegro.tech.hermes.management.domain.detection;

import static java.util.stream.Collectors.groupingBy;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import java.time.Clock;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import pl.allegro.tech.hermes.api.OwnerId;
import pl.allegro.tech.hermes.api.Topic;
import pl.allegro.tech.hermes.api.TopicName;
import pl.allegro.tech.hermes.management.config.detection.InactiveTopicsDetectionProperties;
import pl.allegro.tech.hermes.management.domain.topic.TopicService;

@Component
public class InactiveTopicsDetectionJob {
  private final TopicService topicService;
  private final InactiveTopicsStorageService inactiveTopicsStorageService;
  private final InactiveTopicsDetectionService inactiveTopicsDetectionService;
  private final Optional notifier;
  private final InactiveTopicsDetectionProperties properties;
  private final Clock clock;
  private final MeterRegistry meterRegistry;

  private static final Logger logger = LoggerFactory.getLogger(InactiveTopicsDetectionJob.class);

  public InactiveTopicsDetectionJob(
      TopicService topicService,
      InactiveTopicsStorageService inactiveTopicsStorageService,
      InactiveTopicsDetectionService inactiveTopicsDetectionService,
      Optional notifier,
      InactiveTopicsDetectionProperties properties,
      Clock clock,
      MeterRegistry meterRegistry) {
    this.topicService = topicService;
    this.inactiveTopicsStorageService = inactiveTopicsStorageService;
    this.inactiveTopicsDetectionService = inactiveTopicsDetectionService;
    this.properties = properties;
    this.clock = clock;
    this.meterRegistry = meterRegistry;
    if (notifier.isEmpty()) {
      logger.info("Inactive topics notifier bean is absent");
    }
    this.notifier = notifier;
  }

  public void detectAndNotify() {
    List topics = topicService.getAllTopics();
    List qualifiedTopicNames = topics.stream().map(Topic::getQualifiedName).toList();
    List historicalInactiveTopics = inactiveTopicsStorageService.getInactiveTopics();
    List foundInactiveTopics =
        detectInactiveTopics(qualifiedTopicNames, historicalInactiveTopics);

    Map> groupedByNeedOfNotification =
        foundInactiveTopics.stream()
            .collect(groupingBy(inactiveTopicsDetectionService::shouldBeNotified));

    List topicsToNotify = groupedByNeedOfNotification.getOrDefault(true, List.of());
    List topicsToSkipNotification =
        groupedByNeedOfNotification.getOrDefault(false, List.of());
    List notifiedTopics = notify(enrichWithOwner(topicsToNotify, topics));

    List processedTopics =
        limitHistory(
            Stream.concat(notifiedTopics.stream(), topicsToSkipNotification.stream()).toList());
    measureInactiveTopics(processedTopics);
    inactiveTopicsStorageService.markAsInactive(processedTopics);
  }

  private List detectInactiveTopics(
      List qualifiedTopicNames, List historicalInactiveTopics) {
    Map historicalInactiveTopicsByName =
        groupByName(historicalInactiveTopics);
    return qualifiedTopicNames.stream()
        .map(
            qualifiedTopicName ->
                inactiveTopicsDetectionService.detectInactiveTopic(
                    TopicName.fromQualifiedName(qualifiedTopicName),
                    Optional.ofNullable(historicalInactiveTopicsByName.get(qualifiedTopicName))))
        .map(opt -> opt.orElse(null))
        .filter(Objects::nonNull)
        .toList();
  }

  private Map groupByName(List inactiveTopics) {
    return inactiveTopics.stream()
        .collect(Collectors.toMap(InactiveTopic::qualifiedTopicName, v -> v, (v1, v2) -> v1));
  }

  private List enrichWithOwner(
      List inactiveTopics, List topics) {
    Map ownerByTopicName = new HashMap<>();
    topics.forEach(topic -> ownerByTopicName.put(topic.getQualifiedName(), topic.getOwner()));

    return inactiveTopics.stream()
        .map(
            inactiveTopic ->
                new InactiveTopicWithOwner(
                    inactiveTopic, ownerByTopicName.get(inactiveTopic.qualifiedTopicName())))
        .toList();
  }

  private List notify(List inactiveTopics) {
    if (inactiveTopics.isEmpty()) {
      logger.info("No inactive topics to notify");
      return List.of();
    } else if (notifier.isPresent()) {
      logger.info("Notifying {} inactive topics", inactiveTopics.size());
      NotificationResult result = notifier.get().notify(inactiveTopics);
      Instant now = clock.instant();

      return inactiveTopics.stream()
          .map(InactiveTopicWithOwner::topic)
          .map(
              topic ->
                  result.isSuccess(topic.qualifiedTopicName())
                      ? topic.notificationSent(now)
                      : topic)
          .toList();
    } else {
      logger.info("Skipping notification of {} inactive topics", inactiveTopics.size());
      return inactiveTopics.stream().map(InactiveTopicWithOwner::topic).toList();
    }
  }

  private List limitHistory(List inactiveTopics) {
    return inactiveTopics.stream()
        .map(topic -> topic.limitNotificationsHistory(properties.notificationsHistoryLimit()))
        .toList();
  }

  private void measureInactiveTopics(List processedTopics) {
    processedTopics.stream()
        .collect(
            Collectors.groupingBy(
                topic -> topic.notificationTimestampsMs().size(), Collectors.counting()))
        .forEach(
            (notificationsCount, topicsCount) -> {
              Tags tags = Tags.of("notifications", notificationsCount.toString());
              meterRegistry.gauge("inactive-topics", tags, topicsCount);
            });
  }
}