com.hubspot.singularity.smtp.SingularityMailer Maven / Gradle / Ivy

Go to download
package com.hubspot.singularity.smtp;

import io.dropwizard.lifecycle.Managed;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import com.hubspot.mesos.JavaUtils;
import com.hubspot.mesos.json.MesosFileChunkObject;
import com.hubspot.singularity.ExtendedTaskState;
import com.hubspot.singularity.SingularityMainModule;
import com.hubspot.singularity.SingularityRequest;
import com.hubspot.singularity.SingularityTask;
import com.hubspot.singularity.SingularityTaskCleanup.TaskCleanupType;
import com.hubspot.singularity.SingularityTaskHistoryUpdate;
import com.hubspot.singularity.SingularityTaskHistoryUpdate.SimplifiedTaskState;
import com.hubspot.singularity.SingularityTaskId;
import com.hubspot.singularity.config.EmailConfigurationEnums.EmailDestination;
import com.hubspot.singularity.config.EmailConfigurationEnums.EmailType;
import com.hubspot.singularity.config.SMTPConfiguration;
import com.hubspot.singularity.config.SingularityConfiguration;
import com.hubspot.singularity.data.MetadataManager;
import com.hubspot.singularity.data.SandboxManager;
import com.hubspot.singularity.data.TaskManager;
import com.hubspot.singularity.sentry.SingularityExceptionNotifier;

import de.neuland.jade4j.Jade4J;
import de.neuland.jade4j.template.JadeTemplate;

public class SingularityMailer implements Managed {

  private static final Logger LOG = LoggerFactory.getLogger(SingularityMailer.class);

  private final SingularitySmtpSender smtpSender;
  private final SingularityConfiguration configuration;
  private final Optional maybeSmtpConfiguration;
  private final Optional mailPreparerExecutorService;
  private final SingularityExceptionNotifier exceptionNotifier;

  private final TaskManager taskManager;

  private final JadeTemplate taskTemplate;
  private final JadeTemplate requestInCooldownTemplate;
  private final JadeTemplate requestModifiedTemplate;
  private final JadeTemplate rateLimitedTemplate;

  private final MetadataManager metadataManager;
  private final SandboxManager sandboxManager;

  private final Optional uiHostnameAndPath;

  private final Joiner adminJoiner;

  private static final String TASK_LINK_FORMAT = "%s/task/%s";
  private static final String REQUEST_LINK_FORMAT = "%s/request/%s";

  @Inject
  public SingularityMailer(SingularitySmtpSender smtpSender, SingularityConfiguration configuration,
      TaskManager taskManager,
      SandboxManager sandboxManager,
      MetadataManager metadataManager,
      SingularityExceptionNotifier exceptionNotifier,
      @Named(SingularityMainModule.TASK_TEMPLATE) JadeTemplate taskTemplate,
      @Named(SingularityMainModule.REQUEST_IN_COOLDOWN_TEMPLATE) JadeTemplate requestInCooldownTemplate,
      @Named(SingularityMainModule.REQUEST_MODIFIED_TEMPLATE) JadeTemplate requestModifiedTemplate,
      @Named(SingularityMainModule.RATE_LIMITED_TEMPLATE) JadeTemplate rateLimitedTemplate) {

    this.smtpSender = smtpSender;
    this.maybeSmtpConfiguration = configuration.getSmtpConfiguration();
    this.configuration = configuration;
    this.uiHostnameAndPath = configuration.getUiConfiguration().getBaseUrl();
    this.taskManager = taskManager;
    this.sandboxManager = sandboxManager;
    this.metadataManager = metadataManager;
    this.exceptionNotifier = exceptionNotifier;
    this.adminJoiner = Joiner.on(", ").skipNulls();

    this.requestModifiedTemplate = requestModifiedTemplate;
    this.taskTemplate = taskTemplate;
    this.requestInCooldownTemplate = requestInCooldownTemplate;
    this.rateLimitedTemplate = rateLimitedTemplate;

    if (maybeSmtpConfiguration.isPresent()) {
      this.mailPreparerExecutorService = Optional.of(JavaUtils.newFixedTimingOutThreadPool(maybeSmtpConfiguration.get().getMailMaxThreads(), TimeUnit.SECONDS.toMillis(1), "SingularityMailPreparer-%d"));
    } else {
      this.mailPreparerExecutorService = Optional.absent();
    }
  }

  @Override
  public void start() throws Exception {
  }

  @Override
  public void stop() throws Exception {
    if (mailPreparerExecutorService.isPresent()) {
      MoreExecutors.shutdownAndAwaitTermination(mailPreparerExecutorService.get(), 1, TimeUnit.SECONDS);
    }
  }

  private Optional getTaskLogFile(final SingularityTaskId taskId, final String filename, final Optional task, final Optional directory) {
    if (!task.isPresent() || !directory.isPresent()) {
      LOG.warn("Couldn't retrieve {} for {} because task ({}) or directory ({}) wasn't present", filename, taskId, task.isPresent(), directory.isPresent());
      return Optional.absent();
    }

    final String slaveHostname = task.get().getOffer().getHostname();

    final String fullPath = String.format("%s/%s", directory.get(), filename);

    final Long logLength = Long.valueOf(maybeSmtpConfiguration.get().getTaskLogLength());

    final Optional logChunkObject;

    try {
      logChunkObject = sandboxManager.read(slaveHostname, fullPath, Optional.of(0L), Optional.of(logLength));
    } catch (RuntimeException e) {
      LOG.error("Sandboxmanager failed to read {}/{} on slave {}", directory.get(), filename, slaveHostname, e);
      return Optional.absent();
    }

    if (logChunkObject.isPresent()) {
      return Optional.of(logChunkObject.get().getData().split("[\r\n]+"));
    } else {
      LOG.error("Singularity mailer failed to get {} log for {} task ", filename, taskId.getId());
      return Optional.absent();
    }
  }

  private void populateRequestEmailProperties(Map templateProperties, SingularityRequest request) {
    templateProperties.put("requestId", request.getId());
    templateProperties.put("singularityRequestLink", getSingularityRequestLink(request));

    templateProperties.put("requestScheduled", request.isScheduled());
    templateProperties.put("requestOneOff", request.isOneOff());

    templateProperties.put("taskWillRetry", request.getNumRetriesOnFailure().or(0) > 0);
    templateProperties.put("numRetries", request.getNumRetriesOnFailure().or(0));
  }

  private void populateTaskEmailProperties(Map templateProperties, SingularityTaskId taskId, Collection taskHistory, ExtendedTaskState taskState) {
    Optional task = taskManager.getTask(taskId);
    Optional directory = taskManager.getDirectory(taskId);

    templateProperties.put("singularityTaskLink", getSingularityTaskLink(taskId));
    templateProperties.put("stdout", getTaskLogFile(taskId, "stdout", task, directory).or(new String[0]));
    templateProperties.put("stderr", getTaskLogFile(taskId, "stderr", task, directory).or(new String[0]));
    templateProperties.put("taskId", taskId.getId());
    templateProperties.put("deployId", taskId.getDeployId());

    templateProperties.put("taskDirectory", directory.or("directory missing"));

    if (task.isPresent()) {
      templateProperties.put("slaveHostname", task.get().getOffer().getHostname());
    }

    boolean needsBeenPrefix = taskState == ExtendedTaskState.TASK_LOST || taskState == ExtendedTaskState.TASK_KILLED;

    templateProperties.put("status", String.format("%s%s", needsBeenPrefix ? "has been " : "has ", taskState.getDisplayName()));
    templateProperties.put("taskStateLost", taskState == ExtendedTaskState.TASK_LOST);
    templateProperties.put("taskStateFailed", taskState == ExtendedTaskState.TASK_FAILED);
    templateProperties.put("taskStateFinished", taskState == ExtendedTaskState.TASK_FINISHED);
    templateProperties.put("taskStateKilled", taskState == ExtendedTaskState.TASK_KILLED);
    templateProperties.put("taskStateRunning", taskState == ExtendedTaskState.TASK_RUNNING);

    templateProperties.put("taskUpdates", JadeHelper.getJadeTaskHistory(taskHistory));
    templateProperties.put("taskRan", didTaskRun(taskHistory));
  }

  private Optional getTaskCleanupTypefromSingularityTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate) {
    if (!taskHistoryUpdate.getStatusMessage().isPresent()) {
      return Optional.absent();
    }

    try {
      return Optional.of(TaskCleanupType.valueOf(taskHistoryUpdate.getStatusMessage().get()));
    } catch (IllegalArgumentException iae) {
      LOG.warn("Couldn't parse TaskCleanupType from update {}", taskHistoryUpdate);
      return Optional.absent();
    }
  }

  private Optional getEmailType(ExtendedTaskState taskState, SingularityRequest request, Collection taskHistory) {
    switch (taskState) {
      case TASK_FAILED:
        return Optional.of(EmailType.TASK_FAILED);
      case TASK_FINISHED:
        if (request.isScheduled()) {
          return Optional.of(EmailType.TASK_FINISHED);
        }
        return Optional.of(EmailType.TASK_FINISHED_NON_SCHEDULED_REQUEST);
      case TASK_KILLED:
        Optional cleaningUpdate = SingularityTaskHistoryUpdate.getUpdate(taskHistory, ExtendedTaskState.TASK_CLEANING);

        if (cleaningUpdate.isPresent()) {
          Optional cleanupType = getTaskCleanupTypefromSingularityTaskHistoryUpdate(cleaningUpdate.get());

          if (cleanupType.isPresent()) {
            switch (cleanupType.get()) {
              case DECOMISSIONING:
                return Optional.of(EmailType.TASK_KILLED_DECOMISSIONED);
              case UNHEALTHY_NEW_TASK:
              case OVERDUE_NEW_TASK:
                return Optional.of(EmailType.TASK_KILLED_UNHEALTHY);
              default:
            }
          }
        }

        return Optional.of(EmailType.TASK_KILLED);
      case TASK_LOST:
        return Optional.of(EmailType.TASK_LOST);
      default:
        return Optional.absent();
    }
  }

  public void sendTaskOverdueMail(final SingularityTaskId taskId, final SingularityRequest request, final long runTime, final long expectedRuntime) {
    final Builder templateProperties = ImmutableMap.builder();

    templateProperties.put("runTime", DurationFormatUtils.formatDurationHMS(runTime));
    templateProperties.put("expectedRunTime", DurationFormatUtils.formatDurationHMS(expectedRuntime));
    templateProperties.put("warningThreshold", String.format("%s%%", configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()));

    templateProperties.put("status", "is overdue to finish");

    prepareTaskMail(taskId, request, EmailType.TASK_SCHEDULED_OVERDUE_TO_FINISH, templateProperties.build(), taskManager.getTaskHistoryUpdates(taskId), ExtendedTaskState.TASK_RUNNING);
  }

  public void sendTaskCompletedMail(final SingularityTaskId taskId, final SingularityRequest request, final ExtendedTaskState taskState) {
    if (!maybeSmtpConfiguration.isPresent()) {
      LOG.debug("Not sending task completed mail - no SMTP configuration is present");
      return;
    }

    mailPreparerExecutorService.get().submit(new Runnable() {

      @Override
      public void run() {
        try {
          prepareTaskCompletedMail(taskId, request, taskState);
        } catch (Throwable t) {
          LOG.error("While preparing task completed mail for {}", taskId, t);
          exceptionNotifier.notify(t);
        }
      }
    });
  }

  private void prepareTaskMail(SingularityTaskId taskId, SingularityRequest request, EmailType emailType, Map extraProperties,
      Collection taskHistory, ExtendedTaskState taskState) {

    final Collection emailDestination = getDestination(emailType);

    if (emailDestination.isEmpty()) {
      LOG.debug("Not configured to send task mail for {}", emailType);
      return;
    }

    final Map templateProperties = Maps.newHashMap();
    populateRequestEmailProperties(templateProperties, request);
    populateTaskEmailProperties(templateProperties, taskId, taskHistory, taskState);
    templateProperties.putAll(extraProperties);

    final String subject = getSubjectForTaskHistory(taskId, taskState, emailType, taskHistory);

    final String adminEmails = adminJoiner.join(maybeSmtpConfiguration.get().getAdmins());
    templateProperties.put("adminEmails", adminEmails);

    final String body = Jade4J.render(taskTemplate, templateProperties);

    queueMail(emailDestination, request, emailType, subject, body);
  }

  private void prepareTaskCompletedMail(SingularityTaskId taskId, SingularityRequest request, ExtendedTaskState taskState) {
    final Collection taskHistory = taskManager.getTaskHistoryUpdates(taskId);
    final Optional emailType = getEmailType(taskState, request, taskHistory);

    if (!emailType.isPresent()) {
      LOG.debug("No configured emailType for {} and {}", request, taskState);
      return;
    }

    prepareTaskMail(taskId, request, emailType.get(), Collections. emptyMap(), taskHistory, taskState);
  }

  private List getDestination(EmailType type) {
    List fromMap = maybeSmtpConfiguration.get().getEmailConfiguration().get(type);
    if (fromMap == null) {
      return Collections.emptyList();
    }
    return fromMap;
  }

  public enum RequestMailType {

    PAUSED(EmailType.REQUEST_PAUSED), UNPAUSED(EmailType.REQUEST_UNPAUSED), REMOVED(EmailType.REQUEST_REMOVED);

    private final EmailType emailType;

    private RequestMailType(EmailType emailType) {
      this.emailType = emailType;
    }

    public EmailType getEmailType() {
      return emailType;
    }

  }

  private void sendRequestMail(final SingularityRequest request, final RequestMailType type, final Optional user) {
    if (!maybeSmtpConfiguration.isPresent()) {
      LOG.debug("Not sending request mail - no SMTP configuration is present");
      return;
    }

    mailPreparerExecutorService.get().submit(new Runnable() {

      @Override
      public void run() {
        try {
          prepareRequestMail(request, type, user);
        } catch (Throwable t) {
          LOG.error("While preparing request mail for {} / {}", request, type, t);
          exceptionNotifier.notify(t);
        }
      }
    });
  }

  private void prepareRequestMail(SingularityRequest request, RequestMailType type, Optional user) {
    final List emailDestination = getDestination(type.getEmailType());

    if (emailDestination.isEmpty()) {
      LOG.debug("Not configured to send request cooldown mail for");
      return;
    }

    final String subject = String.format("Request %s has been %s — Singularity", request.getId(), type.name().toLowerCase());
    final Map templateProperties = Maps.newHashMap();
    populateRequestEmailProperties(templateProperties, request);

    templateProperties.put("requestPaused", type == RequestMailType.PAUSED);
    templateProperties.put("requestUnpaused", type == RequestMailType.UNPAUSED);
    templateProperties.put("action", type.name().toLowerCase());
    templateProperties.put("hasUser", user.isPresent());

    if (user.isPresent()) {
      templateProperties.put("user", user.get());
    }

    final String body = Jade4J.render(requestModifiedTemplate, templateProperties);

    queueMail(emailDestination, request, type.getEmailType(), subject, body);
  }

  public void sendRequestPausedMail(SingularityRequest request, Optional user) {
    sendRequestMail(request, RequestMailType.PAUSED, user);
  }

  public void sendRequestUnpausedMail(SingularityRequest request, Optional user) {
    sendRequestMail(request, RequestMailType.UNPAUSED, user);
  }

  public void sendRequestRemovedMail(SingularityRequest request, Optional user) {
    sendRequestMail(request, RequestMailType.REMOVED, user);
  }

  public void sendRequestInCooldownMail(final SingularityRequest request) {
    if (!maybeSmtpConfiguration.isPresent()) {
      LOG.debug("Not sending request in cooldown mail - no SMTP configuration is present");
      return;
    }

    mailPreparerExecutorService.get().submit(new Runnable() {

      @Override
      public void run() {
        try {
          prepareRequestInCooldownMail(request);
        } catch (Throwable t) {
          LOG.error("While preparing request in cooldown mail for {}", request, t);
          exceptionNotifier.notify(t);
        }
      }
    });
  }

  private void prepareRequestInCooldownMail(SingularityRequest request) {
    final List emailDestination = getDestination(EmailType.REQUEST_IN_COOLDOWN);

    if (emailDestination.isEmpty()) {
      LOG.debug("Not configured to send request cooldown mail for");
      return;
    }

    final Map templateProperties = Maps.newHashMap();
    populateRequestEmailProperties(templateProperties, request);

    final String subject = String.format("Request %s has entered system cooldown — Singularity", request.getId());

    templateProperties.put("numFailures", configuration.getCooldownAfterFailures());
    templateProperties.put("cooldownDelayFormat", DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(configuration.getCooldownMinScheduleSeconds())));
    templateProperties.put("cooldownExpiresFormat", DurationFormatUtils.formatDurationHMS(TimeUnit.MINUTES.toMillis(configuration.getCooldownExpiresAfterMinutes())));

    final String body = Jade4J.render(requestInCooldownTemplate, templateProperties);

    queueMail(emailDestination, request, EmailType.REQUEST_IN_COOLDOWN, subject, body);
  }

  private boolean didTaskRun(Collection history) {
    SimplifiedTaskState simplifiedTaskState = SingularityTaskHistoryUpdate.getCurrentState(history);

    return simplifiedTaskState == SimplifiedTaskState.DONE || simplifiedTaskState == SimplifiedTaskState.RUNNING;
  }

  private String getSubjectForTaskHistory(SingularityTaskId taskId, ExtendedTaskState state, EmailType type, Collection history) {
    if (type == EmailType.TASK_SCHEDULED_OVERDUE_TO_FINISH) {
      return String.format("Task is overdue to finish (%s)", taskId.toString());
    }

    if (!didTaskRun(history)) {
      return String.format("Task never started and was %s (%s)", state.getDisplayName(), taskId.toString());
    }

    return String.format("Task %s (%s)", state.getDisplayName(), taskId.toString());
  }

  private String getSingularityTaskLink(SingularityTaskId taskId) {
    if (!uiHostnameAndPath.isPresent()) {
      return "";
    }

    return String.format(TASK_LINK_FORMAT, uiHostnameAndPath.get(), taskId.getId());
  }

  private String getSingularityRequestLink(SingularityRequest request) {
    if (!uiHostnameAndPath.isPresent()) {
      return "";
    }

    return String.format(REQUEST_LINK_FORMAT, uiHostnameAndPath.get(), request.getId());
  }

  private enum RateLimitResult {
    SEND_MAIL, DONT_SEND_MAIL_IN_COOLDOWN, SEND_COOLDOWN_STARTED_MAIL;
  }

  private RateLimitResult checkRateLimitForMail(SingularityRequest request, EmailType emailType) {
    if (maybeSmtpConfiguration.get().getRateLimitAfterNotifications() < 1) {
      LOG.trace("Mail rate limit cooldown disabled");
      return RateLimitResult.SEND_MAIL;
    }

    final String requestId = request.getId();
    final String emailTypeName = emailType.name();

    final long now = System.currentTimeMillis();

    final Optional cooldownMarker = metadataManager.getMailCooldownMarker(requestId, emailTypeName);

    if (cooldownMarker.isPresent()) {
      final long cooldownLeft = maybeSmtpConfiguration.get().getRateLimitCooldownMillis() - (now - Long.parseLong(cooldownMarker.get()));

      if (cooldownLeft > 0) {
        LOG.debug("Not sending {} for {} - mail cooldown has {} time left out of {}", emailTypeName, requestId, cooldownLeft, maybeSmtpConfiguration.get().getRateLimitCooldownMillis());
        return RateLimitResult.DONT_SEND_MAIL_IN_COOLDOWN;
      }

      metadataManager.removeMailCooldown(requestId, emailTypeName);
    }

    metadataManager.saveMailRecord(request, emailType);

    int numInPeriod = 0;

    for (String recentMailRecord : metadataManager.getMailRecords(request.getId(), emailType.name())) {
      if (now - Long.parseLong(recentMailRecord) < maybeSmtpConfiguration.get().getRateLimitPeriodMillis()) {
        numInPeriod++;
      }
    }

    if (numInPeriod > maybeSmtpConfiguration.get().getRateLimitAfterNotifications()) {
      LOG.info("{} for {} sent at least {} times in {}, not sending this mail again for at least {}", emailTypeName, requestId, numInPeriod, maybeSmtpConfiguration.get().getRateLimitAfterNotifications(), maybeSmtpConfiguration.get().getRateLimitCooldownMillis());
      metadataManager.cooldownMail(requestId, emailTypeName);
      return RateLimitResult.SEND_COOLDOWN_STARTED_MAIL;
    }

    return RateLimitResult.SEND_MAIL;
  }

  private Map getRateLimitTemplateProperties(SingularityRequest request, final EmailType emailType) {
    final Builder templateProperties = ImmutableMap.builder();

    templateProperties.put("singularityRequestLink", getSingularityRequestLink(request));
    templateProperties.put("rateLimitAfterNotifications", Integer.toString(maybeSmtpConfiguration.get().getRateLimitAfterNotifications()));
    templateProperties.put("rateLimitPeriodFormat", DurationFormatUtils.formatDurationHMS(maybeSmtpConfiguration.get().getRateLimitPeriodMillis()));
    templateProperties.put("rateLimitCooldownFormat", DurationFormatUtils.formatDurationHMS(maybeSmtpConfiguration.get().getRateLimitCooldownMillis()));
    templateProperties.put("emailType", emailType.name());
    templateProperties.put("requestId", request.getId());

    return templateProperties.build();
  }

  private void queueMail(final Collection destination, final SingularityRequest request, final EmailType emailType, String subject, String body) {
    RateLimitResult result = checkRateLimitForMail(request, emailType);

    if (result == RateLimitResult.DONT_SEND_MAIL_IN_COOLDOWN) {
      return;
    }

    if (result == RateLimitResult.SEND_COOLDOWN_STARTED_MAIL) {
      subject = String.format("%s notifications for %s are being rate limited", emailType.name(), request.getId());
      body = Jade4J.render(rateLimitedTemplate, getRateLimitTemplateProperties(request, emailType));
    }

    final List toList = Lists.newArrayList();
    final List ccList = Lists.newArrayList();

    if (destination.contains(EmailDestination.OWNERS) && request.getOwners().isPresent() && !request.getOwners().get().isEmpty()) {
      toList.addAll(request.getOwners().get());
      if (destination.contains(EmailDestination.ADMINS)) {
        ccList.addAll(maybeSmtpConfiguration.get().getAdmins());
      }
    } else if (destination.contains(EmailDestination.ADMINS)) {
      toList.addAll(maybeSmtpConfiguration.get().getAdmins());
    }

    smtpSender.queueMail(toList, ccList, subject, body);
  }

}