All Downloads are FREE. Search and download functionalities are using the official Maven repository.

akka.actor.CoordinatedShutdown.scala Maven / Gradle / Ivy

There is a newer version: 2.2.6.3
Show newest version
/**
 * Copyright (C) 2016 Lightbend Inc. 
 */
package akka.actor

import scala.concurrent.duration._
import scala.compat.java8.FutureConverters._
import scala.compat.java8.OptionConverters._
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.TimeUnit.MILLISECONDS

import scala.concurrent.ExecutionContext
import scala.concurrent.Future
import scala.concurrent.Promise

import akka.Done
import com.typesafe.config.Config
import scala.concurrent.duration.FiniteDuration
import scala.annotation.tailrec
import com.typesafe.config.ConfigFactory
import akka.pattern.after
import java.util.concurrent.TimeoutException
import scala.util.control.NonFatal
import akka.event.Logging
import akka.dispatch.ExecutionContexts
import java.util.concurrent.Executors
import scala.util.Try
import scala.concurrent.Await
import java.util.concurrent.CountDownLatch
import java.util.concurrent.atomic.AtomicReference
import java.util.function.Supplier
import java.util.concurrent.CompletionStage
import java.util.Optional

object CoordinatedShutdown extends ExtensionId[CoordinatedShutdown] with ExtensionIdProvider {
  val PhaseBeforeServiceUnbind = "before-service-unbind"
  val PhaseServiceUnbind = "service-unbind"
  val PhaseServiceRequestsDone = "service-requests-done"
  val PhaseServiceStop = "service-stop"
  val PhaseBeforeClusterShutdown = "before-cluster-shutdown"
  val PhaseClusterShardingShutdownRegion = "cluster-sharding-shutdown-region"
  val PhaseClusterLeave = "cluster-leave"
  val PhaseClusterExiting = "cluster-exiting"
  val PhaseClusterExitingDone = "cluster-exiting-done"
  val PhaseClusterShutdown = "cluster-shutdown"
  val PhaseBeforeActorSystemTerminate = "before-actor-system-terminate"
  val PhaseActorSystemTerminate = "actor-system-terminate"

   /**
   * Reason for the shutdown, which can be used by tasks in case they need to do
   * different things depending on what caused the shutdown. There are some
   * predefined reasons, but external libraries applications may also define
   * other reasons.
   */
  trait Reason

  /**
   * Scala API: The reason for the shutdown was unknown. Needed for backwards compatibility.
   */
  case object UnknownReason extends Reason

  /**
   * Java API: The reason for the shutdown was unknown. Needed for backwards compatibility.
   */
  def unknownReason: Reason = UnknownReason

  /**
   * Scala API: The shutdown was initiated by ActorSystem.terminate.
   */
  case object ActorSystemTerminateReason extends Reason

  /**
   * Java API: The shutdown was initiated by ActorSystem.terminate.
   */
  def actorSystemTerminateReason: Reason = ActorSystemTerminateReason

  /**
   * Scala API: The shutdown was initiated by a JVM shutdown hook, e.g. triggered by SIGTERM.
   */
  case object JvmExitReason extends Reason

  /**
   * Java API: The shutdown was initiated by a JVM shutdown hook, e.g. triggered by SIGTERM.
   */
  def jvmExitReason: Reason = JvmExitReason

  /**
   * Scala API: The shutdown was initiated by Cluster downing.
   */
  case object ClusterDowningReason extends Reason

  /**
   * Java API: The shutdown was initiated by Cluster downing.
   */
  def clusterDowningReason: Reason = ClusterDowningReason

  /**
   * Scala API: The shutdown was initiated by a failure to join a seed node.
   */
  case object ClusterJoinUnsuccessfulReason extends Reason

  /**
   * Java API: The shutdown was initiated by a failure to join a seed node.
   */
  def clusterJoinUnsuccessfulReason: Reason = ClusterJoinUnsuccessfulReason

  /**
   * Scala API: The shutdown was initiated by a configuration clash within the existing cluster and the joining node
   */
  case object IncompatibleConfigurationDetectedReason extends Reason

  /**
   * Java API: The shutdown was initiated by a configuration clash within the existing cluster and the joining node
   */
  def incompatibleConfigurationDetectedReason: Reason = IncompatibleConfigurationDetectedReason

  /**
   * Scala API: The shutdown was initiated by Cluster leaving.
   */
  case object ClusterLeavingReason extends Reason


  @volatile private var runningJvmHook = false

  override def get(system: ActorSystem): CoordinatedShutdown = super.get(system)

  override def lookup = CoordinatedShutdown

  override def createExtension(system: ExtendedActorSystem): CoordinatedShutdown = {
    val conf = system.settings.config.getConfig("akka.coordinated-shutdown")
    val phases = phasesFromConfig(conf)
    val coord = new CoordinatedShutdown(system, phases)
    initPhaseActorSystemTerminate(system, conf, coord)
    initJvmHook(system, conf, coord)
    coord
  }

  private def initPhaseActorSystemTerminate(system: ActorSystem, conf: Config, coord: CoordinatedShutdown): Unit = {
    val terminateActorSystem = conf.getBoolean("terminate-actor-system")
    val exitJvm = conf.getBoolean("exit-jvm")
    if (terminateActorSystem || exitJvm) {
      coord.addTask(PhaseActorSystemTerminate, "terminate-system") { () ⇒
        if (exitJvm && terminateActorSystem) {
          // In case ActorSystem shutdown takes longer than the phase timeout,
          // exit the JVM forcefully anyway.
          // We must spawn a separate thread to not block current thread,
          // since that would have blocked the shutdown of the ActorSystem.
          /*val timeout = coord.timeout(PhaseActorSystemTerminate)
          val t = new Thread {
            override def run(): Unit = {
              if (Try(Await.ready(system.whenTerminated, timeout)).isFailure && !runningJvmHook)
                System.exit(0)
            }
          }
          t.setName("CoordinatedShutdown-exit")
          t.start()*/
          System.exit(0)
        }

        if (terminateActorSystem) {
          system.terminate().map { _ ⇒
            if (exitJvm && !runningJvmHook) System.exit(0)
            Done
          }(ExecutionContexts.sameThreadExecutionContext)
        } else if (exitJvm) {
          System.exit(0)
          Future.successful(Done)
        } else
          Future.successful(Done)
      }
    }
  }

  private def initJvmHook(system: ActorSystem, conf: Config, coord: CoordinatedShutdown): Unit = {
    val runByJvmShutdownHook = conf.getBoolean("run-by-jvm-shutdown-hook")
    if (runByJvmShutdownHook) {
      coord.addJvmShutdownHook {
        runningJvmHook = true // avoid System.exit from PhaseActorSystemTerminate task
        if (!system.whenTerminated.isCompleted) {
          coord.log.info("Starting coordinated shutdown from JVM shutdown hook")
          try
            //Await.ready(coord.run(), coord.totalTimeout())
            coord.run()
          catch {
            case NonFatal(e) ⇒
              coord.log.warning(
                "CoordinatedShutdown from JVM shutdown failed: {}",
                e.getMessage)
          }
        }
      }
    }
  }

  /**
   * INTERNAL API
   */
  private[akka] final case class Phase(dependsOn: Set[String], timeout: FiniteDuration, recover: Boolean)

  /**
   * INTERNAL API
   */
  private[akka] def phasesFromConfig(conf: Config): Map[String, Phase] = {
    val defaultPhaseTimeout = 5 seconds
    //conf.getDuration(conf.getString("default-phase-timeout"), MILLISECONDS).millis
    /*
    import scala.collection.JavaConverters._
    val defaultPhaseTimeout = conf.getString("default-phase-timeout")
    val phasesConf = conf.getConfig("phases")
    val defaultPhaseConfig = ConfigFactory.parseString(s"""
      timeout = $defaultPhaseTimeout
      recover = true
      depends-on = []
    """)
    phasesConf.root.unwrapped.asScala.toMap.map {
      case (k, _: java.util.Map[_, _]) ⇒
        val c = phasesConf.getConfig(k).withFallback(defaultPhaseConfig)
        val dependsOn = c.getStringList("depends-on").asScala.toSet
        val timeout = c.getDuration("timeout", MILLISECONDS).millis
        val recover = c.getBoolean("recover")
        k → Phase(dependsOn, timeout, recover)
      case (k, v) ⇒
        throw new IllegalArgumentException(s"Expected object value for [$k], got [$v]")
    }
    */

    //SCALA.JS HARDCODED NOW
    Map(
      "before-service-unbind" -> Phase(Set(), defaultPhaseTimeout, true),
      "service-unbind" -> Phase(Set("before-service-unbind"), defaultPhaseTimeout, true),
      "service-requests-done" -> Phase(Set("service-unbind"), defaultPhaseTimeout, true),
      "service-stop" -> Phase(Set("service-requests-done"), defaultPhaseTimeout, true),
      "before-actor-system-terminate" -> Phase(Set("service-stop"), defaultPhaseTimeout, true),
      "actor-system-terminate" -> Phase(Set("before-actor-system-terminate"), 10 seconds, true)
    )
  }

  /**
   * INTERNAL API: https://en.wikipedia.org/wiki/Topological_sorting
   */
  private[akka] def topologicalSort(phases: Map[String, Phase]): List[String] = {
    var result = List.empty[String]
    var unmarked = phases.keySet ++ phases.values.flatMap(_.dependsOn) // in case phase is not defined as key
    var tempMark = Set.empty[String] // for detecting cycles

    while (unmarked.nonEmpty) {
      depthFirstSearch(unmarked.head)
    }

    def depthFirstSearch(u: String): Unit = {
      if (tempMark(u))
        throw new IllegalArgumentException("Cycle detected in graph of phases. It must be a DAG. " +
          s"phase [$u] depends transitively on itself. All dependencies: $phases")
      if (unmarked(u)) {
        tempMark += u
        phases.get(u) match {
          case Some(Phase(dependsOn, _, _)) ⇒ dependsOn.foreach(depthFirstSearch)
          case None                         ⇒
        }
        unmarked -= u // permanent mark
        tempMark -= u
        result = u :: result
      }
    }

    result.reverse
  }

}

final class CoordinatedShutdown private[akka] (
  system: ExtendedActorSystem,
  phases: Map[String, CoordinatedShutdown.Phase]) extends Extension {
  import CoordinatedShutdown._

  /** INTERNAL API */
  private[akka] val log = Logging(system, getClass)
  private val knownPhases = phases.keySet ++ phases.values.flatMap(_.dependsOn)
  /** INTERNAL API */
  private[akka] val orderedPhases = CoordinatedShutdown.topologicalSort(phases)
  private val tasks = new ConcurrentHashMap[String, Vector[(String, () ⇒ Future[Done])]]
  private val runStarted = new AtomicBoolean(false)
  private val runPromise = Promise[Done]()

  private var _jvmHooksLatch = new AtomicReference[CountDownLatch](new CountDownLatch(0))

  /**
   * INTERNAL API
   */
  private[akka] def jvmHooksLatch: CountDownLatch = _jvmHooksLatch.get

  /**
   * Scala API: Add a task to a phase. It doesn't remove previously added tasks.
   * Tasks added to the same phase are executed in parallel without any
   * ordering assumptions. Next phase will not start until all tasks of
   * previous phase have been completed.
   *
   * Tasks should typically be registered as early as possible after system
   * startup. When running the coordinated shutdown tasks that have been registered
   * will be performed but tasks that are added too late will not be run.
   * It is possible to add a task to a later phase by a task in an earlier phase
   * and it will be performed.
   */
  @tailrec def addTask(phase: String, taskName: String)(task: () ⇒ Future[Done]): Unit = {
    require(
      knownPhases(phase),
      s"Unknown phase [$phase], known phases [$knownPhases]. " +
        "All phases (along with their optional dependencies) must be defined in configuration")
    val current = tasks.get(phase)
    if (current == null) {
      if (tasks.putIfAbsent(phase, Vector(taskName → task)) != null)
        addTask(phase, taskName)(task) // CAS failed, retry
    } else {
      if (!tasks.replace(phase, current, current :+ (taskName → task)))
        addTask(phase, taskName)(task) // CAS failed, retry
    }
  }

  /**
   * Java API: Add a task to a phase. It doesn't remove previously added tasks.
   * Tasks added to the same phase are executed in parallel without any
   * ordering assumptions. Next phase will not start until all tasks of
   * previous phase have been completed.
   *
   * Tasks should typically be registered as early as possible after system
   * startup. When running the coordinated shutdown tasks that have been registered
   * will be performed but tasks that are added too late will not be run.
   * It is possible to add a task to a later phase by a task in an earlier phase
   * and it will be performed.
   */
  def addTask(phase: String, taskName: String, task: Supplier[CompletionStage[Done]]): Unit =
    addTask(phase, taskName)(() ⇒ task.get().toScala)

  /**
   * Scala API: Run tasks of all phases. The returned
   * `Future` is completed when all tasks have been completed,
   * or there is a failure when recovery is disabled.
   *
   * It's safe to call this method multiple times. It will only run the once.
   */
  def run(): Future[Done] = run(UnknownReason)

  /**
   * Java API: Run tasks of all phases. The returned
   * `CompletionStage` is completed when all tasks have been completed,
   * or there is a failure when recovery is disabled.
   *
   * It's safe to call this method multiple times. It will only run the once.
   */
  def runAll(): CompletionStage[Done] = run().toJava

  /**
   * Scala API: Run tasks of all phases including and after the given phase.
   * The returned `Future` is completed when all such tasks have been completed,
   * or there is a failure when recovery is disabled.
   *
   * It's safe to call this method multiple times. It will only run the once.
   */
  def run(reason: Reason, fromPhase: Option[String]): Future[Done] = {
    if (runStarted.compareAndSet(false, true)) {
      import system.dispatcher
      val debugEnabled = log.isDebugEnabled
      def loop(remainingPhases: List[String]): Future[Done] = {
        remainingPhases match {
          case Nil ⇒ Future.successful(Done)
          case phase :: remaining ⇒
            val phaseResult = (tasks.get(phase) match {
              case null ⇒
                if (debugEnabled) log.debug("Performing phase [{}] with [0] tasks", phase)
                Future.successful(Done)
              case tasks ⇒
                if (debugEnabled) log.debug(
                  "Performing phase [{}] with [{}] tasks: [{}]",
                  phase, tasks.size, tasks.map { case (taskName, _) ⇒ taskName }.mkString(", "))
                // note that tasks within same phase are performed in parallel
                val recoverEnabled = phases(phase).recover
                val result = Future.sequence(tasks.map {
                  case (taskName, task) ⇒
                    try {
                      val r = task.apply()
                      if (recoverEnabled) r.recover {
                        case NonFatal(e) ⇒
                          log.warning("Task [{}] failed in phase [{}]: {}", taskName, phase, e.getMessage)
                          Done
                      }
                      else r
                    } catch {
                      case NonFatal(e) ⇒
                        // in case task.apply throws
                        if (recoverEnabled) {
                          log.warning("Task [{}] failed in phase [{}]: {}", taskName, phase, e.getMessage)
                          Future.successful(Done)
                        } else
                          Future.failed(e)
                    }
                }).map(_ ⇒ Done)(ExecutionContexts.sameThreadExecutionContext)
                val timeout = phases(phase).timeout
                val deadline = Deadline.now + timeout
                val timeoutFut = try {
                  after(timeout, system.scheduler) {
                    if (phase == CoordinatedShutdown.PhaseActorSystemTerminate && deadline.hasTimeLeft) {
                      // too early, i.e. triggered by system termination
                      result
                    } else if (result.isCompleted)
                      Future.successful(Done)
                    else if (recoverEnabled) {
                      log.warning("Coordinated shutdown phase [{}] timed out after {}", phase, timeout)
                      Future.successful(Done)
                    } else
                      Future.failed(
                        new TimeoutException(s"Coordinated shutdown phase [$phase] timed out after $timeout"))
                  }
                } catch {
                  case _: IllegalStateException ⇒
                    // The call to `after` threw IllegalStateException, triggered by system termination
                    result
                }
                Future.firstCompletedOf(List(result, timeoutFut))
            })
            if (remaining.isEmpty)
              phaseResult // avoid flatMap when system terminated in last phase
            else
              phaseResult.flatMap(_ ⇒ loop(remaining))
        }
      }

      val remainingPhases = fromPhase match {
        case None    ⇒ orderedPhases // all
        case Some(p) ⇒ orderedPhases.dropWhile(_ != p)
      }
      val done = loop(remainingPhases)
      runPromise.completeWith(done)
    }
    runPromise.future
  }

  /**
   * Java API: Run tasks of all phases including and after the given phase.
   * The returned `CompletionStage` is completed when all such tasks have been completed,
   * or there is a failure when recovery is disabled.
   *
   * It's safe to call this method multiple times. It will only run once.
   */
  // def run(fromPhase: Optional[String]): CompletionStage[Done] =
  //   run(fromPhase.asScala).toJava

  /**
   * Scala API: Run tasks of all phases. The returned
   * `Future` is completed when all tasks have been completed,
   * or there is a failure when recovery is disabled.
   *
   * It's safe to call this method multiple times. It will only run the shutdown sequence once.
   */
  def run(reason: Reason): Future[Done] = run(reason, None)

  /**
   * The configured timeout for a given `phase`.
   * For example useful as timeout when actor `ask` requests
   * is used as a task.
   */
  def timeout(phase: String): FiniteDuration =
    phases.get(phase) match {
      case Some(Phase(_, timeout, _)) ⇒ timeout
      case None ⇒
        throw new IllegalArgumentException(s"Unknown phase [$phase]. All phases must be defined in configuration")
    }

  /**
   * Sum of timeouts of all phases that have some task.
   */
  def totalTimeout(): FiniteDuration = {
    import scala.collection.JavaConverters._
    tasks.keySet.asScala.foldLeft(Duration.Zero) {
      case (acc, phase) ⇒ acc + timeout(phase)
    }
  }

  /**
   * Scala API: Add a JVM shutdown hook that will be run when the JVM process
   * begins its shutdown sequence. Added hooks may run in any order
   * concurrently, but they are running before Akka internal shutdown
   * hooks, e.g. those shutting down Artery.
   */
  /*@tailrec*/ def addJvmShutdownHook[T](hook: ⇒ T): Unit = {
    /*
    if (!runStarted.get) {
      val currentLatch = _jvmHooksLatch.get
      val newLatch = new CountDownLatch(currentLatch.getCount.toInt + 1)
      if (_jvmHooksLatch.compareAndSet(currentLatch, newLatch)) {
        try Runtime.getRuntime.addShutdownHook(new Thread {
          override def run(): Unit = {
            try hook finally _jvmHooksLatch.get.countDown()
          }
        }) catch {
          case e: IllegalStateException ⇒
            // Shutdown in progress, if CoordinatedShutdown is created via a JVM shutdown hook (Artery)
            log.warning("Could not addJvmShutdownHook, due to: {}", e.getMessage)
            _jvmHooksLatch.get.countDown()
        }
      } else
        addJvmShutdownHook(hook) // lost CAS, retry
    }
    */
  }

  /**
   * Java API: Add a JVM shutdown hook that will be run when the JVM process
   * begins its shutdown sequence. Added hooks may run in an order
   * concurrently, but they are running before Akka internal shutdown
   * hooks, e.g. those shutting down Artery.
   */
  def addJvmShutdownHook(hook: Runnable): Unit =
    addJvmShutdownHook(hook.run())

}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy