only escalation should cause escalated to be populated in signal

Author: ktoso

Samples/SampleMetrics/main.swift

@@ -80,7 +80,9 @@ struct MetricPrinter {
 
             return .receiveMessage { _ in
                 print("------------------------------------------------------------------------------------------")
-                print(prom.collect())
+                prom.collect { (stringRepr: String) in
+                                print(stringRepr)
+                }
 
                 return .same
             }

Sources/DistributedActors/ActorShell.swift

@@ -319,15 +319,16 @@ internal final class ActorShell<Message>: ActorContext<Message>, AbstractActor {
             switch circumstances {
             // escalation takes precedence over death watch in terms of how we report errors
             case .escalating(let failure):
+                // we only populate `escalation` if the child is escalating
                 let terminated = Signals.ChildTerminated(address: ref.address, escalation: failure)
-                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated, escalation: true)
+                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated)
 
             case .stopped:
                 let terminated = Signals.ChildTerminated(address: ref.address, escalation: nil)
-                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated, escalation: false)
-            case .failed(let failure):
-                let terminated = Signals.ChildTerminated(address: ref.address, escalation: failure)
-                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated, escalation: false)
+                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated)
+            case .failed:
+                let terminated = Signals.ChildTerminated(address: ref.address, escalation: nil)
+                try self.interpretChildTerminatedSignal(who: ref, terminated: terminated)
             }
 
         case .nodeTerminated(let remoteNode):
@@ -396,10 +397,7 @@ internal final class ActorShell<Message>: ActorContext<Message>, AbstractActor {
     internal func fail(_ error: Error) {
         self._myCell.mailbox.setFailed()
         self.behavior = self.behavior.fail(cause: .error(error))
-        // TODO: we could handle here "wait for children to terminate"
 
-        // we only finishTerminating() here and not right away in message handling in order to give the Mailbox
-        // a chance to react to the problem as well; I.e. 1) we throw 2) mailbox sets terminating 3) we get fail() 4) we REALLY terminate
         switch error {
         case DeathPactError.unhandledDeathPact(_, _, let message):
             self.log.error("\(message)") // TODO: configurable logging? in props?
@@ -530,6 +528,8 @@ internal final class ActorShell<Message>: ActorContext<Message>, AbstractActor {
 
         // become stopped, if not already
         switch self.behavior.underlying {
+        case .failed(_, let failure):
+            self.behavior = .stop(reason: .failure(failure))
         case .stop(_, let reason):
             self.behavior = .stop(reason: reason)
         default:
@@ -727,7 +727,7 @@ extension ActorShell {
     }
 
     @inlinable
-    internal func interpretChildTerminatedSignal(who terminatedRef: AddressableActorRef, terminated: Signals.ChildTerminated, escalation: Bool) throws {
+    internal func interpretChildTerminatedSignal(who terminatedRef: AddressableActorRef, terminated: Signals.ChildTerminated) throws {
         #if SACT_TRACE_ACTOR_SHELL
         self.log.info("Received \(terminated)")
         #endif

Sources/DistributedActors/ActorSystemSettings.swift

@@ -55,6 +55,7 @@ public struct ActorSystemSettings {
     /// - SeeAlso: `FaultSupervisionMode` for a detailed discussion of the available modes.
     public var faultSupervisionMode: FaultSupervisionMode = .isolateYetMayLeakMemory
 
+    /// Determines what action should be taken when a failure is escalated to a top level guardian (e.g. `/user` or `/system).
     public var guardianFailureHandling: GuardianFailureHandling = .shutdownActorSystem
 }
 

Sources/DistributedActors/ProcessIsolated/ProcessIsolated.swift

@@ -312,6 +312,7 @@ extension ProcessIsolated {
                     self.system.log.info("\(messagePrefix): RESTART, as decided by: \(restartLogic)")
                     self.control.requestSpawnServant(supervision: servant.supervisionStrategy, args: servant.args)
                 case .restartBackoff:
+                    self.system.log.info("\(messagePrefix): RESTART BACKOFF, as decided by: \(restartLogic)")
                     // TODO: implement backoff for process isolated
                     fatalError("\(messagePrefix): BACKOFF NOT IMPLEMENTED YET")
                 }

Sources/DistributedActors/Props.swift

@@ -15,7 +15,7 @@
 import NIO
 
 // ==== ----------------------------------------------------------------------------------------------------------------
-// MARK: Actor Props
+// MARK: Props
 
 /// `Props` configure an Actors' properties such as mailbox, dispatcher as well as supervision semantics.
 ///

Sources/DistributedActors/Signals.swift

@@ -88,13 +88,42 @@ public enum Signals {
     }
 
     /// Signal sent to a parent actor when an actor it has spawned, i.e. its child, has terminated.
+    /// Upon processing this signal, the parent MAY choose to spawn another child with the _same_ name as the now terminated child --
+    /// a guarantee which is not enjoyed by watching actors from any other actor.
+    ///
+    /// This signal is sent to the parent _always_, i.e. both for the child stopping naturally as well as failing.
+    ///
+    /// ### Death Pacts with Children
     ///
-    /// This signal is sent and can be handled regardless if the child was watched (using `context.watch()`) or not.
     /// If the child is NOT being watched by the parent, this signal will NOT cause the parent (recipient of this signal)
     /// to kill kill itself by throwing an [DeathPactError], as this is reserved only to when a death pact is formed.
     /// In other words, if the parent spawns child actors but does not watch them, this is taken as not caring enough about
     /// their lifetime as to trigger termination itself if one of them terminates.
     ///
+    /// ### Failure Escalation
+    ///
+    /// It is possible, because of the special relationship parent-child actors enjoy, to spawn a child actor using the
+    /// `.escalate` strategy, which means that if the child fails, it will populate the `escalation` failure reason of
+    /// the `ChildTerminated` signal. Propagating failure reasons is not supported through `watch`-ed actors, and is only
+    /// available to parent-child pairs.
+    ///
+    /// This `escalation` failure can by used by the parent to decide if it should also fail, spawn a replacement child,
+    /// or perform any other action, manually. Not that spawning another actor in response to `ChildTerminated` means losing
+    /// the child's mailbox; unlike using the `.restart` supervision strategy, which keeps the mailbox, but instantiates
+    /// a new instance of the child behavior.
+    ///
+    /// It is NOT recommended to perform deep inspection of the escalated failure to perform complex logic, however it
+    /// may be used to determine if a specific error is "very bad" or "not bad enough" and we should start a replacement child.
+    ///
+    /// #### "Bubbling-up" Escalated Failures
+    ///
+    /// Escalated failures which are not handled will cause the parent to crash as well (!).
+    /// This enables spawning a hierarchy of actors, all of which use the `.escalate` strategy, meaning that the entire
+    /// section of the tree will be torn down upon failure of one of the workers. A higher level supervisor may then decide to
+    /// restart one of the higher actors, causing a "sub tree" to be restarted in response to a worker failure. Alternatively,
+    /// this pattern is useful when one wants to bubble up failures all the way to the guardian actors (`/user`, or `/system`),
+    /// in which case the system will issue a configured termination action (see `ActorSystemSettings.guardianFailureHandling`).
+    ///
     /// - Note: Note that `ChildTerminated` IS-A `Terminated` so unless you need to specifically react to a child terminating,
     ///         you may choose to handle all `Terminated` signals the same way.
     ///