Singleton: address feedback in #311

Sources/ActorSingletonPlugin/ActorSingletonProxy.swift

@@ -79,6 +79,7 @@ internal class ActorSingletonProxy<Message> {
                 try self.forwardOrStash(context, message: message)
                 return .same
             }.receiveSpecificSignal(Signals.PostStop.self) { context, _ in
+                // TODO: perhaps we can figure out where `to` is next and hand over gracefully?
                 try self.handOver(context, to: nil)
                 return .same
             }
@@ -160,7 +161,7 @@ internal class ActorSingletonProxy<Message> {
     }
 
     private func updateRef(_ context: ActorContext<Message>, _ newRef: ActorRef<Message>?) {
-        context.log.debug("Updating ref from [\(String(describing: self.ref))] to [\(String(describing: newRef))]")
+        context.log.debug("Updating ref from [\(String(describing: self.ref))] to [\(String(describing: newRef))], flushing \(self.buffer.count) messages.")
         self.ref = newRef
 
         // Unstash messages if we have the singleton

Sources/DistributedActorsTestKit/Cluster/ClusteredNodesTestBase.swift

@@ -278,6 +278,27 @@ extension ClusteredNodesTestBase {
             throw testKit.error("Expected \(reflecting: foundMember.node) on \(reflecting: system.cluster.node) to be seen as: [\(expectedStatus)], but was [\(foundMember.status)]")
         }
     }
+
+    /// Asserts the given node is the leader.
+    ///
+    /// An error is thrown but NOT failing the test; use in pair with `testKit.eventually` to achieve the expected behavior.
+    public func assertLeaderNode(
+        on system: ActorSystem, is expectedNode: UniqueNode?,
+        file: StaticString = #file, line: UInt = #line
+    ) throws {
+        let testKit = self.testKit(system)
+        let p = testKit.spawnTestProbe(expecting: Membership.self)
+        defer {
+            p.stop()
+        }
+        system.cluster.ref.tell(.query(.currentMembership(p.ref)))
+
+        let membership = try p.expectMessage()
+        let leaderNode = membership.leader?.node
+        if leaderNode != expectedNode {
+            throw testKit.error("Expected \(reflecting: expectedNode) to be leader node on \(reflecting: system.cluster.node) but was [\(reflecting: leaderNode)]")
+        }
+    }
 }
 
 // ==== ----------------------------------------------------------------------------------------------------------------

Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift

@@ -18,16 +18,6 @@ import DistributedActorsTestKit
 import XCTest
 
 final class ActorSingletonPluginTests: ClusteredNodesTestBase {
-    var logCaptureHandler: LogCapture!
-
-    override func setUp() {
-        self.logCaptureHandler = LogCapture()
-    }
-
-    override func tearDown() {
-        self.logCaptureHandler.printIfFailed(self.testRun)
-    }
-
     func test_nonCluster() throws {
         // Singleton should work just fine without clustering
         let system = ActorSystem("test") { settings in
@@ -48,6 +38,8 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
 
         // singleton.actor
         let actor = try system.singleton.actor(name: "\(GreeterSingleton.name)-other", GreeterSingleton.self)
+        // TODO: https://github.com/apple/swift-distributed-actors/issues/344
+        //         let string = try probe.expectReply(actor.greet(name: "Charlie", _replyTo: replyProbe.ref))
         actor.ref.tell(.greet(name: "Charlie", _replyTo: replyProbe.ref))
 
         try replyProbe.expectMessage("Hi Charlie!")
@@ -58,9 +50,6 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
         singletonSettings.allocationStrategy = .leadership
 
         let first = self.setUpNode("first") { settings in
-            settings.overrideLogger = self.logCaptureHandler.makeLogger(label: settings.cluster.node.systemName)
-
-            settings.cluster.node.host = "127.0.0.1"
             settings.cluster.node.port = 7111
             settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
 
@@ -69,9 +58,6 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
             settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
         }
         let second = self.setUpNode("second") { settings in
-            settings.overrideLogger = self.logCaptureHandler.makeLogger(label: settings.cluster.node.systemName)
-
-            settings.cluster.node.host = "127.0.0.1"
             settings.cluster.node.port = 8222
             settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
 
@@ -80,9 +66,6 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
             settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
         }
         let third = self.setUpNode("third") { settings in
-            settings.overrideLogger = self.logCaptureHandler.makeLogger(label: settings.cluster.node.systemName)
-
-            settings.cluster.node.host = "127.0.0.1"
             settings.cluster.node.port = 9333
             settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
 
@@ -91,9 +74,6 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
             settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
         }
         let fourth = self.setUpNode("fourth") { settings in
-            settings.overrideLogger = self.logCaptureHandler.makeLogger(label: settings.cluster.node.systemName)
-
-            settings.cluster.node.host = "127.0.0.1"
             settings.cluster.node.port = 7444
             settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
 
@@ -102,28 +82,20 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
             settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
         }
 
-        defer {
-            first.shutdown().wait()
-            second.shutdown().wait()
-            third.shutdown().wait()
-            fourth.shutdown().wait()
-            // self.logCaptureHandler.printLogs()
-        }
-
         first.cluster.join(node: second.cluster.node.node)
         third.cluster.join(node: second.cluster.node.node)
 
         try self.ensureNodes(.up, within: .seconds(10), systems: first, second, third)
 
-        let replyProbe1 = ActorTestKit(first).spawnTestProbe(expecting: String.self)
+        let replyProbe1 = self.testKit(first).spawnTestProbe(expecting: String.self)
         let ref1 = try first.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
         ref1.tell(.greet(name: "Charlie", _replyTo: replyProbe1.ref))
 
-        let replyProbe2 = ActorTestKit(second).spawnTestProbe(expecting: String.self)
+        let replyProbe2 = self.testKit(second).spawnTestProbe(expecting: String.self)
         let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
         ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref))
 
-        let replyProbe3 = ActorTestKit(third).spawnTestProbe(expecting: String.self)
+        let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self)
         let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
         ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref))
 
@@ -135,11 +107,19 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase {
         // Take down the leader
         first.cluster.down(node: first.cluster.node.node)
 
-        try self.assertMemberStatus(on: first, node: first.cluster.node, is: .down)
+        // Make sure that `second` and `third` see `first` as down and become leader-less
+        try self.testKit(second).eventually(within: .seconds(10)) {
+            try self.assertMemberStatus(on: second, node: first.cluster.node, is: .down)
+            try self.assertLeaderNode(on: second, is: nil)
+        }
+        try self.testKit(third).eventually(within: .seconds(10)) {
+            try self.assertMemberStatus(on: third, node: first.cluster.node, is: .down)
+            try self.assertLeaderNode(on: third, is: nil)
+        }
 
         // No leader so singleton is not available, messages sent should be stashed
         ref2.tell(.greet(name: "Charlie-2", _replyTo: replyProbe2.ref))
-        ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe2.ref))
+        ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe3.ref))
 
         // `fourth` will become the new leader and singleton
         fourth.cluster.join(node: second.cluster.node.node)