=test fix log assertions, timeout alignents

ktoso · ktoso · commit 86682207c162 · 2020-01-21T20:27:55.000+09:00
diff --git a/Tests/ActorSingletonPluginTests/ActorSingletonPluginClusteredTests.swift b/Tests/ActorSingletonPluginTests/ActorSingletonPluginClusteredTests.swift
@@ -21,7 +21,7 @@ final class ActorSingletonPluginClusteredTests: ClusteredNodesTestBase {
     func test_singletonByClusterLeadership() throws {
         try shouldNotThrow {
             var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .leadership
+            singletonSettings.allocationStrategy = .byLeadership
 
             let first = self.setUpNode("first") { settings in
                 settings.cluster.node.port = 7111
@@ -75,7 +75,7 @@ final class ActorSingletonPluginClusteredTests: ClusteredNodesTestBase {
     func test_singletonByClusterLeadership_stashMessagesIfNoLeader() throws {
         try shouldNotThrow {
             var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .leadership
+            singletonSettings.allocationStrategy = .byLeadership
 
             let first = self.setUpNode("first") { settings in
                 settings.cluster.node.port = 7111
@@ -134,7 +134,7 @@ final class ActorSingletonPluginClusteredTests: ClusteredNodesTestBase {
     func test_singletonByClusterLeadership_withLeaderChange() throws {
         try shouldNotThrow {
             var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .leadership
+            singletonSettings.allocationStrategy = .byLeadership
 
             let first = self.setUpNode("first") { settings in
                 settings.cluster.node.port = 7111
diff --git a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift
@@ -45,227 +45,4 @@ final class ActorSingletonPluginTests: ActorSystemTestBase {
 
         try replyProbe.expectMessage("Hi Charlie!")
     }
-
-    func test_singletonByClusterLeadership() throws {
-        try shouldNotThrow {
-            var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .byLeadership
-
-            let first = self.setUpNode("first") { settings in
-                settings.cluster.node.port = 7111
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-1")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-            let second = self.setUpNode("second") { settings in
-                settings.cluster.node.port = 8222
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-2")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-            let third = self.setUpNode("third") { settings in
-                settings.cluster.node.port = 9333
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-3")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-
-            first.cluster.join(node: second.cluster.node.node)
-            third.cluster.join(node: second.cluster.node.node)
-
-            // `first` will be the leader (lowest address) and runs the singleton
-            try self.ensureNodes(.up, within: .seconds(10), systems: first, second, third)
-
-            let replyProbe1 = self.testKit(first).spawnTestProbe(expecting: String.self)
-            let ref1 = try first.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref1.tell(.greet(name: "Charlie", _replyTo: replyProbe1.ref))
-
-            let replyProbe2 = self.testKit(second).spawnTestProbe(expecting: String.self)
-            let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref))
-
-            let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self)
-            let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref))
-
-            try replyProbe1.expectMessage("Hello-1 Charlie!")
-            try replyProbe2.expectMessage("Hello-1 Charlie!")
-            try replyProbe3.expectMessage("Hello-1 Charlie!")
-        }
-    }
-
-    func test_singletonByClusterLeadership_stashMessagesIfNoLeader() throws {
-        try shouldNotThrow {
-            var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .byLeadership
-
-            let first = self.setUpNode("first") { settings in
-                settings.cluster.node.port = 7111
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-1")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-            let second = self.setUpNode("second") { settings in
-                settings.cluster.node.port = 8222
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-2")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-            let third = self.setUpNode("third") { settings in
-                settings.cluster.node.port = 9333
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-3")))
-
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-            }
-
-            // No leader so singleton is not available, messages sent should be stashed
-            let replyProbe1 = self.testKit(first).spawnTestProbe(expecting: String.self)
-            let ref1 = try first.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref1.tell(.greet(name: "Charlie", _replyTo: replyProbe1.ref))
-
-            let replyProbe2 = self.testKit(second).spawnTestProbe(expecting: String.self)
-            let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref))
-
-            let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self)
-            let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref))
-
-            try replyProbe1.expectNoMessage(for: .milliseconds(200))
-            try replyProbe2.expectNoMessage(for: .milliseconds(200))
-            try replyProbe3.expectNoMessage(for: .milliseconds(200))
-
-            first.cluster.join(node: second.cluster.node.node)
-            third.cluster.join(node: second.cluster.node.node)
-
-            // `first` becomes the leader (lowest address) and runs the singleton
-            try self.ensureNodes(.up, within: .seconds(10), systems: first, second, third)
-
-            try replyProbe1.expectMessage("Hello-1 Charlie!")
-            try replyProbe2.expectMessage("Hello-1 Charlie!")
-            try replyProbe3.expectMessage("Hello-1 Charlie!")
-        }
-    }
-
-    func test_singletonByClusterLeadership_withLeaderChange() throws {
-        try shouldNotThrow {
-            var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name)
-            singletonSettings.allocationStrategy = .byLeadership
-
-            let first = self.setUpNode("first") { settings in
-                settings.cluster.node.port = 7111
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-1")))
-            }
-            let second = self.setUpNode("second") { settings in
-                settings.cluster.node.port = 8222
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-2")))
-            }
-            let third = self.setUpNode("third") { settings in
-                settings.cluster.node.port = 9333
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-3")))
-            }
-            let fourth = self.setUpNode("fourth") { settings in
-                settings.cluster.node.port = 7444
-                settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3)
-                settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001)
-
-                settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-4")))
-            }
-
-            try self.joinNodes(node: first, with: second)
-            try self.joinNodes(node: third, with: second)
-            try self.ensureNodes(.up, within: .seconds(10), systems: first, second, third)
-
-            let replyProbe1 = self.testKit(first).spawnTestProbe(expecting: String.self)
-            let ref1 = try first.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref1.tell(.greet(name: "Charlie", _replyTo: replyProbe1.ref))
-
-            let replyProbe2 = self.testKit(second).spawnTestProbe(expecting: String.self)
-            let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref))
-
-            let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self)
-            let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self)
-            ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref))
-
-            // `first` has the lowest address so it should be the leader and singleton
-            try replyProbe1.expectMessage("Hello-1 Charlie!")
-            try replyProbe2.expectMessage("Hello-1 Charlie!")
-            try replyProbe3.expectMessage("Hello-1 Charlie!")
-
-            // Take down the leader
-            // first.cluster.down(node: first.cluster.node.node) // FIXME: must also work when the node downs itself and shuts down (!!!) (we do not move to down currently, no default downing impl)
-            second.cluster.down(node: first.cluster.node.node)
-
-            // Ensure the node is seen down
-            try self.ensureNodes(.down, on: second, systems: first)
-            try self.ensureNodes(.down, on: third, systems: first)
-
-            // At the same time, since the node was downed, it is not the leader anymore
-            try self.testKit(third).eventually(within: .seconds(10)) {
-                try self.assertLeaderNode(on: third, is: nil)
-                try self.assertLeaderNode(on: second, is: nil)
-            }
-
-            // `fourth` will become the new leader and singleton
-            try self.joinNodes(node: fourth, with: second)
-            try self.ensureNodes(.up, within: .seconds(10), systems: fourth, second, third)
-
-            try self.testKit(first).eventually(within: .seconds(10)) {
-                // The stashed messages get routed to new singleton running on `fourth`
-
-                ref2.tell(.greet(name: "Charlie-2", _replyTo: replyProbe2.ref))
-                guard let reply2 = try replyProbe2.maybeExpectMessage() else {
-                    pprint("lost msg @ node 2")
-                    throw TestError("No reply to \(replyProbe2) yet, singleton still rebalancing...?")
-                }
-                reply2.shouldEqual("Hello-4 Charlie-2!")
-
-                ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe3.ref))
-                guard let reply3 = try replyProbe3.maybeExpectMessage() else {
-                    pprint("lost msg @ node 3")
-                    throw TestError("No reply to \(replyProbe3) yet, singleton still rebalancing...?")
-                }
-                reply3.shouldEqual("Hello-4 Charlie-3!")
-            }
-        }
-    }
-}
-
-// ==== ----------------------------------------------------------------------------------------------------------------
-// MARK: Test utilities
-
-struct GreeterSingleton: Actorable {
-    static let name = "greeter"
-
-    private let greeting: String
-
-    init(_ greeting: String) {
-        self.greeting = greeting
-    }
-
-    func greet(name: String) -> String {
-        "\(self.greeting) \(name)!"
-    }
 }
diff --git a/Tests/DistributedActorsTests/Cluster/ClusterOnDownActionTests.swift b/Tests/DistributedActorsTests/Cluster/ClusterOnDownActionTests.swift
@@ -20,17 +20,19 @@ import XCTest
 final class ClusterOnDownActionTests: ClusteredNodesTestBase {
     func test_onNodeDowned_performShutdown() throws {
         try shouldNotThrow {
-            let (first, second) = self.setUpPair()
+            let (first, second) = self.setUpPair { settings in
+                settings.cluster.onDownAction = .gracefulShutdown(delay: .milliseconds(300))
+            }
 
             try self.joinNodes(node: first, with: second)
 
             second.cluster.down(node: first.cluster.node.node)
 
-            try self.capturedLogs(of: first).awaitLogContaining(self.testKit(first), text: "Self node was determined [.down]!")
+            try self.capturedLogs(of: first).awaitLogContaining(self.testKit(first), text: "Self node was marked [.down]!")
 
-            try self.testKit(first).eventually(within: .seconds(1)) {
+            try self.testKit(first).eventually(within: .seconds(3)) {
                 guard first.isShuttingDown else {
-                    throw TestError("First system should be shutting down, it was marked down and the default onDownAction should have triggered!")
+                    throw TestError("System \(first) was not shutting down. It was marked down and the default onDownAction should have triggered!")
                 }
             }
         }
@@ -46,7 +48,7 @@ final class ClusterOnDownActionTests: ClusteredNodesTestBase {
 
             second.cluster.down(node: first.cluster.node.node)
 
-            try self.capturedLogs(of: first).awaitLogContaining(self.testKit(first), text: "Self node was determined [.down]!")
+            try self.capturedLogs(of: first).awaitLogContaining(self.testKit(first), text: "Self node was marked [.down]!")
 
             first.isShuttingDown.shouldBeFalse()
         }
diff --git a/Tests/DistributedActorsTests/Cluster/ShootTheOtherNodeClusteredTests.swift b/Tests/DistributedActorsTests/Cluster/ShootTheOtherNodeClusteredTests.swift
@@ -18,10 +18,6 @@ import NIO
 import XCTest
 
 final class ShootTheOtherNodeClusteredTests: ClusteredNodesTestBase {
-    override var alwaysPrintCaptureLogs: Bool {
-        true
-    }
-
     override func configureLogCapture(settings: inout LogCapture.Settings) {
         settings.excludeGrep = [
             "TimerKey",
@@ -50,9 +46,7 @@ final class ShootTheOtherNodeClusteredTests: ClusteredNodesTestBase {
             // we do NOT failTest:, since we are in an eventuallyBlock and are waiting for the logs to happen still
             // the eventually block will escalate the thrown errors if they do not cease within the time limit.
             try self.capturedLogs(of: remote).shouldContain(prefix: "Received .restInPeace", failTest: false)
-            try self.capturedLogs(of: remote).shouldContain(prefix: "Self node was determined [.down]", failTest: false)
+            try self.capturedLogs(of: remote).shouldContain(prefix: "Self node was marked [.down]", failTest: false)
         }
-
-//        try self.assertMemberStatus(on: local, node: remote.cluster.node, is: .down)
     }
 }
