From 1d92a4d993802b2c9707cdc7bb37dfec75dae44f Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Fri, 17 Jan 2020 21:28:59 +0900 Subject: [PATCH 1/3] =singleton #346 stabilize test --- .../ActorSingletonPlugin/ActorSingleton.swift | 6 +- .../ActorSingletonPluginTests.swift | 65 ++++++++++--------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/Sources/ActorSingletonPlugin/ActorSingleton.swift b/Sources/ActorSingletonPlugin/ActorSingleton.swift index 3357d8021..18a2fcfff 100644 --- a/Sources/ActorSingletonPlugin/ActorSingleton.swift +++ b/Sources/ActorSingletonPlugin/ActorSingleton.swift @@ -116,7 +116,7 @@ public struct ActorSingletonSettings { } /// Controls allocation of the node on which the singleton runs. - public var allocationStrategy: AllocationStrategySettings = .leadership + public var allocationStrategy: AllocationStrategySettings = .byLeadership public init(name: String) { self.name = name @@ -126,11 +126,11 @@ public struct ActorSingletonSettings { /// Singleton node allocation strategies. public enum AllocationStrategySettings { /// Singletons will run on the cluster leader - case leadership + case byLeadership func make(_: ClusterSettings, _: ActorSingletonSettings) -> ActorSingletonAllocationStrategy { switch self { - case .leadership: + case .byLeadership: return ActorSingletonAllocationByLeadership() } } diff --git a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift index 25142a4a1..dac05457e 100644 --- a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift +++ b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift @@ -48,7 +48,7 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { func test_singletonByClusterLeadership() throws { try shouldNotThrow { var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name) - singletonSettings.allocationStrategy = .leadership + singletonSettings.allocationStrategy = .byLeadership let first = self.setUpNode("first") { settings in settings.cluster.node.port = 7111 @@ -102,7 +102,7 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { func test_singletonByClusterLeadership_stashMessagesIfNoLeader() throws { try shouldNotThrow { var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name) - singletonSettings.allocationStrategy = .leadership + singletonSettings.allocationStrategy = .byLeadership let first = self.setUpNode("first") { settings in settings.cluster.node.port = 7111 @@ -158,48 +158,42 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { } } - // FIXME: flaky test (https://github.com/apple/swift-distributed-actors/issues/346) func test_singletonByClusterLeadership_withLeaderChange() throws { try shouldNotThrow { var singletonSettings = ActorSingletonSettings(name: GreeterSingleton.name) - singletonSettings.allocationStrategy = .leadership + singletonSettings.allocationStrategy = .byLeadership let first = self.setUpNode("first") { settings in settings.cluster.node.port = 7111 settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3) + settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-1"))) - - settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) } let second = self.setUpNode("second") { settings in settings.cluster.node.port = 8222 settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3) + settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-2"))) - - settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) } let third = self.setUpNode("third") { settings in settings.cluster.node.port = 9333 settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3) + settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-3"))) - - settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) } let fourth = self.setUpNode("fourth") { settings in settings.cluster.node.port = 7444 settings.cluster.autoLeaderElection = .lowestAddress(minNumberOfMembers: 3) + settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) settings += ActorSingleton(settings: singletonSettings, GreeterSingleton.makeBehavior(instance: GreeterSingleton("Hello-4"))) - - settings.serialization.registerCodable(for: GreeterSingleton.Message.self, underId: 10001) } - first.cluster.join(node: second.cluster.node.node) - third.cluster.join(node: second.cluster.node.node) - + try self.joinNodes(node: first, with: second) + try self.joinNodes(node: third, with: second) try self.ensureNodes(.up, within: .seconds(10), systems: first, second, third) let replyProbe1 = self.testKit(first).spawnTestProbe(expecting: String.self) @@ -210,7 +204,7 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self) ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref)) - let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self) + let replyProbe3/**/ = self.testKit(third).spawnTestProbe(expecting: String.self) let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self) ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref)) @@ -222,28 +216,37 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { // Take down the leader first.cluster.down(node: first.cluster.node.node) - // Make sure that `second` and `third` see `first` as down and become leader-less - try self.testKit(second).eventually(within: .seconds(10)) { - try self.assertMemberStatus(on: second, node: first.cluster.node, is: .down) - try self.assertLeaderNode(on: second, is: nil) - } + // Ensure the node is seen down + try self.ensureNodes(.down, on: second, systems: first) + try self.ensureNodes(.down, on: third, systems: first) + + // At the same time, since the node was downed, it is not the leader anymore try self.testKit(third).eventually(within: .seconds(10)) { - try self.assertMemberStatus(on: third, node: first.cluster.node, is: .down) try self.assertLeaderNode(on: third, is: nil) + try self.assertLeaderNode(on: second, is: nil) } - // No leader so singleton is not available, messages sent should be stashed - ref2.tell(.greet(name: "Charlie-2", _replyTo: replyProbe2.ref)) - ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe3.ref)) - // `fourth` will become the new leader and singleton - fourth.cluster.join(node: second.cluster.node.node) - + try self.joinNodes(node: fourth, with: second) try self.ensureNodes(.up, within: .seconds(10), systems: fourth, second, third) - // The stashed messages get routed to new singleton running on `fourth` - try replyProbe2.expectMessage("Hello-4 Charlie-2!") - try replyProbe3.expectMessage("Hello-4 Charlie-3!") + try self.testKit(first).eventually(within: .seconds(10)) { + // The stashed messages get routed to new singleton running on `fourth` + + ref2.tell(.greet(name: "Charlie-2", _replyTo: replyProbe2.ref)) + guard let reply2 = try replyProbe2.maybeExpectMessage() else { + pprint("lost msg @ node 2") + throw TestError("No reply to \(replyProbe2) yet, singleton still rebalancing...?") + } + reply2.shouldEqual("Hello-4 Charlie-2!") + + ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe3.ref)) + guard let reply3 = try replyProbe3.maybeExpectMessage() else { + pprint("lost msg @ node 3") + throw TestError("No reply to \(replyProbe2) yet, singleton still rebalancing...?") + } + reply3.shouldEqual("Hello-4 Charlie-3!") + } } } } From eb5edc1f3434586f1f40cb57e0bd15352a12bfc1 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Sat, 18 Jan 2020 05:46:44 +0900 Subject: [PATCH 2/3] Update Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift Co-Authored-By: Yim Lee --- Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift index dac05457e..148052483 100644 --- a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift +++ b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift @@ -243,7 +243,7 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { ref3.tell(.greet(name: "Charlie-3", _replyTo: replyProbe3.ref)) guard let reply3 = try replyProbe3.maybeExpectMessage() else { pprint("lost msg @ node 3") - throw TestError("No reply to \(replyProbe2) yet, singleton still rebalancing...?") + throw TestError("No reply to \(replyProbe3) yet, singleton still rebalancing...?") } reply3.shouldEqual("Hello-4 Charlie-3!") } From 013234441a2facd7b11248e7a2125be9d9aa036d Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Sat, 18 Jan 2020 05:47:48 +0900 Subject: [PATCH 3/3] Update ActorSingletonPluginTests.swift --- Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift index 148052483..186d098fc 100644 --- a/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift +++ b/Tests/ActorSingletonPluginTests/ActorSingletonPluginTests.swift @@ -204,7 +204,7 @@ final class ActorSingletonPluginTests: ClusteredNodesTestBase { let ref2 = try second.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self) ref2.tell(.greet(name: "Charlie", _replyTo: replyProbe2.ref)) - let replyProbe3/**/ = self.testKit(third).spawnTestProbe(expecting: String.self) + let replyProbe3 = self.testKit(third).spawnTestProbe(expecting: String.self) let ref3 = try third.singleton.ref(name: GreeterSingleton.name, of: GreeterSingleton.Message.self) ref3.tell(.greet(name: "Charlie", _replyTo: replyProbe3.ref))