test set threasholds

Author: blindspotbounty

Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift

@@ -30,7 +30,19 @@ let benchmarks = {
         warmupIterations: 0,
         scalingFactor: .one,
         maxDuration: .seconds(5),
-        maxIterations: 100
+        maxIterations: 100,
+        thresholds: [
+            // Thresholds are wild guess mostly. Have to adjust with time.
+            .wallClock: .init(relative: [.p90: 10]),
+            .cpuTotal: .init(relative: [.p90: 0]),
+            .allocatedResidentMemory: .init(relative: [.p90: 20]),
+            .contextSwitches: .init(relative: [.p90: 10]),
+            .throughput: .init(relative: [.p90: 10]),
+            .objectAllocCount: .init(relative: [.p90: 10]),
+            .retainCount: .init(relative: [.p90: 10]),
+            .releaseCount: .init(relative: [.p90: 10]),
+            .retainReleaseDelta: .init(relative: [.p90: 10]),
+        ]
     )
 
     Benchmark.setup = {
@@ -44,7 +56,7 @@ let benchmarks = {
         uniqueTestTopic = nil
     }
 
-    Benchmark("SwiftKafkaConsumer - basic consumer (messages: \(messageCount))") { benchmark in
+    Benchmark("SwiftKafkaConsumer_basic_consumer_messages_\(messageCount)") { benchmark in
         let uniqueGroupID = UUID().uuidString
         var consumerConfig = KafkaConsumerConfiguration(
             consumptionStrategy: .group(
@@ -112,7 +124,7 @@ let benchmarks = {
         }
     }
 
-    Benchmark("SwiftKafkaConsumer - with offset commit (messages: \(messageCount))") { benchmark in
+    Benchmark("SwiftKafkaConsumer_with_offset_commit_messages_\(messageCount)") { benchmark in
         let uniqueGroupID = UUID().uuidString
         var consumerConfig = KafkaConsumerConfiguration(
             consumptionStrategy: .group(
@@ -183,7 +195,7 @@ let benchmarks = {
         }
     }
 
-    Benchmark("librdkafka - basic consumer (messages: \(messageCount))") { benchmark in
+    Benchmark("librdkafka_basic_consumer_messages_\(messageCount)") { benchmark in
         let uniqueGroupID = UUID().uuidString
         let rdKafkaConsumerConfig: [String: String] = [
             "group.id": uniqueGroupID,
@@ -251,7 +263,7 @@ let benchmarks = {
         benchLog("All read up to ctr: \(ctr), avgRate: (\(Int(avgRateMb))KB/s), timePassed: \(Int(timeIntervalTotal))sec")
     }
 
-    Benchmark("librdkafka - with offset commit (messages: \(messageCount))") { benchmark in
+    Benchmark("librdkafka_with_offset_commit_messages_\(messageCount)") { benchmark in
         let uniqueGroupID = UUID().uuidString
         let rdKafkaConsumerConfig: [String: String] = [
             "group.id": uniqueGroupID,

Benchmarks/Plugins/BenchmarkCheckWrapper/BenchmarkCheckWrapper.swift

@@ -0,0 +1,8 @@
+//
+//  File.swift
+//  
+//
+//  Created by Andrey Potemkin on 26.2.2024.
+//
+
+import Foundation

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 77660159,
+  "cpuTotal" : 80000000,
+  "objectAllocCount" : 4643,
+  "releaseCount" : 30815,
+  "retainCount" : 23791,
+  "retainReleaseDelta" : 2391,
+  "throughput" : 8,
+  "wallClock" : 133693439
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 70254592,
+  "cpuTotal" : 270000000,
+  "objectAllocCount" : 7917,
+  "releaseCount" : 38881,
+  "retainCount" : 27059,
+  "retainReleaseDelta" : 3905,
+  "throughput" : 2,
+  "wallClock" : 634566500
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 42860544,
+  "cpuTotal" : 20000000,
+  "objectAllocCount" : 0,
+  "releaseCount" : 0,
+  "retainCount" : 0,
+  "retainReleaseDelta" : 0,
+  "throughput" : 2,
+  "wallClock" : 620173292
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 48300032,
+  "cpuTotal" : 150000000,
+  "objectAllocCount" : 0,
+  "releaseCount" : 0,
+  "retainCount" : 0,
+  "retainReleaseDelta" : 0,
+  "throughput" : 1,
+  "wallClock" : 827764833
+}

Benchmarks/Thresholds/5.7/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 69468160,
+  "cpuTotal" : 270000000,
+  "objectAllocCount" : 7175,
+  "releaseCount" : 37657,
+  "retainCount" : 26402,
+  "retainReleaseDelta" : 4080,
+  "throughput" : 2,
+  "wallClock" : 683494251
+}

Benchmarks/Thresholds/5.7/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 73007104,
+  "cpuTotal" : 270000000,
+  "objectAllocCount" : 7941,
+  "releaseCount" : 39038,
+  "retainCount" : 27174,
+  "retainReleaseDelta" : 3923,
+  "throughput" : 2,
+  "wallClock" : 644024709
+}

Benchmarks/Thresholds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 46596096,
+  "cpuTotal" : 20000000,
+  "objectAllocCount" : 0,
+  "releaseCount" : 0,
+  "retainCount" : 0,
+  "retainReleaseDelta" : 0,
+  "throughput" : 2,
+  "wallClock" : 627353501
+}

Benchmarks/Thresholds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 44040192,
+  "cpuTotal" : 150000000,
+  "objectAllocCount" : 0,
+  "releaseCount" : 0,
+  "retainCount" : 0,
+  "retainReleaseDelta" : 0,
+  "throughput" : 1,
+  "wallClock" : 835152709
+}