|
| 1 | +import Foundation |
| 2 | + |
| 3 | +public struct BenchmarkRunner { |
| 4 | + let suiteName: String |
| 5 | + var suite: [any RegexBenchmark] = [] |
| 6 | + let samples: Int |
| 7 | + var results: SuiteResult = SuiteResult() |
| 8 | + |
| 9 | + // Outputting |
| 10 | + let startTime = Date() |
| 11 | + let outputPath: String |
| 12 | + |
| 13 | + public init(_ suiteName: String, _ n: Int, _ outputPath: String) { |
| 14 | + self.suiteName = suiteName |
| 15 | + self.samples = n |
| 16 | + self.outputPath = outputPath |
| 17 | + } |
| 18 | + |
| 19 | + public mutating func register(_ new: some RegexBenchmark) { |
| 20 | + suite.append(new) |
| 21 | + } |
| 22 | + |
| 23 | + mutating func measure(benchmark: some RegexBenchmark) -> Time { |
| 24 | + var times: [Time] = [] |
| 25 | + |
| 26 | + // initial run to make sure the regex has been compiled |
| 27 | + // todo: measure compile times, or at least how much this first run |
| 28 | + // differs from the later ones |
| 29 | + benchmark.run() |
| 30 | + |
| 31 | + // fixme: use suspendingclock? |
| 32 | + for _ in 0..<samples { |
| 33 | + let start = Tick.now |
| 34 | + benchmark.run() |
| 35 | + let end = Tick.now |
| 36 | + let time = end.elapsedTime(since: start) |
| 37 | + times.append(time) |
| 38 | + } |
| 39 | + // todo: compute stdev and warn if it's too large |
| 40 | + |
| 41 | + // return median time |
| 42 | + times.sort() |
| 43 | + let median = times[samples/2] |
| 44 | + self.results.add(name: benchmark.name, time: median) |
| 45 | + return median |
| 46 | + } |
| 47 | + |
| 48 | + public mutating func run() { |
| 49 | + print("Running") |
| 50 | + for b in suite { |
| 51 | + print("- \(b.name) \(measure(benchmark: b))") |
| 52 | + } |
| 53 | + } |
| 54 | + |
| 55 | + public func profile() { |
| 56 | + print("Starting") |
| 57 | + for b in suite { |
| 58 | + print("- \(b.name)") |
| 59 | + b.run() |
| 60 | + print("- done") |
| 61 | + } |
| 62 | + } |
| 63 | + |
| 64 | + public mutating func debug() { |
| 65 | + print("Debugging") |
| 66 | + print("========================") |
| 67 | + for b in suite { |
| 68 | + print("- \(b.name) \(measure(benchmark: b))") |
| 69 | + b.debug() |
| 70 | + print("========================") |
| 71 | + } |
| 72 | + } |
| 73 | +} |
| 74 | + |
| 75 | +extension BenchmarkRunner { |
| 76 | + var dateStyle: Date.FormatStyle { |
| 77 | + Date.FormatStyle() |
| 78 | + .year(.twoDigits) |
| 79 | + .month(.twoDigits) |
| 80 | + .day(.twoDigits) |
| 81 | + .hour(.twoDigits(amPM: .omitted)) |
| 82 | + .minute(.twoDigits) |
| 83 | + } |
| 84 | + |
| 85 | + var outputFolderUrl: URL { |
| 86 | + let url = URL(fileURLWithPath: outputPath, isDirectory: true) |
| 87 | + if !FileManager.default.fileExists(atPath: url.path) { |
| 88 | + try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true) |
| 89 | + } |
| 90 | + return url |
| 91 | + } |
| 92 | + |
| 93 | + public func save() throws { |
| 94 | + let now = startTime.formatted(dateStyle) |
| 95 | + let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json") |
| 96 | + print("Saving result to \(resultJsonUrl.path)") |
| 97 | + try results.save(to: resultJsonUrl) |
| 98 | + } |
| 99 | + |
| 100 | + func fetchLatestResult() throws -> (Date, SuiteResult) { |
| 101 | + var pastResults: [Date: SuiteResult] = [:] |
| 102 | + for resultFile in try FileManager.default.contentsOfDirectory( |
| 103 | + at: outputFolderUrl, |
| 104 | + includingPropertiesForKeys: nil |
| 105 | + ) { |
| 106 | + let dateString = resultFile.lastPathComponent.replacingOccurrences( |
| 107 | + of: "-result.json", |
| 108 | + with: "") |
| 109 | + let date = try dateStyle.parse(dateString) |
| 110 | + pastResults.updateValue(try SuiteResult.load(from: resultFile), forKey: date) |
| 111 | + } |
| 112 | + |
| 113 | + let sorted = pastResults |
| 114 | + .sorted(by: {(kv1,kv2) in kv1.0 > kv2.0}) |
| 115 | + return sorted[0] |
| 116 | + } |
| 117 | + |
| 118 | + public func compare() throws { |
| 119 | + // It just compares by the latest result for now, we probably want a CLI |
| 120 | + // flag to set which result we want to compare against |
| 121 | + let (compareDate, compareResult) = try fetchLatestResult() |
| 122 | + let diff = results.compare(with: compareResult) |
| 123 | + let regressions = diff.filter({(_, change) in change.seconds > 0}) |
| 124 | + let improvements = diff.filter({(_, change) in change.seconds < 0}) |
| 125 | + |
| 126 | + print("Comparing against benchmark done on \(compareDate.formatted(dateStyle))") |
| 127 | + print("=== Regressions ====================================================") |
| 128 | + for item in regressions { |
| 129 | + let oldVal = compareResult.results[item.key]! |
| 130 | + let newVal = results.results[item.key]! |
| 131 | + let percentage = item.value.seconds / oldVal.seconds |
| 132 | + print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") |
| 133 | + } |
| 134 | + print("=== Improvements ====================================================") |
| 135 | + for item in improvements { |
| 136 | + let oldVal = compareResult.results[item.key]! |
| 137 | + let newVal = results.results[item.key]! |
| 138 | + let percentage = item.value.seconds / oldVal.seconds |
| 139 | + print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") |
| 140 | + } |
| 141 | + } |
| 142 | +} |
| 143 | + |
| 144 | +struct SuiteResult { |
| 145 | + var results: [String: Time] = [:] |
| 146 | + |
| 147 | + public mutating func add(name: String, time: Time) { |
| 148 | + results.updateValue(time, forKey: name) |
| 149 | + } |
| 150 | + |
| 151 | + public func compare(with other: SuiteResult) -> [String: Time] { |
| 152 | + var output: [String: Time] = [:] |
| 153 | + for item in results { |
| 154 | + if let otherVal = other.results[item.key] { |
| 155 | + let diff = item.value - otherVal |
| 156 | + // note: is this enough time difference? |
| 157 | + if diff.abs() > Time.millisecond { |
| 158 | + output.updateValue(diff, forKey: item.key) |
| 159 | + } |
| 160 | + } |
| 161 | + } |
| 162 | + return output |
| 163 | + } |
| 164 | +} |
| 165 | + |
| 166 | +extension SuiteResult: Codable { |
| 167 | + public func save(to url: URL) throws { |
| 168 | + let encoder = JSONEncoder() |
| 169 | + let data = try encoder.encode(self) |
| 170 | + try data.write(to: url, options: .atomic) |
| 171 | + } |
| 172 | + |
| 173 | + public static func load(from url: URL) throws -> SuiteResult { |
| 174 | + let decoder = JSONDecoder() |
| 175 | + let data = try Data(contentsOf: url) |
| 176 | + return try decoder.decode(SuiteResult.self, from: data) |
| 177 | + } |
| 178 | +} |
0 commit comments