From 8ed3e85e271961f1c24da1ba5844618b695e2d94 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:26:11 +0200
Subject: [PATCH 01/12] Use perflib/err.h unconditionally

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/err.h   | 10 +++++++++-
 source/ssl_poll_perf.c |  4 ++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/source/perflib/err.h b/source/perflib/err.h
index 0bb7a53..eb47e32 100644
--- a/source/perflib/err.h
+++ b/source/perflib/err.h
@@ -11,7 +11,13 @@
 # define OSSL_PERFLIB_ERR_H
 # pragma once
 
-#include <stdarg.h>
+# if !defined(_WIN32)
+
+#  include <err.h>
+
+# else /* _WIN32 */
+
+#  include <stdarg.h>
 
 extern const char *progname;
 
@@ -19,5 +25,7 @@ extern void vwarnx(const char *, va_list);
 extern void errx(int, const char *, ...);
 extern void warnx(const char *, ...);
 
+# endif /* !_WIN32 */
+
 #endif
 
diff --git a/source/ssl_poll_perf.c b/source/ssl_poll_perf.c
index 378058d..dbcca81 100644
--- a/source/ssl_poll_perf.c
+++ b/source/ssl_poll_perf.c
@@ -33,7 +33,6 @@
 #ifdef _WIN32 /* Windows */
 # include <winsock2.h>
 #else /* Linux/Unix */
-# include <err.h>
 # include <sys/socket.h>
 # include <sys/select.h>
 # include <netinet/in.h>
@@ -54,10 +53,11 @@
 #else
 # include <windows.h>
 # include "perflib/basename.h"
-# include "perflib/err.h"
 # include "perflib/getopt.h"
 #endif /* _WIN32 */
 
+#include "perflib/err.h"
+
 /*
  * The code here is based on QUIC poll server found in demos/quic/poll-server
  * in OpenSSL source code repository. Here we take the demo one step further

From 6cb21b80538e75a10d7476a31dbdcdba8c71b09a Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:26:52 +0200
Subject: [PATCH 02/12] perflib/err.c: use program_invocation_name on glibc

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/err.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/source/perflib/err.c b/source/perflib/err.c
index 3a3164d..9dcb21a 100644
--- a/source/perflib/err.c
+++ b/source/perflib/err.c
@@ -13,11 +13,25 @@
 
 const char *progname;
 
+static ossl_inline void *
+get_progname(void)
+{
+    if (progname != NULL)
+        return progname;
+
+#if defined(__GLIBC__)
+    if (program_invocation_name && program_invocation_name[0])
+        return program_invocation_name;
+#endif
+
+    return NULL;
+}
+
 void
 vwarnx(const char *fmt, va_list ap)
 {
-    if (progname != NULL)
-        fprintf(stderr, "%s: ", progname);
+    if (get_progname() != NULL)
+        fprintf(stderr, "%s: ", get_progname());
     vfprintf(stderr, fmt, ap);
     putc('\n', stderr);
 }

From 1f334d139d9db7546c4d5f06b19eab335fb1e5e2 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:28:17 +0200
Subject: [PATCH 03/12] perflib: add vwarn/err/warn

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/err.c | 36 ++++++++++++++++++++++++++++++++++++
 source/perflib/err.h |  5 +++++
 2 files changed, 41 insertions(+)

diff --git a/source/perflib/err.c b/source/perflib/err.c
index 9dcb21a..709a8a0 100644
--- a/source/perflib/err.c
+++ b/source/perflib/err.c
@@ -7,6 +7,7 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -36,6 +37,20 @@ vwarnx(const char *fmt, va_list ap)
     putc('\n', stderr);
 }
 
+void
+vwarn(const char *fmt, va_list ap)
+{
+    int saved_errno = errno;
+
+    if (get_progname() != NULL)
+        fprintf(stderr, "%s: ", get_progname());
+    vfprintf(stderr, fmt, ap);
+    fprintf(stderr, ": ");
+
+    errno = saved_errno;
+    perror(NULL);
+}
+
 void
 errx(int status, const char *fmt, ...)
 {
@@ -47,6 +62,17 @@ errx(int status, const char *fmt, ...)
     exit(status);
 }
 
+void
+err(int status, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    vwarn(fmt, ap);
+    va_end(ap);
+    exit(status);
+}
+
 void
 warnx(const char *fmt, ...)
 {
@@ -56,3 +82,13 @@ warnx(const char *fmt, ...)
     vwarnx(fmt, ap);
     va_end(ap);
 }
+
+void
+warn(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    vwarn(fmt, ap);
+    va_end(ap);
+}
diff --git a/source/perflib/err.h b/source/perflib/err.h
index eb47e32..5e848ee 100644
--- a/source/perflib/err.h
+++ b/source/perflib/err.h
@@ -22,8 +22,13 @@
 extern const char *progname;
 
 extern void vwarnx(const char *, va_list);
+extern void vwarn(const char *, va_list);
+
 extern void errx(int, const char *, ...);
+extern void err(int, const char *, ...);
+
 extern void warnx(const char *, ...);
+extern void warn(const char *, ...);
 
 # endif /* !_WIN32 */
 

From 6e617c1c461ffa437854facf596d3bccee4ed654 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:29:10 +0200
Subject: [PATCH 04/12] perflib/err.h: add WARN/WARNX/ERR/ERRX

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/err.h | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/source/perflib/err.h b/source/perflib/err.h
index 5e848ee..b9721ad 100644
--- a/source/perflib/err.h
+++ b/source/perflib/err.h
@@ -11,6 +11,8 @@
 # define OSSL_PERFLIB_ERR_H
 # pragma once
 
+# include <stdio.h>
+
 # if !defined(_WIN32)
 
 #  include <err.h>
@@ -32,5 +34,26 @@ extern void warn(const char *, ...);
 
 # endif /* !_WIN32 */
 
+# define WARN(...) \
+    do { \
+        fprintf(stderr, "%s:%d(%s): ", __FILE__, __LINE__, __FUNCTION__); \
+        warn(__VA_ARGS__); \
+    } while (0)
+# define WARNX(...) \
+    do { \
+        fprintf(stderr, "%s:%d(%s): ", __FILE__, __LINE__, __FUNCTION__); \
+        warnx(__VA_ARGS__); \
+    } while (0)
+# define ERR(...) \
+    do { \
+        fprintf(stderr, "%s:%d(%s): ", __FILE__, __LINE__, __FUNCTION__); \
+        err(__VA_ARGS__); \
+    } while (0)
+# define ERRX(...) \
+    do { \
+        fprintf(stderr, "%s:%d(%s): ", __FILE__, __LINE__, __FUNCTION__); \
+        errx(__VA_ARGS__); \
+    } while (0)
+
 #endif
 

From abbf780309100dc0f6a18e68349ad90da8a96dc3 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:30:56 +0200
Subject: [PATCH 05/12] perflib: add ability to set thread affinity

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/perfhelper.c |   1 +
 source/perflib/perflib.h    |  58 ++++++-
 source/perflib/threads.c    | 320 +++++++++++++++++++++++++++++++++---
 3 files changed, 355 insertions(+), 24 deletions(-)

diff --git a/source/perflib/perfhelper.c b/source/perflib/perfhelper.c
index 1cbad59..67e25b5 100644
--- a/source/perflib/perfhelper.c
+++ b/source/perflib/perfhelper.c
@@ -9,6 +9,7 @@
 
 #include <string.h>
 #include <openssl/crypto.h>
+#include <openssl/macros.h>
 #include <openssl/bio.h>
 #include <openssl/err.h>
 #include <openssl/ssl.h>
diff --git a/source/perflib/perflib.h b/source/perflib/perflib.h
index e41f816..1048a7a 100644
--- a/source/perflib/perflib.h
+++ b/source/perflib/perflib.h
@@ -21,6 +21,7 @@
 #  include <windows.h>
 
 typedef HANDLE thread_t;
+typedef DWORD_PTR affinity_t;
 
 #  define strcasecmp(_a, _b) _stricmp((_a), (_b))
 
@@ -29,6 +30,7 @@ typedef HANDLE thread_t;
 #  include <pthread.h>
 
 typedef pthread_t thread_t;
+typedef unsigned long affinity_t;
 
 # endif
 
@@ -37,10 +39,60 @@ struct thread_arg_st {
     size_t num;
 };
 
-int perflib_run_thread(thread_t *t, struct thread_arg_st *arg);
+/**
+ * A callback that allows setting CPU affinity for the threads being run.
+ * Gets the set of available CPUs in the cpu_set argument and expected
+ * to update it in accordance with the information provided in num and arg
+ * arguments.
+ *
+ * Currently supported only on glibc on Linux (because of a non-privileged
+ * pthread_attr_setaffinity_np, with a maximum of 1024 CPUs) and Windows
+ * (with a limitation of using only the initial process group).
+ *
+ * @param[in,out] cpu_set       On entering, contains the set of CPUs available
+ *                              to the test.  The callback is supposed
+ *                              to update the set in accordance
+ *                              with the information provided in num, cnt,
+ *                              and arg parameters.
+ * @param[in]     cpu_set_bits  Size of cpu_set_bits, in bits.
+ * @param[in]     num           Index of a thread index being run,
+ *                              counted from 0.
+ * @param[in]     cnt           Total count of available CPUs
+ *                              (popcnt(cpu_set_bits)).
+ * @param         arg           An opaque pointer that a caller has provided
+ *                              along the callback.
+ * @return                      1 on success, 0 on error.
+ */
+typedef int (*perflib_affinity_fn)(affinity_t *cpu_set, size_t cpu_set_bits,
+                                   size_t num, size_t cnt, void *arg);
+
+/**
+ * A simple affinity callback that assigns each thread a single CPU
+ * in a round robin fashion.  arg must be NULL.
+ */
+int perflib_roundrobin_affinity(affinity_t *cpu_set_bits, size_t cpu_set_size,
+                                size_t num, size_t cnt, void *arg);
+
+int perflib_run_thread_ex(thread_t *t, struct thread_arg_st *arg,
+                          perflib_affinity_fn affinity_cb,
+                          void *affinity_cb_arg);
+static ossl_unused ossl_inline int
+perflib_run_thread(thread_t *t, struct thread_arg_st *arg)
+{
+    return perflib_run_thread_ex(t, arg, NULL, NULL);
+}
 int perflib_wait_for_thread(thread_t thread);
-int perflib_run_multi_thread_test(void (*f)(size_t), size_t threadcount,
-                                  OSSL_TIME *duration);
+int perflib_run_multi_thread_test_ex(void (*f)(size_t), size_t threadcount,
+                                     OSSL_TIME *duration,
+                                     perflib_affinity_fn affinity_cb,
+                                     void *affinity_cb_arg);
+static ossl_unused ossl_inline int
+perflib_run_multi_thread_test(void (*f)(size_t), size_t threadcount,
+                              OSSL_TIME *duration)
+{
+    return perflib_run_multi_thread_test_ex(f, threadcount, duration,
+                                            NULL, NULL);
+}
 char *perflib_mk_file_path(const char *dir, const char *file);
 char *perflib_glue_strings(const char *list[], size_t *out_len);
 
diff --git a/source/perflib/threads.c b/source/perflib/threads.c
index a6cd1dc..0c49732 100644
--- a/source/perflib/threads.c
+++ b/source/perflib/threads.c
@@ -7,10 +7,88 @@
  * https://www.openssl.org/source/license.html
  */
 
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+
+#include <stdbool.h>
+#include <string.h>
+
+#include "perflib/err.h"
 #include "perflib/perflib.h"
 
+#define OSSL_NELEM(x) (sizeof(x)/sizeof((x)[0]))
+
+/** affinity_t-typed value with nth bit set. */
+#define AFFINITY_BIT(n) ((affinity_t)1U << (n))
+
+#if defined(__GNUC__)
+
+static ossl_inline unsigned int popcount(affinity_t a)
+{
+    return __builtin_popcountl(a);
+}
+
+#else /* !__GNUC__ */
+
+static ossl_inline unsigned int popcount(affinity_t a)
+{
+    unsigned int ret = 0;
+
+    for (size_t i = 0; i < sizeof(a) * CHAR_BIT; i++)
+        ret += !!(a & AFFINITY_BIT(i));
+
+    return ret;
+}
+
+#endif /* __GNUC__ */
+
+int perflib_roundrobin_affinity(affinity_t *cpu_set_bits, size_t cpu_set_size,
+                                size_t num, size_t cnt, void *arg)
+{
+    enum { BITS_PER_ELEM = sizeof(cpu_set_bits[0]) * CHAR_BIT };
+    size_t set_cnt = 0;
+    size_t i;
+
+    if (arg != NULL) {
+        WARNX("Non-NULL arg");
+
+        return 0;
+    }
+
+    /* Finding (num % cnt)th set bit in the provided mask */
+    for (i = 0; i < cpu_set_size; i++) {
+        if (cpu_set_bits[i / BITS_PER_ELEM] & AFFINITY_BIT(i % BITS_PER_ELEM))
+            set_cnt++;
+
+        if (set_cnt == (num % cnt + 1))
+            break;
+    }
+
+    if (set_cnt != (num % cnt + 1)) {
+        WARNX("Only %zu bits are set in the affinity mask, %zu expected",
+              set_cnt, num % cnt + 1);
+
+        return 0;
+    }
+
+    memset(cpu_set_bits, 0, cpu_set_size / CHAR_BIT);
+
+    cpu_set_bits[i / BITS_PER_ELEM] = AFFINITY_BIT(i % BITS_PER_ELEM);
+
+    return 1;
+}
+
 #if defined(_WIN32)
 
+struct thread_affinity {
+    /*
+     * It is not a "DWORD *", as any sane person would think, but "__int3264",
+     * which is 32-bit wide on 32-bit systems and 64-bit wide on 64-bit ones.
+     */
+    DWORD_PTR affinity;
+};
+
 static DWORD WINAPI thread_run(LPVOID varg)
 {
     struct thread_arg_st *arg = varg;
@@ -20,18 +98,80 @@ static DWORD WINAPI thread_run(LPVOID varg)
     return 0;
 }
 
-int perflib_run_thread(thread_t *t, struct thread_arg_st *arg)
+static int prepare_affinity_args(perflib_affinity_fn affinity_cb,
+                                 void *affinity_cb_arg,
+                                 struct thread_affinity *ta,
+                                 size_t start, size_t count)
 {
-    *t = CreateThread(NULL, 0, thread_run, arg, 0, NULL);
+    HANDLE process;
+    DWORD_PTR dummy;
+    unsigned int cnt;
+
+    if (count == 0)
+        return 1;
+
+    process = GetCurrentProcess();
+    /* TODO: support multiple process groups */
+    if (!GetProcessAffinityMask(process, &ta[0].affinity, &dummy)) {
+        WARNX("Error getting process affinity mask: %lu", GetLastError());
+
+        return 0;
+    }
+
+    cnt = popcount(ta[0].affinity);
+
+    for (size_t i = 1; i < count; i++) {
+        ta[i].affinity = ta[0].affinity;
+
+    for (size_t i = 0; i < count; i++) {
+        if (affinity_cb(&ta[i].affinity, sizeof(DWORD_PTR) * CHAR_BIT,
+                        start + i, cnt, affinity_cb_arg) != 1) {
+            WARNX("Error calling thread affinity callback for thread %zu",
+                  start + i);
+
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+static void cleanup_affinity_arg(struct thread_affinity *ta,
+                                 size_t start, size_t count)
+{
+}
+
+static int perflib_run_thread_(thread_t *t, struct thread_arg_st *arg,
+                               struct thread_affinity *ta)
+{
+    DWORD thread_id;
+
+    *t = CreateThread(NULL, 0, thread_run, arg, CREATE_SUSPENDED, &thread_id);
+
+    if (t == NULL) {
+        WARNX("Error creating thread %zu: %lu", arg->num, GetLastError());
+    } else {
+        if (!SetThreadAffinityMask(t, ta->affinity))
+            WARNX("Error setting thread affinity for thread %zu: %lu",
+                  arg->num, GetLastError());
+        if (ResumeThread(t) < 0)
+            WARNX("Error resuming thread %zu: %lu", arg->num, GetLastError());
+    }
+
     return *t != NULL;
 }
 
+
 int perflib_wait_for_thread(thread_t thread)
 {
     return WaitForSingleObject(thread, INFINITE) == 0;
 }
 
-#else
+#else /* !_WIN32 */
+
+struct thread_affinity {
+    pthread_attr_t attr;
+};
 
 static void *thread_run(void *varg)
 {
@@ -42,9 +182,102 @@ static void *thread_run(void *varg)
     return NULL;
 }
 
-int perflib_run_thread(thread_t *t, struct thread_arg_st *arg)
+# if defined(__linux) && defined(__GLIBC__)
+#  include <errno.h>
+#  include <pthread.h>
+#  include <sched.h>
+#  include <unistd.h>
+
+static int prepare_affinity_args_linux(perflib_affinity_fn affinity_cb,
+                                       void *affinity_cb_arg,
+                                       struct thread_affinity *ta,
+                                       size_t start, size_t count)
+{
+    cpu_set_t process_affinity;
+    unsigned int cnt = 0;
+
+    if (!affinity_cb)
+        return 1;
+
+    /* TODO: support more than 1024 CPUs */
+    if (sched_getaffinity(getpid(), sizeof(process_affinity),
+                          &process_affinity) != 0) {
+        WARN("sched_getaffinity");
+
+        return 0;
+    }
+
+    for (size_t i = 0; i < OSSL_NELEM(process_affinity.__bits); i++)
+        cnt += popcount(process_affinity.__bits[i]);
+
+    for (size_t i = 0; i < count; i++) {
+        cpu_set_t thread_affinity = process_affinity;
+        int ret;
+
+        if (affinity_cb(thread_affinity.__bits,
+                        sizeof(thread_affinity) * CHAR_BIT,
+                        start + i, cnt, affinity_cb_arg) != 1) {
+            WARNX("Error calling thread affinity callback for thread %zu",
+                  start + i);
+
+            return 0;
+        }
+
+        ret = pthread_attr_setaffinity_np(&ta[i].attr, sizeof(thread_affinity),
+                                          &thread_affinity);
+
+        if (ret != 0) {
+            WARNX("Error setting thread affinity afttribute for thread %zu",
+                  start + i);
+
+            return 0;
+        }
+	}
+
+    return 1;
+}
+# endif /* __linux */
+
+static int prepare_affinity_args(perflib_affinity_fn affinity_cb,
+                                 void *affinity_cb_arg,
+                                 struct thread_affinity *ta,
+                                 size_t start, size_t count)
+{
+    if (count == 0)
+        return 1;
+
+    for (size_t i = 0; i < count; i++)
+        pthread_attr_init(&ta[i].attr);
+
+# if defined(__linux) && defined(__GLIBC__)
+	return prepare_affinity_args_linux(affinity_cb, affinity_cb_arg, ta,
+                                       start, count);
+# else /* !(__linux && __GLIBC__) */
+	/*
+     * So far, setting thread affinity is only supported on Linux+glibc on POSIX
+     * systems.
+     */
+    if (affinity_cb) {
+        WARNX("Setting thread affinity is not supported in this environment");
+
+        return 0;
+    }
+
+    return 1;
+# endif
+}
+
+static void cleanup_affinity_arg(struct thread_affinity *ta,
+                                 size_t start, size_t count)
 {
-    return pthread_create(t, NULL, thread_run, arg) == 0;
+    for (size_t i = 0; i < count; i++)
+        pthread_attr_destroy(&ta[i].attr);
+}
+
+int perflib_run_thread_(thread_t *t, struct thread_arg_st *arg,
+                        struct thread_affinity *ta)
+{
+    return pthread_create(t, &ta->attr, thread_run, arg) == 0;
 }
 
 int perflib_wait_for_thread(thread_t thread)
@@ -52,42 +285,87 @@ int perflib_wait_for_thread(thread_t thread)
     return pthread_join(thread, NULL) == 0;
 }
 
-#endif
+#endif /* _WIN32 */
 
-int perflib_run_multi_thread_test(void (*f)(size_t), size_t threadcount,
-                                  OSSL_TIME *duration)
+int perflib_run_thread_ex(thread_t *t, struct thread_arg_st *arg,
+                          perflib_affinity_fn affinity_cb,
+                          void *affinity_cb_arg)
+{
+    struct thread_affinity ta;
+
+    prepare_affinity_args(affinity_cb, affinity_cb_arg, &ta, arg->num, 1);
+
+    return perflib_run_thread_(t, arg, &ta);
+}
+
+int perflib_run_multi_thread_test_ex(void (*f)(size_t), size_t threadcount,
+                                     OSSL_TIME *duration,
+                                     perflib_affinity_fn affinity_cb,
+                                     void *affinity_cb_arg)
 {
     OSSL_TIME start, end;
-    thread_t *threads;
+    thread_t *threads = NULL;
+    int *run_threads = NULL;
+    struct thread_arg_st *args = NULL;
+    struct thread_affinity *ta = NULL;
     size_t i;
-    struct thread_arg_st *args;
+    int ret = 0;
 
     threads = OPENSSL_malloc(sizeof(*threads) * threadcount);
-    if (threads == NULL)
-        return 0;
+    if (threads == NULL) {
+        WARN("Could not allocate threads array");
+
+        goto err;
+    }
+
+    run_threads = OPENSSL_zalloc(sizeof(*run_threads) * threadcount);
+    if (run_threads == NULL) {
+        WARN("Could not allocate run_threads array");
+
+        goto err;
+    }
 
     args = OPENSSL_malloc(sizeof(*args) * threadcount);
     if (args == NULL) {
-        OPENSSL_free(threads);
-        return 0;
+        WARN("Could not allocate args array");
+
+        goto err;
     }
 
+    ta = OPENSSL_malloc(sizeof(*ta) * threadcount);
+    if (ta == NULL) {
+        WARN("Could not allocate args array");
+
+        goto err;
+    }
+    if (!prepare_affinity_args(affinity_cb, affinity_cb_arg,
+                               ta, 0, threadcount))
+        goto err;
+
     start = ossl_time_now();
+    ret = 1;
 
     for (i = 0; i < threadcount; i++) {
         args[i].func = f;
         args[i].num = i;
-        perflib_run_thread(&threads[i], &args[i]);
+        if (!(run_threads[i] = perflib_run_thread_(&threads[i], &args[i],
+                                                   ta + i)))
+            ret = 0;
     }
 
-    for (i = 0; i < threadcount; i++)
-        perflib_wait_for_thread(threads[i]);
+    for (i = 0; i < threadcount; i++) {
+        if (run_threads[i])
+            perflib_wait_for_thread(threads[i]);
+    }
 
     end = ossl_time_now();
-    OPENSSL_free(threads);
-    OPENSSL_free(args);
-
     *duration = ossl_time_subtract(end, start);
 
-    return 1;
+err:
+    OPENSSL_free(ta);
+    OPENSSL_free(args);
+    OPENSSL_free(run_threads);
+    OPENSSL_free(threads);
+
+    return ret;
 }

From 3e158df8b952fdb267606c6031b1cc64af466274 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:32:05 +0200
Subject: [PATCH 06/12] pkeyread: output counts array allocation error to
 stderr

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/pkeyread.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/pkeyread.c b/source/pkeyread.c
index e04c131..fde3200 100644
--- a/source/pkeyread.c
+++ b/source/pkeyread.c
@@ -396,7 +396,7 @@ int main(int argc, char * const argv[])
 
     counts = OPENSSL_malloc(sizeof(size_t) * threadcount);
     if (counts == NULL) {
-        printf("Failed to create counts array\n");
+        fprintf(stderr, "Failed to create counts array\n");
         return EXIT_FAILURE;
     }
 

From ca987dfdd26bd73dc6d99b9a9210a2dd0fa21dd2 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:32:30 +0200
Subject: [PATCH 07/12] pkeyread: tfix

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/pkeyread.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/pkeyread.c b/source/pkeyread.c
index fde3200..a368ffb 100644
--- a/source/pkeyread.c
+++ b/source/pkeyread.c
@@ -420,7 +420,7 @@ int main(int argc, char * const argv[])
             sample_id = k;
             max_time = ossl_time_add(ossl_time_now(), ossl_us2time(timeout_us));
             if (!perflib_run_multi_thread_test(do_f[f], threadcount, &duration)) {
-                fprintf(stderr, "Failed to run the test %s in %s format]\n",
+                fprintf(stderr, "Failed to run the test %s in %s format\n",
                         sample_names[k], format_names[f]);
                 OPENSSL_free(counts);
                 return EXIT_FAILURE;

From 983b8eeb0054a485289cb3e41536b9820edc0e68 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:32:54 +0200
Subject: [PATCH 08/12] README.md: update pkeyread documentation

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 README.md | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index e2d05e3..31b7cc3 100644
--- a/README.md
+++ b/README.md
@@ -121,18 +121,27 @@ lock/unlock pairs are reported as a performance measurement
 
 ## pkeyread
 
-The `pkeyread` test repeatedly calls the [PEM_read_bio_PrivateKey()](https://docs.openssl.org/master/man3/PEM_read_bio_PrivateKey/) function on a
-memory BIO with a private key of desired type, when it is running in pem mode
-(-f pem).  If test is running in der mode (-f der) it calls to
-[d2i_PrivateKey_ex()](https://docs.openssl.org/master/man3/d2i_PrivateKey/) function to repeatedly read private key of desired type.
-It does 10000 repetitions divided evenly among each thread. The number of
-threads to use is provided by option `-t`. The test reports average time per
-call. Use option `-k` to select key type for benchmark.  The list of keys for
-testing is as follows: dh, dhx, dsa, ec, rsa, xkey.  To run benchmark for all
-keys and formats using 4 threads run pkeyread as follows:
+The `pkeyread` test benchmarks reading of private keys of the specified type
+in the specified format.  If PEM format is specified (`-f pem`),
+the [`PEM_read_bio_PrivateKey()`](https://docs.openssl.org/master/man3/PEM_read_bio_PrivateKey/)
+function is called repeatedly on a memory BIO with a private key
+of desired type; in case of DER format specification (`-f der`),
+[`d2i_PrivateKey_ex()`](https://docs.openssl.org/master/man3/d2i_PrivateKey/)
+is called repeatedly to read private key of the desired type.
+The calls are performed repeatedly until the timeout (specified in the `-T`
+option, 5 seconds by default) is reached.
+The number of threads to use is provided by the only positional option.
+By default, the test reports average time per call.  If `-t` ("terse output")
+option is specified, only the number is printed, without key type and format.
+If `-v` ("verbose") option is specified, additional information is printed
+for each run: median, minimum, maximum time among threads, as well as standard
+deviation.
+The mandatory option `-k` selects the key type for benchmark.  The list of keys
+for testing is as follows: dh, dhx, dsa, ec, rsa, x25519.  To run benchmark
+for all keys and formats using 4 threads, run `pkeyread` as follows:
 
 ```sh
-./pkeyread -f all -k all -t 4
+./pkeyread -f all -k all 4
 ```
 
 ## evp_setpeer

From a963f5bcf3759a48d80a402fd15f228d460667d1 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Fri, 26 Sep 2025 01:33:34 +0200
Subject: [PATCH 09/12] pkeyread: add an option to bind threads to cores

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 README.md         |  2 ++
 source/pkeyread.c | 13 ++++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 31b7cc3..bf98ec3 100644
--- a/README.md
+++ b/README.md
@@ -136,6 +136,8 @@ option is specified, only the number is printed, without key type and format.
 If `-v` ("verbose") option is specified, additional information is printed
 for each run: median, minimum, maximum time among threads, as well as standard
 deviation.
+If `-b` ("bind") option is provided, thread affinity is set to the threads
+so the available cores are assigned in a round robin manner.
 The mandatory option `-k` selects the key type for benchmark.  The list of keys
 for testing is as follows: dh, dhx, dsa, ec, rsa, x25519.  To run benchmark
 for all keys and formats using 4 threads, run `pkeyread` as follows:
diff --git a/source/pkeyread.c b/source/pkeyread.c
index a368ffb..65a2b4e 100644
--- a/source/pkeyread.c
+++ b/source/pkeyread.c
@@ -275,7 +275,8 @@ static void usage(char * const argv[])
     fprintf(stderr, "%s -k key_name -f format_name [-t] [-v] [-T time] threadcount\n"
         "\t-t  terse output\n"
         "\t-v  verbose output, includes min, max, stddev, and median times\n"
-        "\t-T  timeout for each test run in seconds, can be fractional"
+        "\t-T  timeout for each test run in seconds, can be fractional\n"
+        "\t-b  Set CPU affinity for the threads (in round robin fashion)\n"
         "\twhere key_name is one of these: ", argv[0]);
     fprintf(stderr, "%s", *key_name);
     do {
@@ -303,6 +304,7 @@ int main(int argc, char * const argv[])
     int key_id, key_id_min, key_id_max, k;
     int format_id, format_id_min, format_id_max, f;
     int verbosity = VERBOSITY_DEFAULT;
+    int bind_threads = 0;
     char *key = NULL;
     char *key_format = NULL;
     void (*do_f[2])(size_t) = {
@@ -313,7 +315,7 @@ int main(int argc, char * const argv[])
     key_id = SAMPLE_INVALID;
     format_id = FORMAT_INVALID;
 
-    while ((ch = getopt(argc, argv, "T:k:f:tv")) != -1) {
+    while ((ch = getopt(argc, argv, "T:k:f:tvb")) != -1) {
         switch (ch) {
         case 'T': {
             double timeout_s;
@@ -342,6 +344,9 @@ int main(int argc, char * const argv[])
         case 'v':
             verbosity = VERBOSITY_VERBOSE;
             break;
+        case 'b':
+            bind_threads = 1;
+            break;
         }
     }
 
@@ -419,7 +424,9 @@ int main(int argc, char * const argv[])
         for (f = format_id_min; f < format_id_max; f++) {
             sample_id = k;
             max_time = ossl_time_add(ossl_time_now(), ossl_us2time(timeout_us));
-            if (!perflib_run_multi_thread_test(do_f[f], threadcount, &duration)) {
+            if (!perflib_run_multi_thread_test_ex(do_f[f], threadcount,
+                &duration, bind_threads ? perflib_roundrobin_affinity : NULL,
+                NULL)) {
                 fprintf(stderr, "Failed to run the test %s in %s format\n",
                         sample_names[k], format_names[f]);
                 OPENSSL_free(counts);

From 7a87ac13987328ea13291b7c165a0e0f87ce71e2 Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Mon, 22 Sep 2025 13:21:15 +0200
Subject: [PATCH 10/12] x509storeissuer update wip

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/CMakeLists.txt    |   14 +-
 source/x509storeissuer.c | 1251 +++++++++++++++++++++++++++++++++++---
 2 files changed, 1167 insertions(+), 98 deletions(-)

diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index b46f21a..1f0290f 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -117,6 +117,12 @@ if(WIN32)
     target_sources(perf PRIVATE perflib/getopt.c perflib/basename.c perflib/err.c)
 endif()
 
+if(WIN32)
+    set(libm_dep)
+else()
+    set(libm_dep PUBLIC m)
+endif()
+
 target_include_directories(perf PUBLIC "${PROJECT_SOURCE_DIR}")
 target_link_libraries(perf PUBLIC OpenSSL::SSL OpenSSL::Crypto)
 
@@ -154,17 +160,13 @@ add_executable(rsasign rsasign.c)
 target_link_libraries(rsasign PRIVATE perf)
 
 add_executable(x509storeissuer x509storeissuer.c)
-target_link_libraries(x509storeissuer PRIVATE perf)
+target_link_libraries(x509storeissuer ${libm_dep} PRIVATE perf)
 
 add_executable(rwlocks rwlocks.c)
 target_link_libraries(rwlocks PRIVATE perf)
 
 add_executable(pkeyread pkeyread.c)
-if(WIN32)
-    target_link_libraries(pkeyread PRIVATE perf)
-else()
-    target_link_libraries(pkeyread PUBLIC m PRIVATE perf)
-endif()
+target_link_libraries(pkeyread ${libm_dep} PRIVATE perf)
 
 add_executable(evp_setpeer evp_setpeer.c)
 target_link_libraries(evp_setpeer PRIVATE perf)
diff --git a/source/x509storeissuer.c b/source/x509storeissuer.c
index 941d432..66c2153 100644
--- a/source/x509storeissuer.c
+++ b/source/x509storeissuer.c
@@ -7,9 +7,14 @@
  * https://www.openssl.org/source/license.html
  */
 
-#include <stdlib.h>
+#include <dirent.h>
+#include <errno.h>
+#include <math.h>
+#include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
 #ifndef _WIN32
 # include <libgen.h>
 # include <unistd.h>
@@ -17,157 +22,1219 @@
 # include <windows.h>
 # include "perflib/getopt.h"
 # include "perflib/basename.h"
-#endif	/* _WIN32 */
+#endif    /* _WIN32 */
 #include <openssl/bio.h>
+#include <openssl/core_names.h>
+#include <openssl/evp.h>
 #include <openssl/x509.h>
+#include "perflib/err.h"
 #include "perflib/perflib.h"
 
+#define NUM_CERTS 1024
+#define NUM_LOAD_CERTS 128
+#define NUM_KEYS 16
+#define KEY_ALGO "rsa:2048"
+#define W_PROBABILITY 50
+#define MAX_WRITERS 0
 #define RUN_TIME 5
+#define QUANTILES 5
+#define NONCE_CFG "file:servercert.pem"
+#define CTX_SHARE_THREADS 1
+
+#define RAND_CYCLE 65536
+
+static size_t num_gen_certs = NUM_CERTS;
+static size_t num_gen_load_certs = NUM_LOAD_CERTS;
+static size_t num_gen_keys = NUM_KEYS;
+static const char *gen_key_algo = KEY_ALGO;
+static size_t timeout_us = RUN_TIME * 1000000;
+static size_t quantiles = QUANTILES;
+static size_t max_writers = MAX_WRITERS;
+static size_t w_probability = W_PROBABILITY * 65536 / 100;
+
+enum verbosity {
+    VERBOSITY_TERSE,
+    VERBOSITY_DEFAULT,
+    VERBOSITY_VERBOSE,
+    VERBOSITY_DEBUG_STATS,
+    VERBOSITY_DEBUG,
+
+    VERBOSITY_MAX__
+};
+
+static enum mode {
+    MODE_R,
+    MODE_RW, /* "MODE_W" is just MODE_RW with 100% write probability */
+} mode = MODE_R;
+
+enum nonce_type {
+    NONCE_GENERATED,
+    NONCE_PATH,
+};
+
+struct call_times {
+    uint64_t duration;
+    uint64_t total_count;
+    uint64_t total_found;
+    uint64_t total_added;
+    uint64_t min_count;
+    uint64_t max_count;
+    double avg;
+    double min;
+    double max;
+    double stddev;
+    double median;
+    size_t min_idx;
+    size_t max_idx;
+};
+
+struct nonce_cfg {
+    enum nonce_type type;
+    const char *path;
+    char **dirs;
+    size_t num_dirs;
+};
+
+struct affinity_cfg {
+    uint32_t stride;
+    uint32_t stripe_step;
+    uint32_t stripe_size;
+};
 
-static int err = 0;
+struct thread_data {
+    //unsigned long rw_cycle[RAND_CYCLE / CHAR_BIT / sizeof(unsigned long)];
+    OSSL_TIME start_time;
+    struct {
+        uint64_t count;
+        uint64_t found;
+        uint64_t added_certs;
+        OSSL_TIME end_time;
+    } *q_data;
+    X509_STORE_CTX *ctx;
+} *thread_data;
+
+static int error = 0;
+static int verbosity = VERBOSITY_DEFAULT;
 static X509_STORE *store = NULL;
-static X509 *x509 = NULL;
+static X509 *x509_nonce = NULL;
+static X509_NAME *x509_name_nonce = NULL;
+static EVP_PKEY **gen_keys = NULL;
+static EVP_PKEY **gen_pubkeys = NULL;
+static X509 **gen_certs = NULL;
 
-static int threadcount;
+static size_t threadcount;
 
-size_t *counts;
 OSSL_TIME max_time;
 
-static void do_x509storeissuer(size_t num)
+#define OSSL_MIN(p, q) ((p) < (q) ? (p) : (q))
+#define OSSL_MAX(p, q) ((p) > (q) ? (p) : (q))
+
+static int stride_stripe_affinity(affinity_t *cpu_set_bits, size_t cpu_set_size,
+                                  size_t num, size_t cnt, void *arg)
 {
-    X509_STORE_CTX *ctx = X509_STORE_CTX_new();
-    X509 *issuer = NULL;
-    OSSL_TIME time;
+    enum { BITS_PER_ELEM = sizeof(cpu_set_bits[0]) * CHAR_BIT };
+    struct affinity_cfg *cfg = (struct affinity_cfg *)arg;
+    size_t set_cnt = 0;
+    uint32_t num_bit;
+    size_t i;
+
+    /* We run everything through "% cnt" to avoid integer overflow */
+    num_bit = (((num % cfg->stripe_size) % cnt) * (cfg->stride % cnt) +
+               ((num / cfg->stripe_size) % cnt) * (cfg->stripe_step % cnt))
+              % cnt;
+
+    /* Finding (num % cnt)th set bit in the provided mask */
+    for (i = 0; i < cpu_set_size; i++) {
+        if (cpu_set_bits[i / BITS_PER_ELEM] & (1UL << (i % BITS_PER_ELEM)))
+            set_cnt++;
+
+        if (set_cnt == (num_bit + 1))
+            break;
+    }
+
+    if (set_cnt != (num_bit + 1)) {
+        WARNX("Only %zu bits are set in the affinity mask, %zu expected",
+              set_cnt, num % cnt + 1);
 
-    if (ctx == NULL || !X509_STORE_CTX_init(ctx, store, x509, NULL)) {
-        printf("Failed to initialise X509_STORE_CTX\n");
-        err = 1;
+        return 0;
+    }
+
+    if (verbosity >= VERBOSITY_DEBUG)
+        fprintf(stderr, "Mapping thread %3" PRIu32 " to CPU %zu\n", num, i);
+
+    memset(cpu_set_bits, 0, cpu_set_size / CHAR_BIT);
+
+    cpu_set_bits[i / BITS_PER_ELEM] = 1 << (i % BITS_PER_ELEM);
+
+    return 1;
+}
+
+static EVP_PKEY_CTX *
+make_pkey_ctx(const char *algstr, char **alg_storage)
+{
+    EVP_PKEY_CTX *ctx = NULL;
+    const char *alg;
+    const char *p = strchr(algstr, ':');
+
+    alg = p ? *alg_storage = OPENSSL_strndup(algstr, p - algstr) : algstr;
+    if (alg == NULL) {
+        warnx("Error getting algorithm name from \"%s\"", algstr);
+        goto err;
+    }
+
+    ctx = EVP_PKEY_CTX_new_from_name(NULL, alg, NULL);
+    if (ctx == NULL) {
+        warnx("Error allocating keygen context");
         goto err;
     }
 
-    counts[num] = 0;
+    if (EVP_PKEY_keygen_init(ctx) <= 0) {
+        warnx("Error initialising keygen contextx");
+        goto err;
+    }
 
-    do {
-        /*
-         * We actually expect this to fail. We've not configured any
-         * certificates inside our store. We're just testing calling this
-         * against an empty store.
-         */
-        if (X509_STORE_CTX_get1_issuer(&issuer, ctx, x509) != 0) {
-            printf("Unexpected result from X509_STORE_CTX_get1_issuer\n");
-            err = 1;
-            X509_free(issuer);
+    /* The bits part */
+    if (p && p[1] >= '0' && p[1] <= '9') {
+        char *endptr = NULL;
+        long bits = strtol(p + 1, &endptr, 0);
+        if (!endptr || (*endptr != '\0' && *endptr != ':')) {
+            warnx("Error while parsing bits from \"%s\"", p + 1);
             goto err;
         }
-        issuer = NULL;
-        counts[num]++;
-        time = ossl_time_now();
+        p = endptr;
+
+        if (bits > 0) {
+            OSSL_PARAM params[] = { OSSL_PARAM_END, OSSL_PARAM_END };
+            size_t val = bits;
+
+            params[0] = OSSL_PARAM_construct_size_t(OSSL_PKEY_PARAM_BITS, &val);
+
+            if (EVP_PKEY_CTX_set_params(ctx, params) <= 0) {
+                warnx("Error setting bits value of %zu to algorithm \"%s\"",
+                      val, alg);
+                goto err;
+            }
+        }
+    }
+
+    /* param=value pairs */
+    while (p && p[0] == ':' && p[1] != '\0') {
+        const char *param_str = p + 1;
+        char *param;
+        char *val;
+        const char *eq;
+        int ret;
+
+        eq = strchr(param_str, '=');
+        p = strchr(param_str, ':');
+
+        if (eq == NULL || (p != NULL && eq > p)) {
+            warnx("Error while parsing a param=value pair from \"%s\"", param_str);
+            goto err;
+        }
+
+        param = OPENSSL_strndup(param_str, eq - param_str);
+        if (param == NULL) {
+            warnx("Error allocating param name string\n");
+            goto err;
+        }
+
+        if (p)
+            val = OPENSSL_strndup(eq + 1, p - eq - 1);
+        else
+            val = OPENSSL_strdup(eq + 1);
+        if (val == NULL) {
+            warnx("Error allocating param value string");
+            OPENSSL_free(param);
+            goto err;
+        }
+
+        if ((ret = EVP_PKEY_CTX_ctrl_str(ctx, param, val)) <= 0) {
+            warnx("Error setting parameter \"%s\" to value \"%s\" for algorithm"
+                  " \"%s\", got %d\n", param, val, alg, ret);
+            OPENSSL_free(val);
+            OPENSSL_free(param);
+            goto err;
+        }
+
+        OPENSSL_free(val);
+        OPENSSL_free(param);
+    }
+
+    return ctx;
+
+err:
+    EVP_PKEY_CTX_free(ctx);
+
+    return NULL;
+}
+
+static EVP_PKEY *
+gen_key(EVP_PKEY_CTX *ctx)
+{
+    EVP_PKEY *res = NULL;
+
+    if (EVP_PKEY_keygen(ctx, &res) <= 0) {
+        warnx("Error generating key");
+
+        return NULL;
+    }
+
+    return res;
+}
+
+static EVP_PKEY *
+get_pubkey(EVP_PKEY *pkey)
+{
+    BIO *bio = NULL;
+    EVP_PKEY *pubkey = NULL;
+
+    bio = BIO_new(BIO_s_mem());
+    if (!bio) {
+        warnx("Error creating memory BIO");
+        goto err;
+    }
+
+    if (!PEM_write_bio_PUBKEY(bio, pkey)) {
+        warnx("Error writing public key to BIO");
+        goto err;
+    }
+
+    if (BIO_seek(bio, 0) < 0) {
+        warnx("Error resetting BIO cursor");
+        goto err;
+    }
+
+    pubkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL);
+    if (pubkey == NULL) {
+        warnx("Error reading pubkey from BIO");
+        goto err;
+    }
+
+    BIO_free(bio);
+
+    return pubkey;
+
+err:
+    EVP_PKEY_free(pubkey);
+    BIO_free(bio);
+
+    return NULL;
+}
+
+static const unsigned char *
+gen_name(void)
+{
+    static const char chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+                                "WXYZabcdefghijklmnopqrstuvwxyz  ";
+    static unsigned char out[64];
+    size_t len;
+
+    len = rand() % (sizeof(out) - 2) + 1;
+
+    for (size_t i = 0; i < len; i++)
+        out[i] = chars[rand() % (sizeof(chars) - 1)];
+
+    out[len] = '\0';
+
+    return out;
+}
+
+static X509 *
+gen_cert(size_t key_id, const unsigned char *sn, const unsigned char *in)
+{
+    EVP_PKEY *pubkey = gen_pubkeys[key_id];
+    EVP_PKEY *pkey = gen_keys[key_id];
+    X509 *cert = NULL;
+    X509_NAME *issuer = NULL;
+    X509_NAME *subject = NULL;
+    EVP_MD_CTX *mctx = NULL;
+    const unsigned char *in_str;
+    const unsigned char *sn_str;
+    int error = 1;
+
+    cert = X509_new();
+    if (!cert) {
+        warnx("Error creating X509 certificate object");
+        goto out;
+    }
+
+    issuer = X509_NAME_new();
+    if (!issuer) {
+        warnx("Error creating X509 issuer name object");
+        goto out;
+    }
+
+    in_str = in ? in : gen_name();
+    if (!X509_NAME_add_entry_by_txt(issuer, "CN", MBSTRING_ASC,
+                                    in_str, -1, -1, 0)) {
+        warnx("Error setting X509 issuer name \"%s\"", (const char *)in_str);
+        goto out;
+    }
+
+    if (!X509_set_issuer_name(cert, issuer)) {
+        warnx("Error setting X509 certificate issuer name:");
+        X509_NAME_print_ex_fp(stderr, issuer, 2, 0);
+        goto out;
+    }
+
+    subject = X509_NAME_new();
+    if (!subject) {
+        warnx("Error creating X509 subject name object");
+        goto out;
+    }
+
+    sn_str = sn ? sn : gen_name();
+    if (!X509_NAME_add_entry_by_txt(subject, "CN", MBSTRING_ASC,
+                                    sn_str, -1, -1, 0)) {
+        warnx("Error setting X509 subject name \"%s\"", (const char *)sn_str);
+        goto out;
+    }
+
+    if (!X509_set_subject_name(cert, subject)) {
+        warnx("Error setting X509 certificate subject name");
+        goto out;
+    }
+
+    if (!X509_set_pubkey(cert, pubkey)) {
+        warnx("Error setting public key for X509 certificate");
+        goto out;
+    }
+
+    mctx = EVP_MD_CTX_new();
+    if (!mctx) {
+        warnx("Error allocating digest context");
+        goto out;
+    }
+
+    if (!EVP_DigestSignInit_ex(mctx, NULL, NULL, NULL, NULL, pkey, NULL)) {
+        warnx("Error initialising digest context");
+        goto out;
+    }
+
+    if (!X509_sign_ctx(cert, mctx)) {
+        warnx("Error signing certificate");
+        goto out;
+    }
+
+    error = 0;
+
+out:
+    EVP_MD_CTX_free(mctx);
+    X509_NAME_free(subject);
+    X509_NAME_free(issuer);
+
+    if (error) {
+        X509_free(cert);
+        cert = NULL;
+    }
+
+    return cert;
+}
+
+static X509 *
+gen_nonce(struct nonce_cfg *cfg)
+{
+    X509 *x509_nonce = X509_new();
+
+    if (!x509_nonce)
+        errx(EXIT_FAILURE, "Error creating X509 nonce object");
+
+    x509_name_nonce = X509_NAME_new();
+    if (!x509_name_nonce)
+        errx(EXIT_FAILURE, "Error creating X509 name nonce object");
+
+    if (!X509_NAME_add_entry_by_txt(x509_name_nonce, "CN", MBSTRING_ASC,
+                                    (unsigned char *) "Test NC CA", -1, -1, 0))
+        errx(EXIT_FAILURE, "Error setting X509 name nonce");
+
+    if (!X509_set_issuer_name(x509_nonce, x509_name_nonce))
+        errx(EXIT_FAILURE, "Error setting X509 nonce name");
+
+    return x509_nonce;
+}
+
+static X509 *
+load_nonce_from_file(const char *path)
+{
+    BIO *bio = BIO_new_file(path, "rb");
+    X509 *x509_nonce = NULL;
+
+    if (bio == NULL) {
+        warnx("Unable to create BIO for reading \"%s\"", path);
+        return NULL;
+    }
+
+    x509_nonce = PEM_read_bio_X509(bio, NULL, NULL, NULL);
+    if (x509_nonce == NULL)
+        warnx("Failed to read certificate \"%s\"", path);
+
+    BIO_free(bio);
+
+    return x509_nonce;
+}
+
+static bool
+is_abs_path(const char *path)
+{
+    if (path == NULL)
+        return false;
+
+#if defined(_WIN32)
+    /*
+     * So, we don't try to concatenate the provided path with the directory
+     * paths if the path start with the following:
+     *  - volume character and a colon ("C:"):  it is either absolute path
+     *    (if followed by a backslash), or a relative path to a current
+     *    directory of that volume (and we don't want to implement any logic
+     *    that handles that);
+     *  - backslash ("\"):  it is an "absolute path" on the "current" drive,
+     *    or (if there are two backslashes in the beginning) an UNC path.
+     */
+    return (isalpha(path[0]) && path[1] == ':') || path[0] == '\\';
+#else /* !_WIN32 */
+    return path[0] == '/';
+#endif
+}
+
+static X509 *
+load_nonce_from_path(struct nonce_cfg *cfg)
+{
+    if (is_abs_path(cfg->path))
+        return load_nonce_from_file(cfg->path);
+
+    for (size_t i = 0; i < cfg->num_dirs; i++) {
+        char *cert;
+        X509 *ret;
+
+        cert = perflib_mk_file_path(cfg->dirs[i], cfg->path);
+        if (cert == NULL) {
+            warnx("Failed to allocate file path for directory \"%s\""
+                  " and path \"%s\"", cfg->dirs[i], cfg->path);
+            continue;
+        }
+
+        ret = load_nonce_from_file(cert);
+        OPENSSL_free(cert);
+
+        if (ret != NULL)
+            return ret;
+    }
+
+    return NULL;
+}
+
+static X509 *
+make_nonce(struct nonce_cfg *cfg)
+{
+    switch (cfg->type) {
+    case NONCE_GENERATED:
+        return gen_nonce(cfg);
+    case NONCE_PATH:
+        return load_nonce_from_path(cfg);
+    default:
+        errx(EXIT_FAILURE, "Unknown nonce type: %lld", (long long ) cfg->type);
+    }
+}
+
+static void
+do_x509storeissuer(size_t num)
+{
+    struct thread_data *td = thread_data + num;
+    X509 *issuer = NULL;
+    OSSL_TIME time;
+    OSSL_TIME duration;
+    OSSL_TIME q_end;
+    size_t q = 0;
+    size_t count = 0;
+    size_t add = 0;
+    size_t found = 0;
+
+    td->start_time = ossl_time_now();
+    duration.t = max_time.t - td->start_time.t;
+    q_end.t = duration.t / quantiles + td->start_time.t;
+
+    do {
+        if (mode == MODE_RW
+            && (!max_writers || (num < max_writers))
+            && (rand() % 65536 < w_probability)) {
+            size_t cert_id = num_gen_load_certs
+                             + ((num_gen_certs - num_gen_load_certs)
+                                * num / threadcount)
+                             + rand() % OSSL_MAX((num_gen_certs
+                                                  - num_gen_load_certs)
+                                                 / threadcount, 1);
+
+            if (!X509_STORE_add_cert(store, gen_certs[cert_id])) {
+                warnx("Failed to add generated certificate %zu to the store",
+                      cert_id);
+            } else {
+                add++;
+            }
+        } else {
+            if (X509_STORE_CTX_get1_issuer(&issuer, td->ctx, x509_nonce) != 0) {
+                found++;
+                X509_free(issuer);
+            }
+            issuer = NULL;
+        }
+
+        count++;
+        if (count % 100 == 0) {
+            time = ossl_time_now();
+            if (time.t >= q_end.t) {
+                td->q_data[q].count = count;
+                td->q_data[q].found = found;
+                td->q_data[q].added_certs = add;
+                td->q_data[q].end_time = time;
+                q_end.t = (duration.t * (++q + 1)) / quantiles + td->start_time.t;
+            }
+        }
     } while (time.t < max_time.t);
 
- err:
-    X509_STORE_CTX_free(ctx);
+    td->q_data[quantiles - 1].count = count;
+    td->q_data[quantiles - 1].end_time = time;
+}
+
+static void
+report_store_size(X509_STORE * const store, const char * const suffix,
+                  int verbosity)
+{
+    if (verbosity >= VERBOSITY_DEBUG_STATS) {
+        STACK_OF(X509_OBJECT) *sk = X509_STORE_get1_objects(store);
+        fprintf(stderr, "Number of certificates in the store %s: %d\n",
+                suffix, sk_X509_OBJECT_num(sk));
+    }
+}
+
+static int
+cmp_double(const void *a_ptr, const void *b_ptr)
+{
+    const double * const a = a_ptr;
+    const double * const b = b_ptr;
+
+    return *a - *b < 0 ? -1 : *a - *b > 0 ? 1 : 0;
+}
+
+static void
+get_calltimes(struct call_times *times, int verbosity)
+{
+    double *call_times;
+
+    for (size_t q = 0; q < quantiles; q++) {
+        for (size_t i = 0; i < threadcount; i++) {
+            uint64_t start_t = q ? thread_data[i].q_data[q - 1].end_time.t
+                                 : thread_data[i].start_time.t;
+            uint64_t count = thread_data[i].q_data[q].count -
+                (q ? thread_data[i].q_data[q - 1].count : 0);
+            uint64_t found = thread_data[i].q_data[q].found -
+                (q ? thread_data[i].q_data[q - 1].found : 0);
+            uint64_t add = thread_data[i].q_data[q].added_certs -
+                (q ? thread_data[i].q_data[q - 1].added_certs : 0);
+
+            times[q].duration += thread_data[i].q_data[q].end_time.t - start_t;
+            times[q].total_count += count;
+            times[q].total_found += found;
+            times[q].total_added += add;
+        }
+    }
+
+    for (size_t q = 0; q < quantiles; q++) {
+        times[quantiles].duration += times[q].duration;
+        times[quantiles].total_count += times[q].total_count;
+        times[quantiles].total_found += times[q].total_found;
+        times[quantiles].total_added += times[q].total_added;
+    }
+
+    for (size_t q = (quantiles == 1); q <= quantiles; q++)
+        times[q].avg = (double) times[q].duration / OSSL_TIME_US / times[q].total_count;
+
+    if (verbosity >= VERBOSITY_VERBOSE) {
+        call_times = OPENSSL_zalloc(threadcount * sizeof(*call_times));
+
+        for (size_t q = (quantiles == 1); q <= quantiles; q++) {
+            double variance = 0;
+
+            for (size_t i = 0; i < threadcount; i++) {
+                uint64_t start_t = q && q != quantiles ? thread_data[i].q_data[q - 1].end_time.t
+                                                       : thread_data[i].start_time.t;
+                uint64_t duration = thread_data[i].q_data[OSSL_MIN(q, quantiles - 1)].end_time.t - start_t;
+                uint64_t count = thread_data[i].q_data[OSSL_MIN(q, quantiles - 1)].count -
+                    (q && q != quantiles ? thread_data[i].q_data[q - 1].count : 0);
+                call_times[i] = (double) duration / OSSL_TIME_US / count;
+            }
+
+            times[q].min = times[q].max = call_times[0];
+            times[q].min_idx = times[q].max_idx = 0;
+
+            for (size_t i = 0; i < threadcount; i++) {
+                if (call_times[i] < times[q].min) {
+                    times[q].min = call_times[i];
+                    times[q].min_idx = i;
+                }
+
+                if (call_times[i] > times[q].max) {
+                    times[q].max = call_times[i];
+                    times[q].max_idx = i;
+                }
+            }
+
+            qsort(call_times, threadcount, sizeof(call_times[0]), cmp_double);
+            times[q].median = call_times[threadcount / 2];
+
+            for (size_t i = 0; i < threadcount; i++) {
+                double dev = call_times[i] - times[q].avg;
+
+                variance += dev * dev;
+            }
+
+            times[q].stddev = sqrt(variance / threadcount);
+        }
+
+        OPENSSL_free(call_times);
+    }
+}
+
+static void
+report_result(int verbosity)
+{
+    struct call_times *times;
+
+    times = OPENSSL_zalloc(sizeof(*times) * (quantiles + 1));
+
+    get_calltimes(times, verbosity);
+
+    switch (verbosity) {
+    case VERBOSITY_TERSE:
+        printf("%lf\n", times[1].avg);
+        break;
+    case VERBOSITY_DEFAULT:
+        printf("Average time per call: %lfus\n", times[1].avg);
+        break;
+    case VERBOSITY_VERBOSE:
+    default:
+        /* if quantiles == 1, we only need to print total runtime info */
+        for (size_t i = (quantiles == 1); i <= quantiles; i++) {
+            if (i < quantiles)
+                printf("Part %8zu", i + 1);
+            else
+                printf("Total runtime");
+
+            printf(": avg: %9.3lf us, median: %9.3lf us"
+                   ", min: %9.3lf us @thread %3zu, max: %9.3lf us @thread %3zu"
+                   ", stddev: %9.3lf us (%8.4lf%%)"
+                   ", hits %9zu of %9zu (%8.4lf%%)"
+                   ", added certs: %zu\n",
+                   times[i].avg, times[i].median,
+                   times[i].min, times[i].min_idx,
+                   times[i].max, times[i].max_idx,
+                   times[i].stddev,
+                   100.0 * times[i].stddev / times[i].avg,
+                   times[i].total_found,
+                   (times[i].total_count - times[i].total_added),
+                   100.0 * times[i].total_found
+                           / (times[i].total_count - times[i].total_added),
+                   times[i].total_added);
+        }
+        break;
+    }
+}
+
+static void
+usage(char * const argv[])
+{
+    fprintf(stderr,
+            "Usage: %s [-t] [-v] [-q N] [-T time] [-G num] [-g num] "
+            "[-k num_keys] [-K keyalg[:bits][:param=value...]] "
+            "[-n nonce_type:type_args] [-m mode] [-w writer_threads] "
+            "[-W percentage] [-C threads] [-s stride] [-r stripe_step] "
+            "[-R stripe_size] certsdir [certsdir...] threadcount\n"
+            "\t-t\tTerse output\n"
+            "\t-v\tVerbose output.  Multiple usage increases verbosity.\n"
+            "\t-q\tGather information about temporal N-quantiles.\n"
+            "\t\tDone only when the output is verbose.  Default: "
+            OPENSSL_MSTR(QUANTILES) "\n"
+            "\t-T\tTimeout for the test run in seconds,\n"
+            "\t\tcan be fractional.  Default: "
+            OPENSSL_MSTR(RUN_TIME) "\n"
+            "\t-G\tNumber of generated certificates.  Default: "
+            OPENSSL_MSTR(NUM_CERTS) "\n"
+            "\t-g\tNumber of initially loaded generated certificates.\n"
+            "\t\tDefault: " OPENSSL_MSTR(NUM_LOAD_CERTS) "\n"
+            "\t-k\tNumber of different keys to be used\n"
+            "\t\tfor the generated certificates.  Default: "
+            OPENSSL_MSTR(NUM_KEYS) "\n"
+            "\t-K\tAlgorithm and key size of the generated keys.\n"
+            "\t\tDefault: " KEY_ALGO "\n"
+            "\t-n\tNonce configuration, supported options:\n"
+            "\t\t\tgen - generated\n"
+            "\t\t\tfile:PATH - load nonce certificate from PATH;\n"
+            "\t\t\tif PATH is relative, the provided certsdir's are searched.\n"
+            "\t\tDefault: " NONCE_CFG "\n"
+            "\t-m\tTest mode, can be one of r, rw.  Default: r\n"
+            "\t-w\tMaximum number of threads that attempt addition\n"
+            "\t\tof the new certificates to the store in rw mode,\n"
+            "\t\t0 is unlimited.  Default: " OPENSSL_MSTR(MAX_WRITERS) "\n"
+            "\t-W\tProbability of a certificate being written\n"
+            "\t\tto the store, instead of being queried,\n"
+            "\t\tin percents.  Default: " OPENSSL_MSTR(W_PROBABILITY) "\n"
+            "\t-C\tNumber of threads that share the same X.509\n"
+            "\t\tstore context object.  Default: "
+            OPENSSL_MSTR(CTX_SHRE_THREADS) "\n"
+            "\t-s stride, -r stripe_step, -R stripe_size\n"
+            "\t\tMap Nth thread to ((N % R) * s + (N / R) * r)th CPU\n"
+            "\t\tin the available process affinity mask (modulus mask size).\n"
+            "\t\tWhen all parameters are 0, no CPU thread affinity setting is\n"
+            "\t\tperformed.  Supported only on Linux/glibc and Windows.\n"
+            "\t\tDefault: 0, 0, 0\n",
+            basename(argv[0]));
+}
+
+static size_t
+parse_timeout(const char * const optarg)
+{
+    char *endptr = NULL;
+    double timeout_s;
+
+    timeout_s = strtod(optarg, &endptr);
+
+    if (endptr == NULL || *endptr != '\0' || timeout_s < 0)
+        errx(EXIT_FAILURE, "incorrect timeout value: \"%s\"");
+
+    if (timeout_s > SIZE_MAX / 1000000)
+        errx(EXIT_FAILURE, "timeout is too large: %f", timeout_s);
+
+    return timeout_s * 1e6;
+}
+
+static double
+parse_probability(const char * const optarg)
+{
+    char *endptr = NULL;
+    double prob;
+
+    prob = strtod(optarg, &endptr);
+
+    if (endptr == NULL || *endptr != '\0' || prob < 0 || prob > 100)
+        errx(EXIT_FAILURE, "incorrect probability value: \"%s\"", optarg);
+
+    return prob;
+}
+
+/**
+ * Parse nonce configuration string. Currently supported formats:
+ *  * "gen" - generate a nonce certificate
+ *  * "file:PATH" - where PATH is either a relative path (that will be then
+ *                  checked against the list of directories provided),
+ *                  or an absolute one.
+ */
+static void
+parse_nonce_cfg(const char * const optarg, struct nonce_cfg *cfg)
+{
+    static const char gen[] = "gen";
+    static const char file_pfx[] = "file:";
+
+    if (strncmp(optarg, gen, sizeof(gen)) == 0) {
+        cfg->type = NONCE_GENERATED;
+    } else if (strncmp(optarg, file_pfx, sizeof(file_pfx) - 1) == 0) {
+        cfg->type = NONCE_PATH;
+        cfg->path = optarg + sizeof(file_pfx) - 1;
+    } else {
+        errx(EXIT_FAILURE, "incorrect nonce configuration: \"%s\"", optarg);
+    }
+}
+
+static long long
+parse_int(const char * const s, long long min, long long max,
+          const char * const what)
+{
+    char *endptr = NULL;
+    long long ret;
+
+    ret = strtoll(s, &endptr, 0);
+    if (endptr == NULL || *endptr != '\0')
+        errx(EXIT_FAILURE, "failed to parse %s as a number: \"%s\"", what, s);
+    if (ret < min || ret > max)
+        errx(EXIT_FAILURE, "provided value of %s is out of the expected"
+                           " %lld..%lld range: %lld", what, min, max, ret);
+
+    return ret;
 }
 
-int main(int argc, char *argv[])
+int
+main(int argc, char *argv[])
 {
     int i;
     OSSL_TIME duration;
-    size_t total_count = 0;
+    size_t ctx_share_cnt = CTX_SHARE_THREADS;
     double avcalltime;
-    int terse = 0;
+    char *alg_name_storage = NULL;
     char *cert = NULL;
     int ret = EXIT_FAILURE;
+    EVP_PKEY_CTX *key_ctx;
     BIO *bio = NULL;
+    X509 *x509 = NULL;
     int opt;
+    int dirs_start;
+    size_t num_certs = 0;
+    size_t num_store_gen_certs = 0;
+    struct nonce_cfg nonce_cfg;
+    struct affinity_cfg affinity_cfg = { 0 };
+    bool set_affinity = false;
 
-    while ((opt = getopt(argc, argv, "t")) != -1) {
+    parse_nonce_cfg(NONCE_CFG, &nonce_cfg);
+
+    while ((opt = getopt(argc, argv, "tvq:T:G:g:k:K:m:n:w:W:S:s:r:R:")) != -1) {
         switch (opt) {
-        case 't':
-            terse = 1;
+        case 't': /* terse */
+            verbosity = VERBOSITY_TERSE;
+            break;
+        case 'v': /* verbose */
+            if (verbosity < VERBOSITY_VERBOSE) {
+                verbosity = VERBOSITY_VERBOSE;
+            } else {
+                if (verbosity < VERBOSITY_MAX__ - 1)
+                    verbosity++;
+            }
+            break;
+        case 'q': /* quantiles */
+            quantiles = parse_int(optarg, 1, INT_MAX,
+                                  "number of quantiles");
+            break;
+        case 'T': /* timeout */
+            timeout_us = parse_timeout(optarg);
+            break;
+        case 'G': /* number of generated certs */
+            num_gen_certs = parse_int(optarg, 0, INT_MAX,
+                                      "number of initially loaded generated"
+                                      " certificates");
+            break;
+        case 'g': /* number of initially loaded generated certs */
+            num_gen_load_certs = parse_int(optarg, 0, INT_MAX,
+                                           "number of initially loaded"
+                                           " generated certificates");
+            break;
+        case 'k': /* number of generated keys */
+            num_gen_keys = parse_int(optarg, 0, INT_MAX,
+                                     "number of generated keys");
+            break;
+        case 'K': /* key type */
+            gen_key_algo = optarg;
+            break;
+        case 'm': /* mode */
+            if (strcasecmp(optarg, "r") == 0) {
+                mode = MODE_R;
+            } else if (strcasecmp(optarg, "rw") == 0) {
+                mode = MODE_RW;
+            } else {
+                errx(EXIT_FAILURE, "Unknown mode: \"%s\"", optarg);
+            }
+            break;
+        case 'n': /* nonce */
+            parse_nonce_cfg(optarg, &nonce_cfg);
+            break;
+        case 'w': /* maximum writers */
+            max_writers = parse_int(optarg, 0, INT_MAX,
+                                    "maximum number of writers");
+        case 'W': /* percent of writes */
+            w_probability = (size_t) (parse_probability(optarg) * 65536 / 100);
+            break;
+        case 'S': /* how many threads share X509_STORE_CTX */
+            ctx_share_cnt = parse_int(optarg, 1, INT_MAX,
+                                      "X509_STORE_CTX share degree");
+            break;
+        case 's': /* stride, s in ((N % R) * s + (N / R) * r) mod ncpus */
+            affinity_cfg.stride = parse_int(optarg, 0, INT_MAX,
+                                            "thread affinity stride");
+            break;
+        case 'r': /* stripe step, r in ((N % R) * s + (N / R) * r) mod ncpus */
+            affinity_cfg.stripe_step = parse_int(optarg, 0, INT_MAX,
+                                                 "thread affinity stripe step");
+            break;
+        case 'R': /* stripe size, R in ((N % R) * s + (N / R) * r) mod ncpus */
+            affinity_cfg.stripe_size = parse_int(optarg, 0, INT_MAX,
+                                                 "thread affinity stripe size");
             break;
         default:
-            printf("Usage: %s [-t] certsdir threadcount\n", basename(argv[0]));
-            printf("-t - terse output\n");
+            warnx("Unknown option: \"-%c\"", opt);
+            usage(argv);
             return EXIT_FAILURE;
         }
     }
 
-    if (argv[optind] == NULL) {
-        printf("certsdir is missing\n");
-        goto err;
-    }
-    cert = perflib_mk_file_path(argv[optind], "servercert.pem");
-    if (cert == NULL) {
-        printf("Failed to allocate cert\n");
-        goto err;
+    if (verbosity < VERBOSITY_VERBOSE)
+        quantiles = 1;
+
+    if (num_gen_certs > 0 && num_gen_keys == 0)
+        errx(EXIT_FAILURE,
+             "Cannot generate certificates without generating keys");
+
+    if (num_gen_certs < num_gen_load_certs)
+        errx(EXIT_FAILURE, "Cannot load more certificates than generate");
+
+    if (num_gen_certs == num_gen_load_certs && mode == MODE_RW)
+        errx(EXIT_FAILURE, "No generated certificates to use after"
+                           " the initially loaded ones, please increase"
+                           " -G to be more than -g");
+
+    if (argv[optind] == NULL)
+        errx(EXIT_FAILURE, "certsdir is missing");
+
+    dirs_start = optind++;
+
+    /*
+     * Store the part of argv containing directories to nonce_cfg so
+     * load_nonce_from_path can use it later.
+     */
+    nonce_cfg.dirs = argv + dirs_start;
+    nonce_cfg.num_dirs = argc - 1 - dirs_start;
+
+    if (optind >= argc)
+        errx(EXIT_FAILURE, "threadcount is missing");
+
+    threadcount = parse_int(argv[argc - 1], 1, INT_MAX, "threadcount");
+
+    thread_data = OPENSSL_zalloc(threadcount * sizeof(*thread_data));
+    if (thread_data == NULL)
+        errx(EXIT_FAILURE, "Failed to create thread_data array");
+
+    for (size_t i = 0; i < threadcount; i++) {
+        thread_data[i].q_data = OPENSSL_zalloc(quantiles *
+                                               sizeof(*(thread_data[i].q_data)));
+        if (thread_data[i].q_data == NULL)
+            errx(EXIT_FAILURE, "Failed to create quantiles array for thread"
+                               " %zu", i);
     }
-    optind++;
 
-    if (argv[optind] == NULL) {
-        printf("threadcount is missing\n");
-        goto err;
+    if (num_gen_keys > 0) {
+        key_ctx = make_pkey_ctx(gen_key_algo, &alg_name_storage);
+        if (key_ctx == NULL)
+            exit(EXIT_FAILURE);
+
+        gen_keys = OPENSSL_zalloc(num_gen_keys * sizeof(*gen_keys));
+        if (gen_keys == NULL)
+            errx(EXIT_FAILURE, "Error allocating generated keys array");
+
+        gen_pubkeys = OPENSSL_zalloc(num_gen_keys * sizeof(*gen_pubkeys));
+        if (gen_pubkeys == NULL)
+            errx(EXIT_FAILURE, "Error allocating public keys array");
+
+        for (size_t i = 0; i < num_gen_keys; i++) {
+            if (verbosity >= VERBOSITY_DEBUG_STATS) {
+                static OSSL_TIME last = { 0 };
+                OSSL_TIME cur = ossl_time_now();
+                if (cur.t - last.t > OSSL_TIME_SECOND) {
+                    fprintf(stderr, "Generating key %zu out of %zu...\n",
+                            i + 1, num_gen_keys);
+                    last.t = cur.t;
+                }
+            }
+            gen_keys[i] = gen_key(key_ctx);
+            if (gen_keys[i] == NULL)
+                exit(EXIT_FAILURE);
+
+            gen_pubkeys[i] = get_pubkey(gen_keys[i]);
+            if (gen_pubkeys[i] == NULL)
+                exit(EXIT_FAILURE);
+        }
+
+        if (verbosity >= VERBOSITY_DEBUG_STATS)
+            fprintf(stderr, "Generated %zu keys\n", num_gen_keys);
     }
-    threadcount = atoi(argv[optind]);
-    if (threadcount < 1) {
-        printf("threadcount must be > 0\n");
-        goto err;
+
+    if (num_gen_certs > 0) {
+        gen_certs = OPENSSL_zalloc(num_gen_certs * sizeof(*gen_certs));
+        if (gen_certs == NULL)
+            errx(EXIT_FAILURE, "Error allocating generated certificates array");
+
+        for (size_t i = 0; i < num_gen_certs; i++) {
+            if (verbosity >= VERBOSITY_DEBUG_STATS) {
+                static OSSL_TIME last = { 0 };
+                OSSL_TIME cur = ossl_time_now();
+                if (cur.t - last.t > OSSL_TIME_SECOND) {
+                    fprintf(stderr, "Generating certificate %zu out of"
+                                    " %zu...\n", i + 1, num_gen_certs);
+                    last.t = cur.t;
+                }
+            }
+            gen_certs[i] = gen_cert(i % num_gen_keys, NULL, NULL);
+            if (gen_certs[i] == NULL)
+                goto err;
+        }
+
+        if (verbosity >= VERBOSITY_DEBUG_STATS)
+            fprintf(stderr, "Generated %zu certificates\n", num_gen_certs);
     }
 
     store = X509_STORE_new();
-    if (store == NULL || !X509_STORE_set_default_paths(store)) {
-        printf("Failed to create X509_STORE\n");
-        goto err;
-    }
+    if (store == NULL || !X509_STORE_set_default_paths(store))
+        errx(EXIT_FAILURE, "Failed to create X509_STORE");
 
-    bio = BIO_new_file(cert, "rb");
-    if (bio == NULL) {
-        printf("Unable to load certificate\n");
-        goto err;
-    }
-    x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL);
-    if (x509 == NULL) {
-        printf("Failed to read certificate\n");
-        goto err;
-    }
-    BIO_free(bio);
-    bio = NULL;
+    for (int i = dirs_start; i < argc - 1; i++) {
+        struct stat st;
+        struct dirent *e;
+        DIR *d = opendir(argv[i]);
 
-    counts = OPENSSL_malloc(sizeof(size_t) * threadcount);
-    if (counts == NULL) {
-        printf("Failed to create counts array\n");
-        goto err;
+        if (d == NULL)
+            err(EXIT_FAILURE, "Could not open \"%s\"", argv[i]);
+
+        while (1) {
+            errno = 0;
+            e = readdir(d);
+
+            if (e == NULL) {
+                if (errno != 0) {
+                    err(EXIT_FAILURE, "An error ocurred while reading directory"
+                                      " \"%s\"", argv[i]);
+                } else {
+                    break;
+                }
+            }
+
+            cert = perflib_mk_file_path(argv[i], e->d_name);
+            if (cert == NULL)
+                errx(EXIT_FAILURE, "Failed to allocate cert name in directory"
+                                   " \"%s\" for entry \"%s\"",
+                                   argv[i], e->d_name);
+
+            if (lstat(cert, &st) < 0) {
+                warn("Got error on lstat(\"%s\")", cert);
+                goto next_file;
+            }
+
+            if (st.st_mode & S_IFMT != S_IFREG) {
+                if (verbosity >= VERBOSITY_DEBUG)
+                    warnx("\"%s\" is not a regular file, skipping", cert);
+                goto next_file;
+            }
+
+            bio = BIO_new_file(cert, "rb");
+            if (bio == NULL)
+                errx(EXIT_FAILURE, "Unable to create BIO for \"%s\"", cert);
+
+            x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL);
+            if (x509 == NULL) {
+                if (verbosity >= VERBOSITY_DEBUG)
+                    warnx("Failed to read certificate from \"%s\", skipping",
+                          cert);
+                goto next_file;
+            } else {
+                if (!X509_STORE_add_cert(store, x509)) {
+                    warnx("Failed to add a certificate from \"%s\""
+                          " to the store\n", cert);
+                    goto next_file;
+                } else {
+                    if (verbosity >= VERBOSITY_DEBUG)
+                        fprintf(stderr, "Successfully added a certificate from"
+                                        " \"%s\" to the store\n", cert);
+                    num_certs++;
+                }
+            }
+
+ next_file:
+            X509_free(x509);
+            x509 = NULL;
+
+            BIO_free(bio);
+            bio = NULL;
+
+            OPENSSL_free(cert);
+            cert = NULL;
+        }
     }
 
-    max_time = ossl_time_add(ossl_time_now(), ossl_seconds2time(RUN_TIME));
+    if (verbosity >= VERBOSITY_DEBUG_STATS)
+        fprintf(stderr, "Added %zu certificates to the store\n", num_certs);
 
-    if (!perflib_run_multi_thread_test(do_x509storeissuer, threadcount, &duration)) {
-        printf("Failed to run the test\n");
-        goto err;
+    num_store_gen_certs = 0;
+    for (size_t i = 0; i < num_gen_load_certs; i++) {
+        if (!X509_STORE_add_cert(store, gen_certs[i])) {
+            warnx("Failed to add generated certificate %zu to the store\n", i);
+        } else {
+            if (verbosity >= VERBOSITY_DEBUG)
+                fprintf(stderr, "Successfully added generated certificate"
+                                " %zu to the store\n", i);
+            num_certs++;
+            num_store_gen_certs++;
+        }
     }
 
-    if (err) {
-        printf("Error during test\n");
-        goto err;
+    if (verbosity >= VERBOSITY_DEBUG_STATS)
+        fprintf(stderr, "Added %zu generated certificates to the store,"
+                        " %zu total\n", num_store_gen_certs, num_certs);
+
+    report_store_size(store, "before the test run", verbosity);
+
+    x509_nonce = make_nonce(&nonce_cfg);
+    if (x509_nonce == NULL)
+        errx(EXIT_FAILURE, "Unable to create the nonce X509 object");
+
+    for (size_t i = 0; i < threadcount; i++) {
+        if (i % ctx_share_cnt) {
+            thread_data[i].ctx = thread_data[i - i % ctx_share_cnt].ctx;
+        } else {
+            thread_data[i].ctx = X509_STORE_CTX_new();
+            if (thread_data[i].ctx == NULL
+                || !X509_STORE_CTX_init(thread_data[i].ctx, store, x509, NULL))
+                errx(EXIT_FAILURE, "Failed to initialise X509_STORE_CTX"
+                     " for thread %zu", i);
+        }
     }
 
-    for (i = 0; i < threadcount; i++)
-        total_count += counts[i];
+    max_time = ossl_time_add(ossl_time_now(), ossl_us2time(timeout_us));
 
-    avcalltime = (double)RUN_TIME * 1e6 * threadcount / total_count;
+    if (affinity_cfg.stride != 0
+        || affinity_cfg.stripe_step != 0
+        || affinity_cfg.stripe_size != 0)
+        set_affinity = true;
 
-    if (terse)
-        printf("%lf\n", avcalltime);
-    else
-        printf("Average time per X509_STORE_CTX_get1_issuer() call: %lfus\n",
-               avcalltime);
+    if (set_affinity && affinity_cfg.stripe_size == 0)
+        affinity_cfg.stripe_size = INT_MAX;
+
+    if (!perflib_run_multi_thread_test_ex(do_x509storeissuer, threadcount,
+                                          &duration,
+                                          set_affinity ? stride_stripe_affinity
+                                                       : NULL,
+                                          &affinity_cfg))
+        errx(EXIT_FAILURE, "Failed to run the test");
+
+    if (error)
+        errx(EXIT_FAILURE, "Error during test");
+
+    report_store_size(store, "after the test run", verbosity);
+
+    report_result(verbosity);
 
     ret = EXIT_SUCCESS;
 
  err:
+    X509_NAME_free(x509_name_nonce);
+    X509_free(x509_nonce);
     X509_STORE_free(store);
-    X509_free(x509);
-    BIO_free(bio);
-    OPENSSL_free(cert);
-    OPENSSL_free(counts);
+    if (gen_certs) {
+        for (size_t i = 0; i < num_gen_certs; i++)
+            X509_free(gen_certs[i]);
+    }
+    OPENSSL_free(gen_certs);
+    if (gen_pubkeys) {
+        for (size_t i = 0; i < num_gen_keys; i++)
+            EVP_PKEY_free(gen_pubkeys[i]);
+    }
+    OPENSSL_free(gen_pubkeys);
+    if (gen_keys) {
+        for (size_t i = 0; i < num_gen_keys; i++)
+            EVP_PKEY_free(gen_keys[i]);
+    }
+    OPENSSL_free(gen_keys);
+    EVP_PKEY_CTX_free(key_ctx);
+    OPENSSL_free(alg_name_storage);
+    if (thread_data != NULL) {
+        for (size_t i = 0; i < threadcount; i++) {
+            if (!(i % ctx_share_cnt))
+                X509_STORE_CTX_free(thread_data[i].ctx);
+            OPENSSL_free(thread_data[i].q_data);
+        }
+    }
+    OPENSSL_free(thread_data);
     return ret;
 }

From 56cbb71e56f8e9ecac8b1353da26bd6a61c866bf Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Tue, 14 Oct 2025 18:19:17 +0200
Subject: [PATCH 11/12] perflib: remove perflib_glue_strings

There are no users of it.

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 source/perflib/perfhelper.c | 26 --------------------------
 source/perflib/perflib.h    |  1 -
 2 files changed, 27 deletions(-)

diff --git a/source/perflib/perfhelper.c b/source/perflib/perfhelper.c
index 67e25b5..c2fdc10 100644
--- a/source/perflib/perfhelper.c
+++ b/source/perflib/perfhelper.c
@@ -32,29 +32,3 @@ char *perflib_mk_file_path(const char *dir, const char *file)
 
     return full_file;
 }
-
-/*
- * Glue an array of strings together and return it as an allocated string.
- * Optionally return the whole length of this string in |out_len|
- */
-char *perflib_glue_strings(const char *list[], size_t *out_len)
-{
-    size_t len = 0;
-    char *p, *ret;
-    int i;
-
-    for (i = 0; list[i] != NULL; i++)
-        len += strlen(list[i]);
-
-    if (out_len != NULL)
-        *out_len = len;
-
-    ret = p = OPENSSL_malloc(len + 1);
-    if (p == NULL)
-        return NULL;
-
-    for (i = 0; list[i] != NULL; i++)
-        p += strlen(strcpy(p, list[i]));
-
-    return ret;
-}
diff --git a/source/perflib/perflib.h b/source/perflib/perflib.h
index 1048a7a..9bab72c 100644
--- a/source/perflib/perflib.h
+++ b/source/perflib/perflib.h
@@ -94,7 +94,6 @@ perflib_run_multi_thread_test(void (*f)(size_t), size_t threadcount,
                                             NULL, NULL);
 }
 char *perflib_mk_file_path(const char *dir, const char *file);
-char *perflib_glue_strings(const char *list[], size_t *out_len);
 
 int perflib_create_ssl_ctx_pair(const SSL_METHOD *sm, const SSL_METHOD *cm,
                                 int min_proto_version, int max_proto_version,

From b32782465543a747e6bdf4f747a131d26ffd867e Mon Sep 17 00:00:00 2001
From: Eugene Syromiatnikov <esyr@openssl.org>
Date: Wed, 15 Oct 2025 16:30:28 +0200
Subject: [PATCH 12/12] README.md update wip

Signed-off-by: Eugene Syromiatnikov <esyr@openssl.org>
---
 README.md | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index bf98ec3..22573d4 100644
--- a/README.md
+++ b/README.md
@@ -37,10 +37,10 @@ labels.
 
 ## randbytes
 
-The randbytes test does 10000 calls of the [RAND_bytes()](https://docs.openssl.org/master/man3/RAND_bytes/) function divided
+The randbytes test does 10000 calls of the [`RAND_bytes()`](https://docs.openssl.org/master/man3/RAND_bytes/) function divided
 evenly among multiple threads. The number of threads to use is provided as
 an argument and the test reports the average time take to execute a block of
-1000 [RAND_bytes()](https://docs.openssl.org/master/man3/RAND_bytes/) calls.
+1000 [`RAND_bytes()`](https://docs.openssl.org/master/man3/RAND_bytes/) calls.
 
 ## handshake
 
@@ -79,43 +79,43 @@ calls.
 
 ## newrawkey
 
-The `newrawkey` test repeatedly calls the [EVP_PKEY_new_raw_public_key_ex()](https://docs.openssl.org/master/man3/EVP_PKEY_new/)
+The `newrawkey` test repeatedly calls the [`EVP_PKEY_new_raw_public_key_ex()`](https://docs.openssl.org/master/man3/EVP_PKEY_new/)
 function. It does 100000 repetitions divided evenly among each thread. The
 number of threads to use is provided as an argument and the test reports the
-average time take to execute a block of 1000 [EVP_PKEY_new_raw_public_key_ex()](https://docs.openssl.org/master/man3/EVP_PKEY_new/)
+average time take to execute a block of 1000 [`EVP_PKEY_new_raw_public_key_ex()`](https://docs.openssl.org/master/man3/EVP_PKEY_new/)
 calls.
 
 Note that this test does not support OpenSSL 1.1.1.
 
 ## rsasign
 
-The `rsasign` test repeatedly calls the [EVP_PKEY_sign_init()/EVP_PKEY_sign()](https://docs.openssl.org/master/man3/EVP_PKEY_sign/)
+The `rsasign` test repeatedly calls the [`EVP_PKEY_sign_init()`/`EVP_PKEY_sign()`](https://docs.openssl.org/master/man3/EVP_PKEY_sign/)
 functions, using a 512 bit RSA key. It does 100000 repetitions divided evenly
 among each thread. The number of threads to use is provided as an argument and
 the test reports the average time take to execute a block of 1000
-[EVP_PKEY_sign_init()/EVP_PKEY_sign()](https://docs.openssl.org/master/man3/EVP_PKEY_sign/) calls.
+[`EVP_PKEY_sign_init()/EVP_PKEY_sign()`](https://docs.openssl.org/master/man3/EVP_PKEY_sign/) calls.
 
 ## x509storeissuer
 
-Runs the function call [X509_STORE_CTX_get1_issuer()](https://docs.openssl.org/master/man3/X509_STORE_set_verify_cb_func/) repeatedly in a loop (which
+Runs the function call [`X509_STORE_CTX_get1_issuer()`](https://docs.openssl.org/master/man3/X509_STORE_set_verify_cb_func/) repeatedly in a loop (which
 is used in certificate chain building as part of a verify operation). The test
 assumes that the default certificates directly exists but is empty. For a
 default configuration this is "/usr/local/ssl/certs". The test takes the number
 of threads to use as an argument and the test reports the average time take to
-execute a block of 1000 [X509_STORE_CTX_get1_issuer()](https://docs.openssl.org/master/man3/X509_STORE_set_verify_cb_func/) calls.
+execute a block of 1000 [`X509_STORE_CTX_get1_issuer()`](https://docs.openssl.org/master/man3/X509_STORE_set_verify_cb_func/) calls.
 
 ## providerdoall
 
-The `providerdoall` test repeatedly calls the [OSSL_PROVIDER_do_all()](https://docs.openssl.org/master/man3/OSSL_PROVIDER) function.
+The `providerdoall` test repeatedly calls the [`OSSL_PROVIDER_do_all()`](https://docs.openssl.org/master/man3/OSSL_PROVIDER) function.
 It does 100000 repetitions divided evenly among each thread. The number of
 threads to use is provided as an argument and the test reports the average time
-take to execute a block of 1000 [OSSL_PROVIDER_do_all()](https://docs.openssl.org/master/man3/OSSL_PROVIDER) calls.
+take to execute a block of 1000 [`OSSL_PROVIDER_do_all()`](https://docs.openssl.org/master/man3/OSSL_PROVIDER) calls.
 
 ## rwlocks
 
 The `rwlocks` test creates the command line specified number of threads, splitting
 them evenly between read and write functions (though this is adjustable via the
-LOCK_WRITERS environment variable).  Threads then iteratively acquire a shared
+`LOCK_WRITERS` environment variable).  Threads then iteratively acquire a shared
 rwlock to read or update some shared data.  The number of read and write
 lock/unlock pairs are reported as a performance measurement
 
@@ -148,7 +148,7 @@ for all keys and formats using 4 threads, run `pkeyread` as follows:
 
 ## evp_setpeer
 
-The `evp_setpeer` test repeatedly calls the [EVP_PKEY_derive_set_peer()](https://docs.openssl.org/master/man3/EVP_PKEY_derive/) function
+The `evp_setpeer` test repeatedly calls the [`EVP_PKEY_derive_set_peer()`](https://docs.openssl.org/master/man3/EVP_PKEY_derive/) function
 on a memory BIO with a private key of desired type.  It does 10000
 repetitions divided evenly among each thread. The last argument will be the
 number of threads run. The test reports average time per call. Use option `-k`
@@ -163,8 +163,8 @@ evp_setpeer as follows:
 ## writeread
 
 Performs an in-memory client and server handshake and measures the average
-time taken for a single sequence of calling [SSL_write_ex()](https://docs.openssl.org/master/man3/SSL_write/) on the client and
-[SSL_write_ex()](https://docs.openssl.org/master/man3/SSL_write/) on the server.  In total 1000000 writes and reads are performed
+time taken for a single sequence of calling [`SSL_write_ex()`](https://docs.openssl.org/master/man3/SSL_write/) on the client and
+[`SSL_write_ex()`](https://docs.openssl.org/master/man3/SSL_write/) on the server.  In total 1000000 writes and reads are performed
 divided evenly among each thread. It take 4 optional and 2 required arguments:
 
 ```
@@ -180,7 +180,7 @@ threadcount - number of concurrent threads to run in test.
 ## ssl_poll_perf
 
 Tool to evaluate performance of QUIC client and server which both use
-[SSL_poll](https://docs.openssl.org/master/man3/SSL_poll/)(3ossl). Application creates two threads, one for client the
+[`SSL_poll()`](https://docs.openssl.org/master/man3/SSL_poll/). Application creates two threads, one for client the
 other for server. Server and client can both accept/create simultanous
 connections. Each connection then can carry multiple unidirectional/bidirectional
 streams. The streams handle HTTP/1.0 GET request/responses only.