sched/uclamp: Optimize sched_uclamp_used static key enabling

JIRA: https://issues.redhat.com/browse/RHEL-110301

commit 4bc4582
Author: Xuewen Yan <[email protected]>
Date:   Wed Feb 19 17:37:47 2025 +0800

    sched/uclamp: Optimize sched_uclamp_used static key enabling

    Repeat calls of static_branch_enable() to an already enabled
    static key introduce overhead, because it calls cpus_read_lock().

    Users may frequently set the uclamp value of tasks, triggering
    the repeat enabling of the sched_uclamp_used static key.

    Optimize this and avoid repeat calls to static_branch_enable()
    by checking whether it's enabled already.

    [ mingo: Rewrote the changelog for legibility ]

    Signed-off-by: Xuewen Yan <[email protected]>
    Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
    Signed-off-by: Ingo Molnar <[email protected]>
    Reviewed-by: Christian Loehle <[email protected]>
    Reviewed-by: Vincent Guittot <[email protected]>
    Link: https://lore.kernel.org/r/[email protected]

Signed-off-by: Phil Auld <[email protected]>

Author: auldp

kernel/sched/core.c

@@ -1918,12 +1918,12 @@ static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write,
 	}
 
 	if (update_root_tg) {
-		static_branch_enable(&sched_uclamp_used);
+		sched_uclamp_enable();
 		uclamp_update_root_tg();
 	}
 
 	if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
-		static_branch_enable(&sched_uclamp_used);
+		sched_uclamp_enable();
 		uclamp_sync_util_min_rt_default();
 	}
 
@@ -9235,7 +9235,7 @@ static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf,
 	if (req.ret)
 		return req.ret;
 
-	static_branch_enable(&sched_uclamp_used);
+	sched_uclamp_enable();
 
 	guard(mutex)(&uclamp_mutex);
 	guard(rcu)();

kernel/sched/sched.h

@@ -3253,6 +3253,18 @@ static inline bool uclamp_is_used(void)
 	return static_branch_likely(&sched_uclamp_used);
 }
 
+/*
+ * Enabling static branches would get the cpus_read_lock(),
+ * check whether uclamp_is_used before enable it to avoid always
+ * calling cpus_read_lock(). Because we never disable this
+ * static key once enable it.
+ */
+static inline void sched_uclamp_enable(void)
+{
+	if (!uclamp_is_used())
+		static_branch_enable(&sched_uclamp_used);
+}
+
 static inline unsigned long uclamp_rq_get(struct rq *rq,
 					  enum uclamp_id clamp_id)
 {
@@ -3332,6 +3344,8 @@ static inline bool uclamp_is_used(void)
 	return false;
 }
 
+static inline void sched_uclamp_enable(void) {}
+
 static inline unsigned long
 uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id)
 {

kernel/sched/syscalls.c

@@ -365,7 +365,7 @@ static int uclamp_validate(struct task_struct *p,
 	 * blocking operation which obviously cannot be done while holding
 	 * scheduler locks.
 	 */
-	static_branch_enable(&sched_uclamp_used);
+	sched_uclamp_enable();
 
 	return 0;
 }