ftrace: Store direct called addresses in their ops

JIRA: https://issues.redhat.com/browse/RHEL-101598

commit dbaccb6
Author: Florent Revest <[email protected]>
Date:   Tue Mar 21 15:04:22 2023 +0100

    ftrace: Store direct called addresses in their ops

    All direct calls are now registered using the register_ftrace_direct API
    so each ops can jump to only one direct-called trampoline.

    By storing the direct called trampoline address directly in the ops we
    can save one hashmap lookup in the direct call ops and implement arm64
    direct calls on top of call ops.

    Link: https://lkml.kernel.org/r/[email protected]

    Signed-off-by: Florent Revest <[email protected]>
    Acked-by: Jiri Olsa <[email protected]>
    Signed-off-by: Steven Rostedt (Google) <[email protected]>

Signed-off-by: Jerome Marchand <[email protected]>

Author: jeromemarchand

include/linux/ftrace.h

@@ -314,6 +314,9 @@ struct ftrace_ops {
 	unsigned long			trampoline_size;
 	struct list_head		list;
 	ftrace_ops_func_t		ops_func;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	unsigned long			direct_call;
+#endif
 #endif
 };
 

kernel/trace/ftrace.c

@@ -2586,9 +2586,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
 static void call_direct_funcs(unsigned long ip, unsigned long pip,
 			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
 {
-	unsigned long addr;
+	unsigned long addr = READ_ONCE(ops->direct_call);
 
-	addr = ftrace_find_rec_direct(ip);
 	if (!addr)
 		return;
 
@@ -5368,6 +5367,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	ops->func = call_direct_funcs;
 	ops->flags = MULTI_FLAGS;
 	ops->trampoline = FTRACE_REGS_ADDR;
+	ops->direct_call = addr;
 
 	err = register_ftrace_function_nolock(ops);
 
@@ -5442,6 +5442,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	/* Enable the tmp_ops to have the same functions as the direct ops */
 	ftrace_ops_init(&tmp_ops);
 	tmp_ops.func_hash = ops->func_hash;
+	tmp_ops.direct_call = addr;
 
 	err = register_ftrace_function_nolock(&tmp_ops);
 	if (err)
@@ -5463,6 +5464,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 			entry->direct = addr;
 		}
 	}
+	/* Prevent store tearing if a trampoline concurrently accesses the value */
+	WRITE_ONCE(ops->direct_call, addr);
 
 	mutex_unlock(&ftrace_lock);