Skip to content

Commit b602d00

Browse files
davem330davem330
committed
Merge branch 'net-sched-retpoline'
Pedro Tammela says: ==================== net/sched: retpoline wrappers for tc In tc all qdics, classifiers and actions can be compiled as modules. This results today in indirect calls in all transitions in the tc hierarchy. Due to CONFIG_RETPOLINE, CPUs with mitigations=on might pay an extra cost on indirect calls. For newer Intel cpus with IBRS the extra cost is nonexistent, but AMD Zen cpus and older x86 cpus still go through the retpoline thunk. Known built-in symbols can be optimized into direct calls, thus avoiding the retpoline thunk. So far, tc has not been leveraging this build information and leaving out a performance optimization for some CPUs. In this series we wire up 'tcf_classify()' and 'tcf_action_exec()' with direct calls when known modules are compiled as built-in as an opt-in optimization. We measured these changes in one AMD Zen 4 cpu (Retpoline), one AMD Zen 3 cpu (Retpoline), one Intel 10th Gen CPU (IBRS), one Intel 3rd Gen cpu (Retpoline) and one Intel Xeon CPU (IBRS) using pktgen with 64b udp packets. Our test setup is a dummy device with clsact and matchall in a kernel compiled with every tc module as built-in. We observed a 3-8% speed up on the retpoline CPUs, when going through 1 tc filter, and a 60-100% speed up when going through 100 filters. For the IBRS cpus we observed a 1-2% degradation in both scenarios, we believe the extra branches check introduced a small overhead therefore we added a static key that bypasses the wrapper on kernels not using the retpoline mitigation, but compiled with CONFIG_RETPOLINE. 1 filter: CPU | before (pps) | after (pps) | diff R9 7950X | 5914980 | 6380227 | +7.8% R9 5950X | 4237838 | 4412241 | +4.1% R9 5950X | 4265287 | 4413757 | +3.4% [*] i5-3337U | 1580565 | 1682406 | +6.4% i5-10210U | 3006074 | 3006857 | +0.0% i5-10210U | 3160245 | 3179945 | +0.6% [*] Xeon 6230R | 3196906 | 3197059 | +0.0% Xeon 6230R | 3190392 | 3196153 | +0.01% [*] 100 filters: CPU | before (pps) | after (pps) | diff R9 7950X | 373598 | 820396 | +119.59% R9 5950X | 313469 | 633303 | +102.03% R9 5950X | 313797 | 633150 | +101.77% [*] i5-3337U | 127454 | 211210 | +65.71% i5-10210U | 389259 | 381765 | -1.9% i5-10210U | 408812 | 412730 | +0.9% [*] Xeon 6230R | 415420 | 406612 | -2.1% Xeon 6230R | 416705 | 405869 | -2.6% [*] [*] In these tests we ran pktgen with clone set to 1000. On the 7950x system we also tested the impact of filters if iteration order placement varied, first by compiling a kernel with the filter under test being the first one in the static iteration and then repeating it with being last (of 15 classifiers existing today). We saw a difference of +0.5-1% in pps between being the first in the iteration vs being the last. Therefore we order the classifiers and actions according to relevance per our current thinking. v5->v6: - Address Eric Dumazet suggestions v4->v5: - Rebase v3->v4: - Address Eric Dumazet suggestions v2->v3: - Address suggestions by Jakub, Paolo and Eric - Dropped RFC tag (I forgot to add it on v2) v1->v2: - Fix build errors found by the bots - Address Kuniyuki Iwashima suggestions ==================== Signed-off-by: David S. Miller <[email protected]>
2 parents 0bdff11 + 9f3101d commit b602d00

38 files changed

+391
-72
lines changed

‎include/net/act_api.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,6 @@ static inline enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
101101
return hw_stats;
102102
}
103103

104-
#ifdef CONFIG_NET_CLS_ACT
105-
106-
#define ACT_P_CREATED 1
107-
#define ACT_P_DELETED 1
108-
109104
typedef void (*tc_action_priv_destructor)(void *priv);
110105

111106
struct tc_action_ops {
@@ -140,6 +135,11 @@ struct tc_action_ops {
140135
struct netlink_ext_ack *extack);
141136
};
142137

138+
#ifdef CONFIG_NET_CLS_ACT
139+
140+
#define ACT_P_CREATED 1
141+
#define ACT_P_DELETED 1
142+
143143
struct tc_action_net {
144144
struct tcf_idrinfo *idrinfo;
145145
const struct tc_action_ops *ops;

‎include/net/tc_wrapper.h

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef __NET_TC_WRAPPER_H
3+
#define __NET_TC_WRAPPER_H
4+
5+
#include <net/pkt_cls.h>
6+
7+
#if IS_ENABLED(CONFIG_RETPOLINE)
8+
9+
#include <linux/cpufeature.h>
10+
#include <linux/static_key.h>
11+
#include <linux/indirect_call_wrapper.h>
12+
13+
#define TC_INDIRECT_SCOPE
14+
15+
extern struct static_key_false tc_skip_wrapper;
16+
17+
/* TC Actions */
18+
#ifdef CONFIG_NET_CLS_ACT
19+
20+
#define TC_INDIRECT_ACTION_DECLARE(fname) \
21+
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
22+
const struct tc_action *a, \
23+
struct tcf_result *res))
24+
25+
TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
26+
TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
27+
TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
28+
TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
29+
TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
30+
TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
31+
TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
32+
TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
33+
TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
34+
TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
35+
TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
36+
TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
37+
TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
38+
TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
39+
TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
40+
TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
41+
TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
42+
TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
43+
TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
44+
TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
45+
46+
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
47+
struct tcf_result *res)
48+
{
49+
if (static_branch_likely(&tc_skip_wrapper))
50+
goto skip;
51+
52+
#if IS_BUILTIN(CONFIG_NET_ACT_GACT)
53+
if (a->ops->act == tcf_gact_act)
54+
return tcf_gact_act(skb, a, res);
55+
#endif
56+
#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
57+
if (a->ops->act == tcf_mirred_act)
58+
return tcf_mirred_act(skb, a, res);
59+
#endif
60+
#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
61+
if (a->ops->act == tcf_pedit_act)
62+
return tcf_pedit_act(skb, a, res);
63+
#endif
64+
#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
65+
if (a->ops->act == tcf_skbedit_act)
66+
return tcf_skbedit_act(skb, a, res);
67+
#endif
68+
#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
69+
if (a->ops->act == tcf_skbmod_act)
70+
return tcf_skbmod_act(skb, a, res);
71+
#endif
72+
#if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
73+
if (a->ops->act == tcf_police_act)
74+
return tcf_police_act(skb, a, res);
75+
#endif
76+
#if IS_BUILTIN(CONFIG_NET_ACT_BPF)
77+
if (a->ops->act == tcf_bpf_act)
78+
return tcf_bpf_act(skb, a, res);
79+
#endif
80+
#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
81+
if (a->ops->act == tcf_connmark_act)
82+
return tcf_connmark_act(skb, a, res);
83+
#endif
84+
#if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
85+
if (a->ops->act == tcf_csum_act)
86+
return tcf_csum_act(skb, a, res);
87+
#endif
88+
#if IS_BUILTIN(CONFIG_NET_ACT_CT)
89+
if (a->ops->act == tcf_ct_act)
90+
return tcf_ct_act(skb, a, res);
91+
#endif
92+
#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
93+
if (a->ops->act == tcf_ctinfo_act)
94+
return tcf_ctinfo_act(skb, a, res);
95+
#endif
96+
#if IS_BUILTIN(CONFIG_NET_ACT_GATE)
97+
if (a->ops->act == tcf_gate_act)
98+
return tcf_gate_act(skb, a, res);
99+
#endif
100+
#if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
101+
if (a->ops->act == tcf_mpls_act)
102+
return tcf_mpls_act(skb, a, res);
103+
#endif
104+
#if IS_BUILTIN(CONFIG_NET_ACT_NAT)
105+
if (a->ops->act == tcf_nat_act)
106+
return tcf_nat_act(skb, a, res);
107+
#endif
108+
#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
109+
if (a->ops->act == tunnel_key_act)
110+
return tunnel_key_act(skb, a, res);
111+
#endif
112+
#if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
113+
if (a->ops->act == tcf_vlan_act)
114+
return tcf_vlan_act(skb, a, res);
115+
#endif
116+
#if IS_BUILTIN(CONFIG_NET_ACT_IFE)
117+
if (a->ops->act == tcf_ife_act)
118+
return tcf_ife_act(skb, a, res);
119+
#endif
120+
#if IS_BUILTIN(CONFIG_NET_ACT_IPT)
121+
if (a->ops->act == tcf_ipt_act)
122+
return tcf_ipt_act(skb, a, res);
123+
#endif
124+
#if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
125+
if (a->ops->act == tcf_simp_act)
126+
return tcf_simp_act(skb, a, res);
127+
#endif
128+
#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
129+
if (a->ops->act == tcf_sample_act)
130+
return tcf_sample_act(skb, a, res);
131+
#endif
132+
133+
skip:
134+
return a->ops->act(skb, a, res);
135+
}
136+
137+
#endif /* CONFIG_NET_CLS_ACT */
138+
139+
/* TC Filters */
140+
#ifdef CONFIG_NET_CLS
141+
142+
#define TC_INDIRECT_FILTER_DECLARE(fname) \
143+
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
144+
const struct tcf_proto *tp, \
145+
struct tcf_result *res))
146+
147+
TC_INDIRECT_FILTER_DECLARE(basic_classify);
148+
TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
149+
TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
150+
TC_INDIRECT_FILTER_DECLARE(fl_classify);
151+
TC_INDIRECT_FILTER_DECLARE(flow_classify);
152+
TC_INDIRECT_FILTER_DECLARE(fw_classify);
153+
TC_INDIRECT_FILTER_DECLARE(mall_classify);
154+
TC_INDIRECT_FILTER_DECLARE(route4_classify);
155+
TC_INDIRECT_FILTER_DECLARE(rsvp_classify);
156+
TC_INDIRECT_FILTER_DECLARE(rsvp6_classify);
157+
TC_INDIRECT_FILTER_DECLARE(tcindex_classify);
158+
TC_INDIRECT_FILTER_DECLARE(u32_classify);
159+
160+
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
161+
struct tcf_result *res)
162+
{
163+
if (static_branch_likely(&tc_skip_wrapper))
164+
goto skip;
165+
166+
#if IS_BUILTIN(CONFIG_NET_CLS_BPF)
167+
if (tp->classify == cls_bpf_classify)
168+
return cls_bpf_classify(skb, tp, res);
169+
#endif
170+
#if IS_BUILTIN(CONFIG_NET_CLS_U32)
171+
if (tp->classify == u32_classify)
172+
return u32_classify(skb, tp, res);
173+
#endif
174+
#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
175+
if (tp->classify == fl_classify)
176+
return fl_classify(skb, tp, res);
177+
#endif
178+
#if IS_BUILTIN(CONFIG_NET_CLS_FW)
179+
if (tp->classify == fw_classify)
180+
return fw_classify(skb, tp, res);
181+
#endif
182+
#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
183+
if (tp->classify == mall_classify)
184+
return mall_classify(skb, tp, res);
185+
#endif
186+
#if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
187+
if (tp->classify == basic_classify)
188+
return basic_classify(skb, tp, res);
189+
#endif
190+
#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
191+
if (tp->classify == cls_cgroup_classify)
192+
return cls_cgroup_classify(skb, tp, res);
193+
#endif
194+
#if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
195+
if (tp->classify == flow_classify)
196+
return flow_classify(skb, tp, res);
197+
#endif
198+
#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
199+
if (tp->classify == route4_classify)
200+
return route4_classify(skb, tp, res);
201+
#endif
202+
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP)
203+
if (tp->classify == rsvp_classify)
204+
return rsvp_classify(skb, tp, res);
205+
#endif
206+
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP6)
207+
if (tp->classify == rsvp6_classify)
208+
return rsvp6_classify(skb, tp, res);
209+
#endif
210+
#if IS_BUILTIN(CONFIG_NET_CLS_TCINDEX)
211+
if (tp->classify == tcindex_classify)
212+
return tcindex_classify(skb, tp, res);
213+
#endif
214+
215+
skip:
216+
return tp->classify(skb, tp, res);
217+
}
218+
219+
static inline void tc_wrapper_init(void)
220+
{
221+
#ifdef CONFIG_X86
222+
if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
223+
static_branch_enable(&tc_skip_wrapper);
224+
#endif
225+
}
226+
227+
#endif /* CONFIG_NET_CLS */
228+
229+
#else
230+
231+
#define TC_INDIRECT_SCOPE static
232+
233+
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
234+
struct tcf_result *res)
235+
{
236+
return a->ops->act(skb, a, res);
237+
}
238+
239+
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
240+
struct tcf_result *res)
241+
{
242+
return tp->classify(skb, tp, res);
243+
}
244+
245+
static inline void tc_wrapper_init(void)
246+
{
247+
}
248+
249+
#endif
250+
251+
#endif /* __NET_TC_WRAPPER_H */

‎net/sched/act_api.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <net/act_api.h>
2424
#include <net/netlink.h>
2525
#include <net/flow_offload.h>
26+
#include <net/tc_wrapper.h>
2627

2728
#ifdef CONFIG_INET
2829
DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count);
@@ -1080,7 +1081,7 @@ int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions,
10801081

10811082
repeat_ttl = 32;
10821083
repeat:
1083-
ret = a->ops->act(skb, a, res);
1084+
ret = tc_act(skb, a, res);
10841085
if (unlikely(ret == TC_ACT_REPEAT)) {
10851086
if (--repeat_ttl != 0)
10861087
goto repeat;

‎net/sched/act_bpf.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
#include <linux/tc_act/tc_bpf.h>
2020
#include <net/tc_act/tc_bpf.h>
21+
#include <net/tc_wrapper.h>
2122

2223
#define ACT_BPF_NAME_LEN 256
2324

@@ -31,8 +32,9 @@ struct tcf_bpf_cfg {
3132

3233
static struct tc_action_ops act_bpf_ops;
3334

34-
static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act,
35-
struct tcf_result *res)
35+
TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb,
36+
const struct tc_action *act,
37+
struct tcf_result *res)
3638
{
3739
bool at_ingress = skb_at_tc_ingress(skb);
3840
struct tcf_bpf *prog = to_bpf(act);

‎net/sched/act_connmark.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,17 @@
2020
#include <net/pkt_cls.h>
2121
#include <uapi/linux/tc_act/tc_connmark.h>
2222
#include <net/tc_act/tc_connmark.h>
23+
#include <net/tc_wrapper.h>
2324

2425
#include <net/netfilter/nf_conntrack.h>
2526
#include <net/netfilter/nf_conntrack_core.h>
2627
#include <net/netfilter/nf_conntrack_zones.h>
2728

2829
static struct tc_action_ops act_connmark_ops;
2930

30-
static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a,
31-
struct tcf_result *res)
31+
TC_INDIRECT_SCOPE int tcf_connmark_act(struct sk_buff *skb,
32+
const struct tc_action *a,
33+
struct tcf_result *res)
3234
{
3335
const struct nf_conntrack_tuple_hash *thash;
3436
struct nf_conntrack_tuple tuple;

‎net/sched/act_csum.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
#include <linux/tc_act/tc_csum.h>
3434
#include <net/tc_act/tc_csum.h>
35+
#include <net/tc_wrapper.h>
3536

3637
static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = {
3738
[TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), },
@@ -563,8 +564,9 @@ static int tcf_csum_ipv6(struct sk_buff *skb, u32 update_flags)
563564
return 0;
564565
}
565566

566-
static int tcf_csum_act(struct sk_buff *skb, const struct tc_action *a,
567-
struct tcf_result *res)
567+
TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb,
568+
const struct tc_action *a,
569+
struct tcf_result *res)
568570
{
569571
struct tcf_csum *p = to_tcf_csum(a);
570572
bool orig_vlan_tag_present = false;

‎net/sched/act_ct.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <net/ipv6_frag.h>
2525
#include <uapi/linux/tc_act/tc_ct.h>
2626
#include <net/tc_act/tc_ct.h>
27+
#include <net/tc_wrapper.h>
2728

2829
#include <net/netfilter/nf_flow_table.h>
2930
#include <net/netfilter/nf_conntrack.h>
@@ -1038,8 +1039,8 @@ static int tcf_ct_act_nat(struct sk_buff *skb,
10381039
#endif
10391040
}
10401041

1041-
static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
1042-
struct tcf_result *res)
1042+
TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
1043+
struct tcf_result *res)
10431044
{
10441045
struct net *net = dev_net(skb->dev);
10451046
enum ip_conntrack_info ctinfo;

‎net/sched/act_ctinfo.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <net/pkt_cls.h>
1919
#include <uapi/linux/tc_act/tc_ctinfo.h>
2020
#include <net/tc_act/tc_ctinfo.h>
21+
#include <net/tc_wrapper.h>
2122

2223
#include <net/netfilter/nf_conntrack.h>
2324
#include <net/netfilter/nf_conntrack_core.h>
@@ -75,8 +76,9 @@ static void tcf_ctinfo_cpmark_set(struct nf_conn *ct, struct tcf_ctinfo *ca,
7576
skb->mark = READ_ONCE(ct->mark) & cp->cpmarkmask;
7677
}
7778

78-
static int tcf_ctinfo_act(struct sk_buff *skb, const struct tc_action *a,
79-
struct tcf_result *res)
79+
TC_INDIRECT_SCOPE int tcf_ctinfo_act(struct sk_buff *skb,
80+
const struct tc_action *a,
81+
struct tcf_result *res)
8082
{
8183
const struct nf_conntrack_tuple_hash *thash = NULL;
8284
struct tcf_ctinfo *ca = to_ctinfo(a);

0 commit comments

Comments
 (0)