Merge branch 'net-better-packing-of-global-vars'

kuba-moo · kuba-moo · commit c60c34a9104e · 2021-11-16T19:07:58.000-08:00
Eric Dumazet says: ==================== net: better packing of global vars First two patches avoid holes in data section, and last patch makes sure some siphash keys are contained in a single cache line. ==================== Link: https://lore.kernel.org/r/20211115172303.3732746-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
@@ -4942,7 +4942,7 @@ void netdev_info(const struct net_device *dev, const char *format, ...);
 
 #define netdev_level_once(level, dev, fmt, ...)			\
 do {								\
-	static bool __print_once __read_mostly;			\
+	static bool __section(".data.once") __print_once;	\
 								\
 	if (!__print_once) {					\
 		__print_once = true;				\
diff --git a/include/linux/once.h b/include/linux/once.h
@@ -38,7 +38,7 @@ void __do_once_done(bool *done, struct static_key_true *once_key,
 #define DO_ONCE(func, ...)						     \
 	({								     \
 		bool ___ret = false;					     \
-		static bool ___done = false;				     \
+		static bool __section(".data.once") ___done = false;	     \
 		static DEFINE_STATIC_KEY_TRUE(___once_key);		     \
 		if (static_branch_unlikely(&___once_key)) {		     \
 			unsigned long ___flags;				     \
diff --git a/include/linux/siphash.h b/include/linux/siphash.h
@@ -21,6 +21,8 @@ typedef struct {
 	u64 key[2];
 } siphash_key_t;
 
+#define siphash_aligned_key_t siphash_key_t __aligned(16)
+
 static inline bool siphash_key_is_zero(const siphash_key_t *key)
 {
 	return !(key->key[0] | key->key[1]);
diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
@@ -1460,7 +1460,7 @@ bool __skb_flow_dissect(const struct net *net,
 }
 EXPORT_SYMBOL(__skb_flow_dissect);
 
-static siphash_key_t hashrnd __read_mostly;
+static siphash_aligned_key_t hashrnd;
 static __always_inline void __flow_hash_secret_init(void)
 {
 	net_get_random_once(&hashrnd, sizeof(hashrnd));
diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c
@@ -19,8 +19,8 @@
 #include <linux/in6.h>
 #include <net/tcp.h>
 
-static siphash_key_t net_secret __read_mostly;
-static siphash_key_t ts_secret __read_mostly;
+static siphash_aligned_key_t net_secret;
+static siphash_aligned_key_t ts_secret;
 
 static __always_inline void net_secret_init(void)
 {
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
@@ -602,7 +602,7 @@ static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash)
 
 static u32 fnhe_hashfun(__be32 daddr)
 {
-	static siphash_key_t fnhe_hash_key __read_mostly;
+	static siphash_aligned_key_t fnhe_hash_key;
 	u64 hval;
 
 	net_get_random_once(&fnhe_hash_key, sizeof(fnhe_hash_key));
diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c
@@ -14,7 +14,7 @@
 #include <net/tcp.h>
 #include <net/route.h>
 
-static siphash_key_t syncookie_secret[2] __read_mostly;
+static siphash_aligned_key_t syncookie_secret[2];
 
 #define COOKIEBITS 24	/* Upper bits store count */
 #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1)
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
@@ -1485,7 +1485,7 @@ static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
 static u32 rt6_exception_hash(const struct in6_addr *dst,
 			      const struct in6_addr *src)
 {
-	static siphash_key_t rt6_exception_key __read_mostly;
+	static siphash_aligned_key_t rt6_exception_key;
 	struct {
 		struct in6_addr dst;
 		struct in6_addr src;
diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c
@@ -20,7 +20,7 @@
 #define COOKIEBITS 24	/* Upper bits store count */
 #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1)
 
-static siphash_key_t syncookie6_secret[2] __read_mostly;
+static siphash_aligned_key_t syncookie6_secret[2];
 
 /* RFC 2460, Section 8.3:
  * [ipv6 tcp] MSS must be computed as the maximum packet size minus 60 [..]
diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
@@ -189,7 +189,7 @@ EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 unsigned int nf_conntrack_max __read_mostly;
 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 seqcount_spinlock_t nf_conntrack_generation __read_mostly;
-static siphash_key_t nf_conntrack_hash_rnd __read_mostly;
+static siphash_aligned_key_t nf_conntrack_hash_rnd;
 
 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 			      unsigned int zoneid,
@@ -482,7 +482,7 @@ EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
  */
 u32 nf_ct_get_id(const struct nf_conn *ct)
 {
-	static __read_mostly siphash_key_t ct_id_seed;
+	static siphash_aligned_key_t ct_id_seed;
 	unsigned long a, b, c, d;
 
 	net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c
@@ -41,7 +41,7 @@ EXPORT_SYMBOL_GPL(nf_ct_expect_hash);
 unsigned int nf_ct_expect_max __read_mostly;
 
 static struct kmem_cache *nf_ct_expect_cachep __read_mostly;
-static siphash_key_t nf_ct_expect_hashrnd __read_mostly;
+static siphash_aligned_key_t nf_ct_expect_hashrnd;
 
 /* nf_conntrack_expect helper functions */
 void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp,
diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c
@@ -2997,7 +2997,7 @@ static const union nf_inet_addr any_addr;
 
 static __be32 nf_expect_get_id(const struct nf_conntrack_expect *exp)
 {
-	static __read_mostly siphash_key_t exp_id_seed;
+	static siphash_aligned_key_t exp_id_seed;
 	unsigned long a, b, c, d;
 
 	net_get_random_once(&exp_id_seed, sizeof(exp_id_seed));
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
@@ -34,7 +34,7 @@ static unsigned int nat_net_id __read_mostly;
 
 static struct hlist_head *nf_nat_bysource __read_mostly;
 static unsigned int nf_nat_htable_size __read_mostly;
-static siphash_key_t nf_nat_hash_rnd __read_mostly;
+static siphash_aligned_key_t nf_nat_hash_rnd;
 
 struct nf_nat_lookup_hook_priv {
 	struct nf_hook_entries __rcu *entries;

Original file line number	Diff line number	Diff line change
`@@ -4942,7 +4942,7 @@ void netdev_info(const struct net_device dev, const char format, ...);`
`4942`	`4942`
`4943`	`4943`	`#define netdev_level_once(level, dev, fmt, ...) \`
`4944`	`4944`	`do { \`
`4945`		`- static bool __print_once __read_mostly; \`
	`4945`	`+ static bool __section(".data.once") __print_once; \`
`4946`	`4946`	`\`
`4947`	`4947`	`if (!__print_once) { \`
`4948`	`4948`	`__print_once = true; \`
Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,8 @@ typedef struct {`
`21`	`21`	`u64 key[2];`
`22`	`22`	`} siphash_key_t;`
`23`	`23`
	`24`	`+#define siphash_aligned_key_t siphash_key_t __aligned(16)`
	`25`	`+`
`24`	`26`	`static inline bool siphash_key_is_zero(const siphash_key_t *key)`
`25`	`27`	`{`
`26`	`28`	`return !(key->key[0] \| key->key[1]);`
Original file line number	Diff line number	Diff line change
`@@ -1460,7 +1460,7 @@ bool __skb_flow_dissect(const struct net *net,`
`1460`	`1460`	`}`
`1461`	`1461`	`EXPORT_SYMBOL(__skb_flow_dissect);`
`1462`	`1462`
`1463`		`-static siphash_key_t hashrnd __read_mostly;`
	`1463`	`+static siphash_aligned_key_t hashrnd;`
`1464`	`1464`	`static __always_inline void __flow_hash_secret_init(void)`
`1465`	`1465`	`{`
`1466`	`1466`	`net_get_random_once(&hashrnd, sizeof(hashrnd));`
Original file line number	Diff line number	Diff line change
`@@ -19,8 +19,8 @@`
`19`	`19`	`#include <linux/in6.h>`
`20`	`20`	`#include <net/tcp.h>`
`21`	`21`
`22`		`-static siphash_key_t net_secret __read_mostly;`
`23`		`-static siphash_key_t ts_secret __read_mostly;`
	`22`	`+static siphash_aligned_key_t net_secret;`
	`23`	`+static siphash_aligned_key_t ts_secret;`
`24`	`24`
`25`	`25`	`static __always_inline void net_secret_init(void)`
`26`	`26`	`{`
Original file line number	Diff line number	Diff line change
`@@ -602,7 +602,7 @@ static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash)`
`602`	`602`
`603`	`603`	`static u32 fnhe_hashfun(__be32 daddr)`
`604`	`604`	`{`
`605`		`- static siphash_key_t fnhe_hash_key __read_mostly;`
	`605`	`+ static siphash_aligned_key_t fnhe_hash_key;`
`606`	`606`	`u64 hval;`
`607`	`607`
`608`	`608`	`net_get_random_once(&fnhe_hash_key, sizeof(fnhe_hash_key));`
Original file line number	Diff line number	Diff line change
`@@ -1485,7 +1485,7 @@ static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)`
`1485`	`1485`	`static u32 rt6_exception_hash(const struct in6_addr *dst,`
`1486`	`1486`	`const struct in6_addr *src)`
`1487`	`1487`	`{`
`1488`		`- static siphash_key_t rt6_exception_key __read_mostly;`
	`1488`	`+ static siphash_aligned_key_t rt6_exception_key;`
`1489`	`1489`	`struct {`
`1490`	`1490`	`struct in6_addr dst;`
`1491`	`1491`	`struct in6_addr src;`