Skip to content

Commit 7785bba

Browse files
klassertklassert
committed
esp: Add a software GRO codepath
This patch adds GRO ifrastructure and callbacks for ESP on ipv4 and ipv6. In case the GRO layer detects an ESP packet, the esp{4,6}_gro_receive() function does a xfrm state lookup and calls the xfrm input layer if it finds a matching state. The packet will be decapsulated and reinjected it into layer 2. Signed-off-by: Steffen Klassert <[email protected]>
1 parent 54ef207 commit 7785bba

File tree

12 files changed

+288
-7
lines changed

12 files changed

+288
-7
lines changed

‎include/net/xfrm.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,7 @@ struct xfrm_spi_skb_cb {
682682

683683
unsigned int daddroff;
684684
unsigned int family;
685+
__be32 seq;
685686
};
686687

687688
#define XFRM_SPI_SKB_CB(__skb) ((struct xfrm_spi_skb_cb *)&((__skb)->cb[0]))

‎net/ipv4/Kconfig‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,19 @@ config INET_ESP
360360

361361
If unsure, say Y.
362362

363+
config INET_ESP_OFFLOAD
364+
tristate "IP: ESP transformation offload"
365+
depends on INET_ESP
366+
select XFRM_OFFLOAD
367+
default n
368+
---help---
369+
Support for ESP transformation offload. This makes sense
370+
only if this system really does IPsec and want to do it
371+
with high throughput. A typical desktop system does not
372+
need it, even if it does IPsec.
373+
374+
If unsure, say N.
375+
363376
config INET_IPCOMP
364377
tristate "IP: IPComp transformation"
365378
select INET_XFRM_TUNNEL

‎net/ipv4/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ obj-$(CONFIG_NET_IPVTI) += ip_vti.o
2929
obj-$(CONFIG_SYN_COOKIES) += syncookies.o
3030
obj-$(CONFIG_INET_AH) += ah4.o
3131
obj-$(CONFIG_INET_ESP) += esp4.o
32+
obj-$(CONFIG_INET_ESP_OFFLOAD) += esp4_offload.o
3233
obj-$(CONFIG_INET_IPCOMP) += ipcomp.o
3334
obj-$(CONFIG_INET_XFRM_TUNNEL) += xfrm4_tunnel.o
3435
obj-$(CONFIG_INET_XFRM_MODE_BEET) += xfrm4_mode_beet.o

‎net/ipv4/esp4_offload.c‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* IPV4 GSO/GRO offload support
3+
* Linux INET implementation
4+
*
5+
* Copyright (C) 2016 secunet Security Networks AG
6+
* Author: Steffen Klassert <[email protected]>
7+
*
8+
* This program is free software; you can redistribute it and/or modify it
9+
* under the terms and conditions of the GNU General Public License,
10+
* version 2, as published by the Free Software Foundation.
11+
*
12+
* ESP GRO support
13+
*/
14+
15+
#include <linux/skbuff.h>
16+
#include <linux/init.h>
17+
#include <net/protocol.h>
18+
#include <crypto/aead.h>
19+
#include <crypto/authenc.h>
20+
#include <linux/err.h>
21+
#include <linux/module.h>
22+
#include <net/ip.h>
23+
#include <net/xfrm.h>
24+
#include <net/esp.h>
25+
#include <linux/scatterlist.h>
26+
#include <linux/kernel.h>
27+
#include <linux/slab.h>
28+
#include <linux/spinlock.h>
29+
#include <net/udp.h>
30+
31+
static struct sk_buff **esp4_gro_receive(struct sk_buff **head,
32+
struct sk_buff *skb)
33+
{
34+
int offset = skb_gro_offset(skb);
35+
struct xfrm_offload *xo;
36+
struct xfrm_state *x;
37+
__be32 seq;
38+
__be32 spi;
39+
int err;
40+
41+
skb_pull(skb, offset);
42+
43+
if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
44+
goto out;
45+
46+
err = secpath_set(skb);
47+
if (err)
48+
goto out;
49+
50+
if (skb->sp->len == XFRM_MAX_DEPTH)
51+
goto out;
52+
53+
x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
54+
(xfrm_address_t *)&ip_hdr(skb)->daddr,
55+
spi, IPPROTO_ESP, AF_INET);
56+
if (!x)
57+
goto out;
58+
59+
skb->sp->xvec[skb->sp->len++] = x;
60+
skb->sp->olen++;
61+
62+
xo = xfrm_offload(skb);
63+
if (!xo) {
64+
xfrm_state_put(x);
65+
goto out;
66+
}
67+
xo->flags |= XFRM_GRO;
68+
69+
XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
70+
XFRM_SPI_SKB_CB(skb)->family = AF_INET;
71+
XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
72+
XFRM_SPI_SKB_CB(skb)->seq = seq;
73+
74+
/* We don't need to handle errors from xfrm_input, it does all
75+
* the error handling and frees the resources on error. */
76+
xfrm_input(skb, IPPROTO_ESP, spi, -2);
77+
78+
return ERR_PTR(-EINPROGRESS);
79+
out:
80+
skb_push(skb, offset);
81+
NAPI_GRO_CB(skb)->same_flow = 0;
82+
NAPI_GRO_CB(skb)->flush = 1;
83+
84+
return NULL;
85+
}
86+
87+
static const struct net_offload esp4_offload = {
88+
.callbacks = {
89+
.gro_receive = esp4_gro_receive,
90+
},
91+
};
92+
93+
static int __init esp4_offload_init(void)
94+
{
95+
return inet_add_offload(&esp4_offload, IPPROTO_ESP);
96+
}
97+
98+
static void __exit esp4_offload_exit(void)
99+
{
100+
inet_del_offload(&esp4_offload, IPPROTO_ESP);
101+
}
102+
103+
module_init(esp4_offload_init);
104+
module_exit(esp4_offload_exit);
105+
MODULE_LICENSE("GPL");
106+
MODULE_AUTHOR("Steffen Klassert <[email protected]>");

‎net/ipv4/xfrm4_input.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ static inline int xfrm4_rcv_encap_finish(struct net *net, struct sock *sk,
4040

4141
int xfrm4_transport_finish(struct sk_buff *skb, int async)
4242
{
43+
struct xfrm_offload *xo = xfrm_offload(skb);
4344
struct iphdr *iph = ip_hdr(skb);
4445

4546
iph->protocol = XFRM_MODE_SKB_CB(skb)->protocol;
@@ -53,6 +54,11 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async)
5354
iph->tot_len = htons(skb->len);
5455
ip_send_check(iph);
5556

57+
if (xo && (xo->flags & XFRM_GRO)) {
58+
skb_mac_header_rebuild(skb);
59+
return 0;
60+
}
61+
5662
NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
5763
dev_net(skb->dev), NULL, skb, skb->dev, NULL,
5864
xfrm4_rcv_encap_finish);

‎net/ipv4/xfrm4_mode_transport.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,16 @@ static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
4343
static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
4444
{
4545
int ihl = skb->data - skb_transport_header(skb);
46+
struct xfrm_offload *xo = xfrm_offload(skb);
4647

4748
if (skb->transport_header != skb->network_header) {
4849
memmove(skb_transport_header(skb),
4950
skb_network_header(skb), ihl);
5051
skb->network_header = skb->transport_header;
5152
}
5253
ip_hdr(skb)->tot_len = htons(skb->len + ihl);
53-
skb_reset_transport_header(skb);
54+
if (!xo || !(xo->flags & XFRM_GRO))
55+
skb_reset_transport_header(skb);
5456
return 0;
5557
}
5658

‎net/ipv6/Kconfig‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,19 @@ config INET6_ESP
7575

7676
If unsure, say Y.
7777

78+
config INET6_ESP_OFFLOAD
79+
tristate "IPv6: ESP transformation offload"
80+
depends on INET6_ESP
81+
select XFRM_OFFLOAD
82+
default n
83+
---help---
84+
Support for ESP transformation offload. This makes sense
85+
only if this system really does IPsec and want to do it
86+
with high throughput. A typical desktop system does not
87+
need it, even if it does IPsec.
88+
89+
If unsure, say N.
90+
7891
config INET6_IPCOMP
7992
tristate "IPv6: IPComp transformation"
8093
select INET6_XFRM_TUNNEL

‎net/ipv6/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ ipv6-objs += $(ipv6-y)
3030

3131
obj-$(CONFIG_INET6_AH) += ah6.o
3232
obj-$(CONFIG_INET6_ESP) += esp6.o
33+
obj-$(CONFIG_INET6_ESP_OFFLOAD) += esp6_offload.o
3334
obj-$(CONFIG_INET6_IPCOMP) += ipcomp6.o
3435
obj-$(CONFIG_INET6_XFRM_TUNNEL) += xfrm6_tunnel.o
3536
obj-$(CONFIG_INET6_TUNNEL) += tunnel6.o

‎net/ipv6/esp6_offload.c‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
* IPV6 GSO/GRO offload support
3+
* Linux INET implementation
4+
*
5+
* Copyright (C) 2016 secunet Security Networks AG
6+
* Author: Steffen Klassert <[email protected]>
7+
*
8+
* This program is free software; you can redistribute it and/or modify it
9+
* under the terms and conditions of the GNU General Public License,
10+
* version 2, as published by the Free Software Foundation.
11+
*
12+
* ESP GRO support
13+
*/
14+
15+
#include <linux/skbuff.h>
16+
#include <linux/init.h>
17+
#include <net/protocol.h>
18+
#include <crypto/aead.h>
19+
#include <crypto/authenc.h>
20+
#include <linux/err.h>
21+
#include <linux/module.h>
22+
#include <net/ip.h>
23+
#include <net/xfrm.h>
24+
#include <net/esp.h>
25+
#include <linux/scatterlist.h>
26+
#include <linux/kernel.h>
27+
#include <linux/slab.h>
28+
#include <linux/spinlock.h>
29+
#include <net/ip6_route.h>
30+
#include <net/ipv6.h>
31+
#include <linux/icmpv6.h>
32+
33+
static struct sk_buff **esp6_gro_receive(struct sk_buff **head,
34+
struct sk_buff *skb)
35+
{
36+
int offset = skb_gro_offset(skb);
37+
struct xfrm_offload *xo;
38+
struct xfrm_state *x;
39+
__be32 seq;
40+
__be32 spi;
41+
int err;
42+
43+
skb_pull(skb, offset);
44+
45+
if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
46+
goto out;
47+
48+
err = secpath_set(skb);
49+
if (err)
50+
goto out;
51+
52+
if (skb->sp->len == XFRM_MAX_DEPTH)
53+
goto out;
54+
55+
x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
56+
(xfrm_address_t *)&ipv6_hdr(skb)->daddr,
57+
spi, IPPROTO_ESP, AF_INET6);
58+
if (!x)
59+
goto out;
60+
61+
skb->sp->xvec[skb->sp->len++] = x;
62+
skb->sp->olen++;
63+
64+
xo = xfrm_offload(skb);
65+
if (!xo) {
66+
xfrm_state_put(x);
67+
goto out;
68+
}
69+
xo->flags |= XFRM_GRO;
70+
71+
XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
72+
XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
73+
XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
74+
XFRM_SPI_SKB_CB(skb)->seq = seq;
75+
76+
/* We don't need to handle errors from xfrm_input, it does all
77+
* the error handling and frees the resources on error. */
78+
xfrm_input(skb, IPPROTO_ESP, spi, -2);
79+
80+
return ERR_PTR(-EINPROGRESS);
81+
out:
82+
skb_push(skb, offset);
83+
NAPI_GRO_CB(skb)->same_flow = 0;
84+
NAPI_GRO_CB(skb)->flush = 1;
85+
86+
return NULL;
87+
}
88+
89+
static const struct net_offload esp6_offload = {
90+
.callbacks = {
91+
.gro_receive = esp6_gro_receive,
92+
},
93+
};
94+
95+
static int __init esp6_offload_init(void)
96+
{
97+
return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
98+
}
99+
100+
static void __exit esp6_offload_exit(void)
101+
{
102+
inet6_del_offload(&esp6_offload, IPPROTO_ESP);
103+
}
104+
105+
module_init(esp6_offload_init);
106+
module_exit(esp6_offload_exit);
107+
MODULE_LICENSE("GPL");
108+
MODULE_AUTHOR("Steffen Klassert <[email protected]>");

‎net/ipv6/xfrm6_input.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ EXPORT_SYMBOL(xfrm6_rcv_spi);
3333

3434
int xfrm6_transport_finish(struct sk_buff *skb, int async)
3535
{
36+
struct xfrm_offload *xo = xfrm_offload(skb);
37+
3638
skb_network_header(skb)[IP6CB(skb)->nhoff] =
3739
XFRM_MODE_SKB_CB(skb)->protocol;
3840

@@ -44,6 +46,11 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async)
4446
ipv6_hdr(skb)->payload_len = htons(skb->len);
4547
__skb_push(skb, skb->data - skb_network_header(skb));
4648

49+
if (xo && (xo->flags & XFRM_GRO)) {
50+
skb_mac_header_rebuild(skb);
51+
return -1;
52+
}
53+
4754
NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING,
4855
dev_net(skb->dev), NULL, skb, skb->dev, NULL,
4956
ip6_rcv_finish);

0 commit comments

Comments
 (0)