Skip to content

Commit 2b43470

Browse files
Björn TöpelAlexei Starovoitov
Björn Töpel
authored and
Alexei Starovoitov
committed
xsk: Introduce AF_XDP buffer allocation API
In order to simplify AF_XDP zero-copy enablement for NIC driver developers, a new AF_XDP buffer allocation API is added. The implementation is based on a single core (single producer/consumer) buffer pool for the AF_XDP UMEM. A buffer is allocated using the xsk_buff_alloc() function, and returned using xsk_buff_free(). If a buffer is disassociated with the pool, e.g. when a buffer is passed to an AF_XDP socket, a buffer is said to be released. Currently, the release function is only used by the AF_XDP internals and not visible to the driver. Drivers using this API should register the XDP memory model with the new MEM_TYPE_XSK_BUFF_POOL type. The API is defined in net/xdp_sock_drv.h. The buffer type is struct xdp_buff, and follows the lifetime of regular xdp_buffs, i.e. the lifetime of an xdp_buff is restricted to a NAPI context. In other words, the API is not replacing xdp_frames. In addition to introducing the API and implementations, the AF_XDP core is migrated to use the new APIs. rfc->v1: Fixed build errors/warnings for m68k and riscv. (kbuild test robot) Added headroom/chunk size getter. (Maxim/Björn) v1->v2: Swapped SoBs. (Maxim) v2->v3: Initialize struct xdp_buff member frame_sz. (Björn) Add API to query the DMA address of a frame. (Maxim) Do DMA sync for CPU till the end of the frame to handle possible growth (frame_sz). (Maxim) Signed-off-by: Björn Töpel <[email protected]> Signed-off-by: Maxim Mikityanskiy <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 89e4a37 commit 2b43470

File tree

12 files changed

+819
-119
lines changed

12 files changed

+819
-119
lines changed

‎include/net/xdp.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ enum xdp_mem_type {
4040
MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
4141
MEM_TYPE_PAGE_POOL,
4242
MEM_TYPE_ZERO_COPY,
43+
MEM_TYPE_XSK_BUFF_POOL,
4344
MEM_TYPE_MAX,
4445
};
4546

@@ -119,7 +120,8 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
119120
int metasize;
120121
int headroom;
121122

122-
if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
123+
if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
124+
xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
123125
return xdp_convert_zc_to_xdp_frame(xdp);
124126

125127
/* Assure headroom is available for storing info */

‎include/net/xdp_sock.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ struct xdp_umem_fq_reuse {
3131
struct xdp_umem {
3232
struct xsk_queue *fq;
3333
struct xsk_queue *cq;
34+
struct xsk_buff_pool *pool;
3435
struct xdp_umem_page *pages;
3536
u64 chunk_mask;
3637
u64 size;
3738
u32 headroom;
3839
u32 chunk_size_nohr;
40+
u32 chunk_size;
3941
struct user_struct *user;
4042
refcount_t users;
4143
struct work_struct work;

‎include/net/xdp_sock_drv.h‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#define _LINUX_XDP_SOCK_DRV_H
88

99
#include <net/xdp_sock.h>
10+
#include <net/xsk_buff_pool.h>
1011

1112
#ifdef CONFIG_XDP_SOCKETS
1213

@@ -101,6 +102,94 @@ static inline u32 xsk_umem_xdp_frame_sz(struct xdp_umem *umem)
101102
return umem->chunk_size_nohr;
102103
}
103104

105+
static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
106+
{
107+
return XDP_PACKET_HEADROOM + umem->headroom;
108+
}
109+
110+
static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
111+
{
112+
return umem->chunk_size;
113+
}
114+
115+
static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
116+
{
117+
return xsk_umem_get_chunk_size(umem) - xsk_umem_get_headroom(umem);
118+
}
119+
120+
static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
121+
struct xdp_rxq_info *rxq)
122+
{
123+
xp_set_rxq_info(umem->pool, rxq);
124+
}
125+
126+
static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
127+
unsigned long attrs)
128+
{
129+
xp_dma_unmap(umem->pool, attrs);
130+
}
131+
132+
static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
133+
unsigned long attrs)
134+
{
135+
return xp_dma_map(umem->pool, dev, attrs, umem->pgs, umem->npgs);
136+
}
137+
138+
static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
139+
{
140+
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
141+
142+
return xp_get_dma(xskb);
143+
}
144+
145+
static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
146+
{
147+
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
148+
149+
return xp_get_frame_dma(xskb);
150+
}
151+
152+
static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
153+
{
154+
return xp_alloc(umem->pool);
155+
}
156+
157+
static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
158+
{
159+
return xp_can_alloc(umem->pool, count);
160+
}
161+
162+
static inline void xsk_buff_free(struct xdp_buff *xdp)
163+
{
164+
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
165+
166+
xp_free(xskb);
167+
}
168+
169+
static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
170+
{
171+
return xp_raw_get_dma(umem->pool, addr);
172+
}
173+
174+
static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
175+
{
176+
return xp_raw_get_data(umem->pool, addr);
177+
}
178+
179+
static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
180+
{
181+
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
182+
183+
xp_dma_sync_for_cpu(xskb);
184+
}
185+
186+
static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
187+
dma_addr_t dma,
188+
size_t size)
189+
{
190+
xp_dma_sync_for_device(umem->pool, dma, size);
191+
}
192+
104193
#else
105194

106195
static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
@@ -212,6 +301,81 @@ static inline u32 xsk_umem_xdp_frame_sz(struct xdp_umem *umem)
212301
return 0;
213302
}
214303

304+
static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
305+
{
306+
return 0;
307+
}
308+
309+
static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
310+
{
311+
return 0;
312+
}
313+
314+
static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
315+
{
316+
return 0;
317+
}
318+
319+
static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
320+
struct xdp_rxq_info *rxq)
321+
{
322+
}
323+
324+
static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
325+
unsigned long attrs)
326+
{
327+
}
328+
329+
static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
330+
unsigned long attrs)
331+
{
332+
return 0;
333+
}
334+
335+
static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
336+
{
337+
return 0;
338+
}
339+
340+
static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
341+
{
342+
return 0;
343+
}
344+
345+
static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
346+
{
347+
return NULL;
348+
}
349+
350+
static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
351+
{
352+
return false;
353+
}
354+
355+
static inline void xsk_buff_free(struct xdp_buff *xdp)
356+
{
357+
}
358+
359+
static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
360+
{
361+
return 0;
362+
}
363+
364+
static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
365+
{
366+
return NULL;
367+
}
368+
369+
static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
370+
{
371+
}
372+
373+
static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
374+
dma_addr_t dma,
375+
size_t size)
376+
{
377+
}
378+
215379
#endif /* CONFIG_XDP_SOCKETS */
216380

217381
#endif /* _LINUX_XDP_SOCK_DRV_H */

‎include/net/xsk_buff_pool.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/* Copyright(c) 2020 Intel Corporation. */
3+
4+
#ifndef XSK_BUFF_POOL_H_
5+
#define XSK_BUFF_POOL_H_
6+
7+
#include <linux/types.h>
8+
#include <linux/dma-mapping.h>
9+
#include <net/xdp.h>
10+
11+
struct xsk_buff_pool;
12+
struct xdp_rxq_info;
13+
struct xsk_queue;
14+
struct xdp_desc;
15+
struct device;
16+
struct page;
17+
18+
struct xdp_buff_xsk {
19+
struct xdp_buff xdp;
20+
dma_addr_t dma;
21+
dma_addr_t frame_dma;
22+
struct xsk_buff_pool *pool;
23+
bool unaligned;
24+
u64 orig_addr;
25+
struct list_head free_list_node;
26+
};
27+
28+
/* AF_XDP core. */
29+
struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
30+
u32 chunk_size, u32 headroom, u64 size,
31+
bool unaligned);
32+
void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq);
33+
void xp_destroy(struct xsk_buff_pool *pool);
34+
void xp_release(struct xdp_buff_xsk *xskb);
35+
u64 xp_get_handle(struct xdp_buff_xsk *xskb);
36+
bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
37+
38+
/* AF_XDP, and XDP core. */
39+
void xp_free(struct xdp_buff_xsk *xskb);
40+
41+
/* AF_XDP ZC drivers, via xdp_sock_buff.h */
42+
void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
43+
int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
44+
unsigned long attrs, struct page **pages, u32 nr_pages);
45+
void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
46+
struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
47+
bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
48+
void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
49+
dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
50+
dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb);
51+
dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb);
52+
void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb);
53+
void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
54+
size_t size);
55+
56+
#endif /* XSK_BUFF_POOL_H_ */

‎include/trace/events/xdp.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,8 @@ TRACE_EVENT(xdp_devmap_xmit,
287287
FN(PAGE_SHARED) \
288288
FN(PAGE_ORDER0) \
289289
FN(PAGE_POOL) \
290-
FN(ZERO_COPY)
290+
FN(ZERO_COPY) \
291+
FN(XSK_BUFF_POOL)
291292

292293
#define __MEM_TYPE_TP_FN(x) \
293294
TRACE_DEFINE_ENUM(MEM_TYPE_##x);

‎net/core/xdp.c‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <net/xdp.h>
1818
#include <net/xdp_priv.h> /* struct xdp_mem_allocator */
1919
#include <trace/events/xdp.h>
20+
#include <net/xdp_sock_drv.h>
2021

2122
#define REG_STATE_NEW 0x0
2223
#define REG_STATE_REGISTERED 0x1
@@ -361,7 +362,7 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
361362
* of xdp_frames/pages in those cases.
362363
*/
363364
static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
364-
unsigned long handle)
365+
unsigned long handle, struct xdp_buff *xdp)
365366
{
366367
struct xdp_mem_allocator *xa;
367368
struct page *page;
@@ -390,6 +391,11 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
390391
xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
391392
xa->zc_alloc->free(xa->zc_alloc, handle);
392393
rcu_read_unlock();
394+
break;
395+
case MEM_TYPE_XSK_BUFF_POOL:
396+
/* NB! Only valid from an xdp_buff! */
397+
xsk_buff_free(xdp);
398+
break;
393399
default:
394400
/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
395401
break;
@@ -398,19 +404,19 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
398404

399405
void xdp_return_frame(struct xdp_frame *xdpf)
400406
{
401-
__xdp_return(xdpf->data, &xdpf->mem, false, 0);
407+
__xdp_return(xdpf->data, &xdpf->mem, false, 0, NULL);
402408
}
403409
EXPORT_SYMBOL_GPL(xdp_return_frame);
404410

405411
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
406412
{
407-
__xdp_return(xdpf->data, &xdpf->mem, true, 0);
413+
__xdp_return(xdpf->data, &xdpf->mem, true, 0, NULL);
408414
}
409415
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
410416

411417
void xdp_return_buff(struct xdp_buff *xdp)
412418
{
413-
__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
419+
__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle, xdp);
414420
}
415421
EXPORT_SYMBOL_GPL(xdp_return_buff);
416422

‎net/xdp/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
# SPDX-License-Identifier: GPL-2.0-only
22
obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o xskmap.o
3+
obj-$(CONFIG_XDP_SOCKETS) += xsk_buff_pool.o
34
obj-$(CONFIG_XDP_SOCKETS_DIAG) += xsk_diag.o

‎net/xdp/xdp_umem.c‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ static void xdp_umem_release(struct xdp_umem *umem)
245245
}
246246

247247
xsk_reuseq_destroy(umem);
248-
248+
xp_destroy(umem->pool);
249249
xdp_umem_unmap_pages(umem);
250250
xdp_umem_unpin_pages(umem);
251251

@@ -390,6 +390,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
390390
umem->size = size;
391391
umem->headroom = headroom;
392392
umem->chunk_size_nohr = chunk_size - headroom;
393+
umem->chunk_size = chunk_size;
393394
umem->npgs = size / PAGE_SIZE;
394395
umem->pgs = NULL;
395396
umem->user = NULL;
@@ -415,11 +416,21 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
415416
}
416417

417418
err = xdp_umem_map_pages(umem);
418-
if (!err)
419-
return 0;
419+
if (err)
420+
goto out_pages;
420421

421-
kvfree(umem->pages);
422+
umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
423+
headroom, size, unaligned_chunks);
424+
if (!umem->pool) {
425+
err = -ENOMEM;
426+
goto out_unmap;
427+
}
428+
return 0;
422429

430+
out_unmap:
431+
xdp_umem_unmap_pages(umem);
432+
out_pages:
433+
kvfree(umem->pages);
423434
out_pin:
424435
xdp_umem_unpin_pages(umem);
425436
out_account:

0 commit comments

Comments
 (0)