Skip to content

Commit 3410d0e

Browse files
Shradha Guptakuba-moo
Shradha Gupta
authored and
kuba-moo
committed
net: mana: Implement get_ringparam/set_ringparam for mana
Currently the values of WQs for RX and TX queues for MANA devices are hardcoded to default sizes. Allow configuring these values for MANA devices as ringparam configuration(get/set) through ethtool_ops. Pre-allocate buffers at the beginning of this operation, to prevent complete network loss in low-memory conditions. Signed-off-by: Shradha Gupta <[email protected]> Reviewed-by: Haiyang Zhang <[email protected]> Reviewed-by: Saurabh Sengar <[email protected]> Link: https://patch.msgid.link/1724688461-12203-1-git-send-email-shradhagupta@linux.microsoft.com Signed-off-by: Jakub Kicinski <[email protected]>
1 parent cf740e3 commit 3410d0e

File tree

3 files changed

+110
-14
lines changed

3 files changed

+110
-14
lines changed

‎drivers/net/ethernet/microsoft/mana/mana_en.c‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,
511511
}
512512

513513
/* Release pre-allocated RX buffers */
514-
static void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
514+
void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
515515
{
516516
struct device *dev;
517517
int i;
@@ -608,7 +608,7 @@ static void mana_get_rxbuf_cfg(int mtu, u32 *datasize, u32 *alloc_size,
608608
*datasize = mtu + ETH_HLEN;
609609
}
610610

611-
static int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu)
611+
int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu)
612612
{
613613
struct device *dev;
614614
struct page *page;
@@ -622,7 +622,7 @@ static int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu)
622622

623623
dev = mpc->ac->gdma_dev->gdma_context->dev;
624624

625-
num_rxb = mpc->num_queues * RX_BUFFERS_PER_QUEUE;
625+
num_rxb = mpc->num_queues * mpc->rx_queue_size;
626626

627627
WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");
628628
mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);
@@ -1909,15 +1909,17 @@ static int mana_create_txq(struct mana_port_context *apc,
19091909
return -ENOMEM;
19101910

19111911
/* The minimum size of the WQE is 32 bytes, hence
1912-
* MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs
1912+
* apc->tx_queue_size represents the maximum number of WQEs
19131913
* the SQ can store. This value is then used to size other queues
19141914
* to prevent overflow.
1915+
* Also note that the txq_size is always going to be MANA_PAGE_ALIGNED,
1916+
* as min val of apc->tx_queue_size is 128 and that would make
1917+
* txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size
1918+
* are always power of two
19151919
*/
1916-
txq_size = MAX_SEND_BUFFERS_PER_QUEUE * 32;
1917-
BUILD_BUG_ON(!MANA_PAGE_ALIGNED(txq_size));
1920+
txq_size = apc->tx_queue_size * 32;
19181921

1919-
cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE;
1920-
cq_size = MANA_PAGE_ALIGN(cq_size);
1922+
cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE;
19211923

19221924
gc = gd->gdma_context;
19231925

@@ -2155,10 +2157,11 @@ static int mana_push_wqe(struct mana_rxq *rxq)
21552157

21562158
static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc)
21572159
{
2160+
struct mana_port_context *mpc = netdev_priv(rxq->ndev);
21582161
struct page_pool_params pprm = {};
21592162
int ret;
21602163

2161-
pprm.pool_size = RX_BUFFERS_PER_QUEUE;
2164+
pprm.pool_size = mpc->rx_queue_size;
21622165
pprm.nid = gc->numa_node;
21632166
pprm.napi = &rxq->rx_cq.napi;
21642167
pprm.netdev = rxq->ndev;
@@ -2190,13 +2193,13 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
21902193

21912194
gc = gd->gdma_context;
21922195

2193-
rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE),
2196+
rxq = kzalloc(struct_size(rxq, rx_oobs, apc->rx_queue_size),
21942197
GFP_KERNEL);
21952198
if (!rxq)
21962199
return NULL;
21972200

21982201
rxq->ndev = ndev;
2199-
rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
2202+
rxq->num_rx_buf = apc->rx_queue_size;
22002203
rxq->rxq_idx = rxq_idx;
22012204
rxq->rxobj = INVALID_MANA_HANDLE;
22022205

@@ -2744,6 +2747,8 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
27442747
apc->ndev = ndev;
27452748
apc->max_queues = gc->max_num_queues;
27462749
apc->num_queues = gc->max_num_queues;
2750+
apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE;
2751+
apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE;
27472752
apc->port_handle = INVALID_MANA_HANDLE;
27482753
apc->pf_filter_handle = INVALID_MANA_HANDLE;
27492754
apc->port_idx = port_idx;

‎drivers/net/ethernet/microsoft/mana/mana_ethtool.c‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,78 @@ static int mana_set_channels(struct net_device *ndev,
369369
return err;
370370
}
371371

372+
static void mana_get_ringparam(struct net_device *ndev,
373+
struct ethtool_ringparam *ring,
374+
struct kernel_ethtool_ringparam *kernel_ring,
375+
struct netlink_ext_ack *extack)
376+
{
377+
struct mana_port_context *apc = netdev_priv(ndev);
378+
379+
ring->rx_pending = apc->rx_queue_size;
380+
ring->tx_pending = apc->tx_queue_size;
381+
ring->rx_max_pending = MAX_RX_BUFFERS_PER_QUEUE;
382+
ring->tx_max_pending = MAX_TX_BUFFERS_PER_QUEUE;
383+
}
384+
385+
static int mana_set_ringparam(struct net_device *ndev,
386+
struct ethtool_ringparam *ring,
387+
struct kernel_ethtool_ringparam *kernel_ring,
388+
struct netlink_ext_ack *extack)
389+
{
390+
struct mana_port_context *apc = netdev_priv(ndev);
391+
u32 new_tx, new_rx;
392+
u32 old_tx, old_rx;
393+
int err;
394+
395+
old_tx = apc->tx_queue_size;
396+
old_rx = apc->rx_queue_size;
397+
398+
if (ring->tx_pending < MIN_TX_BUFFERS_PER_QUEUE) {
399+
NL_SET_ERR_MSG_FMT(extack, "tx:%d less than the min:%d", ring->tx_pending,
400+
MIN_TX_BUFFERS_PER_QUEUE);
401+
return -EINVAL;
402+
}
403+
404+
if (ring->rx_pending < MIN_RX_BUFFERS_PER_QUEUE) {
405+
NL_SET_ERR_MSG_FMT(extack, "rx:%d less than the min:%d", ring->rx_pending,
406+
MIN_RX_BUFFERS_PER_QUEUE);
407+
return -EINVAL;
408+
}
409+
410+
new_rx = roundup_pow_of_two(ring->rx_pending);
411+
new_tx = roundup_pow_of_two(ring->tx_pending);
412+
netdev_info(ndev, "Using nearest power of 2 values for Txq:%d Rxq:%d\n",
413+
new_tx, new_rx);
414+
415+
/* pre-allocating new buffers to prevent failures in mana_attach() later */
416+
apc->rx_queue_size = new_rx;
417+
err = mana_pre_alloc_rxbufs(apc, ndev->mtu);
418+
apc->rx_queue_size = old_rx;
419+
if (err) {
420+
netdev_err(ndev, "Insufficient memory for new allocations\n");
421+
return err;
422+
}
423+
424+
err = mana_detach(ndev, false);
425+
if (err) {
426+
netdev_err(ndev, "mana_detach failed: %d\n", err);
427+
goto out;
428+
}
429+
430+
apc->tx_queue_size = new_tx;
431+
apc->rx_queue_size = new_rx;
432+
433+
err = mana_attach(ndev);
434+
if (err) {
435+
netdev_err(ndev, "mana_attach failed: %d\n", err);
436+
apc->tx_queue_size = old_tx;
437+
apc->rx_queue_size = old_rx;
438+
}
439+
out:
440+
mana_pre_dealloc_rxbufs(apc);
441+
return err;
442+
}
443+
372444
const struct ethtool_ops mana_ethtool_ops = {
373445
.get_ethtool_stats = mana_get_ethtool_stats,
374446
.get_sset_count = mana_get_sset_count,
@@ -380,4 +452,6 @@ const struct ethtool_ops mana_ethtool_ops = {
380452
.set_rxfh = mana_set_rxfh,
381453
.get_channels = mana_get_channels,
382454
.set_channels = mana_set_channels,
455+
.get_ringparam = mana_get_ringparam,
456+
.set_ringparam = mana_set_ringparam,
383457
};

‎include/net/mana/mana.h‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,21 @@ enum TRI_STATE {
3838

3939
#define COMP_ENTRY_SIZE 64
4040

41-
#define RX_BUFFERS_PER_QUEUE 512
41+
/* This Max value for RX buffers is derived from __alloc_page()'s max page
42+
* allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer
43+
* size beyond this value gets rejected by __alloc_page() call.
44+
*/
45+
#define MAX_RX_BUFFERS_PER_QUEUE 8192
46+
#define DEF_RX_BUFFERS_PER_QUEUE 512
47+
#define MIN_RX_BUFFERS_PER_QUEUE 128
4248

43-
#define MAX_SEND_BUFFERS_PER_QUEUE 256
49+
/* This max value for TX buffers is derived as the maximum allocatable
50+
* pages supported on host per guest through testing. TX buffer size beyond
51+
* this value is rejected by the hardware.
52+
*/
53+
#define MAX_TX_BUFFERS_PER_QUEUE 16384
54+
#define DEF_TX_BUFFERS_PER_QUEUE 256
55+
#define MIN_TX_BUFFERS_PER_QUEUE 128
4456

4557
#define EQ_SIZE (8 * MANA_PAGE_SIZE)
4658

@@ -286,7 +298,7 @@ struct mana_recv_buf_oob {
286298
void *buf_va;
287299
bool from_pool; /* allocated from a page pool */
288300

289-
/* SGL of the buffer going to be sent has part of the work request. */
301+
/* SGL of the buffer going to be sent as part of the work request. */
290302
u32 num_sge;
291303
struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES];
292304

@@ -438,6 +450,9 @@ struct mana_port_context {
438450
unsigned int max_queues;
439451
unsigned int num_queues;
440452

453+
unsigned int rx_queue_size;
454+
unsigned int tx_queue_size;
455+
441456
mana_handle_t port_handle;
442457
mana_handle_t pf_filter_handle;
443458

@@ -473,6 +488,8 @@ struct bpf_prog *mana_xdp_get(struct mana_port_context *apc);
473488
void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog);
474489
int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf);
475490
void mana_query_gf_stats(struct mana_port_context *apc);
491+
int mana_pre_alloc_rxbufs(struct mana_port_context *apc, int mtu);
492+
void mana_pre_dealloc_rxbufs(struct mana_port_context *apc);
476493

477494
extern const struct ethtool_ops mana_ethtool_ops;
478495

0 commit comments

Comments
 (0)