Skip to content

Commit 0266a17

Browse files
longlimsftrleon
longlimsft
authored and
rleon
committed
RDMA/mana_ib: Add a driver for Microsoft Azure Network Adapter
Add a RDMA VF driver for Microsoft Azure Network Adapter (MANA). Co-developed-by: Ajay Sharma <[email protected]> Signed-off-by: Ajay Sharma <[email protected]> Reviewed-by: Dexuan Cui <[email protected]> Signed-off-by: Long Li <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Leon Romanovsky <[email protected]>
1 parent 1ec5617 commit 0266a17

File tree

15 files changed

+1793
-0
lines changed

15 files changed

+1793
-0
lines changed

‎MAINTAINERS‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13669,6 +13669,15 @@ F: drivers/scsi/smartpqi/smartpqi*.[ch]
1366913669
F: include/linux/cciss*.h
1367013670
F: include/uapi/linux/cciss*.h
1367113671

13672+
MICROSOFT MANA RDMA DRIVER
13673+
M: Long Li <[email protected]>
13674+
M: Ajay Sharma <[email protected]>
13675+
L: [email protected]
13676+
S: Supported
13677+
F: drivers/infiniband/hw/mana/
13678+
F: include/net/mana
13679+
F: include/uapi/rdma/mana-abi.h
13680+
1367213681
MICROSOFT SURFACE AGGREGATOR TABLET-MODE SWITCH
1367313682
M: Maximilian Luz <[email protected]>
1367413683
L: [email protected]

‎drivers/infiniband/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ source "drivers/infiniband/hw/erdma/Kconfig"
8585
source "drivers/infiniband/hw/hfi1/Kconfig"
8686
source "drivers/infiniband/hw/hns/Kconfig"
8787
source "drivers/infiniband/hw/irdma/Kconfig"
88+
source "drivers/infiniband/hw/mana/Kconfig"
8889
source "drivers/infiniband/hw/mlx4/Kconfig"
8990
source "drivers/infiniband/hw/mlx5/Kconfig"
9091
source "drivers/infiniband/hw/mthca/Kconfig"

‎drivers/infiniband/hw/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ obj-$(CONFIG_INFINIBAND_QIB) += qib/
44
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
55
obj-$(CONFIG_INFINIBAND_EFA) += efa/
66
obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/
7+
obj-$(CONFIG_MANA_INFINIBAND) += mana/
78
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
89
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
910
obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/

‎drivers/infiniband/hw/mana/Kconfig‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# SPDX-License-Identifier: GPL-2.0-only
2+
config MANA_INFINIBAND
3+
tristate "Microsoft Azure Network Adapter support"
4+
depends on NETDEVICES && ETHERNET && PCI && MICROSOFT_MANA
5+
help
6+
This driver provides low-level RDMA support for Microsoft Azure
7+
Network Adapter (MANA). MANA supports RDMA features that can be used
8+
for workloads (e.g. DPDK, MPI etc) that uses RDMA verbs to directly
9+
access hardware from user-mode processes in Microsoft Azure cloud
10+
environment.

‎drivers/infiniband/hw/mana/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# SPDX-License-Identifier: GPL-2.0-only
2+
obj-$(CONFIG_MANA_INFINIBAND) += mana_ib.o
3+
4+
mana_ib-y := device.o main.o wq.o qp.o cq.o mr.o

‎drivers/infiniband/hw/mana/cq.c‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4+
*/
5+
6+
#include "mana_ib.h"
7+
8+
int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
9+
struct ib_udata *udata)
10+
{
11+
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
12+
struct ib_device *ibdev = ibcq->device;
13+
struct mana_ib_create_cq ucmd = {};
14+
struct mana_ib_dev *mdev;
15+
int err;
16+
17+
mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
18+
19+
if (udata->inlen < sizeof(ucmd))
20+
return -EINVAL;
21+
22+
err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
23+
if (err) {
24+
ibdev_dbg(ibdev,
25+
"Failed to copy from udata for create cq, %d\n", err);
26+
return err;
27+
}
28+
29+
if (attr->cqe > MAX_SEND_BUFFERS_PER_QUEUE) {
30+
ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
31+
return -EINVAL;
32+
}
33+
34+
cq->cqe = attr->cqe;
35+
cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
36+
IB_ACCESS_LOCAL_WRITE);
37+
if (IS_ERR(cq->umem)) {
38+
err = PTR_ERR(cq->umem);
39+
ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n",
40+
err);
41+
return err;
42+
}
43+
44+
err = mana_ib_gd_create_dma_region(mdev, cq->umem, &cq->gdma_region);
45+
if (err) {
46+
ibdev_dbg(ibdev,
47+
"Failed to create dma region for create cq, %d\n",
48+
err);
49+
goto err_release_umem;
50+
}
51+
52+
ibdev_dbg(ibdev,
53+
"mana_ib_gd_create_dma_region ret %d gdma_region 0x%llx\n",
54+
err, cq->gdma_region);
55+
56+
/*
57+
* The CQ ID is not known at this time. The ID is generated at create_qp
58+
*/
59+
60+
return 0;
61+
62+
err_release_umem:
63+
ib_umem_release(cq->umem);
64+
return err;
65+
}
66+
67+
int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
68+
{
69+
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
70+
struct ib_device *ibdev = ibcq->device;
71+
struct mana_ib_dev *mdev;
72+
73+
mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
74+
75+
mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
76+
ib_umem_release(cq->umem);
77+
78+
return 0;
79+
}

‎drivers/infiniband/hw/mana/device.c‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4+
*/
5+
6+
#include "mana_ib.h"
7+
#include <net/mana/mana_auxiliary.h>
8+
9+
MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");
10+
MODULE_LICENSE("GPL");
11+
MODULE_IMPORT_NS(NET_MANA);
12+
13+
static const struct ib_device_ops mana_ib_dev_ops = {
14+
.owner = THIS_MODULE,
15+
.driver_id = RDMA_DRIVER_MANA,
16+
.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
17+
18+
.alloc_pd = mana_ib_alloc_pd,
19+
.alloc_ucontext = mana_ib_alloc_ucontext,
20+
.create_cq = mana_ib_create_cq,
21+
.create_qp = mana_ib_create_qp,
22+
.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
23+
.create_wq = mana_ib_create_wq,
24+
.dealloc_pd = mana_ib_dealloc_pd,
25+
.dealloc_ucontext = mana_ib_dealloc_ucontext,
26+
.dereg_mr = mana_ib_dereg_mr,
27+
.destroy_cq = mana_ib_destroy_cq,
28+
.destroy_qp = mana_ib_destroy_qp,
29+
.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
30+
.destroy_wq = mana_ib_destroy_wq,
31+
.disassociate_ucontext = mana_ib_disassociate_ucontext,
32+
.get_port_immutable = mana_ib_get_port_immutable,
33+
.mmap = mana_ib_mmap,
34+
.modify_qp = mana_ib_modify_qp,
35+
.modify_wq = mana_ib_modify_wq,
36+
.query_device = mana_ib_query_device,
37+
.query_gid = mana_ib_query_gid,
38+
.query_port = mana_ib_query_port,
39+
.reg_user_mr = mana_ib_reg_user_mr,
40+
41+
INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
42+
INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
43+
INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
44+
INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
45+
INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,
46+
ib_ind_table),
47+
};
48+
49+
static int mana_ib_probe(struct auxiliary_device *adev,
50+
const struct auxiliary_device_id *id)
51+
{
52+
struct mana_adev *madev = container_of(adev, struct mana_adev, adev);
53+
struct gdma_dev *mdev = madev->mdev;
54+
struct mana_context *mc;
55+
struct mana_ib_dev *dev;
56+
int ret;
57+
58+
mc = mdev->driver_data;
59+
60+
dev = ib_alloc_device(mana_ib_dev, ib_dev);
61+
if (!dev)
62+
return -ENOMEM;
63+
64+
ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops);
65+
66+
dev->ib_dev.phys_port_cnt = mc->num_ports;
67+
68+
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
69+
mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
70+
71+
dev->gdma_dev = mdev;
72+
dev->ib_dev.node_type = RDMA_NODE_IB_CA;
73+
74+
/*
75+
* num_comp_vectors needs to set to the max MSIX index
76+
* when interrupts and event queues are implemented
77+
*/
78+
dev->ib_dev.num_comp_vectors = 1;
79+
dev->ib_dev.dev.parent = mdev->gdma_context->dev;
80+
81+
ret = ib_register_device(&dev->ib_dev, "mana_%d",
82+
mdev->gdma_context->dev);
83+
if (ret) {
84+
ib_dealloc_device(&dev->ib_dev);
85+
return ret;
86+
}
87+
88+
dev_set_drvdata(&adev->dev, dev);
89+
90+
return 0;
91+
}
92+
93+
static void mana_ib_remove(struct auxiliary_device *adev)
94+
{
95+
struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
96+
97+
ib_unregister_device(&dev->ib_dev);
98+
ib_dealloc_device(&dev->ib_dev);
99+
}
100+
101+
static const struct auxiliary_device_id mana_id_table[] = {
102+
{
103+
.name = "mana.rdma",
104+
},
105+
{},
106+
};
107+
108+
MODULE_DEVICE_TABLE(auxiliary, mana_id_table);
109+
110+
static struct auxiliary_driver mana_driver = {
111+
.name = "rdma",
112+
.probe = mana_ib_probe,
113+
.remove = mana_ib_remove,
114+
.id_table = mana_id_table,
115+
};
116+
117+
module_auxiliary_driver(mana_driver);

0 commit comments

Comments
 (0)