Skip to content

Commit 0045e0d

Browse files
Yixing Liujgunthorpe
Yixing Liu
authored and
jgunthorpe
committed
RDMA/hns: Support direct wqe of userspace
The current write wqe mechanism is to write to DDR first, and then notify the hardware through doorbell to read the data. Direct wqe is a mechanism to fill wqe directly into the hardware. In the case of light load, the wqe will be filled into pcie bar space of the hardware, this will reduce one memory access operation and therefore reduce the latency. SIMD instructions allows cpu to write the 512 bits at one time to device memory, thus it can be used for posting direct wqe. Add direct wqe enable switch and address mapping. Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Yixing Liu <[email protected]> Signed-off-by: Wenpeng Liang <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent b1a4da6 commit 0045e0d

File tree

6 files changed

+94
-12
lines changed

6 files changed

+94
-12
lines changed

‎drivers/infiniband/hw/hns/hns_roce_device.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ enum {
182182
HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
183183
HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
184184
HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
185+
HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12),
185186
HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),
186187
HNS_ROCE_CAP_FLAG_STASH = BIT(17),
187188
};
@@ -228,6 +229,7 @@ struct hns_roce_uar {
228229
enum hns_roce_mmap_type {
229230
HNS_ROCE_MMAP_TYPE_DB = 1,
230231
HNS_ROCE_MMAP_TYPE_TPTR,
232+
HNS_ROCE_MMAP_TYPE_DWQE,
231233
};
232234

233235
struct hns_user_mmap_entry {
@@ -627,10 +629,6 @@ struct hns_roce_work {
627629
u32 queue_num;
628630
};
629631

630-
enum {
631-
HNS_ROCE_QP_CAP_DIRECT_WQE = BIT(5),
632-
};
633-
634632
struct hns_roce_qp {
635633
struct ib_qp ibqp;
636634
struct hns_roce_wq rq;
@@ -675,6 +673,7 @@ struct hns_roce_qp {
675673
struct list_head node; /* all qps are on a list */
676674
struct list_head rq_node; /* all recv qps are on a list */
677675
struct list_head sq_node; /* all send qps are on a list */
676+
struct hns_user_mmap_entry *dwqe_mmap_entry;
678677
};
679678

680679
struct hns_roce_ib_iboe {
@@ -1010,6 +1009,7 @@ struct hns_roce_dev {
10101009
u32 func_num;
10111010
u32 is_vf;
10121011
u32 cong_algo_tmpl_id;
1012+
u64 dwqe_page;
10131013
};
10141014

10151015
static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)

‎drivers/infiniband/hw/hns/hns_roce_hw_v2.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1989,7 +1989,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev)
19891989
caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
19901990

19911991
if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1992-
caps->flags |= HNS_ROCE_CAP_FLAG_STASH;
1992+
caps->flags |= HNS_ROCE_CAP_FLAG_STASH |
1993+
HNS_ROCE_CAP_FLAG_DIRECT_WQE;
19931994
caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
19941995
} else {
19951996
caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;

‎drivers/infiniband/hw/hns/hns_roce_main.c‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -310,9 +310,25 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
310310
entry->address = address;
311311
entry->mmap_type = mmap_type;
312312

313-
ret = rdma_user_mmap_entry_insert_exact(
314-
ucontext, &entry->rdma_entry, length,
315-
mmap_type == HNS_ROCE_MMAP_TYPE_DB ? 0 : 1);
313+
switch (mmap_type) {
314+
case HNS_ROCE_MMAP_TYPE_DB:
315+
ret = rdma_user_mmap_entry_insert_exact(
316+
ucontext, &entry->rdma_entry, length, 0);
317+
break;
318+
case HNS_ROCE_MMAP_TYPE_TPTR:
319+
ret = rdma_user_mmap_entry_insert_exact(
320+
ucontext, &entry->rdma_entry, length, 1);
321+
break;
322+
case HNS_ROCE_MMAP_TYPE_DWQE:
323+
ret = rdma_user_mmap_entry_insert_range(
324+
ucontext, &entry->rdma_entry, length, 2,
325+
U32_MAX);
326+
break;
327+
default:
328+
ret = -EINVAL;
329+
break;
330+
}
331+
316332
if (ret) {
317333
kfree(entry);
318334
return NULL;
@@ -439,10 +455,18 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma)
439455

440456
entry = to_hns_mmap(rdma_entry);
441457
pfn = entry->address >> PAGE_SHIFT;
442-
prot = vma->vm_page_prot;
443458

444-
if (entry->mmap_type != HNS_ROCE_MMAP_TYPE_TPTR)
445-
prot = pgprot_device(prot);
459+
switch (entry->mmap_type) {
460+
case HNS_ROCE_MMAP_TYPE_DB:
461+
case HNS_ROCE_MMAP_TYPE_DWQE:
462+
prot = pgprot_device(vma->vm_page_prot);
463+
break;
464+
case HNS_ROCE_MMAP_TYPE_TPTR:
465+
prot = vma->vm_page_prot;
466+
break;
467+
default:
468+
return -EINVAL;
469+
}
446470

447471
ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,
448472
prot, rdma_entry);

‎drivers/infiniband/hw/hns/hns_roce_pd.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
115115
} else {
116116
uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2))
117117
>> PAGE_SHIFT);
118+
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
119+
hr_dev->dwqe_page =
120+
pci_resource_start(hr_dev->pci_dev, 4);
118121
}
119122

120123
return 0;

‎drivers/infiniband/hw/hns/hns_roce_qp.c‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,11 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
379379
return ret;
380380
}
381381

382+
static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
383+
{
384+
rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
385+
}
386+
382387
void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
383388
{
384389
struct xarray *xa = &hr_dev->qp_table_xa;
@@ -780,7 +785,11 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
780785
goto err_inline;
781786
}
782787

788+
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
789+
hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
790+
783791
return 0;
792+
784793
err_inline:
785794
free_rq_inline_buf(hr_qp);
786795

@@ -822,6 +831,35 @@ static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
822831
hns_roce_qp_has_rq(init_attr));
823832
}
824833

834+
static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
835+
struct hns_roce_dev *hr_dev,
836+
struct ib_udata *udata,
837+
struct hns_roce_ib_create_qp_resp *resp)
838+
{
839+
struct hns_roce_ucontext *uctx =
840+
rdma_udata_to_drv_context(udata,
841+
struct hns_roce_ucontext, ibucontext);
842+
struct rdma_user_mmap_entry *rdma_entry;
843+
u64 address;
844+
845+
address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
846+
847+
hr_qp->dwqe_mmap_entry =
848+
hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
849+
HNS_ROCE_DWQE_SIZE,
850+
HNS_ROCE_MMAP_TYPE_DWQE);
851+
852+
if (!hr_qp->dwqe_mmap_entry) {
853+
ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
854+
return -ENOMEM;
855+
}
856+
857+
rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
858+
resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
859+
860+
return 0;
861+
}
862+
825863
static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
826864
struct hns_roce_qp *hr_qp,
827865
struct ib_qp_init_attr *init_attr,
@@ -909,17 +947,29 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
909947
hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
910948

911949
if (udata) {
950+
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
951+
ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
952+
if (ret)
953+
return ret;
954+
}
955+
912956
ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
913957
resp);
914958
if (ret)
915-
return ret;
959+
goto err_remove_qp;
916960
} else {
917961
ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
918962
if (ret)
919963
return ret;
920964
}
921965

922966
return 0;
967+
968+
err_remove_qp:
969+
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
970+
qp_user_mmap_entry_remove(hr_qp);
971+
972+
return ret;
923973
}
924974

925975
static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
@@ -933,6 +983,8 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
933983
hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
934984
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
935985
hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
986+
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
987+
qp_user_mmap_entry_remove(hr_qp);
936988
} else {
937989
if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
938990
hns_roce_free_db(hr_dev, &hr_qp->rdb);

‎include/uapi/rdma/hns-abi.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,12 @@ enum hns_roce_qp_cap_flags {
7777
HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0,
7878
HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1,
7979
HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2,
80+
HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5,
8081
};
8182

8283
struct hns_roce_ib_create_qp_resp {
8384
__aligned_u64 cap_flags;
85+
__aligned_u64 dwqe_mmap_key;
8486
};
8587

8688
struct hns_roce_ib_alloc_ucontext_resp {

0 commit comments

Comments
 (0)