Skip to content

Commit 461bb2e

Browse files
jgunthorpejgunthorpe
committed
IB/uverbs: Add a simple allocator to uverbs_attr_bundle
This is similar in spirit to devm, it keeps track of any allocations linked to this method call and ensures they are all freed when the method exits. Further, if there is space in the internal/onstack buffer then the allocator will hand out that memory and avoid an expensive call to kalloc/kfree in the syscall path. Signed-off-by: Jason Gunthorpe <[email protected]> Reviewed-by: Leon Romanovsky <[email protected]>
1 parent 6a1f444 commit 461bb2e

File tree

2 files changed

+113
-20
lines changed

2 files changed

+113
-20
lines changed

‎drivers/infiniband/core/uverbs_ioctl.c‎

Lines changed: 89 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,18 @@
3535
#include "rdma_core.h"
3636
#include "uverbs.h"
3737

38+
struct bundle_alloc_head {
39+
struct bundle_alloc_head *next;
40+
u8 data[];
41+
};
42+
3843
struct bundle_priv {
44+
/* Must be first */
45+
struct bundle_alloc_head alloc_head;
46+
struct bundle_alloc_head *allocated_mem;
47+
size_t internal_avail;
48+
size_t internal_used;
49+
3950
struct ib_uverbs_attr __user *user_attrs;
4051
struct ib_uverbs_attr *uattrs;
4152
struct uverbs_obj_attr *destroy_attr;
@@ -45,8 +56,53 @@ struct bundle_priv {
4556
* internal_buffer.
4657
*/
4758
struct uverbs_attr_bundle bundle;
59+
u64 internal_buffer[32];
4860
};
4961

62+
/**
63+
* uverbs_alloc() - Quickly allocate memory for use with a bundle
64+
* @bundle: The bundle
65+
* @size: Number of bytes to allocate
66+
* @flags: Allocator flags
67+
*
68+
* The bundle allocator is intended for allocations that are connected with
69+
* processing the system call related to the bundle. The allocated memory is
70+
* always freed once the system call completes, and cannot be freed any other
71+
* way.
72+
*
73+
* This tries to use a small pool of pre-allocated memory for performance.
74+
*/
75+
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
76+
gfp_t flags)
77+
{
78+
struct bundle_priv *pbundle =
79+
container_of(bundle, struct bundle_priv, bundle);
80+
size_t new_used;
81+
void *res;
82+
83+
if (check_add_overflow(size, pbundle->internal_used, &new_used))
84+
return ERR_PTR(-EINVAL);
85+
86+
if (new_used > pbundle->internal_avail) {
87+
struct bundle_alloc_head *buf;
88+
89+
buf = kvmalloc(struct_size(buf, data, size), flags);
90+
if (!buf)
91+
return ERR_PTR(-ENOMEM);
92+
buf->next = pbundle->allocated_mem;
93+
pbundle->allocated_mem = buf;
94+
return buf->data;
95+
}
96+
97+
res = (void *)pbundle->internal_buffer + pbundle->internal_used;
98+
pbundle->internal_used =
99+
ALIGN(new_used, sizeof(*pbundle->internal_buffer));
100+
if (flags & __GFP_ZERO)
101+
memset(res, 0, size);
102+
return res;
103+
}
104+
EXPORT_SYMBOL(_uverbs_alloc);
105+
50106
static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
51107
u16 len)
52108
{
@@ -129,17 +185,15 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,
129185
if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
130186
void *p;
131187

132-
p = kvmalloc(uattr->len, GFP_KERNEL);
133-
if (!p)
134-
return -ENOMEM;
188+
p = uverbs_alloc(&pbundle->bundle, uattr->len);
189+
if (IS_ERR(p))
190+
return PTR_ERR(p);
135191

136192
e->ptr_attr.ptr = p;
137193

138194
if (copy_from_user(p, u64_to_user_ptr(uattr->data),
139-
uattr->len)) {
140-
kvfree(p);
195+
uattr->len))
141196
return -EFAULT;
142-
}
143197
} else {
144198
e->ptr_attr.data = uattr->data;
145199
}
@@ -234,10 +288,6 @@ static int uverbs_finalize_attrs(struct bundle_priv *pbundle,
234288
spec->u.obj.access, commit);
235289
if (!ret)
236290
ret = current_ret;
237-
} else if (spec->type == UVERBS_ATTR_TYPE_PTR_IN &&
238-
spec->alloc_and_copy &&
239-
!uverbs_attr_ptr_is_inline(attr)) {
240-
kvfree(attr->ptr_attr.ptr);
241291
}
242292
}
243293
}
@@ -372,7 +422,18 @@ static int uverbs_handle_method(size_t num_uattrs,
372422
return ret ? ret : finalize_ret;
373423
}
374424

375-
#define UVERBS_OPTIMIZE_USING_STACK_SZ 256
425+
static void bundle_destroy(struct bundle_priv *pbundle)
426+
{
427+
struct bundle_alloc_head *memblock;
428+
429+
for (memblock = pbundle->allocated_mem; memblock;) {
430+
struct bundle_alloc_head *tmp = memblock;
431+
432+
memblock = memblock->next;
433+
kvfree(tmp);
434+
}
435+
}
436+
376437
static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
377438
struct ib_uverbs_file *file,
378439
struct ib_uverbs_ioctl_hdr *hdr,
@@ -382,11 +443,11 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
382443
const struct uverbs_method_spec *method_spec;
383444
long err = 0;
384445
unsigned int i;
446+
struct bundle_priv onstack_pbundle;
385447
struct bundle_priv *ctx;
386448
struct uverbs_attr *curr_attr;
387449
unsigned long *curr_bitmap;
388450
size_t ctx_size;
389-
uintptr_t data[UVERBS_OPTIMIZE_USING_STACK_SZ / sizeof(uintptr_t)];
390451

391452
if (hdr->driver_id != ib_dev->driver_id)
392453
return -EINVAL;
@@ -399,7 +460,7 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
399460
if (!method_spec)
400461
return -EPROTONOSUPPORT;
401462

402-
ctx_size = sizeof(*ctx) +
463+
ctx_size = sizeof(*ctx) - sizeof(ctx->internal_buffer) +
403464
sizeof(struct uverbs_attr_bundle_hash) * method_spec->num_buckets +
404465
sizeof(*ctx->uattrs) * hdr->num_attrs +
405466
sizeof(*ctx->bundle.hash[0].attrs) *
@@ -408,17 +469,26 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
408469
(method_spec->num_child_attrs / BITS_PER_LONG +
409470
method_spec->num_buckets);
410471

411-
if (ctx_size <= UVERBS_OPTIMIZE_USING_STACK_SZ)
412-
ctx = (void *)data;
413-
if (!ctx)
472+
if (ctx_size <= sizeof(onstack_pbundle)) {
473+
ctx = &onstack_pbundle;
474+
ctx->internal_avail =
475+
sizeof(onstack_pbundle) -
476+
offsetof(struct bundle_priv, internal_buffer);
477+
ctx->allocated_mem = NULL;
478+
} else {
414479
ctx = kmalloc(ctx_size, GFP_KERNEL);
415-
if (!ctx)
416-
return -ENOMEM;
480+
if (!ctx)
481+
return -ENOMEM;
482+
ctx->internal_avail = 0;
483+
ctx->alloc_head.next = NULL;
484+
ctx->allocated_mem = &ctx->alloc_head;
485+
}
417486

418487
ctx->uattrs = (void *)(ctx + 1) +
419488
(sizeof(ctx->bundle.hash[0]) * method_spec->num_buckets);
420489
curr_attr = (void *)(ctx->uattrs + hdr->num_attrs);
421490
curr_bitmap = (void *)(curr_attr + method_spec->num_child_attrs);
491+
ctx->internal_used = ALIGN(ctx_size, sizeof(*ctx->internal_buffer));
422492

423493
/*
424494
* We just fill the pointers and num_attrs here. The data itself will be
@@ -462,8 +532,7 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
462532
err = -EINVAL;
463533
}
464534
out:
465-
if (ctx != (void *)data)
466-
kfree(ctx);
535+
bundle_destroy(ctx);
467536
return err;
468537
}
469538

‎include/rdma/uverbs_ioctl.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,6 +651,20 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
651651
size_t idx, u64 allowed_bits);
652652
int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx,
653653
const void *from, size_t size);
654+
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
655+
gfp_t flags);
656+
657+
static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
658+
size_t size)
659+
{
660+
return _uverbs_alloc(bundle, size, GFP_KERNEL);
661+
}
662+
663+
static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle,
664+
size_t size)
665+
{
666+
return _uverbs_alloc(bundle, size, GFP_KERNEL | __GFP_ZERO);
667+
}
654668
#else
655669
static inline int
656670
uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
@@ -669,6 +683,16 @@ static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle,
669683
{
670684
return -EINVAL;
671685
}
686+
static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
687+
size_t size)
688+
{
689+
return ERR_PTR(-EINVAL);
690+
}
691+
static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle,
692+
size_t size)
693+
{
694+
return ERR_PTR(-EINVAL);
695+
}
672696
#endif
673697

674698
/* =================================================

0 commit comments

Comments
 (0)