Skip to content

Commit b54a140

Browse files
axboeaxboe
committed
io_uring/rsrc: add io_rsrc_node_lookup() helper
There are lots of spots open-coding this functionality, add a generic helper that does the node lookup in a speculation safe way. Signed-off-by: Jens Axboe <[email protected]>
1 parent 3597f27 commit b54a140

File tree

12 files changed

+57
-59
lines changed

12 files changed

+57
-59
lines changed

‎io_uring/cancel.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -240,10 +240,12 @@ static int __io_sync_cancel(struct io_uring_task *tctx,
240240
/* fixed must be grabbed every time since we drop the uring_lock */
241241
if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
242242
(cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
243-
if (unlikely(fd >= ctx->file_table.data.nr))
243+
struct io_rsrc_node *node;
244+
245+
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
246+
if (unlikely(!node))
244247
return -EBADF;
245-
fd = array_index_nospec(fd, ctx->file_table.data.nr);
246-
cd->file = io_file_from_index(&ctx->file_table, fd);
248+
cd->file = io_slot_file(node);
247249
if (!cd->file)
248250
return -EBADF;
249251
}

‎io_uring/filetable.c‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
5858
u32 slot_index)
5959
__must_hold(&req->ctx->uring_lock)
6060
{
61-
struct io_rsrc_node *node;
61+
struct io_rsrc_node *node, *old_node;
6262

6363
if (io_is_uring_fops(file))
6464
return -EBADF;
@@ -71,9 +71,9 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
7171
if (!node)
7272
return -ENOMEM;
7373

74-
slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr);
75-
if (ctx->file_table.data.nodes[slot_index])
76-
io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]);
74+
old_node = io_rsrc_node_lookup(&ctx->file_table.data, slot_index);
75+
if (old_node)
76+
io_put_rsrc_node(old_node);
7777
else
7878
io_file_bitmap_set(&ctx->file_table, slot_index);
7979

@@ -123,15 +123,17 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
123123

124124
int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
125125
{
126+
struct io_rsrc_node *node;
127+
126128
if (unlikely(!ctx->file_table.data.nr))
127129
return -ENXIO;
128130
if (offset >= ctx->file_table.data.nr)
129131
return -EINVAL;
130132

131-
offset = array_index_nospec(offset, ctx->file_table.data.nr);
132-
if (!ctx->file_table.data.nodes[offset])
133+
node = io_rsrc_node_lookup(&ctx->file_table.data, offset);
134+
if (!node)
133135
return -EBADF;
134-
io_put_rsrc_node(ctx->file_table.data.nodes[offset]);
136+
io_put_rsrc_node(node);
135137
ctx->file_table.data.nodes[offset] = NULL;
136138
io_file_bitmap_clear(&ctx->file_table, offset);
137139
return 0;

‎io_uring/filetable.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node)
5252
static inline struct file *io_file_from_index(struct io_file_table *table,
5353
int index)
5454
{
55-
struct io_rsrc_node *node = table->data.nodes[index];
55+
struct io_rsrc_node *node = io_rsrc_node_lookup(&table->data, index);
5656

5757
if (node)
5858
return io_slot_file(node);

‎io_uring/io_uring.c‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1879,16 +1879,12 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
18791879
struct file *file = NULL;
18801880

18811881
io_ring_submit_lock(ctx, issue_flags);
1882-
if (unlikely((unsigned int)fd >= ctx->file_table.data.nr))
1883-
goto out;
1884-
fd = array_index_nospec(fd, ctx->file_table.data.nr);
1885-
node = ctx->file_table.data.nodes[fd];
1882+
node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
18861883
if (node) {
18871884
io_req_assign_rsrc_node(req, node);
18881885
req->flags |= io_slot_flags(node);
18891886
file = io_slot_file(node);
18901887
}
1891-
out:
18921888
io_ring_submit_unlock(ctx, issue_flags);
18931889
return file;
18941890
}

‎io_uring/msg_ring.c‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -172,22 +172,24 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)
172172
return __io_msg_ring_data(target_ctx, msg, issue_flags);
173173
}
174174

175-
static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
175+
static int io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
176176
{
177177
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
178178
struct io_ring_ctx *ctx = req->ctx;
179-
struct file *file = NULL;
180-
int idx = msg->src_fd;
179+
struct io_rsrc_node *node;
180+
int ret = -EBADF;
181181

182182
io_ring_submit_lock(ctx, issue_flags);
183-
if (likely(idx < ctx->file_table.data.nr)) {
184-
idx = array_index_nospec(idx, ctx->file_table.data.nr);
185-
file = io_file_from_index(&ctx->file_table, idx);
186-
if (file)
187-
get_file(file);
183+
node = io_rsrc_node_lookup(&ctx->file_table.data, msg->src_fd);
184+
if (node) {
185+
msg->src_file = io_slot_file(node);
186+
if (msg->src_file)
187+
get_file(msg->src_file);
188+
req->flags |= REQ_F_NEED_CLEANUP;
189+
ret = 0;
188190
}
189191
io_ring_submit_unlock(ctx, issue_flags);
190-
return file;
192+
return ret;
191193
}
192194

193195
static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags)
@@ -256,20 +258,17 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
256258
struct io_ring_ctx *target_ctx = req->file->private_data;
257259
struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
258260
struct io_ring_ctx *ctx = req->ctx;
259-
struct file *src_file = msg->src_file;
260261

261262
if (msg->len)
262263
return -EINVAL;
263264
if (target_ctx == ctx)
264265
return -EINVAL;
265266
if (target_ctx->flags & IORING_SETUP_R_DISABLED)
266267
return -EBADFD;
267-
if (!src_file) {
268-
src_file = io_msg_grab_file(req, issue_flags);
269-
if (!src_file)
270-
return -EBADF;
271-
msg->src_file = src_file;
272-
req->flags |= REQ_F_NEED_CLEANUP;
268+
if (!msg->src_file) {
269+
int ret = io_msg_grab_file(req, issue_flags);
270+
if (unlikely(ret))
271+
return ret;
273272
}
274273

275274
if (io_msg_need_remote(target_ctx))

‎io_uring/net.c‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1343,13 +1343,11 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
13431343
if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
13441344
struct io_ring_ctx *ctx = req->ctx;
13451345
struct io_rsrc_node *node;
1346-
int idx;
13471346

13481347
ret = -EFAULT;
13491348
io_ring_submit_lock(ctx, issue_flags);
1350-
if (sr->buf_index < ctx->buf_table.nr) {
1351-
idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr);
1352-
node = ctx->buf_table.nodes[idx];
1349+
node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
1350+
if (node) {
13531351
io_req_assign_rsrc_node(sr->notif, node);
13541352
ret = 0;
13551353
}

‎io_uring/nop.c‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,11 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
6262
if (nop->flags & IORING_NOP_FIXED_BUFFER) {
6363
struct io_ring_ctx *ctx = req->ctx;
6464
struct io_rsrc_node *node;
65-
int idx;
6665

6766
ret = -EFAULT;
6867
io_ring_submit_lock(ctx, issue_flags);
69-
if (nop->buffer < ctx->buf_table.nr) {
70-
idx = array_index_nospec(nop->buffer, ctx->buf_table.nr);
71-
node = READ_ONCE(ctx->buf_table.nodes[idx]);
68+
node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
69+
if (node) {
7270
io_req_assign_rsrc_node(req, node);
7371
ret = 0;
7472
}

‎io_uring/rsrc.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
181181
return -EINVAL;
182182

183183
for (done = 0; done < nr_args; done++) {
184+
struct io_rsrc_node *node;
184185
u64 tag = 0;
185186

186187
if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
@@ -195,9 +196,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
195196
if (fd == IORING_REGISTER_FILES_SKIP)
196197
continue;
197198

198-
i = array_index_nospec(up->offset + done, ctx->file_table.data.nr);
199-
if (ctx->file_table.data.nodes[i]) {
200-
io_put_rsrc_node(ctx->file_table.data.nodes[i]);
199+
i = up->offset + done;
200+
node = io_rsrc_node_lookup(&ctx->file_table.data, i);
201+
if (node) {
202+
io_put_rsrc_node(node);
201203
ctx->file_table.data.nodes[i] = NULL;
202204
io_file_bitmap_clear(&ctx->file_table, i);
203205
}
@@ -958,9 +960,9 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
958960
goto out_unlock;
959961

960962
for (i = 0; i < nbufs; i++) {
961-
struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i];
962-
struct io_rsrc_node *dst_node;
963+
struct io_rsrc_node *dst_node, *src_node;
963964

965+
src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i);
964966
if (src_node == rsrc_empty_node) {
965967
dst_node = rsrc_empty_node;
966968
} else {

‎io_uring/rsrc.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,14 @@ int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
7070
extern const struct io_rsrc_node empty_node;
7171
#define rsrc_empty_node (struct io_rsrc_node *) &empty_node
7272

73+
static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data,
74+
int index)
75+
{
76+
if (index < data->nr)
77+
return data->nodes[array_index_nospec(index, data->nr)];
78+
return NULL;
79+
}
80+
7381
static inline void io_put_rsrc_node(struct io_rsrc_node *node)
7482
{
7583
if (node != rsrc_empty_node && !--node->refs)

‎io_uring/rw.c‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -332,17 +332,15 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe
332332
struct io_ring_ctx *ctx = req->ctx;
333333
struct io_rsrc_node *node;
334334
struct io_async_rw *io;
335-
u16 index;
336335
int ret;
337336

338337
ret = io_prep_rw(req, sqe, ddir, false);
339338
if (unlikely(ret))
340339
return ret;
341340

342-
if (unlikely(req->buf_index >= ctx->buf_table.nr))
341+
node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
342+
if (!node)
343343
return -EFAULT;
344-
index = array_index_nospec(req->buf_index, ctx->buf_table.nr);
345-
node = ctx->buf_table.nodes[index];
346344
io_req_assign_rsrc_node(req, node);
347345

348346
io = req->async_data;

0 commit comments

Comments
 (0)