Skip to content

Commit 15c14a3

Browse files
Brian VazquezAlexei Starovoitov
Brian Vazquez
authored and
Alexei Starovoitov
committed
bpf: Add bpf_map_{value_size, update_value, map_copy_value} functions
This commit moves reusable code from map_lookup_elem and map_update_elem to avoid code duplication in kernel/bpf/syscall.c. Signed-off-by: Brian Vazquez <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]> Acked-by: John Fastabend <[email protected]> Acked-by: Yonghong Song <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 83e4b88 commit 15c14a3

File tree

1 file changed

+152
-128
lines changed

1 file changed

+152
-128
lines changed

‎kernel/bpf/syscall.c‎

Lines changed: 152 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,154 @@ static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
129129
return map;
130130
}
131131

132+
static u32 bpf_map_value_size(struct bpf_map *map)
133+
{
134+
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
135+
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
136+
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
137+
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
138+
return round_up(map->value_size, 8) * num_possible_cpus();
139+
else if (IS_FD_MAP(map))
140+
return sizeof(u32);
141+
else
142+
return map->value_size;
143+
}
144+
145+
static void maybe_wait_bpf_programs(struct bpf_map *map)
146+
{
147+
/* Wait for any running BPF programs to complete so that
148+
* userspace, when we return to it, knows that all programs
149+
* that could be running use the new map value.
150+
*/
151+
if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
152+
map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
153+
synchronize_rcu();
154+
}
155+
156+
static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
157+
void *value, __u64 flags)
158+
{
159+
int err;
160+
161+
/* Need to create a kthread, thus must support schedule */
162+
if (bpf_map_is_dev_bound(map)) {
163+
return bpf_map_offload_update_elem(map, key, value, flags);
164+
} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
165+
map->map_type == BPF_MAP_TYPE_SOCKHASH ||
166+
map->map_type == BPF_MAP_TYPE_SOCKMAP ||
167+
map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
168+
return map->ops->map_update_elem(map, key, value, flags);
169+
} else if (IS_FD_PROG_ARRAY(map)) {
170+
return bpf_fd_array_map_update_elem(map, f.file, key, value,
171+
flags);
172+
}
173+
174+
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
175+
* inside bpf map update or delete otherwise deadlocks are possible
176+
*/
177+
preempt_disable();
178+
__this_cpu_inc(bpf_prog_active);
179+
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
180+
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
181+
err = bpf_percpu_hash_update(map, key, value, flags);
182+
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
183+
err = bpf_percpu_array_update(map, key, value, flags);
184+
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
185+
err = bpf_percpu_cgroup_storage_update(map, key, value,
186+
flags);
187+
} else if (IS_FD_ARRAY(map)) {
188+
rcu_read_lock();
189+
err = bpf_fd_array_map_update_elem(map, f.file, key, value,
190+
flags);
191+
rcu_read_unlock();
192+
} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
193+
rcu_read_lock();
194+
err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
195+
flags);
196+
rcu_read_unlock();
197+
} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
198+
/* rcu_read_lock() is not needed */
199+
err = bpf_fd_reuseport_array_update_elem(map, key, value,
200+
flags);
201+
} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
202+
map->map_type == BPF_MAP_TYPE_STACK) {
203+
err = map->ops->map_push_elem(map, value, flags);
204+
} else {
205+
rcu_read_lock();
206+
err = map->ops->map_update_elem(map, key, value, flags);
207+
rcu_read_unlock();
208+
}
209+
__this_cpu_dec(bpf_prog_active);
210+
preempt_enable();
211+
maybe_wait_bpf_programs(map);
212+
213+
return err;
214+
}
215+
216+
static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
217+
__u64 flags)
218+
{
219+
void *ptr;
220+
int err;
221+
222+
if (bpf_map_is_dev_bound(map)) {
223+
err = bpf_map_offload_lookup_elem(map, key, value);
224+
return err;
225+
}
226+
227+
preempt_disable();
228+
this_cpu_inc(bpf_prog_active);
229+
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
230+
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
231+
err = bpf_percpu_hash_copy(map, key, value);
232+
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
233+
err = bpf_percpu_array_copy(map, key, value);
234+
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
235+
err = bpf_percpu_cgroup_storage_copy(map, key, value);
236+
} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
237+
err = bpf_stackmap_copy(map, key, value);
238+
} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
239+
err = bpf_fd_array_map_lookup_elem(map, key, value);
240+
} else if (IS_FD_HASH(map)) {
241+
err = bpf_fd_htab_map_lookup_elem(map, key, value);
242+
} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
243+
err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
244+
} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
245+
map->map_type == BPF_MAP_TYPE_STACK) {
246+
err = map->ops->map_peek_elem(map, value);
247+
} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
248+
/* struct_ops map requires directly updating "value" */
249+
err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
250+
} else {
251+
rcu_read_lock();
252+
if (map->ops->map_lookup_elem_sys_only)
253+
ptr = map->ops->map_lookup_elem_sys_only(map, key);
254+
else
255+
ptr = map->ops->map_lookup_elem(map, key);
256+
if (IS_ERR(ptr)) {
257+
err = PTR_ERR(ptr);
258+
} else if (!ptr) {
259+
err = -ENOENT;
260+
} else {
261+
err = 0;
262+
if (flags & BPF_F_LOCK)
263+
/* lock 'ptr' and copy everything but lock */
264+
copy_map_value_locked(map, value, ptr, true);
265+
else
266+
copy_map_value(map, value, ptr);
267+
/* mask lock, since value wasn't zero inited */
268+
check_and_init_map_lock(map, value);
269+
}
270+
rcu_read_unlock();
271+
}
272+
273+
this_cpu_dec(bpf_prog_active);
274+
preempt_enable();
275+
maybe_wait_bpf_programs(map);
276+
277+
return err;
278+
}
279+
132280
static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
133281
{
134282
/* We really just want to fail instead of triggering OOM killer
@@ -827,7 +975,7 @@ static int map_lookup_elem(union bpf_attr *attr)
827975
void __user *uvalue = u64_to_user_ptr(attr->value);
828976
int ufd = attr->map_fd;
829977
struct bpf_map *map;
830-
void *key, *value, *ptr;
978+
void *key, *value;
831979
u32 value_size;
832980
struct fd f;
833981
int err;
@@ -859,75 +1007,14 @@ static int map_lookup_elem(union bpf_attr *attr)
8591007
goto err_put;
8601008
}
8611009

862-
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
863-
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
864-
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
865-
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
866-
value_size = round_up(map->value_size, 8) * num_possible_cpus();
867-
else if (IS_FD_MAP(map))
868-
value_size = sizeof(u32);
869-
else
870-
value_size = map->value_size;
1010+
value_size = bpf_map_value_size(map);
8711011

8721012
err = -ENOMEM;
8731013
value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
8741014
if (!value)
8751015
goto free_key;
8761016

877-
if (bpf_map_is_dev_bound(map)) {
878-
err = bpf_map_offload_lookup_elem(map, key, value);
879-
goto done;
880-
}
881-
882-
preempt_disable();
883-
this_cpu_inc(bpf_prog_active);
884-
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
885-
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
886-
err = bpf_percpu_hash_copy(map, key, value);
887-
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
888-
err = bpf_percpu_array_copy(map, key, value);
889-
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
890-
err = bpf_percpu_cgroup_storage_copy(map, key, value);
891-
} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
892-
err = bpf_stackmap_copy(map, key, value);
893-
} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
894-
err = bpf_fd_array_map_lookup_elem(map, key, value);
895-
} else if (IS_FD_HASH(map)) {
896-
err = bpf_fd_htab_map_lookup_elem(map, key, value);
897-
} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
898-
err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
899-
} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
900-
map->map_type == BPF_MAP_TYPE_STACK) {
901-
err = map->ops->map_peek_elem(map, value);
902-
} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
903-
/* struct_ops map requires directly updating "value" */
904-
err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
905-
} else {
906-
rcu_read_lock();
907-
if (map->ops->map_lookup_elem_sys_only)
908-
ptr = map->ops->map_lookup_elem_sys_only(map, key);
909-
else
910-
ptr = map->ops->map_lookup_elem(map, key);
911-
if (IS_ERR(ptr)) {
912-
err = PTR_ERR(ptr);
913-
} else if (!ptr) {
914-
err = -ENOENT;
915-
} else {
916-
err = 0;
917-
if (attr->flags & BPF_F_LOCK)
918-
/* lock 'ptr' and copy everything but lock */
919-
copy_map_value_locked(map, value, ptr, true);
920-
else
921-
copy_map_value(map, value, ptr);
922-
/* mask lock, since value wasn't zero inited */
923-
check_and_init_map_lock(map, value);
924-
}
925-
rcu_read_unlock();
926-
}
927-
this_cpu_dec(bpf_prog_active);
928-
preempt_enable();
929-
930-
done:
1017+
err = bpf_map_copy_value(map, key, value, attr->flags);
9311018
if (err)
9321019
goto free_value;
9331020

@@ -946,16 +1033,6 @@ static int map_lookup_elem(union bpf_attr *attr)
9461033
return err;
9471034
}
9481035

949-
static void maybe_wait_bpf_programs(struct bpf_map *map)
950-
{
951-
/* Wait for any running BPF programs to complete so that
952-
* userspace, when we return to it, knows that all programs
953-
* that could be running use the new map value.
954-
*/
955-
if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
956-
map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
957-
synchronize_rcu();
958-
}
9591036

9601037
#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
9611038

@@ -1011,61 +1088,8 @@ static int map_update_elem(union bpf_attr *attr)
10111088
if (copy_from_user(value, uvalue, value_size) != 0)
10121089
goto free_value;
10131090

1014-
/* Need to create a kthread, thus must support schedule */
1015-
if (bpf_map_is_dev_bound(map)) {
1016-
err = bpf_map_offload_update_elem(map, key, value, attr->flags);
1017-
goto out;
1018-
} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
1019-
map->map_type == BPF_MAP_TYPE_SOCKHASH ||
1020-
map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1021-
map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1022-
err = map->ops->map_update_elem(map, key, value, attr->flags);
1023-
goto out;
1024-
} else if (IS_FD_PROG_ARRAY(map)) {
1025-
err = bpf_fd_array_map_update_elem(map, f.file, key, value,
1026-
attr->flags);
1027-
goto out;
1028-
}
1091+
err = bpf_map_update_value(map, f, key, value, attr->flags);
10291092

1030-
/* must increment bpf_prog_active to avoid kprobe+bpf triggering from
1031-
* inside bpf map update or delete otherwise deadlocks are possible
1032-
*/
1033-
preempt_disable();
1034-
__this_cpu_inc(bpf_prog_active);
1035-
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1036-
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
1037-
err = bpf_percpu_hash_update(map, key, value, attr->flags);
1038-
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
1039-
err = bpf_percpu_array_update(map, key, value, attr->flags);
1040-
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
1041-
err = bpf_percpu_cgroup_storage_update(map, key, value,
1042-
attr->flags);
1043-
} else if (IS_FD_ARRAY(map)) {
1044-
rcu_read_lock();
1045-
err = bpf_fd_array_map_update_elem(map, f.file, key, value,
1046-
attr->flags);
1047-
rcu_read_unlock();
1048-
} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
1049-
rcu_read_lock();
1050-
err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
1051-
attr->flags);
1052-
rcu_read_unlock();
1053-
} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
1054-
/* rcu_read_lock() is not needed */
1055-
err = bpf_fd_reuseport_array_update_elem(map, key, value,
1056-
attr->flags);
1057-
} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1058-
map->map_type == BPF_MAP_TYPE_STACK) {
1059-
err = map->ops->map_push_elem(map, value, attr->flags);
1060-
} else {
1061-
rcu_read_lock();
1062-
err = map->ops->map_update_elem(map, key, value, attr->flags);
1063-
rcu_read_unlock();
1064-
}
1065-
__this_cpu_dec(bpf_prog_active);
1066-
preempt_enable();
1067-
maybe_wait_bpf_programs(map);
1068-
out:
10691093
free_value:
10701094
kfree(value);
10711095
free_key:

0 commit comments

Comments
 (0)