Skip to content

Commit a540d01

Browse files
steffenlarsensteffenlarsen
authored
[SYCL][PTX] Implements missing atomic builtins (#1173)
- Builtins for atomic load and store (`int`, `unsigned int`, `long`, `unsigned long`, `long long`, and `unsigned long long`) - Added `long`, `unsigned long`, `long long`, and `unsigned long long` variants of the following existing builtins: - Atomic add - Atomic sub - Atomic and - Atomic or - Atomic xor - Atomic exhange - Atomic compare-and-exchange - Atomic max - Atomic min Notice: `long` is currently used in place of `long long` due to a problem with the width of `long long` being defined as 128-bit. Signed-off-by: Steffen Larsen <[email protected]>
1 parent 03326f7 commit a540d01

File tree

19 files changed

+571
-0
lines changed

19 files changed

+571
-0
lines changed

‎libclc/generic/include/spirv/atomic/atomic_decl.inc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_
2828
#ifdef cl_khr_int64_base_atomics
2929
__CLC_DECLARE_ATOMIC_ADDRSPACE(long, l, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
3030
__CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
31+
__CLC_DECLARE_ATOMIC_ADDRSPACE(long, x, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
32+
__CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, y, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
3133
#endif
3234
#endif
3335

‎libclc/generic/include/spirv/atomic/atomic_load.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10+
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11+
_CLC_DECL TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
12+
volatile AS const TYPE *, enum Scope, enum MemorySemanticsMask);
13+
14+
#define DECL_AS(TYPE, TYPE_MANGLED) \
15+
DECL(TYPE, TYPE_MANGLED, global, AS1) \
16+
DECL(TYPE, TYPE_MANGLED, local, AS3)
17+
18+
DECL_AS(int, i)
19+
DECL_AS(unsigned int, j)
20+
21+
#ifdef cl_khr_int64_base_atomics
22+
DECL_AS(long, l)
23+
DECL_AS(unsigned long, m)
24+
DECL_AS(long, x)
25+
DECL_AS(unsigned long, y)
26+
#endif
27+
28+
#undef DECL_AS
29+
#undef DECL

‎libclc/generic/include/spirv/atomic/atomic_store.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10+
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11+
_CLC_DECL void _Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE##TYPE_MANGLED( \
12+
volatile AS TYPE *, enum Scope, enum MemorySemanticsMask, TYPE);
13+
14+
#define DECL_AS(TYPE, TYPE_MANGLED) \
15+
DECL(TYPE, TYPE_MANGLED, global, AS1) \
16+
DECL(TYPE, TYPE_MANGLED, local, AS3)
17+
18+
DECL_AS(int, i)
19+
DECL_AS(unsigned int, j)
20+
21+
#ifdef cl_khr_int64_base_atomics
22+
DECL_AS(long, l)
23+
DECL_AS(unsigned long, m)
24+
DECL_AS(long, x)
25+
DECL_AS(unsigned long, y)
26+
#endif
27+
28+
#undef DECL_AS
29+
#undef DECL

‎libclc/generic/include/spirv/spirv.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,8 @@
235235
#include <spirv/atomic/atomic_sub.h>
236236
#include <spirv/atomic/atomic_xchg.h>
237237
#include <spirv/atomic/atomic_xor.h>
238+
#include <spirv/atomic/atomic_load.h>
239+
#include <spirv/atomic/atomic_store.h>
238240

239241
/* cl_khr extension atomics are omitted from __spirv */
240242

‎libclc/generic/libspirv/SOURCES‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ atomic/atomic_or.cl
1212
atomic/atomic_sub.cl
1313
atomic/atomic_xchg.cl
1414
atomic/atomic_xor.cl
15+
atomic/atomic_load.cl
16+
atomic/atomic_store.cl
1517
common/degrees.cl
1618
common/mix.cl
1719
common/radians.cl

‎libclc/generic/libspirv/atomic/atomic_add.cl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_add_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_add_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_add_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_add_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_add_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_add_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_add_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_add_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_and.cl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_and_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_and_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_and_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_and_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_and_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_and_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_and_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_and_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_cmpxchg.cl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,24 @@ _CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1mN5__spv5ScopeENS1_19Memor
5050
volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
5151
return __sync_val_compare_and_swap_8(p, cmp, val);
5252
}
53+
54+
_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS3xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
55+
volatile local long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
56+
return __sync_val_compare_and_swap_8(p, cmp, val);
57+
}
58+
59+
_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS1xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
60+
volatile global long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
61+
return __sync_val_compare_and_swap_8(p, cmp, val);
62+
}
63+
64+
_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS3yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
65+
volatile local ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
66+
return __sync_val_compare_and_swap_8(p, cmp, val);
67+
}
68+
69+
_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
70+
volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
71+
return __sync_val_compare_and_swap_8(p, cmp, val);
72+
}
5373
#endif

‎libclc/generic/libspirv/atomic/atomic_load.cl‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include <spirv/spirv.h>
10+
11+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
12+
13+
#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
14+
TYPE __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *);
15+
16+
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
17+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
18+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, acquire) \
19+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
20+
_CLC_DEF TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
21+
volatile AS const TYPE *p, enum Scope scope, enum MemorySemanticsMask semantics) { \
22+
if (semantics == Acquire) { \
23+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_acquire(p); \
24+
} \
25+
if (semantics == SequentiallyConsistent) { \
26+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_seq_cst(p); \
27+
} \
28+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_unordered(p); \
29+
}
30+
31+
#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
32+
IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
33+
IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
34+
35+
IMPL_AS(int, i, , 4)
36+
IMPL_AS(unsigned int, j, u, 4)
37+
38+
#ifdef cl_khr_int64_base_atomics
39+
IMPL_AS(long, l, , 8)
40+
IMPL_AS(unsigned long, m, u, 8)
41+
IMPL_AS(long, x, , 8)
42+
IMPL_AS(unsigned long, y, u, 8)
43+
#endif
44+
45+
#undef FDECL
46+
#undef IMPL_AS
47+
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_max.cl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,9 @@ IMPL(long, l, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
3131
IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
3232
IMPL(long, l, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
3333
IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
34+
IMPL(long, x, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
35+
IMPL(unsigned long, y, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
36+
IMPL(long, x, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
37+
IMPL(unsigned long, y, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
3438
#endif
3539
#undef IMPL

0 commit comments

Comments
 (0)