[LIBCLC][CUDA] Apply always_inline to all atomics (#5710)

Fixes: #5429

Interestingly enough, the performance penalty comes here not from performing the call, but from clang not being able to optimise away all the cases that atomics define, but don't need at call site.

Author: jchlanda

libclc/ptx-nvidiacl/libspirv/atomic/atomic_cmpxchg.cl

@@ -76,10 +76,12 @@ Memory order is stored in the lowest 5 bits */
 
 #define __CLC_NVVM_ATOMIC_CAS(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV,    \
                               OP, OP_MANGLED)                                  \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, \
                              OP_MANGLED, __global, AS1, _global_)              \
-  __CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, \
-                             OP_MANGLED, __local, AS3, _shared_)
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, \
+                                 OP, OP_MANGLED, __local, AS3, _shared_)
 
 __CLC_NVVM_ATOMIC_CAS(int, i, int, i, cas, CompareExchange)
 __CLC_NVVM_ATOMIC_CAS(long, l, long, l, cas, CompareExchange)

libclc/ptx-nvidiacl/libspirv/atomic/atomic_helpers.h

@@ -78,9 +78,11 @@ Memory order is stored in the lowest 5 bits */
 
 #define __CLC_NVVM_ATOMIC(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP,    \
                           NAME_MANGLED)                                        \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP,     \
                          NAME_MANGLED, __global, AS1, _global_)                \
-  __CLC_NVVM_ATOMIC_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP,     \
-                         NAME_MANGLED, __local, AS3, _shared_)
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, \
+                             NAME_MANGLED, __local, AS3, _shared_)
 
 #endif

libclc/ptx-nvidiacl/libspirv/atomic/atomic_inc_dec_helpers.h

@@ -27,8 +27,9 @@
   }
 
 #define __CLC_NVVM_ATOMIC_INCDEC(TYPE, TYPE_MANGLED, OP_MANGLED, VAL)          \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_INCDEC_IMPL(TYPE, TYPE_MANGLED, OP_MANGLED, VAL, __global, \
-                                AS1)                                           \
+                                AS1) __attribute__((always_inline))            \
   __CLC_NVVM_ATOMIC_INCDEC_IMPL(TYPE, TYPE_MANGLED, OP_MANGLED, VAL, __local,  \
                                 AS3)
 

libclc/ptx-nvidiacl/libspirv/atomic/atomic_load.cl

@@ -64,10 +64,11 @@ Memory order is stored in the lowest 5 bits */
   }
 
 #define __CLC_NVVM_ATOMIC_LOAD(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV)   \
-  __CLC_NVVM_ATOMIC_LOAD_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV,    \
-                              __global, AS1, _global_)                         \
-  __CLC_NVVM_ATOMIC_LOAD_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV,    \
-                              __local, AS3, _shared_)
+  __attribute__((always_inline)) __CLC_NVVM_ATOMIC_LOAD_IMPL(                  \
+      TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, __global, AS1, _global_)   \
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_LOAD_IMPL(TYPE, TYPE_MANGLED, TYPE_NV,                 \
+                                  TYPE_MANGLED_NV, __local, AS3, _shared_)
 
 __CLC_NVVM_ATOMIC_LOAD(int, i, int, i)
 __CLC_NVVM_ATOMIC_LOAD(uint, j, int, i)

libclc/ptx-nvidiacl/libspirv/atomic/atomic_max.cl

@@ -70,10 +70,12 @@ __CLC_NVVM_ATOMIC(unsigned long, m, unsigned long, ul, max,
 
 #define __CLC_NVVM_ATOMIC_MAX(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,  \
                               OP_MANGLED)                                      \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_MAX_IMPL(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,   \
                              OP_MANGLED, __global, AS1)                        \
-  __CLC_NVVM_ATOMIC_MAX_IMPL(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,   \
-                             OP_MANGLED, __local, AS3)
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_MAX_IMPL(TYPE, TYPE_MANGLED, TYPE_INT,                 \
+                                 TYPE_INT_MANGLED, OP_MANGLED, __local, AS3)
 
 __CLC_NVVM_ATOMIC_MAX(float, f, int, i, FMaxEXT)
 __CLC_NVVM_ATOMIC_MAX(double, d, long, l, FMaxEXT)

libclc/ptx-nvidiacl/libspirv/atomic/atomic_min.cl

@@ -68,10 +68,12 @@ __CLC_NVVM_ATOMIC(ulong, m, ulong, ul, min, _Z18__spirv_AtomicUMin)
 
 #define __CLC_NVVM_ATOMIC_MIN(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,  \
                               OP_MANGLED)                                      \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_MIN_IMPL(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,   \
                              OP_MANGLED, __global, AS1)                        \
-  __CLC_NVVM_ATOMIC_MIN_IMPL(TYPE, TYPE_MANGLED, TYPE_INT, TYPE_INT_MANGLED,   \
-                             OP_MANGLED, __local, AS3)
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_MIN_IMPL(TYPE, TYPE_MANGLED, TYPE_INT,                 \
+                                 TYPE_INT_MANGLED, OP_MANGLED, __local, AS3)
 
 __CLC_NVVM_ATOMIC_MIN(float, f, int, i, FMinEXT)
 __CLC_NVVM_ATOMIC_MIN(double, d, long, l, FMinEXT)

libclc/ptx-nvidiacl/libspirv/atomic/atomic_store.cl

@@ -65,10 +65,11 @@ Memory order is stored in the lowest 5 bits */
   }
 
 #define __CLC_NVVM_ATOMIC_STORE(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV)  \
-  __CLC_NVVM_ATOMIC_STORE_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV,   \
-                               __global, AS1, _global_)                        \
-  __CLC_NVVM_ATOMIC_STORE_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV,   \
-                               __local, AS3, _shared_)
+  __attribute__((always_inline)) __CLC_NVVM_ATOMIC_STORE_IMPL(                 \
+      TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, __global, AS1, _global_)   \
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_STORE_IMPL(TYPE, TYPE_MANGLED, TYPE_NV,                \
+                                   TYPE_MANGLED_NV, __local, AS3, _shared_)
 
 __CLC_NVVM_ATOMIC_STORE(int, i, int, i)
 __CLC_NVVM_ATOMIC_STORE(uint, j, int, i)

libclc/ptx-nvidiacl/libspirv/atomic/atomic_sub.cl

@@ -24,8 +24,10 @@
   }
 
 #define __CLC_NVVM_ATOMIC_SUB(TYPE, TYPE_MANGLED, OP_MANGLED)                  \
+  __attribute__((always_inline))                                               \
   __CLC_NVVM_ATOMIC_SUB_IMPL(TYPE, TYPE_MANGLED, OP_MANGLED, __global, AS1)    \
-  __CLC_NVVM_ATOMIC_SUB_IMPL(TYPE, TYPE_MANGLED, OP_MANGLED, __local, AS3)
+      __attribute__((always_inline))                                           \
+      __CLC_NVVM_ATOMIC_SUB_IMPL(TYPE, TYPE_MANGLED, OP_MANGLED, __local, AS3)
 
 __CLC_NVVM_ATOMIC_SUB(int, i, ISub)
 __CLC_NVVM_ATOMIC_SUB(unsigned int, j, ISub)