Skip to content

Commit 580fa1b

Browse files
Andrew Murraywilldeacon
Andrew Murray
authored and
willdeacon
committed
arm64: Use correct ll/sc atomic constraints
The A64 ISA accepts distinct (but overlapping) ranges of immediates for: * add arithmetic instructions ('I' machine constraint) * sub arithmetic instructions ('J' machine constraint) * 32-bit logical instructions ('K' machine constraint) * 64-bit logical instructions ('L' machine constraint) ... but we currently use the 'I' constraint for many atomic operations using sub or logical instructions, which is not always valid. When CONFIG_ARM64_LSE_ATOMICS is not set, this allows invalid immediates to be passed to instructions, potentially resulting in a build failure. When CONFIG_ARM64_LSE_ATOMICS is selected the out-of-line ll/sc atomics always use a register as they have no visibility of the value passed by the caller. This patch adds a constraint parameter to the ATOMIC_xx and __CMPXCHG_CASE macros so that we can pass appropriate constraints for each case, with uses updated accordingly. Unfortunately prior to GCC 8.1.0 the 'K' constraint erroneously accepted '4294967295', so we must instead force the use of a register. Signed-off-by: Andrew Murray <[email protected]> Signed-off-by: Will Deacon <[email protected]>
1 parent 8f35eaa commit 580fa1b

File tree

1 file changed

+47
-42
lines changed

1 file changed

+47
-42
lines changed

‎arch/arm64/include/asm/atomic_ll_sc.h‎

Lines changed: 47 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
* (the optimize attribute silently ignores these options).
2727
*/
2828

29-
#define ATOMIC_OP(op, asm_op) \
29+
#define ATOMIC_OP(op, asm_op, constraint) \
3030
__LL_SC_INLINE void \
3131
__LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v)) \
3232
{ \
@@ -40,11 +40,11 @@ __LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v)) \
4040
" stxr %w1, %w0, %2\n" \
4141
" cbnz %w1, 1b" \
4242
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
43-
: "Ir" (i)); \
43+
: #constraint "r" (i)); \
4444
} \
4545
__LL_SC_EXPORT(arch_atomic_##op);
4646

47-
#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
47+
#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
4848
__LL_SC_INLINE int \
4949
__LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v)) \
5050
{ \
@@ -59,14 +59,14 @@ __LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v)) \
5959
" cbnz %w1, 1b\n" \
6060
" " #mb \
6161
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
62-
: "Ir" (i) \
62+
: #constraint "r" (i) \
6363
: cl); \
6464
\
6565
return result; \
6666
} \
6767
__LL_SC_EXPORT(arch_atomic_##op##_return##name);
6868

69-
#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
69+
#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
7070
__LL_SC_INLINE int \
7171
__LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v)) \
7272
{ \
@@ -81,7 +81,7 @@ __LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v)) \
8181
" cbnz %w2, 1b\n" \
8282
" " #mb \
8383
: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \
84-
: "Ir" (i) \
84+
: #constraint "r" (i) \
8585
: cl); \
8686
\
8787
return result; \
@@ -99,8 +99,8 @@ __LL_SC_EXPORT(arch_atomic_fetch_##op##name);
9999
ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\
100100
ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__)
101101

102-
ATOMIC_OPS(add, add)
103-
ATOMIC_OPS(sub, sub)
102+
ATOMIC_OPS(add, add, I)
103+
ATOMIC_OPS(sub, sub, J)
104104

105105
#undef ATOMIC_OPS
106106
#define ATOMIC_OPS(...) \
@@ -110,17 +110,17 @@ ATOMIC_OPS(sub, sub)
110110
ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\
111111
ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__)
112112

113-
ATOMIC_OPS(and, and)
114-
ATOMIC_OPS(andnot, bic)
115-
ATOMIC_OPS(or, orr)
116-
ATOMIC_OPS(xor, eor)
113+
ATOMIC_OPS(and, and, )
114+
ATOMIC_OPS(andnot, bic, )
115+
ATOMIC_OPS(or, orr, )
116+
ATOMIC_OPS(xor, eor, )
117117

118118
#undef ATOMIC_OPS
119119
#undef ATOMIC_FETCH_OP
120120
#undef ATOMIC_OP_RETURN
121121
#undef ATOMIC_OP
122122

123-
#define ATOMIC64_OP(op, asm_op) \
123+
#define ATOMIC64_OP(op, asm_op, constraint) \
124124
__LL_SC_INLINE void \
125125
__LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v)) \
126126
{ \
@@ -134,11 +134,11 @@ __LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v)) \
134134
" stxr %w1, %0, %2\n" \
135135
" cbnz %w1, 1b" \
136136
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
137-
: "Ir" (i)); \
137+
: #constraint "r" (i)); \
138138
} \
139139
__LL_SC_EXPORT(arch_atomic64_##op);
140140

141-
#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
141+
#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
142142
__LL_SC_INLINE s64 \
143143
__LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\
144144
{ \
@@ -153,14 +153,14 @@ __LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\
153153
" cbnz %w1, 1b\n" \
154154
" " #mb \
155155
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
156-
: "Ir" (i) \
156+
: #constraint "r" (i) \
157157
: cl); \
158158
\
159159
return result; \
160160
} \
161161
__LL_SC_EXPORT(arch_atomic64_##op##_return##name);
162162

163-
#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
163+
#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
164164
__LL_SC_INLINE s64 \
165165
__LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v)) \
166166
{ \
@@ -175,7 +175,7 @@ __LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v)) \
175175
" cbnz %w2, 1b\n" \
176176
" " #mb \
177177
: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \
178-
: "Ir" (i) \
178+
: #constraint "r" (i) \
179179
: cl); \
180180
\
181181
return result; \
@@ -193,8 +193,8 @@ __LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
193193
ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \
194194
ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__)
195195

196-
ATOMIC64_OPS(add, add)
197-
ATOMIC64_OPS(sub, sub)
196+
ATOMIC64_OPS(add, add, I)
197+
ATOMIC64_OPS(sub, sub, J)
198198

199199
#undef ATOMIC64_OPS
200200
#define ATOMIC64_OPS(...) \
@@ -204,10 +204,10 @@ ATOMIC64_OPS(sub, sub)
204204
ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \
205205
ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__)
206206

207-
ATOMIC64_OPS(and, and)
208-
ATOMIC64_OPS(andnot, bic)
209-
ATOMIC64_OPS(or, orr)
210-
ATOMIC64_OPS(xor, eor)
207+
ATOMIC64_OPS(and, and, L)
208+
ATOMIC64_OPS(andnot, bic, )
209+
ATOMIC64_OPS(or, orr, L)
210+
ATOMIC64_OPS(xor, eor, L)
211211

212212
#undef ATOMIC64_OPS
213213
#undef ATOMIC64_FETCH_OP
@@ -237,7 +237,7 @@ __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
237237
}
238238
__LL_SC_EXPORT(arch_atomic64_dec_if_positive);
239239

240-
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
240+
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \
241241
__LL_SC_INLINE u##sz \
242242
__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
243243
unsigned long old, \
@@ -265,29 +265,34 @@ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
265265
"2:" \
266266
: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
267267
[v] "+Q" (*(u##sz *)ptr) \
268-
: [old] "Kr" (old), [new] "r" (new) \
268+
: [old] #constraint "r" (old), [new] "r" (new) \
269269
: cl); \
270270
\
271271
return oldval; \
272272
} \
273273
__LL_SC_EXPORT(__cmpxchg_case_##name##sz);
274274

275-
__CMPXCHG_CASE(w, b, , 8, , , , )
276-
__CMPXCHG_CASE(w, h, , 16, , , , )
277-
__CMPXCHG_CASE(w, , , 32, , , , )
278-
__CMPXCHG_CASE( , , , 64, , , , )
279-
__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory")
280-
__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory")
281-
__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory")
282-
__CMPXCHG_CASE( , , acq_, 64, , a, , "memory")
283-
__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory")
284-
__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory")
285-
__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory")
286-
__CMPXCHG_CASE( , , rel_, 64, , , l, "memory")
287-
__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
288-
__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
289-
__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
290-
__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
275+
/*
276+
* Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
277+
* handle the 'K' constraint for the value 4294967295 - thus we use no
278+
* constraint for 32 bit operations.
279+
*/
280+
__CMPXCHG_CASE(w, b, , 8, , , , , )
281+
__CMPXCHG_CASE(w, h, , 16, , , , , )
282+
__CMPXCHG_CASE(w, , , 32, , , , , )
283+
__CMPXCHG_CASE( , , , 64, , , , , L)
284+
__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", )
285+
__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", )
286+
__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", )
287+
__CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L)
288+
__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", )
289+
__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", )
290+
__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", )
291+
__CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L)
292+
__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", )
293+
__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", )
294+
__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", )
295+
__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L)
291296

292297
#undef __CMPXCHG_CASE
293298

0 commit comments

Comments
 (0)