Skip to content

Commit 4f07118

Browse files
davem330davem330
committed
[SPARC64]: More fully work around Spitfire Errata 51.
It appears that a memory barrier soon after a mispredicted branch, not just in the delay slot, can cause the hang condition of this cpu errata. So move them out-of-line, and explicitly put them into a "branch always, predict taken" delay slot which should fully kill this problem. Signed-off-by: David S. Miller <[email protected]>
1 parent 442464a commit 4f07118

File tree

14 files changed

+143
-54
lines changed

14 files changed

+143
-54
lines changed

‎arch/sparc64/kernel/pci_iommu.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu,
466466
if (!limit)
467467
break;
468468
udelay(1);
469-
membar("#LoadLoad");
469+
rmb();
470470
}
471471
if (!limit)
472472
printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout "

‎arch/sparc64/kernel/process.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ void cpu_idle(void)
103103
* other cpus see our increasing idleness for the buddy
104104
* redistribution algorithm. -DaveM
105105
*/
106-
membar("#StoreStore | #StoreLoad");
106+
membar_storeload_storestore();
107107
}
108108
}
109109

‎arch/sparc64/kernel/sbus.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long
147147
if (!limit)
148148
break;
149149
udelay(1);
150-
membar("#LoadLoad");
150+
rmb();
151151
}
152152
if (!limit)
153153
printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "

‎arch/sparc64/kernel/signal32.c‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -877,11 +877,12 @@ static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs,
877877
unsigned long page = (unsigned long)
878878
page_address(pte_page(*ptep));
879879

880-
__asm__ __volatile__(
881-
" membar #StoreStore\n"
882-
" flush %0 + %1"
883-
: : "r" (page), "r" (address & (PAGE_SIZE - 1))
884-
: "memory");
880+
wmb();
881+
__asm__ __volatile__("flush %0 + %1"
882+
: /* no outputs */
883+
: "r" (page),
884+
"r" (address & (PAGE_SIZE - 1))
885+
: "memory");
885886
}
886887
pte_unmap(ptep);
887888
preempt_enable();
@@ -1292,11 +1293,12 @@ static void setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs,
12921293
unsigned long page = (unsigned long)
12931294
page_address(pte_page(*ptep));
12941295

1295-
__asm__ __volatile__(
1296-
" membar #StoreStore\n"
1297-
" flush %0 + %1"
1298-
: : "r" (page), "r" (address & (PAGE_SIZE - 1))
1299-
: "memory");
1296+
wmb();
1297+
__asm__ __volatile__("flush %0 + %1"
1298+
: /* no outputs */
1299+
: "r" (page),
1300+
"r" (address & (PAGE_SIZE - 1))
1301+
: "memory");
13001302
}
13011303
pte_unmap(ptep);
13021304
preempt_enable();

‎arch/sparc64/kernel/smp.c‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ void __init smp_callin(void)
144144
current->active_mm = &init_mm;
145145

146146
while (!cpu_isset(cpuid, smp_commenced_mask))
147-
membar("#LoadLoad");
147+
rmb();
148148

149149
cpu_set(cpuid, cpu_online_map);
150150
}
@@ -184,11 +184,11 @@ static inline long get_delta (long *rt, long *master)
184184
for (i = 0; i < NUM_ITERS; i++) {
185185
t0 = tick_ops->get_tick();
186186
go[MASTER] = 1;
187-
membar("#StoreLoad");
187+
membar_storeload();
188188
while (!(tm = go[SLAVE]))
189-
membar("#LoadLoad");
189+
rmb();
190190
go[SLAVE] = 0;
191-
membar("#StoreStore");
191+
wmb();
192192
t1 = tick_ops->get_tick();
193193

194194
if (t1 - t0 < best_t1 - best_t0)
@@ -221,7 +221,7 @@ void smp_synchronize_tick_client(void)
221221
go[MASTER] = 1;
222222

223223
while (go[MASTER])
224-
membar("#LoadLoad");
224+
rmb();
225225

226226
local_irq_save(flags);
227227
{
@@ -273,21 +273,21 @@ static void smp_synchronize_one_tick(int cpu)
273273

274274
/* wait for client to be ready */
275275
while (!go[MASTER])
276-
membar("#LoadLoad");
276+
rmb();
277277

278278
/* now let the client proceed into his loop */
279279
go[MASTER] = 0;
280-
membar("#StoreLoad");
280+
membar_storeload();
281281

282282
spin_lock_irqsave(&itc_sync_lock, flags);
283283
{
284284
for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
285285
while (!go[MASTER])
286-
membar("#LoadLoad");
286+
rmb();
287287
go[MASTER] = 0;
288-
membar("#StoreStore");
288+
wmb();
289289
go[SLAVE] = tick_ops->get_tick();
290-
membar("#StoreLoad");
290+
membar_storeload();
291291
}
292292
}
293293
spin_unlock_irqrestore(&itc_sync_lock, flags);
@@ -927,11 +927,11 @@ void smp_capture(void)
927927
smp_processor_id());
928928
#endif
929929
penguins_are_doing_time = 1;
930-
membar("#StoreStore | #LoadStore");
930+
membar_storestore_loadstore();
931931
atomic_inc(&smp_capture_registry);
932932
smp_cross_call(&xcall_capture, 0, 0, 0);
933933
while (atomic_read(&smp_capture_registry) != ncpus)
934-
membar("#LoadLoad");
934+
rmb();
935935
#ifdef CAPTURE_DEBUG
936936
printk("done\n");
937937
#endif
@@ -947,7 +947,7 @@ void smp_release(void)
947947
smp_processor_id());
948948
#endif
949949
penguins_are_doing_time = 0;
950-
membar("#StoreStore | #StoreLoad");
950+
membar_storeload_storestore();
951951
atomic_dec(&smp_capture_registry);
952952
}
953953
}
@@ -970,9 +970,9 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs)
970970
save_alternate_globals(global_save);
971971
prom_world(1);
972972
atomic_inc(&smp_capture_registry);
973-
membar("#StoreLoad | #StoreStore");
973+
membar_storeload_storestore();
974974
while (penguins_are_doing_time)
975-
membar("#LoadLoad");
975+
rmb();
976976
restore_alternate_globals(global_save);
977977
atomic_dec(&smp_capture_registry);
978978
prom_world(0);

‎arch/sparc64/kernel/sparc64_ksyms.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,3 +406,12 @@ EXPORT_SYMBOL(xor_vis_4);
406406
EXPORT_SYMBOL(xor_vis_5);
407407

408408
EXPORT_SYMBOL(prom_palette);
409+
410+
/* memory barriers */
411+
EXPORT_SYMBOL(mb);
412+
EXPORT_SYMBOL(rmb);
413+
EXPORT_SYMBOL(wmb);
414+
EXPORT_SYMBOL(membar_storeload);
415+
EXPORT_SYMBOL(membar_storeload_storestore);
416+
EXPORT_SYMBOL(membar_storeload_loadload);
417+
EXPORT_SYMBOL(membar_storestore_loadstore);

‎arch/sparc64/lib/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
1212
U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
1313
U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
1414
copy_in_user.o user_fixup.o memmove.o \
15-
mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
15+
mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o mb.o
1616

1717
lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
1818
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o

‎arch/sparc64/lib/debuglocks.c‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,15 @@ void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller)
6161
: "=r" (val)
6262
: "r" (&(lock->lock))
6363
: "memory");
64-
membar("#StoreLoad | #StoreStore");
64+
membar_storeload_storestore();
6565
if (val) {
6666
while (lock->lock) {
6767
if (!--stuck) {
6868
if (shown++ <= 2)
6969
show(str, lock, caller);
7070
stuck = INIT_STUCK;
7171
}
72-
membar("#LoadLoad");
72+
rmb();
7373
}
7474
goto again;
7575
}
@@ -90,7 +90,7 @@ int _do_spin_trylock(spinlock_t *lock, unsigned long caller)
9090
: "=r" (val)
9191
: "r" (&(lock->lock))
9292
: "memory");
93-
membar("#StoreLoad | #StoreStore");
93+
membar_storeload_storestore();
9494
if (!val) {
9595
lock->owner_pc = ((unsigned int)caller);
9696
lock->owner_cpu = cpu;
@@ -107,7 +107,7 @@ void _do_spin_unlock(spinlock_t *lock)
107107
{
108108
lock->owner_pc = 0;
109109
lock->owner_cpu = NO_PROC_ID;
110-
membar("#StoreStore | #LoadStore");
110+
membar_storestore_loadstore();
111111
lock->lock = 0;
112112
current->thread.smp_lock_count--;
113113
}
@@ -129,7 +129,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller)
129129
show_read(str, rw, caller);
130130
stuck = INIT_STUCK;
131131
}
132-
membar("#LoadLoad");
132+
rmb();
133133
}
134134
/* Try once to increment the counter. */
135135
__asm__ __volatile__(
@@ -142,7 +142,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller)
142142
"2:" : "=r" (val)
143143
: "0" (&(rw->lock))
144144
: "g1", "g7", "memory");
145-
membar("#StoreLoad | #StoreStore");
145+
membar_storeload_storestore();
146146
if (val)
147147
goto wlock_again;
148148
rw->reader_pc[cpu] = ((unsigned int)caller);
@@ -201,7 +201,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller)
201201
show_write(str, rw, caller);
202202
stuck = INIT_STUCK;
203203
}
204-
membar("#LoadLoad");
204+
rmb();
205205
}
206206

207207
/* Try to acuire the write bit. */
@@ -256,7 +256,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller)
256256
show_write(str, rw, caller);
257257
stuck = INIT_STUCK;
258258
}
259-
membar("#LoadLoad");
259+
rmb();
260260
}
261261
goto wlock_again;
262262
}

‎arch/sparc64/lib/mb.S‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/* mb.S: Out of line memory barriers.
2+
*
3+
* Copyright (C) 2005 David S. Miller ([email protected])
4+
*/
5+
6+
/* These are here in an effort to more fully work around
7+
* Spitfire Errata #51. Essentially, if a memory barrier
8+
* occurs soon after a mispredicted branch, the chip can stop
9+
* executing instructions until a trap occurs. Therefore, if
10+
* interrupts are disabled, the chip can hang forever.
11+
*
12+
* It used to be believed that the memory barrier had to be
13+
* right in the delay slot, but a case has been traced
14+
* recently wherein the memory barrier was one instruction
15+
* after the branch delay slot and the chip still hung. The
16+
* offending sequence was the following in sym_wakeup_done()
17+
* of the sym53c8xx_2 driver:
18+
*
19+
* call sym_ccb_from_dsa, 0
20+
* movge %icc, 0, %l0
21+
* brz,pn %o0, .LL1303
22+
* mov %o0, %l2
23+
* membar #LoadLoad
24+
*
25+
* The branch has to be mispredicted for the bug to occur.
26+
* Therefore, we put the memory barrier explicitly into a
27+
* "branch always, predicted taken" delay slot to avoid the
28+
* problem case.
29+
*/
30+
31+
.text
32+
33+
99: retl
34+
nop
35+
36+
.globl mb
37+
mb: ba,pt %xcc, 99b
38+
membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad
39+
.size mb, .-mb
40+
41+
.globl rmb
42+
rmb: ba,pt %xcc, 99b
43+
membar #LoadLoad
44+
.size rmb, .-rmb
45+
46+
.globl wmb
47+
wmb: ba,pt %xcc, 99b
48+
membar #StoreStore
49+
.size wmb, .-wmb
50+
51+
.globl membar_storeload
52+
membar_storeload:
53+
ba,pt %xcc, 99b
54+
membar #StoreLoad
55+
.size membar_storeload, .-membar_storeload
56+
57+
.globl membar_storeload_storestore
58+
membar_storeload_storestore:
59+
ba,pt %xcc, 99b
60+
membar #StoreLoad | #StoreStore
61+
.size membar_storeload_storestore, .-membar_storeload_storestore
62+
63+
.globl membar_storeload_loadload
64+
membar_storeload_loadload:
65+
ba,pt %xcc, 99b
66+
membar #StoreLoad | #LoadLoad
67+
.size membar_storeload_loadload, .-membar_storeload_loadload
68+
69+
.globl membar_storestore_loadstore
70+
membar_storestore_loadstore:
71+
ba,pt %xcc, 99b
72+
membar #StoreStore | #LoadStore
73+
.size membar_storestore_loadstore, .-membar_storestore_loadstore

‎arch/sparc64/solaris/misc.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -737,7 +737,8 @@ MODULE_LICENSE("GPL");
737737
extern u32 tl0_solaris[8];
738738
#define update_ttable(x) \
739739
tl0_solaris[3] = (((long)(x) - (long)tl0_solaris - 3) >> 2) | 0x40000000; \
740-
__asm__ __volatile__ ("membar #StoreStore; flush %0" : : "r" (&tl0_solaris[3]))
740+
wmb(); \
741+
__asm__ __volatile__ ("flush %0" : : "r" (&tl0_solaris[3]))
741742
#else
742743
#endif
743744

@@ -761,7 +762,8 @@ int init_module(void)
761762
entry64_personality_patch |=
762763
(offsetof(struct task_struct, personality) +
763764
(sizeof(unsigned long) - 1));
764-
__asm__ __volatile__("membar #StoreStore; flush %0"
765+
wmb();
766+
__asm__ __volatile__("flush %0"
765767
: : "r" (&entry64_personality_patch));
766768
return 0;
767769
}

0 commit comments

Comments
 (0)