diff options
| author | David S. Miller <davem@sunset.davemloft.net> | 2007-10-15 16:41:44 -0700 |
|---|---|---|
| committer | David S. Miller <davem@sunset.davemloft.net> | 2007-10-17 16:24:55 -0700 |
| commit | 24f287e412ae90de8d281543c8b1043b6ed6c019 (patch) | |
| tree | eb69803d187d35fd9e90c1428952c0ed5a0970c1 /arch/sparc64/lib/atomic.S | |
| parent | d85714d81cc0408daddb68c10f7fd69eafe7c213 (diff) | |
[SPARC64]: Implement atomic backoff.
When the cpu count is high and contention hits an atomic object, the
processors can synchronize such that some cpus continually get knocked
out and cannot complete the atomic update.
So implement an exponential backoff when SMP.
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc64/lib/atomic.S')
| -rw-r--r-- | arch/sparc64/lib/atomic.S | 38 |
1 files changed, 27 insertions, 11 deletions
diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S index 9633750167d..70ac4186f62 100644 --- a/arch/sparc64/lib/atomic.S +++ b/arch/sparc64/lib/atomic.S @@ -1,10 +1,10 @@ -/* $Id: atomic.S,v 1.4 2001/11/18 00:12:56 davem Exp $ - * atomic.S: These things are too big to do inline. +/* atomic.S: These things are too big to do inline. * - * Copyright (C) 1999 David S. Miller (davem@redhat.com) + * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) */ #include <asm/asi.h> +#include <asm/backoff.h> .text @@ -16,27 +16,31 @@ .globl atomic_add .type atomic_add,#function atomic_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 add %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_add, .-atomic_add .globl atomic_sub .type atomic_sub,#function atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 sub %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_sub, .-atomic_sub /* On SMP we need to use memory barriers to ensure @@ -60,89 +64,101 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ .globl atomic_add_ret .type atomic_add_ret,#function atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: lduw [%o1], %g1 add %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f add %g7, %o0, %g7 sra %g7, 0, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_add_ret, .-atomic_add_ret .globl atomic_sub_ret .type atomic_sub_ret,#function atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: lduw [%o1], %g1 sub %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f sub %g7, %o0, %g7 sra %g7, 0, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_sub_ret, .-atomic_sub_ret .globl atomic64_add .type atomic64_add,#function atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 add %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_add, .-atomic64_add .globl atomic64_sub .type atomic64_sub,#function atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 sub %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_sub, .-atomic64_sub .globl atomic64_add_ret .type atomic64_add_ret,#function atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: ldx [%o1], %g1 add %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f add %g7, %o0, %g7 mov %g7, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_add_ret, .-atomic64_add_ret .globl atomic64_sub_ret .type atomic64_sub_ret,#function atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: ldx [%o1], %g1 sub %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f sub %g7, %o0, %g7 mov %g7, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_sub_ret, .-atomic64_sub_ret |