aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--arch/sparc64/lib/atomic.S38
-rw-r--r--arch/sparc64/lib/bitops.S30
-rw-r--r--include/asm-sparc64/backoff.h28
3 files changed, 76 insertions, 20 deletions
diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
index 9633750167d..70ac4186f62 100644
--- a/arch/sparc64/lib/atomic.S
+++ b/arch/sparc64/lib/atomic.S
@@ -1,10 +1,10 @@
-/* $Id: atomic.S,v 1.4 2001/11/18 00:12:56 davem Exp $
- * atomic.S: These things are too big to do inline.
+/* atomic.S: These things are too big to do inline.
*
- * Copyright (C) 1999 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net)
*/
#include <asm/asi.h>
+#include <asm/backoff.h>
.text
@@ -16,27 +16,31 @@
.globl atomic_add
.type atomic_add,#function
atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
1: lduw [%o1], %g1
add %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %icc, 1b
+ bne,pn %icc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic_add, .-atomic_add
.globl atomic_sub
.type atomic_sub,#function
atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
1: lduw [%o1], %g1
sub %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %icc, 1b
+ bne,pn %icc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic_sub, .-atomic_sub
/* On SMP we need to use memory barriers to ensure
@@ -60,89 +64,101 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
.globl atomic_add_ret
.type atomic_add_ret,#function
atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
ATOMIC_PRE_BARRIER
1: lduw [%o1], %g1
add %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %icc, 1b
+ bne,pn %icc, 2f
add %g7, %o0, %g7
sra %g7, 0, %o0
ATOMIC_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic_add_ret, .-atomic_add_ret
.globl atomic_sub_ret
.type atomic_sub_ret,#function
atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
ATOMIC_PRE_BARRIER
1: lduw [%o1], %g1
sub %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %icc, 1b
+ bne,pn %icc, 2f
sub %g7, %o0, %g7
sra %g7, 0, %o0
ATOMIC_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic_sub_ret, .-atomic_sub_ret
.globl atomic64_add
.type atomic64_add,#function
atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
1: ldx [%o1], %g1
add %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic64_add, .-atomic64_add
.globl atomic64_sub
.type atomic64_sub,#function
atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
1: ldx [%o1], %g1
sub %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic64_sub, .-atomic64_sub
.globl atomic64_add_ret
.type atomic64_add_ret,#function
atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
ATOMIC_PRE_BARRIER
1: ldx [%o1], %g1
add %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
add %g7, %o0, %g7
mov %g7, %o0
ATOMIC_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic64_add_ret, .-atomic64_add_ret
.globl atomic64_sub_ret
.type atomic64_sub_ret,#function
atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+ BACKOFF_SETUP(%o2)
ATOMIC_PRE_BARRIER
1: ldx [%o1], %g1
sub %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
sub %g7, %o0, %g7
mov %g7, %o0
ATOMIC_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o2, %o3, 1b)
.size atomic64_sub_ret, .-atomic64_sub_ret
diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 892431a8213..6b015a6eefb 100644
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -1,10 +1,10 @@
-/* $Id: bitops.S,v 1.3 2001/11/18 00:12:56 davem Exp $
- * bitops.S: Sparc64 atomic bit operations.
+/* bitops.S: Sparc64 atomic bit operations.
*
- * Copyright (C) 2000 David S. Miller (davem@redhat.com)
+ * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
*/
#include <asm/asi.h>
+#include <asm/backoff.h>
.text
@@ -29,6 +29,7 @@
.globl test_and_set_bit
.type test_and_set_bit,#function
test_and_set_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %o2
@@ -40,18 +41,20 @@ test_and_set_bit: /* %o0=nr, %o1=addr */
or %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
and %g7, %o2, %g2
clr %o0
movrne %g2, 1, %o0
BITOP_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size test_and_set_bit, .-test_and_set_bit
.globl test_and_clear_bit
.type test_and_clear_bit,#function
test_and_clear_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %o2
@@ -63,18 +66,20 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */
andn %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
and %g7, %o2, %g2
clr %o0
movrne %g2, 1, %o0
BITOP_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size test_and_clear_bit, .-test_and_clear_bit
.globl test_and_change_bit
.type test_and_change_bit,#function
test_and_change_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %o2
@@ -86,18 +91,20 @@ test_and_change_bit: /* %o0=nr, %o1=addr */
xor %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
and %g7, %o2, %g2
clr %o0
movrne %g2, 1, %o0
BITOP_POST_BARRIER
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size test_and_change_bit, .-test_and_change_bit
.globl set_bit
.type set_bit,#function
set_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
srlx %o0, 6, %g1
mov 1, %o2
sllx %g1, 3, %g3
@@ -108,15 +115,17 @@ set_bit: /* %o0=nr, %o1=addr */
or %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size set_bit, .-set_bit
.globl clear_bit
.type clear_bit,#function
clear_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
srlx %o0, 6, %g1
mov 1, %o2
sllx %g1, 3, %g3
@@ -127,15 +136,17 @@ clear_bit: /* %o0=nr, %o1=addr */
andn %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size clear_bit, .-clear_bit
.globl change_bit
.type change_bit,#function
change_bit: /* %o0=nr, %o1=addr */
+ BACKOFF_SETUP(%o3)
srlx %o0, 6, %g1
mov 1, %o2
sllx %g1, 3, %g3
@@ -146,8 +157,9 @@ change_bit: /* %o0=nr, %o1=addr */
xor %g7, %o2, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
- bne,pn %xcc, 1b
+ bne,pn %xcc, 2f
nop
retl
nop
+2: BACKOFF_SPIN(%o3, %o4, 1b)
.size change_bit, .-change_bit
diff --git a/include/asm-sparc64/backoff.h b/include/asm-sparc64/backoff.h
new file mode 100644
index 00000000000..0e32f8b62fd
--- /dev/null
+++ b/include/asm-sparc64/backoff.h
@@ -0,0 +1,28 @@
+#ifndef _SPARC64_BACKOFF_H
+#define _SPARC64_BACKOFF_H
+
+#define BACKOFF_LIMIT (4 * 1024)
+
+#ifdef CONFIG_SMP
+
+#define BACKOFF_SETUP(reg) \
+ mov 1, reg
+
+#define BACKOFF_SPIN(reg, tmp, label) \
+ mov reg, tmp; \
+88: brnz,pt tmp, 88b; \
+ sub tmp, 1, tmp; \
+ cmp reg, BACKOFF_LIMIT; \
+ bg,pn %xcc, label; \
+ nop; \
+ ba,pt %xcc, label; \
+ sllx reg, 1, reg;
+
+#else
+
+#define BACKOFF_SETUP(reg)
+#define BACKOFF_SPIN(reg, tmp, label)
+
+#endif
+
+#endif /* _SPARC64_BACKOFF_H */