diff options
Diffstat (limited to 'arch/arm/kernel/entry-armv.S')
| -rw-r--r-- | arch/arm/kernel/entry-armv.S | 96 |
1 files changed, 58 insertions, 38 deletions
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index d645897652c..29dec080a60 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -339,16 +339,6 @@ __pabt_svc: str r1, [sp] @ save the "real" r0 copied @ from the exception stack -#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) -#ifndef CONFIG_MMU -#warning "NPTL on non MMU needs fixing" -#else - @ make sure our user space atomic helper is aborted - cmp r2, #TASK_SIZE - bichs r3, r3, #PSR_Z_BIT -#endif -#endif - @ @ We are now ready to fill in the remaining blanks on the stack: @ @@ -372,9 +362,25 @@ __pabt_svc: zero_fp .endm + .macro kuser_cmpxchg_check +#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) +#ifndef CONFIG_MMU +#warning "NPTL on non MMU needs fixing" +#else + @ Make sure our user space atomic helper is restarted + @ if it was interrupted in a critical region. Here we + @ perform a quick test inline since it should be false + @ 99.9999% of the time. The rest is done out of line. + cmp r2, #TASK_SIZE + blhs kuser_cmpxchg_fixup +#endif +#endif + .endm + .align 5 __dabt_usr: usr_entry + kuser_cmpxchg_check @ @ Call the processor-specific abort handler: @@ -404,6 +410,7 @@ __dabt_usr: .align 5 __irq_usr: usr_entry + kuser_cmpxchg_check #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_off @@ -446,9 +453,9 @@ __und_usr: @ @ r0 - instruction @ -1: ldrt r0, [r4] adr r9, ret_from_exception adr lr, __und_usr_unknown +1: ldrt r0, [r4] @ @ fallthrough to call_fpe @ @@ -669,7 +676,7 @@ __kuser_helper_start: * * Clobbered: * - * the Z flag might be lost + * none * * Definition and user space usage example: * @@ -730,9 +737,6 @@ __kuser_memory_barrier: @ 0xffff0fa0 * * - This routine already includes memory barriers as needed. * - * - A failure might be transient, i.e. it is possible, although unlikely, - * that "failure" be returned even if *ptr == oldval. - * * For example, a user space atomic_add implementation could look like this: * * #define atomic_add(ptr, val) \ @@ -769,46 +773,62 @@ __kuser_cmpxchg: @ 0xffff0fc0 #elif __LINUX_ARM_ARCH__ < 6 +#ifdef CONFIG_MMU + /* - * Theory of operation: - * - * We set the Z flag before loading oldval. If ever an exception - * occurs we can not be sure the loaded value will still be the same - * when the exception returns, therefore the user exception handler - * will clear the Z flag whenever the interrupted user code was - * actually from the kernel address space (see the usr_entry macro). - * - * The post-increment on the str is used to prevent a race with an - * exception happening just after the str instruction which would - * clear the Z flag although the exchange was done. + * The only thing that can break atomicity in this cmpxchg + * implementation is either an IRQ or a data abort exception + * causing another process/thread to be scheduled in the middle + * of the critical sequence. To prevent this, code is added to + * the IRQ and data abort exception handlers to set the pc back + * to the beginning of the critical section if it is found to be + * within that critical section (see kuser_cmpxchg_fixup). */ -#ifdef CONFIG_MMU - teq ip, ip @ set Z flag - ldr ip, [r2] @ load current val - add r3, r2, #1 @ prepare store ptr - teqeq ip, r0 @ compare with oldval if still allowed - streq r1, [r3, #-1]! @ store newval if still allowed - subs r0, r2, r3 @ if r2 == r3 the str occured +1: ldr r3, [r2] @ load current val + subs r3, r3, r0 @ compare with oldval +2: streq r1, [r2] @ store newval if eq + rsbs r0, r3, #0 @ set return val and C flag + usr_ret lr + + .text +kuser_cmpxchg_fixup: + @ Called from kuser_cmpxchg_check macro. + @ r2 = address of interrupted insn (must be preserved). + @ sp = saved regs. r7 and r8 are clobbered. + @ 1b = first critical insn, 2b = last critical insn. + @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. + mov r7, #0xffff0fff + sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) + subs r8, r2, r7 + rsbcss r8, r8, #(2b - 1b) + strcs r7, [sp, #S_PC] + mov pc, lr + .previous + #else #warning "NPTL on non MMU needs fixing" mov r0, #-1 adds r0, r0, #0 -#endif usr_ret lr +#endif #else #ifdef CONFIG_SMP mcr p15, 0, r0, c7, c10, 5 @ dmb #endif - ldrex r3, [r2] +1: ldrex r3, [r2] subs r3, r3, r0 strexeq r3, r1, [r2] + teqeq r3, #1 + beq 1b rsbs r0, r3, #0 + /* beware -- each __kuser slot must be 8 instructions max */ #ifdef CONFIG_SMP - mcr p15, 0, r0, c7, c10, 5 @ dmb -#endif + b __kuser_memory_barrier +#else usr_ret lr +#endif #endif @@ -829,7 +849,7 @@ __kuser_cmpxchg: @ 0xffff0fc0 * * Clobbered: * - * the Z flag might be lost + * none * * Definition and user space usage example: * |