diff options
| -rw-r--r-- | arch/arm/kernel/smp.c | 123 | ||||
| -rw-r--r-- | arch/arm/mm/init.c | 2 | ||||
| -rw-r--r-- | arch/arm/mm/mm-armv.c | 2 | ||||
| -rw-r--r-- | include/asm-arm/system.h | 12 | ||||
| -rw-r--r-- | include/asm-arm/tlbflush.h | 28 |
5 files changed, 156 insertions, 11 deletions
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 34892758f09..a931409c8fe 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -502,3 +502,126 @@ int __init setup_profiling_timer(unsigned int multiplier) { return -EINVAL; } + +static int +on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait, + cpumask_t mask) +{ + int ret = 0; + + preempt_disable(); + + ret = smp_call_function_on_cpu(func, info, retry, wait, mask); + if (cpu_isset(smp_processor_id(), mask)) + func(info); + + preempt_enable(); + + return ret; +} + +/**********************************************************************/ + +/* + * TLB operations + */ +struct tlb_args { + struct vm_area_struct *ta_vma; + unsigned long ta_start; + unsigned long ta_end; +}; + +static inline void ipi_flush_tlb_all(void *ignored) +{ + local_flush_tlb_all(); +} + +static inline void ipi_flush_tlb_mm(void *arg) +{ + struct mm_struct *mm = (struct mm_struct *)arg; + + local_flush_tlb_mm(mm); +} + +static inline void ipi_flush_tlb_page(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_page(ta->ta_vma, ta->ta_start); +} + +static inline void ipi_flush_tlb_kernel_page(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_kernel_page(ta->ta_start); +} + +static inline void ipi_flush_tlb_range(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); +} + +static inline void ipi_flush_tlb_kernel_range(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end); +} + +void flush_tlb_all(void) +{ + on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + cpumask_t mask = mm->cpu_vm_mask; + + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +{ + cpumask_t mask = vma->vm_mm->cpu_vm_mask; + struct tlb_args ta; + + ta.ta_vma = vma; + ta.ta_start = uaddr; + + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask); +} + +void flush_tlb_kernel_page(unsigned long kaddr) +{ + struct tlb_args ta; + + ta.ta_start = kaddr; + + on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1); +} + +void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + cpumask_t mask = vma->vm_mm->cpu_vm_mask; + struct tlb_args ta; + + ta.ta_vma = vma; + ta.ta_start = start; + ta.ta_end = end; + + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + struct tlb_args ta; + + ta.ta_start = start; + ta.ta_end = end; + + on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1); +} diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 6dcb23d64bf..edffa47a4b2 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -437,7 +437,7 @@ void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc) memtable_init(mi); if (mdesc->map_io) mdesc->map_io(); - flush_tlb_all(); + local_flush_tlb_all(); /* * initialise the zones within each node diff --git a/arch/arm/mm/mm-armv.c b/arch/arm/mm/mm-armv.c index 052ab443ec4..c3bd503b43a 100644 --- a/arch/arm/mm/mm-armv.c +++ b/arch/arm/mm/mm-armv.c @@ -682,7 +682,7 @@ void __init memtable_init(struct meminfo *mi) } flush_cache_all(); - flush_tlb_all(); + local_flush_tlb_all(); top_pmd = pmd_off_k(0xffff0000); } diff --git a/include/asm-arm/system.h b/include/asm-arm/system.h index 3d0d2860b6d..cdf49f442fd 100644 --- a/include/asm-arm/system.h +++ b/include/asm-arm/system.h @@ -290,7 +290,6 @@ do { \ }) #ifdef CONFIG_SMP -#error SMP not supported #define smp_mb() mb() #define smp_rmb() rmb() @@ -304,6 +303,8 @@ do { \ #define smp_wmb() barrier() #define smp_read_barrier_depends() do { } while(0) +#endif /* CONFIG_SMP */ + #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) /* * On the StrongARM, "swp" is terminally broken since it bypasses the @@ -316,9 +317,16 @@ do { \ * * We choose (1) since its the "easiest" to achieve here and is not * dependent on the processor type. + * + * NOTE that this solution won't work on an SMP system, so explcitly + * forbid it here. */ +#ifdef CONFIG_SMP +#error SMP is not supported on SA1100/SA110 +#else #define swp_is_buggy #endif +#endif static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) { @@ -361,8 +369,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size return ret; } -#endif /* CONFIG_SMP */ - #endif /* __ASSEMBLY__ */ #define arch_align_stack(x) (x) diff --git a/include/asm-arm/tlbflush.h b/include/asm-arm/tlbflush.h index 8a864b11856..9387a5e1ffe 100644 --- a/include/asm-arm/tlbflush.h +++ b/include/asm-arm/tlbflush.h @@ -235,7 +235,7 @@ extern struct cpu_tlb_fns cpu_tlb; #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) -static inline void flush_tlb_all(void) +static inline void local_flush_tlb_all(void) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; @@ -253,7 +253,7 @@ static inline void flush_tlb_all(void) asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); } -static inline void flush_tlb_mm(struct mm_struct *mm) +static inline void local_flush_tlb_mm(struct mm_struct *mm) { const int zero = 0; const int asid = ASID(mm); @@ -282,7 +282,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm) } static inline void -flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; @@ -313,7 +313,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (uaddr)); } -static inline void flush_tlb_kernel_page(unsigned long kaddr) +static inline void local_flush_tlb_kernel_page(unsigned long kaddr) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; @@ -384,8 +384,24 @@ static inline void clean_pmd_entry(pmd_t *pmd) /* * Convert calls to our calling convention. */ -#define flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) -#define flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) +#define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) +#define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) + +#ifndef CONFIG_SMP +#define flush_tlb_all local_flush_tlb_all +#define flush_tlb_mm local_flush_tlb_mm +#define flush_tlb_page local_flush_tlb_page +#define flush_tlb_kernel_page local_flush_tlb_kernel_page +#define flush_tlb_range local_flush_tlb_range +#define flush_tlb_kernel_range local_flush_tlb_kernel_range +#else +extern void flush_tlb_all(void); +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); +extern void flush_tlb_kernel_page(unsigned long kaddr); +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +#endif /* * if PG_dcache_dirty is set for the page, we need to ensure that any |