From 826681043d7184b4d650cab5b007b9a86b628eb5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 17 May 2009 16:20:18 +0100 Subject: [ARM] smp: fix cpumask usage in ARM SMP code The ARM SMP code wasn't properly updated for the cpumask changes, which results in smp_timer_broadcast() broadcasting ticks to non-online CPUs. Signed-off-by: Russell King --- arch/arm/kernel/smp.c | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 7801aac3c04..6014dfd22af 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -326,14 +326,14 @@ void __init smp_prepare_boot_cpu(void) per_cpu(cpu_data, cpu).idle = current; } -static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg) +static void send_ipi_message(const struct cpumask *mask, enum ipi_msg_type msg) { unsigned long flags; unsigned int cpu; local_irq_save(flags); - for_each_cpu_mask(cpu, callmap) { + for_each_cpu(cpu, mask) { struct ipi_data *ipi = &per_cpu(ipi_data, cpu); spin_lock(&ipi->lock); @@ -344,19 +344,19 @@ static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg) /* * Call the platform specific cross-CPU call function. */ - smp_cross_call(callmap); + smp_cross_call(mask); local_irq_restore(flags); } -void arch_send_call_function_ipi(cpumask_t mask) +void arch_send_call_function_ipi_mask(const struct cpumask *mask) { send_ipi_message(mask, IPI_CALL_FUNC); } void arch_send_call_function_single_ipi(int cpu) { - send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE); + send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); } void show_ipi_list(struct seq_file *p) @@ -498,17 +498,10 @@ asmlinkage void __exception do_IPI(struct pt_regs *regs) void smp_send_reschedule(int cpu) { - send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE); + send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -void smp_send_timer(void) -{ - cpumask_t mask = cpu_online_map; - cpu_clear(smp_processor_id(), mask); - send_ipi_message(mask, IPI_TIMER); -} - -void smp_timer_broadcast(cpumask_t mask) +void smp_timer_broadcast(const struct cpumask *mask) { send_ipi_message(mask, IPI_TIMER); } @@ -517,7 +510,7 @@ void smp_send_stop(void) { cpumask_t mask = cpu_online_map; cpu_clear(smp_processor_id(), mask); - send_ipi_message(mask, IPI_CPU_STOP); + send_ipi_message(&mask, IPI_CPU_STOP); } /* @@ -528,20 +521,17 @@ int setup_profiling_timer(unsigned int multiplier) return -EINVAL; } -static int -on_each_cpu_mask(void (*func)(void *), void *info, int wait, cpumask_t mask) +static void +on_each_cpu_mask(void (*func)(void *), void *info, int wait, + const struct cpumask *mask) { - int ret = 0; - preempt_disable(); - ret = smp_call_function_mask(mask, func, info, wait); - if (cpu_isset(smp_processor_id(), mask)) + smp_call_function_many(mask, func, info, wait); + if (cpumask_test_cpu(smp_processor_id(), mask)) func(info); preempt_enable(); - - return ret; } /**********************************************************************/ @@ -602,20 +592,17 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { - cpumask_t mask = mm->cpu_vm_mask; - - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { - cpumask_t mask = vma->vm_mm->cpu_vm_mask; struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); } void flush_tlb_kernel_page(unsigned long kaddr) @@ -630,14 +617,13 @@ void flush_tlb_kernel_page(unsigned long kaddr) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - cpumask_t mask = vma->vm_mm->cpu_vm_mask; struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = start; ta.ta_end = end; - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) -- cgit v1.2.3 From bc28248ee25e5c239cbe6afca35a100b08401de5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 17 May 2009 18:58:34 +0100 Subject: [ARM] smp: move core localtimer support out of platform specific files Signed-off-by: Russell King --- arch/arm/kernel/smp.c | 56 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 6014dfd22af..91130e218ae 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -32,6 +34,7 @@ #include #include #include +#include /* * as from 2.5, kernels no longer have an init_tasks structure @@ -274,9 +277,9 @@ asmlinkage void __cpuinit secondary_start_kernel(void) local_fiq_enable(); /* - * Setup local timer for this CPU. + * Setup the percpu timer for this CPU. */ - local_timer_setup(); + percpu_timer_setup(); calibrate_delay(); @@ -383,10 +386,16 @@ void show_local_irqs(struct seq_file *p) seq_putc(p, '\n'); } +/* + * Timer (local or broadcast) support + */ +static DEFINE_PER_CPU(struct clock_event_device, percpu_clockevent); + static void ipi_timer(void) { + struct clock_event_device *evt = &__get_cpu_var(percpu_clockevent); irq_enter(); - local_timer_interrupt(); + evt->event_handler(evt); irq_exit(); } @@ -405,6 +414,42 @@ asmlinkage void __exception do_local_timer(struct pt_regs *regs) } #endif +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST +static void smp_timer_broadcast(const struct cpumask *mask) +{ + send_ipi_message(mask, IPI_TIMER); +} + +static void broadcast_timer_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ +} + +static void local_timer_setup(struct clock_event_device *evt) +{ + evt->name = "dummy_timer"; + evt->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_DUMMY; + evt->rating = 400; + evt->mult = 1; + evt->set_mode = broadcast_timer_set_mode; + evt->broadcast = smp_timer_broadcast; + + clockevents_register_device(evt); +} +#endif + +void __cpuinit percpu_timer_setup(void) +{ + unsigned int cpu = smp_processor_id(); + struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); + + evt->cpumask = cpumask_of(cpu); + + local_timer_setup(evt); +} + static DEFINE_SPINLOCK(stop_lock); /* @@ -501,11 +546,6 @@ void smp_send_reschedule(int cpu) send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -void smp_timer_broadcast(const struct cpumask *mask) -{ - send_ipi_message(mask, IPI_TIMER); -} - void smp_send_stop(void) { cpumask_t mask = cpu_online_map; -- cgit v1.2.3 From e03cdade0ca945a04e982525e50fef275190b77b Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 28 May 2009 14:16:52 +0100 Subject: [ARM] smp: use new cpumask functions Convert cpu_*_mask bit twiddling to the new set_cpu_*() API. Signed-off-by: Russell King --- arch/arm/kernel/smp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 91130e218ae..0d8097fa4ca 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -166,7 +166,7 @@ int __cpuexit __cpu_disable(void) * Take this CPU offline. Once we clear this, we can't return, * and we must not schedule until we're ready to give up the cpu. */ - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); /* * OK - migrate IRQs away from this CPU @@ -288,7 +288,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void) /* * OK, now it's safe to let the boot CPU continue */ - cpu_set(cpu, cpu_online_map); + set_cpu_online(cpu, true); /* * OK, it's off to the idle thread for us @@ -462,7 +462,7 @@ static void ipi_cpu_stop(unsigned int cpu) dump_stack(); spin_unlock(&stop_lock); - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); local_fiq_disable(); local_irq_disable(); -- cgit v1.2.3 From faa7bc51c11d5bbe440ac04710fd7a3208782000 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Sat, 30 May 2009 14:00:14 +0100 Subject: Check whether the TLB operations need broadcasting on SMP systems ARMv7 SMP hardware can handle the TLB maintenance operations broadcasting in hardware so that the software can avoid the costly IPIs. This patch adds the necessary checks (the MMFR3 CPUID register) to avoid the broadcasting if already supported by the hardware. (this patch is based on the work done by Tony Thompson @ ARM) Signed-off-by: Catalin Marinas --- arch/arm/kernel/smp.c | 69 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 26 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 6014dfd22af..ece658e773a 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -32,6 +32,7 @@ #include #include #include +#include /* * as from 2.5, kernels no longer have an init_tasks structure @@ -545,6 +546,12 @@ struct tlb_args { unsigned long ta_end; }; +/* all SMP configurations have the extended CPUID registers */ +static inline int tlb_ops_need_broadcast(void) +{ + return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; +} + static inline void ipi_flush_tlb_all(void *ignored) { local_flush_tlb_all(); @@ -587,51 +594,61 @@ static inline void ipi_flush_tlb_kernel_range(void *arg) void flush_tlb_all(void) { - on_each_cpu(ipi_flush_tlb_all, NULL, 1); + if (tlb_ops_need_broadcast()) + on_each_cpu(ipi_flush_tlb_all, NULL, 1); + else + local_flush_tlb_all(); } void flush_tlb_mm(struct mm_struct *mm) { - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); + else + local_flush_tlb_mm(mm); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { - struct tlb_args ta; - - ta.ta_vma = vma; - ta.ta_start = uaddr; - - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = uaddr; + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); + } else + local_flush_tlb_page(vma, uaddr); } void flush_tlb_kernel_page(unsigned long kaddr) { - struct tlb_args ta; - - ta.ta_start = kaddr; - - on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = kaddr; + on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); + } else + local_flush_tlb_kernel_page(kaddr); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - struct tlb_args ta; - - ta.ta_vma = vma; - ta.ta_start = start; - ta.ta_end = end; - - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); + } else + local_flush_tlb_range(vma, start, end); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - struct tlb_args ta; - - ta.ta_start = start; - ta.ta_end = end; - - on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); + } else + local_flush_tlb_kernel_range(start, end); } -- cgit v1.2.3