From 298476220d1f793ca0ac6c9e5dc817e1ad3e9851 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 27 Sep 2006 14:57:44 +0900 Subject: sh: Add control register barriers. Currently when making changes to control registers, we typically need some time for changes to take effect (8 nops, generally). However, for sh4a we simply need to do an icbi.. This is a simple patch for implementing a general purpose ctrl_barrier() which functions as a control register write barrier. There's some additional documentation in the patch itself, but it's pretty self explanatory. There were also some places where we were not doing the barrier, which didn't seem to have any adverse effects on legacy parts, but certainly did on sh4a. It's safer to have the barrier in place for legacy parts as well in these cases, though this does make flush_tlb_all() more expensive (by an order of 8 nops). We can ifdef around the flush_tlb_all() case for now if it's clear that all legacy parts won't have a problem with this. Signed-off-by: Paul Mundt --- arch/sh/mm/fault.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch/sh/mm/fault.c') diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 775f86cd3fe..364181f27b7 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -369,12 +369,13 @@ void flush_tlb_all(void) * Flush all the TLB. * * Write to the MMU control register's bit: - * TF-bit for SH-3, TI-bit for SH-4. + * TF-bit for SH-3, TI-bit for SH-4. * It's same position, bit #2. */ local_irq_save(flags); status = ctrl_inl(MMUCR); - status |= 0x04; + status |= 0x04; ctrl_outl(status, MMUCR); + ctrl_barrier(); local_irq_restore(flags); } -- cgit v1.2.3 From 26ff6c11ef38e08990c1e417c299246e6ab18ff7 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 27 Sep 2006 15:13:36 +0900 Subject: sh: page table alloc cleanups and page fault optimizations. Cleanup of page table allocators, using generic folded PMD and PUD helpers. TLB flushing operations are moved to a more sensible spot. The page fault handler is also optimized slightly, we no longer waste cycles on IRQ disabling for flushing of the page from the ITLB, since we're already under CLI protection by the initial exception handler. Signed-off-by: Paul Mundt --- arch/sh/mm/fault.c | 202 +++++++++++------------------------------------------ 1 file changed, 40 insertions(+), 162 deletions(-) (limited to 'arch/sh/mm/fault.c') diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 364181f27b7..7a03ffe6dad 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -1,33 +1,20 @@ -/* $Id: fault.c,v 1.14 2004/01/13 05:52:11 kkojima Exp $ +/* + * Page fault handler for SH with an MMU. * - * linux/arch/sh/mm/fault.c * Copyright (C) 1999 Niibe Yutaka * Copyright (C) 2003 Paul Mundt * * Based on linux/arch/i386/mm/fault.c: * Copyright (C) 1995 Linus Torvalds + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. */ - -#include -#include #include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include - #include -#include -#include -#include #include -#include #include extern void die(const char *,struct pt_regs *,long); @@ -187,14 +174,25 @@ do_sigbus: goto no_context; } +#ifdef CONFIG_SH_STORE_QUEUES /* - * Called with interrupt disabled. + * This is a special case for the SH-4 store queues, as pages for this + * space still need to be faulted in before it's possible to flush the + * store queue cache for writeout to the remapped region. + */ +#define P3_ADDR_MAX (P4SEG_STORE_QUE + 0x04000000) +#else +#define P3_ADDR_MAX P4SEG +#endif + +/* + * Called with interrupts disabled. */ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, unsigned long address) { - unsigned long addrmax = P4SEG; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t entry; @@ -207,31 +205,36 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, kgdb_bus_err_hook(); #endif -#ifdef CONFIG_SH_STORE_QUEUES - addrmax = P4SEG_STORE_QUE + 0x04000000; -#endif - - if (address >= P3SEG && address < addrmax) { + /* + * We don't take page faults for P1, P2, and parts of P4, these + * are always mapped, whether it be due to legacy behaviour in + * 29-bit mode, or due to PMB configuration in 32-bit mode. + */ + if (address >= P3SEG && address < P3_ADDR_MAX) pgd = pgd_offset_k(address); - mm = NULL; - } else if (address >= TASK_SIZE) - return 1; - else if (!(mm = current->mm)) - return 1; - else - pgd = pgd_offset(mm, address); + else { + if (unlikely(address >= TASK_SIZE || !current->mm)) + return 1; + + pgd = pgd_offset(current->mm, address); + } - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (pud_none_or_clear_bad(pud)) + return 1; + pmd = pmd_offset(pud, address); if (pmd_none_or_clear_bad(pmd)) return 1; + if (mm) pte = pte_offset_map_lock(mm, pmd, address, &ptl); else pte = pte_offset_kernel(pmd, address); entry = *pte; - if (pte_none(entry) || pte_not_present(entry) - || (writeaccess && !pte_write(entry))) + if (unlikely(pte_none(entry) || pte_not_present(entry))) + goto unlock; + if (unlikely(writeaccess && !pte_write(entry))) goto unlock; if (writeaccess) @@ -243,13 +246,7 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, * ITLB is not affected by "ldtlb" instruction. * So, we need to flush the entry by ourselves. */ - - { - unsigned long flags; - local_irq_save(flags); - __flush_tlb_page(get_asid(), address&PAGE_MASK); - local_irq_restore(flags); - } + __flush_tlb_page(get_asid(), address & PAGE_MASK); #endif set_pte(pte, entry); @@ -260,122 +257,3 @@ unlock: pte_unmap_unlock(pte, ptl); return ret; } - -void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - if (vma->vm_mm && vma->vm_mm->context != NO_CONTEXT) { - unsigned long flags; - unsigned long asid; - unsigned long saved_asid = MMU_NO_ASID; - - asid = vma->vm_mm->context & MMU_CONTEXT_ASID_MASK; - page &= PAGE_MASK; - - local_irq_save(flags); - if (vma->vm_mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - __flush_tlb_page(asid, page); - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - local_irq_restore(flags); - } -} - -void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - - if (mm->context != NO_CONTEXT) { - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - mm->context = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm); - } else { - unsigned long asid = mm->context&MMU_CONTEXT_ASID_MASK; - unsigned long saved_asid = MMU_NO_ASID; - - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - if (mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - while (start < end) { - __flush_tlb_page(asid, start); - start += PAGE_SIZE; - } - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - } - local_irq_restore(flags); - } -} - -void flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - flush_tlb_all(); - } else { - unsigned long asid = init_mm.context&MMU_CONTEXT_ASID_MASK; - unsigned long saved_asid = get_asid(); - - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - set_asid(asid); - while (start < end) { - __flush_tlb_page(asid, start); - start += PAGE_SIZE; - } - set_asid(saved_asid); - } - local_irq_restore(flags); -} - -void flush_tlb_mm(struct mm_struct *mm) -{ - /* Invalidate all TLB of this process. */ - /* Instead of invalidating each TLB, we get new MMU context. */ - if (mm->context != NO_CONTEXT) { - unsigned long flags; - - local_irq_save(flags); - mm->context = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm); - local_irq_restore(flags); - } -} - -void flush_tlb_all(void) -{ - unsigned long flags, status; - - /* - * Flush all the TLB. - * - * Write to the MMU control register's bit: - * TF-bit for SH-3, TI-bit for SH-4. - * It's same position, bit #2. - */ - local_irq_save(flags); - status = ctrl_inl(MMUCR); - status |= 0x04; - ctrl_outl(status, MMUCR); - ctrl_barrier(); - local_irq_restore(flags); -} -- cgit v1.2.3 From f647d33f879d258de4ab2559975bd6eebda2033e Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 27 Sep 2006 15:30:24 +0900 Subject: sh: Fix split ptlock for user mappings in __do_page_fault(). There was a bug that got introduced when the split ptlock changes went in where mm could be unintialized for user mappings, this fixes it up.. Signed-off-by: Paul Mundt --- arch/sh/mm/fault.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/sh/mm/fault.c') diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 7a03ffe6dad..dc461d2bc18 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -210,10 +210,11 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, * are always mapped, whether it be due to legacy behaviour in * 29-bit mode, or due to PMB configuration in 32-bit mode. */ - if (address >= P3SEG && address < P3_ADDR_MAX) + if (address >= P3SEG && address < P3_ADDR_MAX) { pgd = pgd_offset_k(address); - else { - if (unlikely(address >= TASK_SIZE || !current->mm)) + mm = NULL; + } else { + if (unlikely(address >= TASK_SIZE || !(mm = current->mm))) return 1; pgd = pgd_offset(current->mm, address); -- cgit v1.2.3 From 0f08f338083cc1d68788ccbccc44bd0502fc57ae Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 27 Sep 2006 17:03:56 +0900 Subject: sh: More cosmetic cleanups and trivial fixes. Nothing exciting here, just trivial fixes.. Signed-off-by: Paul Mundt --- arch/sh/mm/fault.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'arch/sh/mm/fault.c') diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index dc461d2bc18..c69fd603226 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -13,6 +13,8 @@ */ #include #include +#include +#include #include #include #include @@ -188,15 +190,16 @@ do_sigbus: /* * Called with interrupts disabled. */ -asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, - unsigned long address) +asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs, + unsigned long writeaccess, + unsigned long address) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t entry; - struct mm_struct *mm; + struct mm_struct *mm = current->mm; spinlock_t *ptl; int ret = 1; @@ -214,10 +217,10 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess, pgd = pgd_offset_k(address); mm = NULL; } else { - if (unlikely(address >= TASK_SIZE || !(mm = current->mm))) + if (unlikely(address >= TASK_SIZE || !mm)) return 1; - pgd = pgd_offset(current->mm, address); + pgd = pgd_offset(mm, address); } pud = pud_offset(pgd, address); -- cgit v1.2.3