diff options
Diffstat (limited to 'arch/sparc64')
| -rw-r--r-- | arch/sparc64/kernel/Makefile | 4 | ||||
| -rw-r--r-- | arch/sparc64/kernel/ebus.c | 2 | ||||
| -rw-r--r-- | arch/sparc64/kernel/iommu.c (renamed from arch/sparc64/kernel/pci_iommu.c) | 304 | ||||
| -rw-r--r-- | arch/sparc64/kernel/isa.c | 2 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci.c | 62 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci_fire.c | 24 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci_psycho.c | 32 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci_sabre.c | 35 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci_schizo.c | 42 | ||||
| -rw-r--r-- | arch/sparc64/kernel/pci_sun4v.c | 168 | ||||
| -rw-r--r-- | arch/sparc64/kernel/sbus.c | 568 |
11 files changed, 404 insertions, 839 deletions
diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile index b66876bf410..40d2f3aae91 100644 --- a/arch/sparc64/kernel/Makefile +++ b/arch/sparc64/kernel/Makefile @@ -8,14 +8,14 @@ EXTRA_CFLAGS := -Werror extra-y := head.o init_task.o vmlinux.lds obj-y := process.o setup.o cpu.o idprom.o \ - traps.o auxio.o una_asm.o sysfs.o \ + traps.o auxio.o una_asm.o sysfs.o iommu.o \ irq.o ptrace.o time.o sys_sparc.o signal.o \ unaligned.o central.o pci.o starfire.o semaphore.o \ power.o sbus.o iommu_common.o sparc64_ksyms.o chmc.o \ visemul.o prom.o of_device.o hvapi.o sstate.o mdesc.o obj-$(CONFIG_STACKTRACE) += stacktrace.o -obj-$(CONFIG_PCI) += ebus.o isa.o pci_common.o pci_iommu.o \ +obj-$(CONFIG_PCI) += ebus.o isa.o pci_common.o \ pci_psycho.o pci_sabre.o pci_schizo.o \ pci_sun4v.o pci_sun4v_asm.o pci_fire.o obj-$(CONFIG_SMP) += smp.o trampoline.o hvtramp.o diff --git a/arch/sparc64/kernel/ebus.c b/arch/sparc64/kernel/ebus.c index 6d2956179cd..bc9ae36f7a4 100644 --- a/arch/sparc64/kernel/ebus.c +++ b/arch/sparc64/kernel/ebus.c @@ -391,6 +391,8 @@ static void __init fill_ebus_device(struct device_node *dp, struct linux_ebus_de sd = &dev->ofdev.dev.archdata; sd->prom_node = dp; sd->op = &dev->ofdev; + sd->iommu = dev->bus->ofdev.dev.parent->archdata.iommu; + sd->stc = dev->bus->ofdev.dev.parent->archdata.stc; dev->ofdev.node = dp; dev->ofdev.dev.parent = &dev->bus->ofdev.dev; diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/iommu.c index 70d2364fdfe..b35a62167e9 100644 --- a/arch/sparc64/kernel/pci_iommu.c +++ b/arch/sparc64/kernel/iommu.c @@ -1,28 +1,32 @@ -/* pci_iommu.c: UltraSparc PCI controller IOM/STC support. +/* iommu.c: Generic sparc64 IOMMU support. * * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 1999, 2000 Jakub Jelinek (jakub@redhat.com) */ #include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/mm.h> +#include <linux/module.h> #include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> + +#ifdef CONFIG_PCI #include <linux/pci.h> +#endif -#include <asm/oplib.h> +#include <asm/iommu.h> #include "iommu_common.h" -#include "pci_impl.h" -#define PCI_STC_CTXMATCH_ADDR(STC, CTX) \ +#define STC_CTXMATCH_ADDR(STC, CTX) \ ((STC)->strbuf_ctxmatch_base + ((CTX) << 3)) +#define STC_FLUSHFLAG_INIT(STC) \ + (*((STC)->strbuf_flushflag) = 0UL) +#define STC_FLUSHFLAG_SET(STC) \ + (*((STC)->strbuf_flushflag) != 0UL) -/* Accessing IOMMU and Streaming Buffer registers. - * REG parameter is a physical address. All registers - * are 64-bits in size. - */ -#define pci_iommu_read(__reg) \ +#define iommu_read(__reg) \ ({ u64 __ret; \ __asm__ __volatile__("ldxa [%1] %2, %0" \ : "=r" (__ret) \ @@ -30,7 +34,7 @@ : "memory"); \ __ret; \ }) -#define pci_iommu_write(__reg, __val) \ +#define iommu_write(__reg, __val) \ __asm__ __volatile__("stxa %0, [%1] %2" \ : /* no outputs */ \ : "r" (__val), "r" (__reg), \ @@ -40,19 +44,19 @@ static void __iommu_flushall(struct iommu *iommu) { if (iommu->iommu_flushinv) { - pci_iommu_write(iommu->iommu_flushinv, ~(u64)0); + iommu_write(iommu->iommu_flushinv, ~(u64)0); } else { unsigned long tag; int entry; - tag = iommu->iommu_flush + (0xa580UL - 0x0210UL); + tag = iommu->iommu_tags; for (entry = 0; entry < 16; entry++) { - pci_iommu_write(tag, 0); + iommu_write(tag, 0); tag += 8; } /* Ensure completion of previous PIO writes. */ - (void) pci_iommu_read(iommu->write_complete_reg); + (void) iommu_read(iommu->write_complete_reg); } } @@ -80,7 +84,7 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) } /* Based largely upon the ppc64 iommu allocator. */ -static long pci_arena_alloc(struct iommu *iommu, unsigned long npages) +static long arena_alloc(struct iommu *iommu, unsigned long npages) { struct iommu_arena *arena = &iommu->arena; unsigned long n, i, start, end, limit; @@ -121,7 +125,7 @@ again: return n; } -static void pci_arena_free(struct iommu_arena *arena, unsigned long base, unsigned long npages) +static void arena_free(struct iommu_arena *arena, unsigned long base, unsigned long npages) { unsigned long i; @@ -129,7 +133,8 @@ static void pci_arena_free(struct iommu_arena *arena, unsigned long base, unsign __clear_bit(i, arena->map); } -void pci_iommu_table_init(struct iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask) +int iommu_table_init(struct iommu *iommu, int tsbsize, + u32 dma_offset, u32 dma_addr_mask) { unsigned long i, tsbbase, order, sz, num_tsb_entries; @@ -146,8 +151,8 @@ void pci_iommu_table_init(struct iommu *iommu, int tsbsize, u32 dma_offset, u32 sz = (sz + 7UL) & ~7UL; iommu->arena.map = kzalloc(sz, GFP_KERNEL); if (!iommu->arena.map) { - prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n"); - prom_halt(); + printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n"); + return -ENOMEM; } iommu->arena.limit = num_tsb_entries; @@ -156,8 +161,8 @@ void pci_iommu_table_init(struct iommu *iommu, int tsbsize, u32 dma_offset, u32 */ iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0); if (!iommu->dummy_page) { - prom_printf("PCI_IOMMU: Error, gfp(dummy_page) failed.\n"); - prom_halt(); + printk(KERN_ERR "IOMMU: Error, gfp(dummy_page) failed.\n"); + goto out_free_map; } memset((void *)iommu->dummy_page, 0, PAGE_SIZE); iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); @@ -166,20 +171,32 @@ void pci_iommu_table_init(struct iommu *iommu, int tsbsize, u32 dma_offset, u32 order = get_order(tsbsize); tsbbase = __get_free_pages(GFP_KERNEL, order); if (!tsbbase) { - prom_printf("PCI_IOMMU: Error, gfp(tsb) failed.\n"); - prom_halt(); + printk(KERN_ERR "IOMMU: Error, gfp(tsb) failed.\n"); + goto out_free_dummy_page; } iommu->page_table = (iopte_t *)tsbbase; for (i = 0; i < num_tsb_entries; i++) iopte_make_dummy(iommu, &iommu->page_table[i]); + + return 0; + +out_free_dummy_page: + free_page(iommu->dummy_page); + iommu->dummy_page = 0UL; + +out_free_map: + kfree(iommu->arena.map); + iommu->arena.map = NULL; + + return -ENOMEM; } static inline iopte_t *alloc_npages(struct iommu *iommu, unsigned long npages) { long entry; - entry = pci_arena_alloc(iommu, npages); + entry = arena_alloc(iommu, npages); if (unlikely(entry < 0)) return NULL; @@ -188,7 +205,7 @@ static inline iopte_t *alloc_npages(struct iommu *iommu, unsigned long npages) static inline void free_npages(struct iommu *iommu, dma_addr_t base, unsigned long npages) { - pci_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); + arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); } static int iommu_alloc_ctx(struct iommu *iommu) @@ -219,11 +236,8 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) } } -/* Allocate and map kernel buffer of size SIZE using consistent mode - * DMA for PCI device PDEV. Return non-NULL cpu-side address if - * successful and set *DMA_ADDRP to the PCI side dma address. - */ -static void *pci_4u_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) +static void *dma_4u_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addrp, gfp_t gfp) { struct iommu *iommu; iopte_t *iopte; @@ -241,7 +255,7 @@ static void *pci_4u_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr return NULL; memset((char *)first_page, 0, PAGE_SIZE << order); - iommu = pdev->dev.archdata.iommu; + iommu = dev->archdata.iommu; spin_lock_irqsave(&iommu->lock, flags); iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); @@ -268,15 +282,15 @@ static void *pci_4u_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr return ret; } -/* Free and unmap a consistent DMA translation. */ -static void pci_4u_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma) +static void dma_4u_free_coherent(struct device *dev, size_t size, + void *cpu, dma_addr_t dvma) { struct iommu *iommu; iopte_t *iopte; unsigned long flags, order, npages; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; - iommu = pdev->dev.archdata.iommu; + iommu = dev->archdata.iommu; iopte = iommu->page_table + ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); @@ -291,10 +305,8 @@ static void pci_4u_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, free_pages((unsigned long)cpu, order); } -/* Map a single buffer at PTR of SZ bytes for PCI DMA - * in streaming mode. - */ -static dma_addr_t pci_4u_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direction) +static dma_addr_t dma_4u_map_single(struct device *dev, void *ptr, size_t sz, + enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; @@ -304,10 +316,10 @@ static dma_addr_t pci_4u_map_single(struct pci_dev *pdev, void *ptr, size_t sz, u32 bus_addr, ret; unsigned long iopte_protection; - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; - if (unlikely(direction == PCI_DMA_NONE)) + if (unlikely(direction == DMA_NONE)) goto bad_no_ctx; oaddr = (unsigned long)ptr; @@ -332,7 +344,7 @@ static dma_addr_t pci_4u_map_single(struct pci_dev *pdev, void *ptr, size_t sz, iopte_protection = IOPTE_STREAMING(ctx); else iopte_protection = IOPTE_CONSISTENT(ctx); - if (direction != PCI_DMA_TODEVICE) + if (direction != DMA_TO_DEVICE) iopte_protection |= IOPTE_WRITE; for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) @@ -345,10 +357,12 @@ bad: bad_no_ctx: if (printk_ratelimit()) WARN_ON(1); - return PCI_DMA_ERROR_CODE; + return DMA_ERROR_CODE; } -static void pci_strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, u32 vaddr, unsigned long ctx, unsigned long npages, int direction) +static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, + u32 vaddr, unsigned long ctx, unsigned long npages, + enum dma_data_direction direction) { int limit; @@ -358,22 +372,22 @@ static void pci_strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, u32 vad u64 val; flushreg = strbuf->strbuf_ctxflush; - matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx); + matchreg = STC_CTXMATCH_ADDR(strbuf, ctx); - pci_iommu_write(flushreg, ctx); - val = pci_iommu_read(matchreg); + iommu_write(flushreg, ctx); + val = iommu_read(matchreg); val &= 0xffff; if (!val) goto do_flush_sync; while (val) { if (val & 0x1) - pci_iommu_write(flushreg, ctx); + iommu_write(flushreg, ctx); val >>= 1; } - val = pci_iommu_read(matchreg); + val = iommu_read(matchreg); if (unlikely(val)) { - printk(KERN_WARNING "pci_strbuf_flush: ctx flush " + printk(KERN_WARNING "strbuf_flush: ctx flush " "timeout matchreg[%lx] ctx[%lx]\n", val, ctx); goto do_page_flush; @@ -383,7 +397,7 @@ static void pci_strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, u32 vad do_page_flush: for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE) - pci_iommu_write(strbuf->strbuf_pflush, vaddr); + iommu_write(strbuf->strbuf_pflush, vaddr); } do_flush_sync: @@ -391,15 +405,15 @@ do_flush_sync: * the streaming cache, no flush-flag synchronization needs * to be performed. */ - if (direction == PCI_DMA_TODEVICE) + if (direction == DMA_TO_DEVICE) return; - PCI_STC_FLUSHFLAG_INIT(strbuf); - pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa); - (void) pci_iommu_read(iommu->write_complete_reg); + STC_FLUSHFLAG_INIT(strbuf); + iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa); + (void) iommu_read(iommu->write_complete_reg); limit = 100000; - while (!PCI_STC_FLUSHFLAG_SET(strbuf)) { + while (!STC_FLUSHFLAG_SET(strbuf)) { limit--; if (!limit) break; @@ -407,37 +421,32 @@ do_flush_sync: rmb(); } if (!limit) - printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout " + printk(KERN_WARNING "strbuf_flush: flushflag timeout " "vaddr[%08x] ctx[%lx] npages[%ld]\n", vaddr, ctx, npages); } -/* Unmap a single streaming mode DMA translation. */ -static void pci_4u_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) +static void dma_4u_unmap_single(struct device *dev, dma_addr_t bus_addr, + size_t sz, enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; iopte_t *base; unsigned long flags, npages, ctx, i; - if (unlikely(direction == PCI_DMA_NONE)) { + if (unlikely(direction == DMA_NONE)) { if (printk_ratelimit()) WARN_ON(1); return; } - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; base = iommu->page_table + ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); -#ifdef DEBUG_PCI_IOMMU - if (IOPTE_IS_DUMMY(iommu, base)) - printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n", - bus_addr, sz, __builtin_return_address(0)); -#endif bus_addr &= IO_PAGE_MASK; spin_lock_irqsave(&iommu->lock, flags); @@ -449,8 +458,8 @@ static void pci_4u_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ /* Step 1: Kick data out of streaming buffers if necessary. */ if (strbuf->strbuf_enabled) - pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, - npages, direction); + strbuf_flush(strbuf, iommu, bus_addr, ctx, + npages, direction); /* Step 2: Clear out TSB entries. */ for (i = 0; i < npages; i++) @@ -467,7 +476,8 @@ static void pci_4u_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ (__pa(page_address((SG)->page)) + (SG)->offset) static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, - int nused, int nelems, unsigned long iopte_protection) + int nused, int nelems, + unsigned long iopte_protection) { struct scatterlist *dma_sg = sg; struct scatterlist *sg_end = sg + nelems; @@ -539,12 +549,8 @@ static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, } } -/* Map a set of buffers described by SGLIST with NELEMS array - * elements in streaming mode for PCI DMA. - * When making changes here, inspect the assembly output. I was having - * hard time to keep this routine out of using stack slots for holding variables. - */ -static int pci_4u_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; @@ -557,19 +563,20 @@ static int pci_4u_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n /* Fast path single entry scatterlists. */ if (nelems == 1) { sglist->dma_address = - pci_4u_map_single(pdev, - (page_address(sglist->page) + sglist->offset), + dma_4u_map_single(dev, + (page_address(sglist->page) + + sglist->offset), sglist->length, direction); - if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE)) + if (unlikely(sglist->dma_address == DMA_ERROR_CODE)) return 0; sglist->dma_length = sglist->length; return 1; } - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; - - if (unlikely(direction == PCI_DMA_NONE)) + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + if (unlikely(direction == DMA_NONE)) goto bad_no_ctx; /* Step 1: Prepare scatter list. */ @@ -609,7 +616,7 @@ static int pci_4u_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n iopte_protection = IOPTE_STREAMING(ctx); else iopte_protection = IOPTE_CONSISTENT(ctx); - if (direction != PCI_DMA_TODEVICE) + if (direction != DMA_TO_DEVICE) iopte_protection |= IOPTE_WRITE; fill_sg(base, sglist, used, nelems, iopte_protection); @@ -628,8 +635,8 @@ bad_no_ctx: return 0; } -/* Unmap a set of streaming mode DMA translations. */ -static void pci_4u_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; @@ -637,14 +644,14 @@ static void pci_4u_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in unsigned long flags, ctx, i, npages; u32 bus_addr; - if (unlikely(direction == PCI_DMA_NONE)) { + if (unlikely(direction == DMA_NONE)) { if (printk_ratelimit()) WARN_ON(1); } - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; - + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + bus_addr = sglist->dma_address & IO_PAGE_MASK; for (i = 1; i < nelems; i++) @@ -657,11 +664,6 @@ static void pci_4u_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in base = iommu->page_table + ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); -#ifdef DEBUG_PCI_IOMMU - if (IOPTE_IS_DUMMY(iommu, base)) - printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0)); -#endif - spin_lock_irqsave(&iommu->lock, flags); /* Record the context, if any. */ @@ -671,7 +673,7 @@ static void pci_4u_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in /* Step 1: Kick data out of streaming buffers if necessary. */ if (strbuf->strbuf_enabled) - pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); + strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); /* Step 2: Clear out the TSB entries. */ for (i = 0; i < npages; i++) @@ -684,17 +686,16 @@ static void pci_4u_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in spin_unlock_irqrestore(&iommu->lock, flags); } -/* Make physical memory consistent for a single - * streaming mode DMA translation after a transfer. - */ -static void pci_4u_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) +static void dma_4u_sync_single_for_cpu(struct device *dev, + dma_addr_t bus_addr, size_t sz, + enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; unsigned long flags, ctx, npages; - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; if (!strbuf->strbuf_enabled) return; @@ -717,23 +718,22 @@ static void pci_4u_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_ } /* Step 2: Kick data out of streaming buffers. */ - pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); + strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); spin_unlock_irqrestore(&iommu->lock, flags); } -/* Make physical memory consistent for a set of streaming - * mode DMA translations after a transfer. - */ -static void pci_4u_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static void dma_4u_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sglist, int nelems, + enum dma_data_direction direction) { struct iommu *iommu; struct strbuf *strbuf; unsigned long flags, ctx, npages, i; u32 bus_addr; - iommu = pdev->dev.archdata.iommu; - strbuf = pdev->dev.archdata.stc; + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; if (!strbuf->strbuf_enabled) return; @@ -759,65 +759,51 @@ static void pci_4u_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist i--; npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> IO_PAGE_SHIFT; - pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); + strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); spin_unlock_irqrestore(&iommu->lock, flags); } -const struct pci_iommu_ops pci_sun4u_iommu_ops = { - .alloc_consistent = pci_4u_alloc_consistent, - .free_consistent = pci_4u_free_consistent, - .map_single = pci_4u_map_single, - .unmap_single = pci_4u_unmap_single, - .map_sg = pci_4u_map_sg, - .unmap_sg = pci_4u_unmap_sg, - .dma_sync_single_for_cpu = pci_4u_dma_sync_single_for_cpu, - .dma_sync_sg_for_cpu = pci_4u_dma_sync_sg_for_cpu, +const struct dma_ops sun4u_dma_ops = { + .alloc_coherent = dma_4u_alloc_coherent, + .free_coherent = dma_4u_free_coherent, + .map_single = dma_4u_map_single, + .unmap_single = dma_4u_unmap_single, + .map_sg = dma_4u_map_sg, + .unmap_sg = dma_4u_unmap_sg, + .sync_single_for_cpu = dma_4u_sync_single_for_cpu, + .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu, }; -static void ali_sound_dma_hack(struct pci_dev *pdev, int set_bit) -{ - struct pci_dev *ali_isa_bridge; - u8 val; +const struct dma_ops *dma_ops = &sun4u_dma_ops; +EXPORT_SYMBOL(dma_ops); - /* ALI sound chips generate 31-bits of DMA, a special register - * determines what bit 31 is emitted as. - */ - ali_isa_bridge = pci_get_device(PCI_VENDOR_ID_AL, - PCI_DEVICE_ID_AL_M1533, - NULL); - - pci_read_config_byte(ali_isa_bridge, 0x7e, &val); - if (set_bit) - val |= 0x01; - else - val &= ~0x01; - pci_write_config_byte(ali_isa_bridge, 0x7e, val); - pci_dev_put(ali_isa_bridge); -} - -int pci_dma_supported(struct pci_dev *pdev, u64 device_mask) +int dma_supported(struct device *dev, u64 device_mask) { - u64 dma_addr_mask; + struct iommu *iommu = dev->archdata.iommu; + u64 dma_addr_mask = iommu->dma_addr_mask; - if (pdev == NULL) { - dma_addr_mask = 0xffffffff; - } else { - struct iommu *iommu = pdev->dev.archdata.iommu; + if (device_mask >= (1UL << 32UL)) + return 0; - dma_addr_mask = iommu->dma_addr_mask; + if ((device_mask & dma_addr_mask) == dma_addr_mask) + return 1; - if (pdev->vendor == PCI_VENDOR_ID_AL && - pdev->device == PCI_DEVICE_ID_AL_M5451 && - device_mask == 0x7fffffff) { - ali_sound_dma_hack(pdev, - (dma_addr_mask & 0x80000000) != 0); - return 1; - } - } +#ifdef CONFIG_PCI + if (dev->bus == &pci_bus_type) + return pci_dma_supported(to_pci_dev(dev), device_mask); +#endif - if (device_mask >= (1UL << 32UL)) - return 0; + return 0; +} +EXPORT_SYMBOL(dma_supported); - return (device_mask & dma_addr_mask) == dma_addr_mask; +int dma_set_mask(struct device *dev, u64 dma_mask) +{ +#ifdef CONFIG_PCI + if (dev->bus == &pci_bus_type) + return pci_set_dma_mask(to_pci_dev(dev), dma_mask); +#endif + return -EINVAL; } +EXPORT_SYMBOL(dma_set_mask); diff --git a/arch/sparc64/kernel/isa.c b/arch/sparc64/kernel/isa.c index 1a1043fcf97..0f19dce1c90 100644 --- a/arch/sparc64/kernel/isa.c +++ b/arch/sparc64/kernel/isa.c @@ -90,6 +90,8 @@ static void __init isa_fill_devices(struct sparc_isa_bridge *isa_br) sd = &isa_dev->ofdev.dev.archdata; sd->prom_node = dp; sd->op = &isa_dev->ofdev; + sd->iommu = isa_br->ofdev.dev.parent->archdata.iommu; + sd->stc = isa_br->ofdev.dev.parent->archdata.stc; isa_dev->ofdev.node = dp; isa_dev->ofdev.dev.parent = &isa_br->ofdev.dev; diff --git a/arch/sparc64/kernel/pci.c b/arch/sparc64/kernel/pci.c index 77449a00575..3d93e9203ba 100644 --- a/arch/sparc64/kernel/pci.c +++ b/arch/sparc64/kernel/pci.c @@ -283,12 +283,6 @@ int __init pcic_present(void) return pci_controller_scan(pci_is_controller); } -const struct pci_iommu_ops *pci_iommu_ops; -EXPORT_SYMBOL(pci_iommu_ops); - -extern const struct pci_iommu_ops pci_sun4u_iommu_ops, - pci_sun4v_iommu_ops; - /* Find each controller in the system, attach and initialize * software state structure for each and link into the * pci_pbm_root. Setup the controller enough such @@ -296,11 +290,6 @@ extern const struct pci_iommu_ops pci_sun4u_iommu_ops, */ static void __init pci_controller_probe(void) { - if (tlb_type == hypervisor) - pci_iommu_ops = &pci_sun4v_iommu_ops; - else - pci_iommu_ops = &pci_sun4u_iommu_ops; - printk("PCI: Probing for controllers.\n"); pci_controller_scan(pci_controller_init); @@ -406,6 +395,10 @@ struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, sd->op = of_find_device_by_node(node); sd->msi_num = 0xffffffff; + sd = &sd->op->dev.archdata; + sd->iommu = pbm->iommu; + sd->stc = &pbm->stc; + type = of_get_property(node, "device_type", NULL); if (type == NULL) type = ""; @@ -1226,4 +1219,51 @@ struct device_node *pci_device_to_OF_node(struct pci_dev *pdev) } EXPORT_SYMBOL(pci_device_to_OF_node); +static void ali_sound_dma_hack(struct pci_dev *pdev, int set_bit) +{ + struct pci_dev *ali_isa_bridge; + u8 val; + + /* ALI sound chips generate 31-bits of DMA, a special register + * determines what bit 31 is emitted as. + */ + ali_isa_bridge = pci_get_device(PCI_VENDOR_ID_AL, + PCI_DEVICE_ID_AL_M1533, + NULL); + + pci_read_config_byte(ali_isa_bridge, 0x7e, &val); + if (set_bit) + val |= 0x01; + else + val &= ~0x01; + pci_write_config_byte(ali_isa_bridge, 0x7e, val); + pci_dev_put(ali_isa_bridge); +} + +int pci_dma_supported(struct pci_dev *pdev, u64 device_mask) +{ + u64 dma_addr_mask; + + if (pdev == NULL) { + dma_addr_mask = 0xffffffff; + } else { + struct iommu *iommu = pdev->dev.archdata.iommu; + + dma_addr_mask = iommu->dma_addr_mask; + + if (pdev->vendor == PCI_VENDOR_ID_AL && + pdev->device == PCI_DEVICE_ID_AL_M5451 && + device_mask == 0x7fffffff) { + ali_sound_dma_hack(pdev, + (dma_addr_mask & 0x80000000) != 0); + return 1; + } + } + + if (device_mask >= (1UL << 32UL)) + return 0; + + return (device_mask & dma_addr_mask) == dma_addr_mask; +} + #endif /* !(CONFIG_PCI) */ diff --git a/arch/sparc64/kernel/pci_fire.c b/arch/sparc64/kernel/pci_fire.c index 7f5d473901c..14d67fe21ab 100644 --- a/arch/sparc64/kernel/pci_fire.c +++ b/arch/sparc64/kernel/pci_fire.c @@ -39,12 +39,12 @@ static void pci_fire_scan_bus(struct pci_pbm_info *pbm) #define FIRE_IOMMU_FLUSH 0x40100UL #define FIRE_IOMMU_FLUSHINV 0x40108UL -static void pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) +static int pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) { struct iommu *iommu = pbm->iommu; u32 vdma[2], dma_mask; u64 control; - int tsbsize; + int tsbsize, err; /* No virtual-dma property on these guys, use largest size. */ vdma[0] = 0xc0000000; /* base */ @@ -68,7 +68,9 @@ static void pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) */ fire_write(iommu->iommu_flushinv, ~(u64)0); - pci_iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + if (err) + return err; fire_write(iommu->iommu_tsbbase, __pa(iommu->page_table) | 0x7UL); @@ -78,6 +80,8 @@ static void pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) 0x00000002 /* Bypass enable */ | 0x00000001 /* Translation enable */); fire_write(iommu->iommu_control, control); + + return 0; } /* Based at pbm->controller_regs */ @@ -167,8 +171,8 @@ static void pci_fire_hw_init(struct pci_pbm_info *pbm) fire_write(pbm->pbm_regs + FIRE_PEC_IENAB, ~(u64)0); } -static void pci_fire_pbm_init(struct pci_controller_info *p, - struct device_node *dp, u32 portid) +static int pci_fire_pbm_init(struct pci_controller_info *p, + struct device_node *dp, u32 portid) { const struct linux_prom64_registers *regs; struct pci_pbm_info *pbm; @@ -203,7 +207,8 @@ static void pci_fire_pbm_init(struct pci_controller_info *p, pci_get_pbm_props(pbm); pci_fire_hw_init(pbm); - pci_fire_pbm_iommu_init(pbm); + + return pci_fire_pbm_iommu_init(pbm); } static inline int portid_compare(u32 x, u32 y) @@ -222,7 +227,8 @@ void fire_pci_init(struct device_node *dp, const char *model_name) for (pbm = pci_pbm_root; pbm; pbm = pbm->next) { if (portid_compare(pbm->portid, portid)) { - pci_fire_pbm_init(pbm->parent, dp, portid); + if (pci_fire_pbm_init(pbm->parent, dp, portid)) + goto fatal_memory_error; return; } } @@ -250,7 +256,9 @@ void fire_pci_init(struct device_node *dp, const char *model_name) */ pci_memspace_mask = 0x7fffffffUL; - pci_fire_pbm_init(p, dp, portid); + if (pci_fire_pbm_init(p, dp, portid)) + goto fatal_memory_error; + return; fatal_memory_error: diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c index 598393a2df1..b6b4cfea5b5 100644 --- a/arch/sparc64/kernel/pci_psycho.c +++ b/arch/sparc64/kernel/pci_psycho.c @@ -813,16 +813,19 @@ static void psycho_scan_bus(struct pci_pbm_info *pbm) psycho_register_error_handlers(pbm); } -static void psycho_iommu_init(struct pci_pbm_info *pbm) +static int psycho_iommu_init(struct pci_pbm_info *pbm) { struct iommu *iommu = pbm->iommu; unsigned long i; u64 control; + int err; /* Register addresses. */ iommu->iommu_control = pbm->controller_regs + PSYCHO_IOMMU_CONTROL; iommu->iommu_tsbbase = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE; iommu->iommu_flush = pbm->controller_regs + PSYCHO_IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL); + /* PSYCHO's IOMMU lacks ctx flushing. */ iommu->iommu_ctxflush = 0; @@ -845,7 +848,9 @@ static void psycho_iommu_init(struct pci_pbm_info *pbm) /* Leave diag mode enabled for full-flushing done * in pci_iommu.c */ - pci_iommu_table_init(iommu, IO_TSB_SIZE, 0xc0000000, 0xffffffff); + err = iommu_table_init(iommu, IO_TSB_SIZE, 0xc0000000, 0xffffffff); + if (err) + return err; psycho_write(pbm->controller_regs + PSYCHO_IOMMU_TSBBASE, __pa(iommu->page_table)); @@ -858,6 +863,8 @@ static void psycho_iommu_init(struct pci_pbm_info *pbm) /* If necessary, hook us up for starfire IRQ translations. */ if (this_is_starfire) starfire_hookup(pbm->portid); + + return 0; } #define PSYCHO_IRQ_RETRY 0x1a00UL @@ -1031,15 +1038,12 @@ void psycho_init(struct device_node *dp, char *model_name) } p = kzalloc(sizeof(struct pci_controller_info), GFP_ATOMIC); - if (!p) { - prom_printf("PSYCHO: Fatal memory allocation error.\n"); - prom_halt(); - } + if (!p) + goto fatal_memory_error; iommu = kzalloc(sizeof(struct iommu), GFP_ATOMIC); - if (!iommu) { - prom_printf("PSYCHO: Fatal memory allocation error.\n"); - prom_halt(); - } + if (!iommu) + goto fatal_memory_error; + p->pbm_A.iommu = p->pbm_B.iommu = iommu; p->pbm_A.portid = upa_portid; @@ -1062,8 +1066,14 @@ void psycho_init(struct device_node *dp, char *model_name) psycho_controller_hwinit(&p->pbm_A); - psycho_iommu_init(&p->pbm_A); + if (psycho_iommu_init(&p->pbm_A)) + goto fatal_memory_error; is_pbm_a = ((pr_regs[0].phys_addr & 0x6000) == 0x2000); psycho_pbm_init(p, dp, is_pbm_a); + return; + +fatal_memory_error: + prom_printf("PSYCHO: Fatal memory allocation error.\n"); + prom_halt(); } diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c index 22e1be5c748..fba67c3d880 100644 --- a/arch/sparc64/kernel/pci_sabre.c +++ b/arch/sparc64/kernel/pci_sabre.c @@ -672,18 +672,20 @@ static void sabre_scan_bus(struct pci_pbm_info *pbm) sabre_register_error_handlers(pbm); } -static void sabre_iommu_init(struct pci_pbm_info *pbm, - int tsbsize, unsigned long dvma_offset, - u32 dma_mask) +static int sabre_iommu_init(struct pci_pbm_info *pbm, + int tsbsize, unsigned long dvma_offset, + u32 dma_mask) { struct iommu *iommu = pbm->iommu; unsigned long i; u64 control; + int err; /* Register addresses. */ iommu->iommu_control = pbm->controller_regs + SABRE_IOMMU_CONTROL; iommu->iommu_tsbbase = pbm->controller_regs + SABRE_IOMMU_TSBBASE; iommu->iommu_flush = pbm->controller_regs + SABRE_IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL); iommu->write_complete_reg = pbm->controller_regs + SABRE_WRSYNC; /* Sabre's IOMMU lacks ctx flushing. */ iommu->iommu_ctxflush = 0; @@ -701,7 +703,10 @@ static void sabre_iommu_init(struct pci_pbm_info *pbm, /* Leave diag mode enabled for full-flushing done * in pci_iommu.c */ - pci_iommu_table_init(iommu, tsbsize * 1024 * 8, dvma_offset, dma_mask); + err = iommu_table_init(iommu, tsbsize * 1024 * 8, + dvma_offset, dma_mask); + if (err) + return err; sabre_write(pbm->controller_regs + SABRE_IOMMU_TSBBASE, __pa(iommu->page_table)); @@ -722,6 +727,8 @@ static void sabre_iommu_init(struct pci_pbm_info *pbm, break; } sabre_write(pbm->controller_regs + SABRE_IOMMU_CONTROL, control); + + return 0; } static void sabre_pbm_init(struct pci_controller_info *p, struct pci_pbm_info *pbm, struct device_node *dp) @@ -775,16 +782,12 @@ void sabre_init(struct device_node *dp, char *model_name) } p = kzalloc(sizeof(*p), GFP_ATOMIC); - if (!p) { - prom_printf("SABRE: Error, kmalloc(pci_controller_info) failed.\n"); - prom_halt(); - } + if (!p) + goto fatal_memory_error; iommu = kzalloc(sizeof(*iommu), GFP_ATOMIC); - if (!iommu) { - prom_printf("SABRE: Error, kmalloc(pci_iommu) failed.\n"); - prom_halt(); - } + if (!iommu) + goto fatal_memory_error; pbm = &p->pbm_A; pbm->iommu = iommu; @@ -847,10 +850,16 @@ void sabre_init(struct device_node *dp, char *model_name) prom_halt(); } - sabre_iommu_init(pbm, tsbsize, vdma[0], dma_mask); + if (sabre_iommu_init(pbm, tsbsize, vdma[0], dma_mask)) + goto fatal_memory_error; /* * Look for APB underneath. */ sabre_pbm_init(p, pbm, dp); + return; + +fatal_memory_error: + prom_printf("SABRE: Fatal memory allocation error.\n"); + prom_halt(); } diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c index ae76898bbe2..3c30bfa1f3a 100644 --- a/arch/sparc64/kernel/pci_schizo.c +++ b/arch/sparc64/kernel/pci_schizo.c @@ -1148,14 +1148,14 @@ static void schizo_pbm_strbuf_init(struct pci_pbm_info *pbm) #define SCHIZO_IOMMU_FLUSH (0x00210UL) #define SCHIZO_IOMMU_CTXFLUSH (0x00218UL) -static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) +static int schizo_pbm_iommu_init(struct pci_pbm_info *pbm) { struct iommu *iommu = pbm->iommu; unsigned long i, tagbase, database; struct property *prop; u32 vdma[2], dma_mask; + int tsbsize, err; u64 control; - int tsbsize; prop = of_find_property(pbm->prom_node, "virtual-dma", NULL); if (prop) { @@ -1195,6 +1195,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) iommu->iommu_control = pbm->pbm_regs + SCHIZO_IOMMU_CONTROL; iommu->iommu_tsbbase = pbm->pbm_regs + SCHIZO_IOMMU_TSBBASE; iommu->iommu_flush = pbm->pbm_regs + SCHIZO_IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL); iommu->iommu_ctxflush = pbm->pbm_regs + SCHIZO_IOMMU_CTXFLUSH; /* We use the main control/status register of SCHIZO as the write @@ -1219,7 +1220,9 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) /* Leave diag mode enabled for full-flushing done * in pci_iommu.c */ - pci_iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask); + if (err) + return err; schizo_write(iommu->iommu_tsbbase, __pa(iommu->page_table)); @@ -1236,6 +1239,8 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) control |= SCHIZO_IOMMU_CTRL_ENAB; schizo_write(iommu->iommu_control, control); + + return 0; } #define SCHIZO_PCI_IRQ_RETRY (0x1a00UL) @@ -1328,14 +1333,14 @@ static void schizo_pbm_hw_init(struct pci_pbm_info *pbm) } } -static void schizo_pbm_init(struct pci_controller_info *p, - struct device_node *dp, u32 portid, - int chip_type) +static int schizo_pbm_init(struct pci_controller_info *p, + struct device_node *dp, u32 portid, + int chip_type) { const struct linux_prom64_registers *regs; struct pci_pbm_info *pbm; const char *chipset_name; - int is_pbm_a; + int is_pbm_a, err; switch (chip_type) { case PBM_CHIP_TYPE_TOMATILLO: @@ -1406,8 +1411,13 @@ static void schizo_pbm_init(struct pci_controller_info *p, pci_get_pbm_props(pbm); - schizo_pbm_iommu_init(pbm); + err = schizo_pbm_iommu_init(pbm); + if (err) + return err; + schizo_pbm_strbuf_init(pbm); + + return 0; } static inline int portid_compare(u32 x, u32 y, int chip_type) @@ -1431,34 +1441,38 @@ static void __schizo_init(struct device_node *dp, char *model_name, int chip_typ for (pbm = pci_pbm_root; pbm; pbm = pbm->next) { if (portid_compare(pbm->portid, portid, chip_type)) { - schizo_pbm_init(pbm->parent, dp, portid, chip_type); + if (schizo_pbm_init(pbm->parent, dp, + portid, chip_type)) + goto fatal_memory_error; return; } } p = kzalloc(sizeof(struct pci_controller_info), GFP_ATOMIC); if (!p) - goto memfail; + goto fatal_memory_error; iommu = kzalloc(sizeof(struct iommu), GFP_ATOMIC); if (!iommu) - goto memfail; + goto fatal_memory_error; p->pbm_A.iommu = iommu; iommu = kzalloc(sizeof(struct iommu), GFP_ATOMIC); if (!iommu) - goto memfail; + goto fatal_memory_error; p->pbm_B.iommu = iommu; /* Like PSYCHO we have a 2GB aligned area for memory space. */ pci_memspace_mask = 0x7fffffffUL; - schizo_pbm_init(p, dp, portid, chip_type); + if (schizo_pbm_init(p, dp, portid, chip_type)) + goto fatal_memory_error; + return; -memfail: +fatal_memory_error: prom_printf("SCHIZO: Fatal memory allocation error.\n"); prom_halt(); } diff --git a/arch/sparc64/kernel/pci_sun4v.c b/arch/sparc64/kernel/pci_sun4v.c index 639cf06ca37..466f4aa8fc8 100644 --- a/arch/sparc64/kernel/pci_sun4v.c +++ b/arch/sparc64/kernel/pci_sun4v.c @@ -33,30 +33,30 @@ static unsigned long vpci_minor = 1; #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) struct iommu_batch { - struct pci_dev *pdev; /* Device mapping is for. */ + struct device *dev; /* Device mapping is for. */ unsigned long prot; /* IOMMU page protections */ unsigned long entry; /* Index into IOTSB. */ u64 *pglist; /* List of physical pages */ unsigned long npages; /* Number of pages in list. */ }; -static DEFINE_PER_CPU(struct iommu_batch, pci_iommu_batch); +static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); /* Interrupts must be disabled. */ -static inline void pci_iommu_batch_start(struct pci_dev *pdev, unsigned long prot, unsigned long entry) +static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) { - struct iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + struct iommu_batch *p = &__get_cpu_var(iommu_batch); - p->pdev = pdev; + p->dev = dev; p->prot = prot; p->entry = entry; p->npages = 0; } /* Interrupts must be disabled. */ -static long pci_iommu_batch_flush(struct iommu_batch *p) +static long iommu_batch_flush(struct iommu_batch *p) { - struct pci_pbm_info *pbm = p->pdev->dev.archdata.host_controller; + struct pci_pbm_info *pbm = p->dev->archdata.host_controller; unsigned long devhandle = pbm->devhandle; unsigned long prot = p->prot; unsigned long entry = p->entry; @@ -70,7 +70,7 @@ static long pci_iommu_batch_flush(struct iommu_batch *p) npages, prot, __pa(pglist)); if (unlikely(num < 0)) { if (printk_ratelimit()) - printk("pci_iommu_batch_flush: IOMMU map of " + printk("iommu_batch_flush: IOMMU map of " "[%08lx:%08lx:%lx:%lx:%lx] failed with " "status %ld\n", devhandle, HV_PCI_TSBID(0, entry), @@ -90,30 +90,30 @@ static long pci_iommu_batch_flush(struct iommu_batch *p) } /* Interrupts must be disabled. */ -static inline long pci_iommu_batch_add(u64 phys_page) +static inline long iommu_batch_add(u64 phys_page) { - struct iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + struct iommu_batch *p = &__get_cpu_var(iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); p->pglist[p->npages++] = phys_page; if (p->npages == PGLIST_NENTS) - return pci_iommu_batch_flush(p); + return iommu_batch_flush(p); return 0; } /* Interrupts must be disabled. */ -static inline long pci_iommu_batch_end(void) +static inline long iommu_batch_end(void) { - struct iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + struct iommu_batch *p = &__get_cpu_var(iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); - return pci_iommu_batch_flush(p); + return iommu_batch_flush(p); } -static long pci_arena_alloc(struct iommu_arena *arena, unsigned long npages) +static long arena_alloc(struct iommu_arena *arena, unsigned long npages) { unsigned long n, i, start, end, limit; int pass; @@ -152,7 +152,8 @@ again: return n; } -static void pci_arena_free(struct iommu_arena *arena, unsigned long base, unsigned long npages) +static void arena_free(struct iommu_arena *arena, unsigned long base, + unsigned long npages) { unsigned long i; @@ -160,7 +161,8 @@ static void pci_arena_free(struct iommu_arena *arena, unsigned long base, unsign __clear_bit(i, arena->map); } -static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) +static void *dma_4v_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addrp, gfp_t gfp) { struct iommu *iommu; unsigned long flags, order, first_page, npages, n; @@ -180,10 +182,10 @@ static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr memset((char *)first_page, 0, PAGE_SIZE << order); - iommu = pdev->dev.archdata.iommu; + iommu = dev->archdata.iommu; spin_lock_irqsave(&iommu->lock, flags); - entry = pci_arena_alloc(&iommu->arena, npages); + entry = arena_alloc(&iommu->arena, npages); spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(entry < 0L)) @@ -196,18 +198,18 @@ static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr local_irq_save(flags); - pci_iommu_batch_start(pdev, - (HV_PCI_MAP_ATTR_READ | - HV_PCI_MAP_ATTR_WRITE), - entry); + iommu_batch_start(dev, + (HV_PCI_MAP_ATTR_READ | + HV_PCI_MAP_ATTR_WRITE), + entry); for (n = 0; n < npages; n++) { - long err = pci_iommu_batch_add(first_page + (n * PAGE_SIZE)); + long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); if (unlikely(err < 0L)) goto iommu_map_fail; } - if (unlikely(pci_iommu_batch_end() < 0L)) + if (unlikely(iommu_batch_end() < 0L)) goto iommu_map_fail; local_irq_restore(flags); @@ -217,7 +219,7 @@ static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); spin_unlock_irqrestore(&iommu->lock, flags); arena_alloc_fail: @@ -225,7 +227,8 @@ arena_alloc_fail: return NULL; } -static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma) +static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, + dma_addr_t dvma) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -233,14 +236,14 @@ static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, u32 devhandle; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; - iommu = pdev->dev.archdata.iommu; - pbm = pdev->dev.archdata.host_controller; + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); spin_lock_irqsave(&iommu->lock, flags); - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); do { unsigned long num; @@ -258,7 +261,8 @@ static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, free_pages((unsigned long)cpu, order); } -static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direction) +static dma_addr_t dma_4v_map_single(struct device *dev, void *ptr, size_t sz, + enum dma_data_direction direction) { struct iommu *iommu; unsigned long flags, npages, oaddr; @@ -267,9 +271,9 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, unsigned long prot; long entry; - iommu = pdev->dev.archdata.iommu; + iommu = dev->archdata.iommu; - if (unlikely(direction == PCI_DMA_NONE)) + if (unlikely(direction == DMA_NONE)) goto bad; oaddr = (unsigned long)ptr; @@ -277,7 +281,7 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, npages >>= IO_PAGE_SHIFT; spin_lock_irqsave(&iommu->lock, flags); - entry = pci_arena_alloc(&iommu->arena, npages); + entry = arena_alloc(&iommu->arena, npages); spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(entry < 0L)) @@ -288,19 +292,19 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, ret = bus_addr | (oaddr & ~IO_PAGE_MASK); base_paddr = __pa(oaddr & IO_PAGE_MASK); prot = HV_PCI_MAP_ATTR_READ; - if (direction != PCI_DMA_TODEVICE) + if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; local_irq_save(flags); - pci_iommu_batch_start(pdev, prot, entry); + iommu_batch_start(dev, prot, entry); for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { - long err = pci_iommu_batch_add(base_paddr); + long err = iommu_batch_add(base_paddr); if (unlikely(err < 0L)) goto iommu_map_fail; } - if (unlikely(pci_iommu_batch_end() < 0L)) + if (unlikely(iommu_batch_end() < 0L)) goto iommu_map_fail; local_irq_restore(flags); @@ -310,18 +314,19 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, bad: if (printk_ratelimit()) WARN_ON(1); - return PCI_DMA_ERROR_CODE; + return DMA_ERROR_CODE; iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); spin_unlock_irqrestore(&iommu->lock, flags); - return PCI_DMA_ERROR_CODE; + return DMA_ERROR_CODE; } -static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) +static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr, + size_t sz, enum dma_data_direction direction) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -329,14 +334,14 @@ static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ long entry; u32 devhandle; - if (unlikely(direction == PCI_DMA_NONE)) { + if (unlikely(direction == DMA_NONE)) { if (printk_ratelimit()) WARN_ON(1); return; } - iommu = pdev->dev.archdata.iommu; - pbm = pdev->dev.archdata.host_controller; + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); @@ -346,7 +351,7 @@ static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ spin_lock_irqsave(&iommu->lock, flags); entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); do { unsigned long num; @@ -363,7 +368,7 @@ static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ #define SG_ENT_PHYS_ADDRESS(SG) \ (__pa(page_address((SG)->page)) + (SG)->offset) -static inline long fill_sg(long entry, struct pci_dev *pdev, +static inline long fill_sg(long entry, struct device *dev, struct scatterlist *sg, int nused, int nelems, unsigned long prot) { @@ -374,7 +379,7 @@ static inline long fill_sg(long entry, struct pci_dev *pdev, local_irq_save(flags); - pci_iommu_batch_start(pdev, prot, entry); + iommu_batch_start(dev, prot, entry); for (i = 0; i < nused; i++) { unsigned long pteval = ~0UL; @@ -415,7 +420,7 @@ static inline long fill_sg(long entry, struct pci_dev *pdev, while (len > 0) { long err; - err = pci_iommu_batch_add(pteval); + err = iommu_batch_add(pteval); if (unlikely(err < 0L)) goto iommu_map_failed; @@ -446,7 +451,7 @@ static inline long fill_sg(long entry, struct pci_dev *pdev, dma_sg++; } - if (unlikely(pci_iommu_batch_end() < 0L)) + if (unlikely(iommu_batch_end() < 0L)) goto iommu_map_failed; local_irq_restore(flags); @@ -457,7 +462,8 @@ iommu_map_failed: return -1L; } -static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction) { struct iommu *iommu; unsigned long flags, npages, prot; @@ -469,18 +475,19 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n /* Fast path single entry scatterlists. */ if (nelems == 1) { sglist->dma_address = - pci_4v_map_single(pdev, - (page_address(sglist->page) + sglist->offset), + dma_4v_map_single(dev, + (page_address(sglist->page) + + sglist->offset), sglist->length, direction); - if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE)) + if (unlikely(sglist->dma_address == DMA_ERROR_CODE)) return 0; sglist->dma_length = sglist->length; return 1; } - iommu = pdev->dev.archdata.iommu; + iommu = dev->archdata.iommu; - if (unlikely(direction == PCI_DMA_NONE)) + if (unlikely(direction == DMA_NONE)) goto bad; /* Step 1: Prepare scatter list. */ @@ -488,7 +495,7 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n /* Step 2: Allocate a cluster and context, if necessary. */ spin_lock_irqsave(&iommu->lock, flags); - entry = pci_arena_alloc(&iommu->arena, npages); + entry = arena_alloc(&iommu->arena, npages); spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(entry < 0L)) @@ -510,10 +517,10 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n /* Step 4: Create the mappings. */ prot = HV_PCI_MAP_ATTR_READ; - if (direction != PCI_DMA_TODEVICE) + if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; - err = fill_sg(entry, pdev, sglist, used, nelems, prot); + err = fill_sg(entry, dev, sglist, used, nelems, prot); if (unlikely(err < 0L)) goto iommu_map_failed; @@ -526,13 +533,14 @@ bad: iommu_map_failed: spin_lock_irqsave(&iommu->lock, flags); - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); spin_unlock_irqrestore(&iommu->lock, flags); return 0; } -static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -540,13 +548,13 @@ static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in long entry; u32 devhandle, bus_addr; - if (unlikely(direction == PCI_DMA_NONE)) { + if (unlikely(direction == DMA_NONE)) { if (printk_ratelimit()) WARN_ON(1); } - iommu = pdev->dev.archdata.iommu; - pbm = pdev->dev.archdata.host_controller; + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; bus_addr = sglist->dma_address & IO_PAGE_MASK; @@ -562,7 +570,7 @@ static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in spin_lock_irqsave(&iommu->lock, flags); - pci_arena_free(&iommu->arena, entry, npages); + arena_free(&iommu->arena, entry, npages); do { unsigned long num; @@ -576,25 +584,29 @@ static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, in spin_unlock_irqrestore(&iommu->lock, flags); } -static void pci_4v_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) +static void dma_4v_sync_single_for_cpu(struct device *dev, + dma_addr_t bus_addr, size_t sz, + enum dma_data_direction direction) { /* Nothing to do... */ } -static void pci_4v_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) +static void dma_4v_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sglist, int nelems, + enum dma_data_direction direction) { /* Nothing to do... */ } -const struct pci_iommu_ops pci_sun4v_iommu_ops = { - .alloc_consistent = pci_4v_alloc_consistent, - .free_consistent = pci_4v_free_consistent, - .map_single = pci_4v_map_single, - .unmap_single = pci_4v_unmap_single, - .map_sg = pci_4v_map_sg, - .unmap_sg = pci_4v_unmap_sg, - .dma_sync_single_for_cpu = pci_4v_dma_sync_single_for_cpu, - .dma_sync_sg_for_cpu = pci_4v_dma_sync_sg_for_cpu, +const struct dma_ops sun4v_dma_ops = { + .alloc_coherent = dma_4v_alloc_coherent, + .free_coherent = dma_4v_free_coherent, + .map_single = dma_4v_map_single, + .unmap_single = dma_4v_unmap_single, + .map_sg = dma_4v_map_sg, + .unmap_sg = dma_4v_unmap_sg, + .sync_single_for_cpu = dma_4v_sync_single_for_cpu, + .sync_sg_for_cpu = dma_4v_sync_sg_for_cpu, }; static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm) @@ -1186,6 +1198,8 @@ void __init sun4v_pci_init(struct device_node *dp, char *model_name) } printk("SUN4V_PCI: Registered hvapi major[%lu] minor[%lu]\n", vpci_major, vpci_minor); + + dma_ops = &sun4v_dma_ops; } prop = of_find_property(dp, "reg", NULL); @@ -1206,7 +1220,7 @@ void __init sun4v_pci_init(struct device_node *dp, char *model_name) if (!page) goto fatal_memory_error; - per_cpu(pci_iommu_batch, i).pglist = (u64 *) page; + per_cpu(iommu_batch, i).pglist = (u64 *) page; } p = kzalloc(sizeof(struct pci_controller_info), GFP_ATOMIC); diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c index a1fd9bcc0b8..d1fb13ba02b 100644 --- a/arch/sparc64/kernel/sbus.c +++ b/arch/sparc64/kernel/sbus.c @@ -26,11 +26,6 @@ #define MAP_BASE ((u32)0xc0000000) -struct sbus_info { - struct iommu iommu; - struct strbuf strbuf; -}; - /* Offsets from iommu_regs */ #define SYSIO_IOMMUREG_BASE 0x2400UL #define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */ @@ -44,19 +39,6 @@ struct sbus_info { #define IOMMU_DRAM_VALID (1UL << 30UL) -static void __iommu_flushall(struct iommu *iommu) -{ - unsigned long tag; - int entry; - - tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL); - for (entry = 0; entry < 16; entry++) { - upa_writeq(0, tag); - tag += 8UL; - } - upa_readq(iommu->write_complete_reg); -} - /* Offsets from strbuf_regs */ #define SYSIO_STRBUFREG_BASE 0x2800UL #define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */ @@ -69,511 +51,10 @@ static void __iommu_flushall(struct iommu *iommu) #define STRBUF_TAG_VALID 0x02UL -static void sbus_strbuf_flush(struct iommu *iommu, struct strbuf *strbuf, u32 base, unsigned long npages, int direction) -{ - unsigned long n; - int limit; - - n = npages; - while (n--) - upa_writeq(base + (n << IO_PAGE_SHIFT), strbuf->strbuf_pflush); - - /* If the device could not have possibly put dirty data into - * the streaming cache, no flush-flag synchronization needs - * to be performed. - */ - if (direction == SBUS_DMA_TODEVICE) - return; - - *(strbuf->strbuf_flushflag) = 0UL; - - /* Whoopee cushion! */ - upa_writeq(strbuf->strbuf_flushflag_pa, strbuf->strbuf_fsync); - upa_readq(iommu->write_complete_reg); - - limit = 100000; - while (*(strbuf->strbuf_flushflag) == 0UL) { - limit--; - if (!limit) - break; - udelay(1); - rmb(); - } - if (!limit) - printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " - "vaddr[%08x] npages[%ld]\n", - base, npages); -} - -/* Based largely upon the ppc64 iommu allocator. */ -static long sbus_arena_alloc(struct iommu *iommu, unsigned long npages) -{ - struct iommu_arena *arena = &iommu->arena; - unsigned long n, i, start, end, limit; - int pass; - - limit = arena->limit; - start = arena->hint; - pass = 0; - -again: - n = find_next_zero_bit(arena->map, limit, start); - end = n + npages; - if (unlikely(end >= limit)) { - if (likely(pass < 1)) { - limit = start; - start = 0; - __iommu_flushall(iommu); - pass++; - goto again; - } else { - /* Scanned the whole thing, give up. */ - return -1; - } - } - - for (i = n; i < end; i++) { - if (test_bit(i, arena->map)) { - start = i + 1; - goto again; - } - } - - for (i = n; i < end; i++) - __set_bit(i, arena->map); - - arena->hint = end; - - return n; -} - -static void sbus_arena_free(struct iommu_arena *arena, unsigned long base, unsigned long npages) -{ - unsigned long i; - - for (i = base; i < (base + npages); i++) - __clear_bit(i, arena->map); -} - -static void sbus_iommu_table_init(struct iommu *iommu, unsigned int tsbsize) -{ - unsigned long tsbbase, order, sz, num_tsb_entries; - - num_tsb_entries = tsbsize / sizeof(iopte_t); - - /* Setup initial software IOMMU state. */ - spin_lock_init(&iommu->lock); - iommu->page_table_map_base = MAP_BASE; - - /* Allocate and initialize the free area map. */ - sz = num_tsb_entries / 8; - sz = (sz + 7UL) & ~7UL; - iommu->arena.map = kzalloc(sz, GFP_KERNEL); - if (!iommu->arena.map) { - prom_printf("SBUS_IOMMU: Error, kmalloc(arena.map) failed.\n"); - prom_halt(); - } - iommu->arena.limit = num_tsb_entries; - - /* Now allocate and setup the IOMMU page table itself. */ - order = get_order(tsbsize); - tsbbase = __get_free_pages(GFP_KERNEL, order); - if (!tsbbase) { - prom_printf("IOMMU: Error, gfp(tsb) failed.\n"); - prom_halt(); - } - iommu->page_table = (iopte_t *)tsbbase; - memset(iommu->page_table, 0, tsbsize); -} - -static inline iopte_t *alloc_npages(struct iommu *iommu, unsigned long npages) -{ - long entry; - - entry = sbus_arena_alloc(iommu, npages); - if (unlikely(entry < 0)) - return NULL; - - return iommu->page_table + entry; -} - -static inline void free_npages(struct iommu *iommu, dma_addr_t base, unsigned long npages) -{ - sbus_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); -} - -void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr) -{ - struct sbus_info *info; - struct iommu *iommu; - iopte_t *iopte; - unsigned long flags, order, first_page; - void *ret; - int npages; - - size = IO_PAGE_ALIGN(size); - order = get_order(size); - if (order >= 10) - return NULL; - - first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order); - if (first_page == 0UL) - return NULL; - memset((char *)first_page, 0, PAGE_SIZE << order); - - info = sdev->bus->iommu; - iommu = &info->iommu; - - spin_lock_irqsave(&iommu->lock, flags); - iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); - spin_unlock_irqrestore(&iommu->lock, flags); - - if (unlikely(iopte == NULL)) { - free_pages(first_page, order); - return NULL; - } - - *dvma_addr = (iommu->page_table_map_base + - ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); - ret = (void *) first_page; - npages = size >> IO_PAGE_SHIFT; - first_page = __pa(first_page); - while (npages--) { - iopte_val(*iopte) = (IOPTE_VALID | IOPTE_CACHE | - IOPTE_WRITE | - (first_page & IOPTE_PAGE)); - iopte++; - first_page += IO_PAGE_SIZE; - } - - return ret; -} - -void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma) -{ - struct sbus_info *info; - struct iommu *iommu; - iopte_t *iopte; - unsigned long flags, order, npages; - - npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; - info = sdev->bus->iommu; - iommu = &info->iommu; - iopte = iommu->page_table + - ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); - - spin_lock_irqsave(&iommu->lock, flags); - - free_npages(iommu, dvma - iommu->page_table_map_base, npages); - - spin_unlock_irqrestore(&iommu->lock, flags); - - order = get_order(size); - if (order < 10) - free_pages((unsigned long)cpu, order); -} - -dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t sz, int direction) -{ - struct sbus_info *info; - struct iommu *iommu; - iopte_t *base; - unsigned long flags, npages, oaddr; - unsigned long i, base_paddr; - u32 bus_addr, ret; - unsigned long iopte_protection; - - info = sdev->bus->iommu; - iommu = &info->iommu; - - if (unlikely(direction == SBUS_DMA_NONE)) - BUG(); - - oaddr = (unsigned long)ptr; - npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); - npages >>= IO_PAGE_SHIFT; - - spin_lock_irqsave(&iommu->lock, flags); - base = alloc_npages(iommu, npages); - spin_unlock_irqrestore(&iommu->lock, flags); - - if (unlikely(!base)) - BUG(); - - bus_addr = (iommu->page_table_map_base + - ((base - iommu->page_table) << IO_PAGE_SHIFT)); - ret = bus_addr | (oaddr & ~IO_PAGE_MASK); - base_paddr = __pa(oaddr & IO_PAGE_MASK); - - iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; - if (direction != SBUS_DMA_TODEVICE) - iopte_protection |= IOPTE_WRITE; - - for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) - iopte_val(*base) = iopte_protection | base_paddr; - - return ret; -} - -void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) -{ - struct sbus_info *info = sdev->bus->iommu; - struct iommu *iommu = &info->iommu; - struct strbuf *strbuf = &info->strbuf; - iopte_t *base; - unsigned long flags, npages, i; - - if (unlikely(direction == SBUS_DMA_NONE)) - BUG(); - - npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); - npages >>= IO_PAGE_SHIFT; - base = iommu->page_table + - ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); - - bus_addr &= IO_PAGE_MASK; - - spin_lock_irqsave(&iommu->lock, flags); - sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); - for (i = 0; i < npages; i++) - iopte_val(base[i]) = 0UL; - free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); - spin_unlock_irqrestore(&iommu->lock, flags); -} - -#define SG_ENT_PHYS_ADDRESS(SG) \ - (__pa(page_address((SG)->page)) + (SG)->offset) - -static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, - int nused, int nelems, unsigned long iopte_protection) -{ - struct scatterlist *dma_sg = sg; - struct scatterlist *sg_end = sg + nelems; - int i; - - for (i = 0; i < nused; i++) { - unsigned long pteval = ~0UL; - u32 dma_npages; - - dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + - dma_sg->dma_length + - ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; - do { - unsigned long offset; - signed int len; - - /* If we are here, we know we have at least one - * more page to map. So walk forward until we - * hit a page crossing, and begin creating new - * mappings from that spot. - */ - for (;;) { - unsigned long tmp; - - tmp = SG_ENT_PHYS_ADDRESS(sg); - len = sg->length; - if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { - pteval = tmp & IO_PAGE_MASK; - offset = tmp & (IO_PAGE_SIZE - 1UL); - break; - } - if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { - pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; - offset = 0UL; - len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); - break; - } - sg++; - } - - pteval = iopte_protection | (pteval & IOPTE_PAGE); - while (len > 0) { - *iopte++ = __iopte(pteval); - pteval += IO_PAGE_SIZE; - len -= (IO_PAGE_SIZE - offset); - offset = 0; - dma_npages--; - } - - pteval = (pteval & IOPTE_PAGE) + len; - sg++; - - /* Skip over any tail mappings we've fully mapped, - * adjusting pteval along the way. Stop when we - * detect a page crossing event. - */ - while (sg < sg_end && - (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && - (pteval == SG_ENT_PHYS_ADDRESS(sg)) && - ((pteval ^ - (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { - pteval += sg->length; - sg++; - } - if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) - pteval = ~0UL; - } while (dma_npages != 0); - dma_sg++; - } -} - -int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) -{ - struct sbus_info *info; - struct iommu *iommu; - unsigned long flags, npages, iopte_protection; - iopte_t *base; - u32 dma_base; - struct scatterlist *sgtmp; - int used; - - /* Fast path single entry scatterlists. */ - if (nelems == 1) { - sglist->dma_address = - sbus_map_single(sdev, - (page_address(sglist->page) + sglist->offset), - sglist->length, direction); - sglist->dma_length = sglist->length; - return 1; - } - - info = sdev->bus->iommu; - iommu = &info->iommu; - - if (unlikely(direction == SBUS_DMA_NONE)) - BUG(); - - npages = prepare_sg(sglist, nelems); - - spin_lock_irqsave(&iommu->lock, flags); - base = alloc_npages(iommu, npages); - spin_unlock_irqrestore(&iommu->lock, flags); - - if (unlikely(base == NULL)) - BUG(); - - dma_base = iommu->page_table_map_base + - ((base - iommu->page_table) << IO_PAGE_SHIFT); - - /* Normalize DVMA addresses. */ - used = nelems; - - sgtmp = sglist; - while (used && sgtmp->dma_length) { - sgtmp->dma_address += dma_base; - sgtmp++; - used--; - } - used = nelems - used; - - iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; - if (direction != SBUS_DMA_TODEVICE) - iopte_protection |= IOPTE_WRITE; - - fill_sg(base, sglist, used, nelems, iopte_protection); - -#ifdef VERIFY_SG - verify_sglist(sglist, nelems, base, npages); -#endif - - return used; -} - -void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) -{ - struct sbus_info *info; - struct iommu *iommu; - struct strbuf *strbuf; - iopte_t *base; - unsigned long flags, i, npages; - u32 bus_addr; - - if (unlikely(direction == SBUS_DMA_NONE)) - BUG(); - - info = sdev->bus->iommu; - iommu = &info->iommu; - strbuf = &info->strbuf; - - bus_addr = sglist->dma_address & IO_PAGE_MASK; - - for (i = 1; i < nelems; i++) - if (sglist[i].dma_length == 0) - break; - i--; - npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - - bus_addr) >> IO_PAGE_SHIFT; - - base = iommu->page_table + - ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); - - spin_lock_irqsave(&iommu->lock, flags); - sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); - for (i = 0; i < npages; i++) - iopte_val(base[i]) = 0UL; - free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); - spin_unlock_irqrestore(&iommu->lock, flags); -} - -void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) -{ - struct sbus_info *info; - struct iommu *iommu; - struct strbuf *strbuf; - unsigned long flags, npages; - - info = sdev->bus->iommu; - iommu = &info->iommu; - strbuf = &info->strbuf; - - npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); - npages >>= IO_PAGE_SHIFT; - bus_addr &= IO_PAGE_MASK; - - spin_lock_irqsave(&iommu->lock, flags); - sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); - spin_unlock_irqrestore(&iommu->lock, flags); -} - -void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) -{ -} - -void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) -{ - struct sbus_info *info; - struct iommu *iommu; - struct strbuf *strbuf; - unsigned long flags, npages, i; - u32 bus_addr; - - info = sdev->bus->iommu; - iommu = &info->iommu; - strbuf = &info->strbuf; - - bus_addr = sglist[0].dma_address & IO_PAGE_MASK; - for (i = 0; i < nelems; i++) { - if (!sglist[i].dma_length) - break; - } - i--; - npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - - bus_addr) >> IO_PAGE_SHIFT; - - spin_lock_irqsave(&iommu->lock, flags); - sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); - spin_unlock_irqrestore(&iommu->lock, flags); -} - -void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) -{ -} - /* Enable 64-bit DVMA mode for the given device. */ void sbus_set_sbus64(struct sbus_dev *sdev, int bursts) { - struct sbus_info *info = sdev->bus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sdev->ofdev.dev.archdata.iommu; int slot = sdev->slot; unsigned long cfg_reg; u64 val; @@ -713,8 +194,7 @@ static unsigned long sysio_imap_to_iclr(unsigned long imap) unsigned int sbus_build_irq(void *buscookie, unsigned int ino) { struct sbus_bus *sbus = (struct sbus_bus *)buscookie; - struct sbus_info *info = sbus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sbus->ofdev.dev.archdata.iommu; unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; unsigned long imap, iclr; int sbus_level = 0; @@ -776,8 +256,7 @@ unsigned int sbus_build_irq(void *buscookie, unsigned int ino) static irqreturn_t sysio_ue_handler(int irq, void *dev_id) { struct sbus_bus *sbus = dev_id; - struct sbus_info *info = sbus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sbus->ofdev.dev.archdata.iommu; unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; unsigned long afsr_reg, afar_reg; unsigned long afsr, afar, error_bits; @@ -849,8 +328,7 @@ static irqreturn_t sysio_ue_handler(int irq, void *dev_id) static irqreturn_t sysio_ce_handler(int irq, void *dev_id) { struct sbus_bus *sbus = dev_id; - struct sbus_info *info = sbus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sbus->ofdev.dev.archdata.iommu; unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; unsigned long afsr_reg, afar_reg; unsigned long afsr, afar, error_bits; @@ -927,8 +405,7 @@ static irqreturn_t sysio_ce_handler(int irq, void *dev_id) static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id) { struct sbus_bus *sbus = dev_id; - struct sbus_info *info = sbus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sbus->ofdev.dev.archdata.iommu; unsigned long afsr_reg, afar_reg, reg_base; unsigned long afsr, afar, error_bits; int reported; @@ -995,8 +472,7 @@ static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id) static void __init sysio_register_error_handlers(struct sbus_bus *sbus) { - struct sbus_info *info = sbus->iommu; - struct iommu *iommu = &info->iommu; + struct iommu *iommu = sbus->ofdev.dev.archdata.iommu; unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; unsigned int irq; u64 control; @@ -1041,7 +517,6 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) { const struct linux_prom64_registers *pr; struct device_node *dp; - struct sbus_info *info; struct iommu *iommu; struct strbuf *strbuf; unsigned long regs, reg_base; @@ -1054,25 +529,28 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) pr = of_get_property(dp, "reg", NULL); if (!pr) { - prom_printf("sbus_iommu_init: Cannot map SYSIO control registers.\n"); + prom_printf("sbus_iommu_init: Cannot map SYSIO " + "control registers.\n"); prom_halt(); } regs = pr->phys_addr; - info = kzalloc(sizeof(*info), GFP_ATOMIC); - if (info == NULL) { - prom_printf("sbus_iommu_init: Fatal error, " - "kmalloc(info) failed\n"); - prom_halt(); - } + iommu = kzalloc(sizeof(*iommu), GFP_ATOMIC); + if (!iommu) + goto fatal_memory_error; + strbuf = kzalloc(sizeof(*strbuf), GFP_ATOMIC); + if (!strbuf) + goto fatal_memory_error; - iommu = &info->iommu; - strbuf = &info->strbuf; + sbus->ofdev.dev.archdata.iommu = iommu; + sbus->ofdev.dev.archdata.stc = strbuf; reg_base = regs + SYSIO_IOMMUREG_BASE; iommu->iommu_control = reg_base + IOMMU_CONTROL; iommu->iommu_tsbbase = reg_base + IOMMU_TSBBASE; iommu->iommu_flush = reg_base + IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_control + + (IOMMU_TAGDIAG - IOMMU_CONTROL); reg_base = regs + SYSIO_STRBUFREG_BASE; strbuf->strbuf_control = reg_base + STRBUF_CONTROL; @@ -1093,14 +571,12 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) */ iommu->write_complete_reg = regs + 0x2000UL; - /* Link into SYSIO software state. */ - sbus->iommu = info; - printk("SYSIO: UPA portID %x, at %016lx\n", sbus->portid, regs); /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ - sbus_iommu_table_init(iommu, IO_TSB_SIZE); + if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff)) + goto fatal_memory_error; control = upa_readq(iommu->iommu_control); control = ((7UL << 16UL) | @@ -1157,6 +633,10 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) starfire_hookup(sbus->portid); sysio_register_error_handlers(sbus); + return; + +fatal_memory_error: + prom_printf("sbus_iommu_init: Fatal memory allocation error.\n"); } void sbus_fill_device_irq(struct sbus_dev *sdev) |