From 70a7d3cc1308a55104fbe505d76f2aca8a4cf53e Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 22 Dec 2008 10:26:05 -0800 Subject: swiotlb: add hwdev to swiotlb_phys_to_bus() / swiotlb_sg_to_bus() Impact: extend functions with a (yet unused) parameter, update callsites Some architectures need it - in preparation for highmem swiotlb. Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 53 ++++++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index fa2dc4e5f9b..3657da8ebbc 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -126,7 +126,7 @@ void * __weak swiotlb_alloc(unsigned order, unsigned long nslabs) return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order); } -dma_addr_t __weak swiotlb_phys_to_bus(phys_addr_t paddr) +dma_addr_t __weak swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr) { return paddr; } @@ -136,9 +136,10 @@ phys_addr_t __weak swiotlb_bus_to_phys(dma_addr_t baddr) return baddr; } -static dma_addr_t swiotlb_virt_to_bus(volatile void *address) +static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, + volatile void *address) { - return swiotlb_phys_to_bus(virt_to_phys(address)); + return swiotlb_phys_to_bus(hwdev, virt_to_phys(address)); } static void *swiotlb_bus_to_virt(dma_addr_t address) @@ -151,35 +152,23 @@ int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size) return 0; } -static dma_addr_t swiotlb_sg_to_bus(struct scatterlist *sg) +static dma_addr_t swiotlb_sg_to_bus(struct device *hwdev, struct scatterlist *sg) { - return swiotlb_phys_to_bus(page_to_phys(sg_page(sg)) + sg->offset); + return swiotlb_phys_to_bus(hwdev, page_to_phys(sg_page(sg)) + sg->offset); } static void swiotlb_print_info(unsigned long bytes) { phys_addr_t pstart, pend; - dma_addr_t bstart, bend; pstart = virt_to_phys(io_tlb_start); pend = virt_to_phys(io_tlb_end); - bstart = swiotlb_phys_to_bus(pstart); - bend = swiotlb_phys_to_bus(pend); - printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n", bytes >> 20, io_tlb_start, io_tlb_end); - if (pstart != bstart || pend != bend) - printk(KERN_INFO "software IO TLB at phys %#llx - %#llx" - " bus %#llx - %#llx\n", - (unsigned long long)pstart, - (unsigned long long)pend, - (unsigned long long)bstart, - (unsigned long long)bend); - else - printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n", - (unsigned long long)pstart, - (unsigned long long)pend); + printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n", + (unsigned long long)pstart, + (unsigned long long)pend); } /* @@ -406,7 +395,7 @@ map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, i struct swiotlb_phys_addr slot_buf; mask = dma_get_seg_boundary(hwdev); - start_dma_addr = swiotlb_virt_to_bus(io_tlb_start) & mask; + start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start) & mask; offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; @@ -585,7 +574,9 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_mask = hwdev->coherent_dma_mask; ret = (void *)__get_free_pages(flags, order); - if (ret && !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(ret), size)) { + if (ret && + !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(hwdev, ret), + size)) { /* * The allocated memory isn't reachable by the device. * Fall back on swiotlb_map_single(). @@ -609,7 +600,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, } memset(ret, 0, size); - dev_addr = swiotlb_virt_to_bus(ret); + dev_addr = swiotlb_virt_to_bus(hwdev, ret); /* Confirm address can be DMA'd by device */ if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) { @@ -669,7 +660,7 @@ dma_addr_t swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, int dir, struct dma_attrs *attrs) { - dma_addr_t dev_addr = swiotlb_virt_to_bus(ptr); + dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, ptr); void *map; struct swiotlb_phys_addr buffer; @@ -694,7 +685,7 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, map = io_tlb_overflow_buffer; } - dev_addr = swiotlb_virt_to_bus(map); + dev_addr = swiotlb_virt_to_bus(hwdev, map); /* * Ensure that the address returned is DMA'ble @@ -840,7 +831,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - dev_addr = swiotlb_sg_to_bus(sg); + dev_addr = swiotlb_sg_to_bus(hwdev, sg); if (range_needs_mapping(sg_virt(sg), sg->length) || address_needs_mapping(hwdev, dev_addr, sg->length)) { void *map; @@ -856,7 +847,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, sgl[0].dma_length = 0; return 0; } - sg->dma_address = swiotlb_virt_to_bus(map); + sg->dma_address = swiotlb_virt_to_bus(hwdev, map); } else sg->dma_address = dev_addr; sg->dma_length = sg->length; @@ -886,7 +877,7 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - if (sg->dma_address != swiotlb_sg_to_bus(sg)) + if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg)) unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), sg->dma_length, dir); else if (dir == DMA_FROM_DEVICE) @@ -919,7 +910,7 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - if (sg->dma_address != swiotlb_sg_to_bus(sg)) + if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg)) sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), sg->dma_length, dir, target); else if (dir == DMA_FROM_DEVICE) @@ -944,7 +935,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, int swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) { - return (dma_addr == swiotlb_virt_to_bus(io_tlb_overflow_buffer)); + return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer)); } /* @@ -956,7 +947,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) int swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return swiotlb_virt_to_bus(io_tlb_end - 1) <= mask; + return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask; } EXPORT_SYMBOL(swiotlb_map_single); -- cgit v1.2.3 From bc40ac66988a7721f2a244b6df65f8c13d16479c Mon Sep 17 00:00:00 2001 From: Becky Bruce Date: Mon, 22 Dec 2008 10:26:08 -0800 Subject: swiotlb: store phys address in io_tlb_orig_addr array Impact: refactor code, cleanup When we enable swiotlb for platforms that support HIGHMEM, we can no longer store the virtual address of the original dma buffer, because that buffer might not have a permament mapping. Change the swiotlb code to instead store the physical address of the original buffer. Signed-off-by: Becky Bruce Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 120 +++++++++++++++------------------------------------------- 1 file changed, 30 insertions(+), 90 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 3657da8ebbc..98a7a4450e0 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -89,10 +89,7 @@ static unsigned int io_tlb_index; * We need to save away the original address corresponding to a mapped entry * for the sync operations. */ -static struct swiotlb_phys_addr { - struct page *page; - unsigned int offset; -} *io_tlb_orig_addr; +static phys_addr_t *io_tlb_orig_addr; /* * Protect the above data structures in the map and unmap calls @@ -204,7 +201,7 @@ swiotlb_init_with_default_size(size_t default_size) for (i = 0; i < io_tlb_nslabs; i++) io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); io_tlb_index = 0; - io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr)); + io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t)); /* * Get the overflow emergency buffer @@ -278,12 +275,14 @@ swiotlb_late_init_with_default_size(size_t default_size) io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); io_tlb_index = 0; - io_tlb_orig_addr = (struct swiotlb_phys_addr *)__get_free_pages(GFP_KERNEL, - get_order(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr))); + io_tlb_orig_addr = (phys_addr_t *) + __get_free_pages(GFP_KERNEL, + get_order(io_tlb_nslabs * + sizeof(phys_addr_t))); if (!io_tlb_orig_addr) goto cleanup3; - memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(struct swiotlb_phys_addr)); + memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t)); /* * Get the overflow emergency buffer @@ -298,8 +297,8 @@ swiotlb_late_init_with_default_size(size_t default_size) return 0; cleanup4: - free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs * - sizeof(char *))); + free_pages((unsigned long)io_tlb_orig_addr, + get_order(io_tlb_nslabs * sizeof(phys_addr_t))); io_tlb_orig_addr = NULL; cleanup3: free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * @@ -330,59 +329,11 @@ static int is_swiotlb_buffer(char *addr) return addr >= io_tlb_start && addr < io_tlb_end; } -static struct swiotlb_phys_addr swiotlb_bus_to_phys_addr(char *dma_addr) -{ - int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; - struct swiotlb_phys_addr buffer = io_tlb_orig_addr[index]; - buffer.offset += (long)dma_addr & ((1 << IO_TLB_SHIFT) - 1); - buffer.page += buffer.offset >> PAGE_SHIFT; - buffer.offset &= PAGE_SIZE - 1; - return buffer; -} - -static void -__sync_single(struct swiotlb_phys_addr buffer, char *dma_addr, size_t size, int dir) -{ - if (PageHighMem(buffer.page)) { - size_t len, bytes; - char *dev, *host, *kmp; - - len = size; - while (len != 0) { - unsigned long flags; - - bytes = len; - if ((bytes + buffer.offset) > PAGE_SIZE) - bytes = PAGE_SIZE - buffer.offset; - local_irq_save(flags); /* protects KM_BOUNCE_READ */ - kmp = kmap_atomic(buffer.page, KM_BOUNCE_READ); - dev = dma_addr + size - len; - host = kmp + buffer.offset; - if (dir == DMA_FROM_DEVICE) - memcpy(host, dev, bytes); - else - memcpy(dev, host, bytes); - kunmap_atomic(kmp, KM_BOUNCE_READ); - local_irq_restore(flags); - len -= bytes; - buffer.page++; - buffer.offset = 0; - } - } else { - void *v = page_address(buffer.page) + buffer.offset; - - if (dir == DMA_TO_DEVICE) - memcpy(dma_addr, v, size); - else - memcpy(v, dma_addr, size); - } -} - /* * Allocates bounce buffer and returns its kernel virtual address. */ static void * -map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, int dir) +map_single(struct device *hwdev, phys_addr_t phys, size_t size, int dir) { unsigned long flags; char *dma_addr; @@ -392,7 +343,6 @@ map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, i unsigned long mask; unsigned long offset_slots; unsigned long max_slots; - struct swiotlb_phys_addr slot_buf; mask = dma_get_seg_boundary(hwdev); start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start) & mask; @@ -477,15 +427,10 @@ found: * This is needed when we sync the memory. Then we sync the buffer if * needed. */ - slot_buf = buffer; - for (i = 0; i < nslots; i++) { - slot_buf.page += slot_buf.offset >> PAGE_SHIFT; - slot_buf.offset &= PAGE_SIZE - 1; - io_tlb_orig_addr[index+i] = slot_buf; - slot_buf.offset += 1 << IO_TLB_SHIFT; - } + for (i = 0; i < nslots; i++) + io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT); if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) - __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE); + memcpy(dma_addr, phys_to_virt(phys), size); return dma_addr; } @@ -499,17 +444,17 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir) unsigned long flags; int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; - struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr); + phys_addr_t phys = io_tlb_orig_addr[index]; /* * First, sync the memory before unmapping the entry */ - if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)) + if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) /* * bounce... copy the data back into the original buffer * and * delete the bounce buffer. */ - __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE); + memcpy(phys_to_virt(phys), dma_addr, size); /* * Return the buffer to the free list by setting the corresponding @@ -541,18 +486,21 @@ static void sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir, int target) { - struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr); + int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; + phys_addr_t phys = io_tlb_orig_addr[index]; + + phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1)); switch (target) { case SYNC_FOR_CPU: if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) - __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE); + memcpy(phys_to_virt(phys), dma_addr, size); else BUG_ON(dir != DMA_TO_DEVICE); break; case SYNC_FOR_DEVICE: if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) - __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE); + memcpy(dma_addr, phys_to_virt(phys), size); else BUG_ON(dir != DMA_FROM_DEVICE); break; @@ -591,10 +539,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, * swiotlb_map_single(), which will grab memory from * the lowest available address range. */ - struct swiotlb_phys_addr buffer; - buffer.page = virt_to_page(NULL); - buffer.offset = 0; - ret = map_single(hwdev, buffer, size, DMA_FROM_DEVICE); + ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE); if (!ret) return NULL; } @@ -662,7 +607,6 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, { dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, ptr); void *map; - struct swiotlb_phys_addr buffer; BUG_ON(dir == DMA_NONE); /* @@ -677,9 +621,7 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, /* * Oh well, have to allocate and map a bounce buffer. */ - buffer.page = virt_to_page(ptr); - buffer.offset = (unsigned long)ptr & ~PAGE_MASK; - map = map_single(hwdev, buffer, size, dir); + map = map_single(hwdev, virt_to_phys(ptr), size, dir); if (!map) { swiotlb_full(hwdev, size, dir, 1); map = io_tlb_overflow_buffer; @@ -824,20 +766,18 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, int dir, struct dma_attrs *attrs) { struct scatterlist *sg; - struct swiotlb_phys_addr buffer; - dma_addr_t dev_addr; int i; BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - dev_addr = swiotlb_sg_to_bus(hwdev, sg); - if (range_needs_mapping(sg_virt(sg), sg->length) || + void *addr = sg_virt(sg); + dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, addr); + + if (range_needs_mapping(addr, sg->length) || address_needs_mapping(hwdev, dev_addr, sg->length)) { - void *map; - buffer.page = sg_page(sg); - buffer.offset = sg->offset; - map = map_single(hwdev, buffer, sg->length, dir); + void *map = map_single(hwdev, sg_phys(sg), + sg->length, dir); if (!map) { /* Don't panic here, we expect map_sg users to do proper error handling. */ -- cgit v1.2.3 From fb05a37929e0cd99016b4f5e5a5ef077fb10a947 Mon Sep 17 00:00:00 2001 From: Becky Bruce Date: Mon, 22 Dec 2008 10:26:09 -0800 Subject: swiotlb: add support for systems with highmem Impact: extend code for highmem - existing users unaffected On highmem systems, the original dma buffer might not have a virtual mapping - we need to kmap it in to perform the bounce. Extract the code that does the actual copy into a function that does the kmap if highmem is enabled, and default to the normal swiotlb memcpy if not. [ ported by Jeremy Fitzhardinge ] Signed-off-by: Becky Bruce Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 68 ++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 98a7a4450e0..785046e4c3a 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -14,6 +14,7 @@ * 04/07/.. ak Better overflow handling. Assorted fixes. * 05/09/10 linville Add support for syncing ranges, support syncing for * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. + * 08/12/11 beckyb Add highmem support */ #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -149,11 +151,6 @@ int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size) return 0; } -static dma_addr_t swiotlb_sg_to_bus(struct device *hwdev, struct scatterlist *sg) -{ - return swiotlb_phys_to_bus(hwdev, page_to_phys(sg_page(sg)) + sg->offset); -} - static void swiotlb_print_info(unsigned long bytes) { phys_addr_t pstart, pend; @@ -329,6 +326,47 @@ static int is_swiotlb_buffer(char *addr) return addr >= io_tlb_start && addr < io_tlb_end; } +/* + * Bounce: copy the swiotlb buffer back to the original dma location + */ +static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size, + enum dma_data_direction dir) +{ + unsigned long pfn = PFN_DOWN(phys); + + if (PageHighMem(pfn_to_page(pfn))) { + /* The buffer does not have a mapping. Map it in and copy */ + unsigned int offset = phys & ~PAGE_MASK; + char *buffer; + unsigned int sz = 0; + unsigned long flags; + + while (size) { + sz = min(PAGE_SIZE - offset, size); + + local_irq_save(flags); + buffer = kmap_atomic(pfn_to_page(pfn), + KM_BOUNCE_READ); + if (dir == DMA_TO_DEVICE) + memcpy(dma_addr, buffer + offset, sz); + else + memcpy(buffer + offset, dma_addr, sz); + kunmap_atomic(buffer, KM_BOUNCE_READ); + local_irq_restore(flags); + + size -= sz; + pfn++; + dma_addr += sz; + offset = 0; + } + } else { + if (dir == DMA_TO_DEVICE) + memcpy(dma_addr, phys_to_virt(phys), size); + else + memcpy(phys_to_virt(phys), dma_addr, size); + } +} + /* * Allocates bounce buffer and returns its kernel virtual address. */ @@ -430,7 +468,7 @@ found: for (i = 0; i < nslots; i++) io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT); if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) - memcpy(dma_addr, phys_to_virt(phys), size); + swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); return dma_addr; } @@ -450,11 +488,7 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir) * First, sync the memory before unmapping the entry */ if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) - /* - * bounce... copy the data back into the original buffer * and - * delete the bounce buffer. - */ - memcpy(phys_to_virt(phys), dma_addr, size); + swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); /* * Return the buffer to the free list by setting the corresponding @@ -494,13 +528,13 @@ sync_single(struct device *hwdev, char *dma_addr, size_t size, switch (target) { case SYNC_FOR_CPU: if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) - memcpy(phys_to_virt(phys), dma_addr, size); + swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); else BUG_ON(dir != DMA_TO_DEVICE); break; case SYNC_FOR_DEVICE: if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) - memcpy(dma_addr, phys_to_virt(phys), size); + swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); else BUG_ON(dir != DMA_FROM_DEVICE); break; @@ -817,11 +851,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg)) + if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg))) unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), sg->dma_length, dir); else if (dir == DMA_FROM_DEVICE) - dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length); + dma_mark_clean(sg_virt(sg), sg->dma_length); } } EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); @@ -850,11 +884,11 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg)) + if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg))) sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), sg->dma_length, dir, target); else if (dir == DMA_FROM_DEVICE) - dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length); + dma_mark_clean(sg_virt(sg), sg->dma_length); } } -- cgit v1.2.3 From ac86ccc637458dea53077823f98bc3e1b874d478 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sun, 28 Dec 2008 15:02:06 +0900 Subject: swiotlb: remove unnecessary declaration Impact: cleanup Signed-off-by: FUJITA Tomonori Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 785046e4c3a..5a7d0a4a9da 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -777,8 +776,6 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, SYNC_FOR_DEVICE); } -void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int, - struct dma_attrs *); /* * Map a set of buffers described by scatterlist in streaming mode for DMA. * This is the scatter-gather version of the above swiotlb_map_single -- cgit v1.2.3 From 874d6a9555516cbac828131d573ce3d16ce47164 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sun, 28 Dec 2008 15:02:07 +0900 Subject: swiotlb: clean up EXPORT_SYMBOL usage Impact: cleanup swiotlb uses EXPORT_SYMBOL in an inconsistent way. Some functions use EXPORT_SYMBOL at the end of functions. Some use it at the end of swiotlb.c. This cleans up swiotlb to use EXPORT_SYMBOL in a consistent way (at the end of functions). Signed-off-by: FUJITA Tomonori Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 5a7d0a4a9da..ef3cfe58a15 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -593,6 +593,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, *dma_handle = dev_addr; return ret; } +EXPORT_SYMBOL(swiotlb_alloc_coherent); void swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, @@ -605,6 +606,7 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE); } +EXPORT_SYMBOL(swiotlb_free_coherent); static void swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) @@ -677,6 +679,7 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir) { return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL); } +EXPORT_SYMBOL(swiotlb_map_single); /* * Unmap a single streaming mode DMA translation. The dma_addr and size must @@ -706,6 +709,8 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size, { return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL); } +EXPORT_SYMBOL(swiotlb_unmap_single); + /* * Make physical memory consistent for a single streaming mode DMA translation * after a transfer. @@ -735,6 +740,7 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); } +EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); void swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, @@ -742,6 +748,7 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); } +EXPORT_SYMBOL(swiotlb_sync_single_for_device); /* * Same as above, but for a sub-range of the mapping. @@ -767,6 +774,7 @@ swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr, swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, SYNC_FOR_CPU); } +EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu); void swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, @@ -775,6 +783,7 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, SYNC_FOR_DEVICE); } +EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); /* * Map a set of buffers described by scatterlist in streaming mode for DMA. @@ -833,6 +842,7 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, { return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); } +EXPORT_SYMBOL(swiotlb_map_sg); /* * Unmap a set of streaming mode DMA translations. Again, cpu read rules @@ -863,6 +873,7 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, { return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); } +EXPORT_SYMBOL(swiotlb_unmap_sg); /* * Make physical memory consistent for a set of streaming mode DMA translations @@ -895,6 +906,7 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); } +EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); void swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, @@ -902,12 +914,14 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); } +EXPORT_SYMBOL(swiotlb_sync_sg_for_device); int swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) { return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer)); } +EXPORT_SYMBOL(swiotlb_dma_mapping_error); /* * Return whether the given device DMA address mask can be supported @@ -920,18 +934,4 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) { return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask; } - -EXPORT_SYMBOL(swiotlb_map_single); -EXPORT_SYMBOL(swiotlb_unmap_single); -EXPORT_SYMBOL(swiotlb_map_sg); -EXPORT_SYMBOL(swiotlb_unmap_sg); -EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); -EXPORT_SYMBOL(swiotlb_sync_single_for_device); -EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu); -EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); -EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); -EXPORT_SYMBOL(swiotlb_sync_sg_for_device); -EXPORT_SYMBOL(swiotlb_dma_mapping_error); -EXPORT_SYMBOL(swiotlb_alloc_coherent); -EXPORT_SYMBOL(swiotlb_free_coherent); EXPORT_SYMBOL(swiotlb_dma_supported); -- cgit v1.2.3 From ea319518ba3de282c13ae1cf4bf2215c5e03e67e Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 26 Dec 2008 15:08:55 +0100 Subject: locking, percpu counters: introduce separate lock classes Impact: fix lockdep false positives Classify percpu_counter instances similar to regular lock objects -- that is, per instantiation site. The networking code has increased its use of percpu_counters, which leads to false positives if they are treated as a single class. Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar --- lib/percpu_counter.c | 18 ++++-------------- lib/proportions.c | 6 +++--- 2 files changed, 7 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index a8663890a88..c7fe2e4e8ed 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -71,11 +71,11 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc) } EXPORT_SYMBOL(__percpu_counter_sum); -static struct lock_class_key percpu_counter_irqsafe; - -int percpu_counter_init(struct percpu_counter *fbc, s64 amount) +int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, + struct lock_class_key *key) { spin_lock_init(&fbc->lock); + lockdep_set_class(&fbc->lock, key); fbc->count = amount; fbc->counters = alloc_percpu(s32); if (!fbc->counters) @@ -87,17 +87,7 @@ int percpu_counter_init(struct percpu_counter *fbc, s64 amount) #endif return 0; } -EXPORT_SYMBOL(percpu_counter_init); - -int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount) -{ - int err; - - err = percpu_counter_init(fbc, amount); - if (!err) - lockdep_set_class(&fbc->lock, &percpu_counter_irqsafe); - return err; -} +EXPORT_SYMBOL(__percpu_counter_init); void percpu_counter_destroy(struct percpu_counter *fbc) { diff --git a/lib/proportions.c b/lib/proportions.c index 4f387a643d7..7367f2b727d 100644 --- a/lib/proportions.c +++ b/lib/proportions.c @@ -83,11 +83,11 @@ int prop_descriptor_init(struct prop_descriptor *pd, int shift) pd->index = 0; pd->pg[0].shift = shift; mutex_init(&pd->mutex); - err = percpu_counter_init_irq(&pd->pg[0].events, 0); + err = percpu_counter_init(&pd->pg[0].events, 0); if (err) goto out; - err = percpu_counter_init_irq(&pd->pg[1].events, 0); + err = percpu_counter_init(&pd->pg[1].events, 0); if (err) percpu_counter_destroy(&pd->pg[0].events); @@ -191,7 +191,7 @@ int prop_local_init_percpu(struct prop_local_percpu *pl) spin_lock_init(&pl->lock); pl->shift = 0; pl->period = 0; - return percpu_counter_init_irq(&pl->events, 0); + return percpu_counter_init(&pl->events, 0); } void prop_local_destroy_percpu(struct prop_local_percpu *pl) -- cgit v1.2.3 From 79ff56ebd3edfb16f8badc558cb439b203a3298f Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Tue, 30 Dec 2008 20:18:00 -0800 Subject: swiotlb: add missing __init annotations Impact: cleanup, reduce kernel size a bit The current kernel build warns: WARNING: vmlinux.o(.text+0x11458): Section mismatch in reference from the function swiotlb_alloc_boot() to the function .init.text:__alloc_bootmem_low() The function swiotlb_alloc_boot() references the function __init __alloc_bootmem_low(). This is often because swiotlb_alloc_boot lacks a __init annotation or the annotation of __alloc_bootmem_low is wrong. WARNING: vmlinux.o(.text+0x1011f2): Section mismatch in reference from the function swiotlb_late_init_with_default_size() to the function .init.text:__alloc_bootmem_low() The function swiotlb_late_init_with_default_size() references the function __init __alloc_bootmem_low(). This is often because swiotlb_late_init_with_default_size lacks a __init annotation or the annotation of __alloc_bootmem_low is wrong. and indeed the functions calling __alloc_bootmem_low() can be marked __init as well. Signed-off-by: Roland Dreier Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index fa2dc4e5f9b..b6d0aae4fd3 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -116,7 +116,7 @@ setup_io_tlb_npages(char *str) __setup("swiotlb=", setup_io_tlb_npages); /* make io_tlb_overflow tunable too? */ -void * __weak swiotlb_alloc_boot(size_t size, unsigned long nslabs) +void * __weak __init swiotlb_alloc_boot(size_t size, unsigned long nslabs) { return alloc_bootmem_low_pages(size); } -- cgit v1.2.3 From 52942b6b16c6ebb25f4dd4df0208d840ba0cbc5c Mon Sep 17 00:00:00 2001 From: Jesper Juhl Date: Sat, 3 Jan 2009 00:16:03 +0100 Subject: swiotlb: Don't include linux/swiotlb.h twice in lib/swiotlb.c There's no point in including the linux/swiotlb.h header twice in lib/swiotlb.c - this patch gets rid of the unneeded include. Signed-off-by: Jesper Juhl Signed-off-by: Ingo Molnar --- lib/swiotlb.c | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index b6d0aae4fd3..7f5e21b9c16 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From 8e6bdb7f8b478323525444ecee7a1477ff5b6639 Mon Sep 17 00:00:00 2001 From: Wu Fengguang Date: Thu, 27 Nov 2008 11:42:22 +0100 Subject: trivial: radix-tree: document wrap-around issue of radix_tree_next_hole() And some 80-line cleanups. Signed-off-by: Wu Fengguang Signed-off-by: Jiri Kosina --- lib/radix-tree.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/radix-tree.c b/lib/radix-tree.c index be86b32bc87..a7f5217dd67 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -640,13 +640,14 @@ EXPORT_SYMBOL(radix_tree_tag_get); * * Returns: the index of the hole if found, otherwise returns an index * outside of the set specified (in which case 'return - index >= max_scan' - * will be true). + * will be true). In rare cases of index wrap-around, 0 will be returned. * * radix_tree_next_hole may be called under rcu_read_lock. However, like - * radix_tree_gang_lookup, this will not atomically search a snapshot of the - * tree at a single point in time. For example, if a hole is created at index - * 5, then subsequently a hole is created at index 10, radix_tree_next_hole - * covering both indexes may return 10 if called under rcu_read_lock. + * radix_tree_gang_lookup, this will not atomically search a snapshot of + * the tree at a single point in time. For example, if a hole is created + * at index 5, then subsequently a hole is created at index 10, + * radix_tree_next_hole covering both indexes may return 10 if called + * under rcu_read_lock. */ unsigned long radix_tree_next_hole(struct radix_tree_root *root, unsigned long index, unsigned long max_scan) -- cgit v1.2.3 From 210272a28465a7a31bcd580d2f9529f924965aa5 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 16 Oct 2008 14:57:54 -0600 Subject: driver core: Remove completion from struct klist_node Removing the completion from klist_node reduces its size from 64 bytes to 28 on x86-64. To maintain the semantics of klist_remove(), we add a single list of klist nodes which are pending deletion and scan them. Signed-off-by: Matthew Wilcox Signed-off-by: Greg Kroah-Hartman --- lib/klist.c | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/klist.c b/lib/klist.c index bbdd3015c2c..573d6068a42 100644 --- a/lib/klist.c +++ b/lib/klist.c @@ -36,6 +36,7 @@ #include #include +#include /* * Use the lowest bit of n_klist to mark deleted nodes and exclude @@ -108,7 +109,6 @@ static void add_tail(struct klist *k, struct klist_node *n) static void klist_node_init(struct klist *k, struct klist_node *n) { INIT_LIST_HEAD(&n->n_node); - init_completion(&n->n_removed); kref_init(&n->n_ref); knode_set_klist(n, k); if (k->get) @@ -171,13 +171,34 @@ void klist_add_before(struct klist_node *n, struct klist_node *pos) } EXPORT_SYMBOL_GPL(klist_add_before); +struct klist_waiter { + struct list_head list; + struct klist_node *node; + struct task_struct *process; + int woken; +}; + +static DEFINE_SPINLOCK(klist_remove_lock); +static LIST_HEAD(klist_remove_waiters); + static void klist_release(struct kref *kref) { + struct klist_waiter *waiter, *tmp; struct klist_node *n = container_of(kref, struct klist_node, n_ref); WARN_ON(!knode_dead(n)); list_del(&n->n_node); - complete(&n->n_removed); + spin_lock(&klist_remove_lock); + list_for_each_entry_safe(waiter, tmp, &klist_remove_waiters, list) { + if (waiter->node != n) + continue; + + waiter->woken = 1; + mb(); + wake_up_process(waiter->process); + list_del(&waiter->list); + } + spin_unlock(&klist_remove_lock); knode_set_klist(n, NULL); } @@ -217,8 +238,24 @@ EXPORT_SYMBOL_GPL(klist_del); */ void klist_remove(struct klist_node *n) { + struct klist_waiter waiter; + + waiter.node = n; + waiter.process = current; + waiter.woken = 0; + spin_lock(&klist_remove_lock); + list_add(&waiter.list, &klist_remove_waiters); + spin_unlock(&klist_remove_lock); + klist_del(n); - wait_for_completion(&n->n_removed); + + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (waiter.woken) + break; + schedule(); + } + __set_current_state(TASK_RUNNING); } EXPORT_SYMBOL_GPL(klist_remove); -- cgit v1.2.3 From c65b9145f40da99cad000f81823265dc70e5fcf9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 13 Nov 2008 13:20:00 +0900 Subject: uevent: don't pass envp_ext[] as format string in kobject_uevent_env() kobject_uevent_env() uses envp_ext[] as verbatim format string which can cause problems ranging from unexpectedly mangled string to oops if a string in envp_ext[] contains substring which can be interpreted as format. Fix it. Signed-off-by: Tejun Heo Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- lib/kobject_uevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 3f914725bda..ca215bc2329 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -165,7 +165,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, /* keys passed in from the caller */ if (envp_ext) { for (i = 0; envp_ext[i]; i++) { - retval = add_uevent_var(env, envp_ext[i]); + retval = add_uevent_var(env, "%s", envp_ext[i]); if (retval) goto exit; } -- cgit v1.2.3 From e0d7bf5d580c20ff14d0200b6ab47bc77f99b152 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sun, 16 Nov 2008 18:23:27 +0800 Subject: kobject: return the result of uevent sending by netlink We need to return the result of uevent sending by netlink to caller, when uevent_helper is disabled and CONFIG_NET is defined. Signed-off-by: Ming Lei Cc: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- lib/kobject_uevent.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index ca215bc2329..318328ddbd1 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -225,8 +225,10 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, } NETLINK_CB(skb).dst_group = 1; - netlink_broadcast(uevent_sock, skb, 0, 1, GFP_KERNEL); - } + retval = netlink_broadcast(uevent_sock, skb, 0, 1, + GFP_KERNEL); + } else + retval = -ENOMEM; } #endif -- cgit v1.2.3 From 2e5ba26a483218b51b7b327fb235b348890aea15 Mon Sep 17 00:00:00 2001 From: Wu Fengguang Date: Thu, 11 Dec 2008 07:43:17 +0800 Subject: dynamic_printk: reduce one level of indentation Cleanup pr_debug_write() to reduce one level of indentation. Cc: Marcel Holtmann Acked-by: Jason Baron Signed-off-by: Wu Fengguang Signed-off-by: Greg Kroah-Hartman --- lib/dynamic_printk.c | 58 +++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/dynamic_printk.c b/lib/dynamic_printk.c index 8e30295e856..165a19763dc 100644 --- a/lib/dynamic_printk.c +++ b/lib/dynamic_printk.c @@ -277,40 +277,34 @@ static ssize_t pr_debug_write(struct file *file, const char __user *buf, dynamic_enabled = DYNAMIC_ENABLED_NONE; } err = 0; - } else { - if (elem) { - if (value && (elem->enable == 0)) { - dynamic_printk_enabled |= - (1LL << elem->hash1); - dynamic_printk_enabled2 |= - (1LL << elem->hash2); - elem->enable = 1; - num_enabled++; - dynamic_enabled = DYNAMIC_ENABLED_SOME; - err = 0; - printk(KERN_DEBUG - "debugging enabled for module %s\n", - elem->name); - } else if (!value && (elem->enable == 1)) { - elem->enable = 0; - num_enabled--; - if (disabled_hash(elem->hash1, true)) - dynamic_printk_enabled &= + } else if (elem) { + if (value && (elem->enable == 0)) { + dynamic_printk_enabled |= (1LL << elem->hash1); + dynamic_printk_enabled2 |= (1LL << elem->hash2); + elem->enable = 1; + num_enabled++; + dynamic_enabled = DYNAMIC_ENABLED_SOME; + err = 0; + printk(KERN_DEBUG + "debugging enabled for module %s\n", + elem->name); + } else if (!value && (elem->enable == 1)) { + elem->enable = 0; + num_enabled--; + if (disabled_hash(elem->hash1, true)) + dynamic_printk_enabled &= ~(1LL << elem->hash1); - if (disabled_hash(elem->hash2, false)) - dynamic_printk_enabled2 &= + if (disabled_hash(elem->hash2, false)) + dynamic_printk_enabled2 &= ~(1LL << elem->hash2); - if (num_enabled) - dynamic_enabled = - DYNAMIC_ENABLED_SOME; - else - dynamic_enabled = - DYNAMIC_ENABLED_NONE; - err = 0; - printk(KERN_DEBUG - "debugging disabled for module " - "%s\n", elem->name); - } + if (num_enabled) + dynamic_enabled = DYNAMIC_ENABLED_SOME; + else + dynamic_enabled = DYNAMIC_ENABLED_NONE; + err = 0; + printk(KERN_DEBUG + "debugging disabled for module %s\n", + elem->name); } } } -- cgit v1.2.3 From 94b324864ef2a8e461f3933ab99638255299e9f0 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Tue, 6 Jan 2009 10:44:37 -0800 Subject: swiotlb: struct device - replace bus_id with dev_name(), dev_set_name() Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman --- lib/swiotlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 7f5e21b9c16..c2a4e640145 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -647,7 +647,7 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) * the damage, or panic when the transfer is too big. */ printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " - "device %s\n", size, dev ? dev->bus_id : "?"); + "device %s\n", size, dev ? dev_name(dev) : "?"); if (size > io_tlb_overflow && do_panic) { if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) -- cgit v1.2.3 From b61312d353da1871778711040464b10f5cd904df Mon Sep 17 00:00:00 2001 From: Viktor Rosendahl Date: Tue, 6 Jan 2009 14:40:42 -0800 Subject: oops handling: ensure that any oops is flushed to the mtdoops console This used to work unpatched with older kernels, during the development phase of mtdoops. Before commit e3e8a75d2acfc61ebf25524666a0a2c6abb0620c a space was printed with console_loglevel set to 15, which probably flushed the oops message as a side effect. This is another patch from the Nokia N810 kernel. Signed-off-by: Viktor Rosendahl Cc: Alan Cox Cc: Ingo Molnar Cc: David Woodhouse Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/bust_spinlocks.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c index 486da62b2b0..9681d54b95d 100644 --- a/lib/bust_spinlocks.c +++ b/lib/bust_spinlocks.c @@ -12,6 +12,7 @@ #include #include #include +#include void __attribute__((weak)) bust_spinlocks(int yes) @@ -22,6 +23,7 @@ void __attribute__((weak)) bust_spinlocks(int yes) #ifdef CONFIG_VT unblank_screen(); #endif + console_unblank(); if (--oops_in_progress == 0) wake_up_klogd(); } -- cgit v1.2.3 From f1883f86dea84fe47a71a39fc1afccc005915ed8 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Tue, 6 Jan 2009 14:40:45 -0800 Subject: Remove remaining unwinder code Signed-off-by: Alexey Dobriyan Cc: Gabor Gombas Cc: Jan Beulich Cc: Andi Kleen Cc: Ingo Molnar , Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/fault-inject.c | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/fault-inject.c b/lib/fault-inject.c index a50a311554c..f97af55bdd9 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From 40bc1f2dbc29ab88176a650e51f2246526105093 Mon Sep 17 00:00:00 2001 From: Harvey Harrison Date: Tue, 6 Jan 2009 14:40:49 -0800 Subject: lib: fix sparse shadowed variable warning pos is always set before being used, no need to declare a second one inside the if() block. lib/prio_heap.c:34:7: warning: symbol 'pos' shadows an earlier one lib/prio_heap.c:30:6: originally declared here Signed-off-by: Harvey Harrison Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/prio_heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/prio_heap.c b/lib/prio_heap.c index 471944a54e2..a7af6f85eca 100644 --- a/lib/prio_heap.c +++ b/lib/prio_heap.c @@ -31,7 +31,7 @@ void *heap_insert(struct ptr_heap *heap, void *p) if (heap->size < heap->max) { /* Heap insertion */ - int pos = heap->size++; + pos = heap->size++; while (pos > 0 && heap->gt(p, ptrs[(pos-1)/2])) { ptrs[pos] = ptrs[(pos-1)/2]; pos = (pos-1)/2; -- cgit v1.2.3 From 8cef7d57a4a47ef7166acde05eea0bc4f723691c Mon Sep 17 00:00:00 2001 From: Harvey Harrison Date: Tue, 6 Jan 2009 14:40:50 -0800 Subject: lib: radix_tree.c make percpu variable static radix_tree_preloads is unused outside of this file, make it static. Noticed by sparse: lib/radix-tree.c:84:1: warning: symbol 'per_cpu__radix_tree_preloads' was not declared. Should it be static? Signed-off-by: Harvey Harrison Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/radix-tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/radix-tree.c b/lib/radix-tree.c index be86b32bc87..8d3fb0bd128 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -81,7 +81,7 @@ struct radix_tree_preload { int nr; struct radix_tree_node *nodes[RADIX_TREE_MAX_PATH]; }; -DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, }; +static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, }; static inline gfp_t root_gfp_mask(struct radix_tree_root *root) { -- cgit v1.2.3 From 30079677428ae1349d5063b96f677270dfd54309 Mon Sep 17 00:00:00 2001 From: Harvey Harrison Date: Tue, 6 Jan 2009 14:40:50 -0800 Subject: lib: proportions.c trivial sparse lock annotation Suppresses sparse warning: lib/proportions.c:159:16: warning: context imbalance in 'prop_get_global': wrong count at exit lib/proportions.c:159:16: context 'RCU': wanted 0, got 1 lib/proportions.c:164:2: warning: context imbalance in 'prop_put_global': unexpected unlock lib/proportions.c:164:2: context 'RCU': wanted 0, got -1 Signed-off-by: Harvey Harrison Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/proportions.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/proportions.c b/lib/proportions.c index 4f387a643d7..3fda810faf0 100644 --- a/lib/proportions.c +++ b/lib/proportions.c @@ -147,6 +147,7 @@ out: * this is used to track the active references. */ static struct prop_global *prop_get_global(struct prop_descriptor *pd) +__acquires(RCU) { int index; @@ -160,6 +161,7 @@ static struct prop_global *prop_get_global(struct prop_descriptor *pd) } static void prop_put_global(struct prop_descriptor *pd, struct prop_global *pg) +__releases(RCU) { rcu_read_unlock(); } -- cgit v1.2.3 From e899aa823ad74860a17b541b0ad38f513728923d Mon Sep 17 00:00:00 2001 From: Pavel Machek Date: Tue, 6 Jan 2009 14:40:53 -0800 Subject: strict_strto* is not strict enough It decodes "\n" as 0, which is bad, because stray echo into backlight will turn your backlight off, etc... Signed-off-by: Pavel Machek Cc: Yi Yang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/vsprintf.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 98d632277ca..0fbd0121d91 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -170,6 +170,8 @@ int strict_strtoul(const char *cp, unsigned int base, unsigned long *res) return -EINVAL; val = simple_strtoul(cp, &tail, base); + if (tail == cp) + return -EINVAL; if ((*tail == '\0') || ((len == (size_t)(tail - cp) + 1) && (*tail == '\n'))) { *res = val; @@ -241,6 +243,8 @@ int strict_strtoull(const char *cp, unsigned int base, unsigned long long *res) return -EINVAL; val = simple_strtoull(cp, &tail, base); + if (tail == cp) + return -EINVAL; if ((*tail == '\0') || ((len == (size_t)(tail - cp) + 1) && (*tail == '\n'))) { *res = val; -- cgit v1.2.3 From 179f7ebff6be45738c6e2fa68c8d2cc5c2c6308e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 6 Jan 2009 14:41:04 -0800 Subject: percpu_counter: FBC_BATCH should be a variable For NR_CPUS >= 16 values, FBC_BATCH is 2*NR_CPUS Considering more and more distros are using high NR_CPUS values, it makes sense to use a more sensible value for FBC_BATCH, and get rid of NR_CPUS. A sensible value is 2*num_online_cpus(), with a minimum value of 32 (This minimum value helps branch prediction in __percpu_counter_add()) We already have a hotcpu notifier, so we can adjust FBC_BATCH dynamically. We rename FBC_BATCH to percpu_counter_batch since its not a constant anymore. Signed-off-by: Eric Dumazet Acked-by: David S. Miller Acked-by: Peter Zijlstra Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/percpu_counter.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index b255b939bc1..a60bd804609 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -9,10 +9,8 @@ #include #include -#ifdef CONFIG_HOTPLUG_CPU static LIST_HEAD(percpu_counters); static DEFINE_MUTEX(percpu_counters_lock); -#endif void percpu_counter_set(struct percpu_counter *fbc, s64 amount) { @@ -111,13 +109,24 @@ void percpu_counter_destroy(struct percpu_counter *fbc) } EXPORT_SYMBOL(percpu_counter_destroy); -#ifdef CONFIG_HOTPLUG_CPU +int percpu_counter_batch __read_mostly = 32; +EXPORT_SYMBOL(percpu_counter_batch); + +static void compute_batch_value(void) +{ + int nr = num_online_cpus(); + + percpu_counter_batch = max(32, nr*2); +} + static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { +#ifdef CONFIG_HOTPLUG_CPU unsigned int cpu; struct percpu_counter *fbc; + compute_batch_value(); if (action != CPU_DEAD) return NOTIFY_OK; @@ -134,13 +143,14 @@ static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, spin_unlock_irqrestore(&fbc->lock, flags); } mutex_unlock(&percpu_counters_lock); +#endif return NOTIFY_OK; } static int __init percpu_counter_startup(void) { + compute_batch_value(); hotcpu_notifier(percpu_counter_hotcpu_callback, 0); return 0; } module_init(percpu_counter_startup); -#endif -- cgit v1.2.3 From 8feae13110d60cc6287afabc2887366b0eb226c2 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 8 Jan 2009 12:04:47 +0000 Subject: NOMMU: Make VMAs per MM as for MMU-mode linux Make VMAs per mm_struct as for MMU-mode linux. This solves two problems: (1) In SYSV SHM where nattch for a segment does not reflect the number of shmat's (and forks) done. (2) In mmap() where the VMA's vm_mm is set to point to the parent mm by an exec'ing process when VM_EXECUTABLE is specified, regardless of the fact that a VMA might be shared and already have its vm_mm assigned to another process or a dead process. A new struct (vm_region) is introduced to track a mapped region and to remember the circumstances under which it may be shared and the vm_list_struct structure is discarded as it's no longer required. This patch makes the following additional changes: (1) Regions are now allocated with alloc_pages() rather than kmalloc() and with no recourse to __GFP_COMP, so the pages are not composite. Instead, each page has a reference on it held by the region. Anything else that is interested in such a page will have to get a reference on it to retain it. When the pages are released due to unmapping, each page is passed to put_page() and will be freed when the page usage count reaches zero. (2) Excess pages are trimmed after an allocation as the allocation must be made as a power-of-2 quantity of pages. (3) VMAs are added to the parent MM's R/B tree and mmap lists. As an MM may end up with overlapping VMAs within the tree, the VMA struct address is appended to the sort key. (4) Non-anonymous VMAs are now added to the backing inode's prio list. (5) Holes may be punched in anonymous VMAs with munmap(), releasing parts of the backing region. The VMA and region structs will be split if necessary. (6) sys_shmdt() only releases one attachment to a SYSV IPC shared memory segment instead of all the attachments at that addresss. Multiple shmat()'s return the same address under NOMMU-mode instead of different virtual addresses as under MMU-mode. (7) Core dumping for ELF-FDPIC requires fewer exceptions for NOMMU-mode. (8) /proc/maps is now the global list of mapped regions, and may list bits that aren't actually mapped anywhere. (9) /proc/meminfo gains a line (tagged "MmapCopy") that indicates the amount of RAM currently allocated by mmap to hold mappable regions that can't be mapped directly. These are copies of the backing device or file if not anonymous. These changes make NOMMU mode more similar to MMU mode. The downside is that NOMMU mode requires some extra memory to track things over NOMMU without this patch (VMAs are no longer shared, and there are now region structs). Signed-off-by: David Howells Tested-by: Mike Frysinger Acked-by: Paul Mundt --- lib/Kconfig.debug | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2e75478e9c6..d0a32aab03f 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -512,6 +512,13 @@ config DEBUG_VIRTUAL If unsure, say N. +config DEBUG_NOMMU_REGIONS + bool "Debug the global anon/private NOMMU mapping region tree" + depends on DEBUG_KERNEL && !MMU + help + This option causes the global tree of anonymous and private mapping + regions to be regularly checked for invalid topology. + config DEBUG_WRITECOUNT bool "Debug filesystem writers count" depends on DEBUG_KERNEL -- cgit v1.2.3 From b53907c0100a353a7ac53bed260e735e5ccbbbcc Mon Sep 17 00:00:00 2001 From: Wu Fengguang Date: Wed, 7 Jan 2009 18:09:11 -0800 Subject: generic swap(): lib/sort.c: rename swap to swap_func This is to avoid name clashes for the introduction of a global swap() macro. Signed-off-by: Wu Fengguang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/sort.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/sort.c b/lib/sort.c index 6abbaf3d585..926d00429ed 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -32,11 +32,11 @@ static void generic_swap(void *a, void *b, int size) * @base: pointer to data to sort * @num: number of elements * @size: size of each element - * @cmp: pointer to comparison function - * @swap: pointer to swap function or NULL + * @cmp_func: pointer to comparison function + * @swap_func: pointer to swap function or NULL * * This function does a heapsort on the given array. You may provide a - * swap function optimized to your element type. + * swap_func function optimized to your element type. * * Sorting time is O(n log n) both on average and worst-case. While * qsort is about 20% faster on average, it suffers from exploitable @@ -45,37 +45,39 @@ static void generic_swap(void *a, void *b, int size) */ void sort(void *base, size_t num, size_t size, - int (*cmp)(const void *, const void *), - void (*swap)(void *, void *, int size)) + int (*cmp_func)(const void *, const void *), + void (*swap_func)(void *, void *, int size)) { /* pre-scale counters for performance */ int i = (num/2 - 1) * size, n = num * size, c, r; - if (!swap) - swap = (size == 4 ? u32_swap : generic_swap); + if (!swap_func) + swap_func = (size == 4 ? u32_swap : generic_swap); /* heapify */ for ( ; i >= 0; i -= size) { for (r = i; r * 2 + size < n; r = c) { c = r * 2 + size; - if (c < n - size && cmp(base + c, base + c + size) < 0) + if (c < n - size && + cmp_func(base + c, base + c + size) < 0) c += size; - if (cmp(base + r, base + c) >= 0) + if (cmp_func(base + r, base + c) >= 0) break; - swap(base + r, base + c, size); + swap_func(base + r, base + c, size); } } /* sort */ for (i = n - size; i > 0; i -= size) { - swap(base, base + i, size); + swap_func(base, base + i, size); for (r = 0; r * 2 + size < i; r = c) { c = r * 2 + size; - if (c < i - size && cmp(base + c, base + c + size) < 0) + if (c < i - size && + cmp_func(base + c, base + c + size) < 0) c += size; - if (cmp(base + r, base + c) >= 0) + if (cmp_func(base + r, base + c) >= 0) break; - swap(base + r, base + c, size); + swap_func(base + r, base + c, size); } } } -- cgit v1.2.3