diff options
Diffstat (limited to 'linux-core/i915_gem.c')
| -rw-r--r-- | linux-core/i915_gem.c | 876 |
1 files changed, 339 insertions, 537 deletions
diff --git a/linux-core/i915_gem.c b/linux-core/i915_gem.c index 63f4b91d..9d935be2 100644 --- a/linux-core/i915_gem.c +++ b/linux-core/i915_gem.c @@ -30,33 +30,26 @@ #include "drm_compat.h" #include "i915_drm.h" #include "i915_drv.h" +#include <linux/swap.h> -#define WATCH_COHERENCY 0 -#define WATCH_BUF 0 -#define WATCH_EXEC 0 -#define WATCH_LRU 0 -#define WATCH_RELOC 0 -#define WATCH_INACTIVE 0 -#define WATCH_PWRITE 0 - -#if WATCH_BUF | WATCH_EXEC | WATCH_PWRITE -static void -i915_gem_dump_object(struct drm_gem_object *obj, int len, - const char *where, uint32_t mark); -#endif - static int i915_gem_object_set_domain(struct drm_gem_object *obj, uint32_t read_domains, uint32_t write_domain); +static int +i915_gem_object_set_domain_range(struct drm_gem_object *obj, + uint64_t offset, + uint64_t size, + uint32_t read_domains, + uint32_t write_domain); int i915_gem_set_domain(struct drm_gem_object *obj, struct drm_file *file_priv, uint32_t read_domains, uint32_t write_domain); - -static void -i915_gem_clflush_object(struct drm_gem_object *obj); +static int i915_gem_object_get_page_list(struct drm_gem_object *obj); +static void i915_gem_object_free_page_list(struct drm_gem_object *obj); +static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); int i915_gem_do_init(struct drm_device *dev, unsigned long start, unsigned long end) @@ -69,8 +62,8 @@ int i915_gem_do_init(struct drm_device *dev, unsigned long start, return -EINVAL; } - drm_memrange_init(&dev_priv->mm.gtt_space, start, - end - start); + drm_mm_init(&dev_priv->mm.gtt_space, start, + end - start); dev->gtt_total = (uint32_t) (end - start); @@ -134,22 +127,35 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_pread *args = data; struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; ssize_t read; loff_t offset; int ret; obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) + return -EBADF; + obj_priv = obj->driver_private; + + /* Bounds check source. + * + * XXX: This could use review for overflow issues... + */ + if (args->offset > obj->size || args->size > obj->size || + args->offset + args->size > obj->size) { + drm_gem_object_unreference(obj); return -EINVAL; + } mutex_lock(&dev->struct_mutex); - ret = i915_gem_set_domain(obj, file_priv, - I915_GEM_DOMAIN_CPU, 0); - if (ret) { + + ret = i915_gem_object_set_domain_range(obj, args->offset, args->size, + I915_GEM_DOMAIN_CPU, 0); + if (ret != 0) { drm_gem_object_unreference(obj); mutex_unlock(&dev->struct_mutex); - return ret; } + offset = args->offset; read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr, @@ -171,18 +177,12 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, #include "drm_compat.h" -/** - * Writes data to the object referenced by handle. - * - * On error, the contents of the buffer that were to be modified are undefined. - */ -int -i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) +static int +i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj, + struct drm_i915_gem_pwrite *args, + struct drm_file *file_priv) { - struct drm_i915_gem_pwrite *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj_priv = obj->driver_private; ssize_t remain; loff_t offset; char __user *user_data; @@ -192,18 +192,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, unsigned long pfn; unsigned long unwritten; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); - if (obj == NULL) - return -EINVAL; - - /** Bounds check destination. - * - * XXX: This could use review for overflow issues... - */ - if (args->offset > obj->size || args->size > obj->size || - args->offset + args->size > obj->size) - return -EFAULT; - user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; if (!access_ok(VERIFY_READ, user_data, remain)) @@ -213,7 +201,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, mutex_lock(&dev->struct_mutex); ret = i915_gem_object_pin(obj, 0); if (ret) { - drm_gem_object_unreference(obj); mutex_unlock(&dev->struct_mutex); return ret; } @@ -221,14 +208,13 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT); if (ret) goto fail; - + obj_priv = obj->driver_private; offset = obj_priv->gtt_offset + args->offset; obj_priv->dirty = 1; - + while (remain > 0) { - - /** Operation in this page + /* Operation in this page * * i = page number * o = offset within page @@ -241,7 +227,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, l = PAGE_SIZE - o; pfn = (dev->agp->base >> PAGE_SHIFT) + i; - + #ifdef DRM_KMAP_ATOMIC_PROT_PFN /* kmap_atomic can't map IO pages on non-HIGHMEM kernels */ @@ -251,7 +237,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n", i, o, l, pfn, vaddr); #endif - unwritten = __copy_from_user_inatomic_nocache(vaddr + o, user_data, l); + unwritten = __copy_from_user_inatomic_nocache(vaddr + o, + user_data, l); kunmap_atomic(vaddr, KM_USER0); if (unwritten) @@ -259,7 +246,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, { vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE); #if WATCH_PWRITE - DRM_INFO("pwrite slow i %d o %d l %d pfn %ld vaddr %p\n", + DRM_INFO("pwrite slow i %d o %d l %d " + "pfn %ld vaddr %p\n", i, o, l, pfn, vaddr); #endif if (vaddr == NULL) { @@ -288,14 +276,96 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, #endif fail: - i915_gem_object_unpin (obj); - drm_gem_object_unreference(obj); + i915_gem_object_unpin(obj); mutex_unlock(&dev->struct_mutex); + return ret; +} + +int +i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj, + struct drm_i915_gem_pwrite *args, + struct drm_file *file_priv) +{ + int ret; + loff_t offset; + ssize_t written; + + mutex_lock(&dev->struct_mutex); + + ret = i915_gem_set_domain(obj, file_priv, + I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU); + if (ret) { + mutex_unlock(&dev->struct_mutex); + return ret; + } + + offset = args->offset; + + written = vfs_write(obj->filp, + (char __user *)(uintptr_t) args->data_ptr, + args->size, &offset); + if (written != args->size) { + mutex_unlock(&dev->struct_mutex); + if (written < 0) + return written; + else + return -EINVAL; + } + + mutex_unlock(&dev->struct_mutex); + + return 0; +} + +/** + * Writes data to the object referenced by handle. + * + * On error, the contents of the buffer that were to be modified are undefined. + */ +int +i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_gem_pwrite *args = data; + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + if (obj == NULL) + return -EBADF; + obj_priv = obj->driver_private; + + /* Bounds check destination. + * + * XXX: This could use review for overflow issues... + */ + if (args->offset > obj->size || args->size > obj->size || + args->offset + args->size > obj->size) { + drm_gem_object_unreference(obj); + return -EINVAL; + } + + /* We can only do the GTT pwrite on untiled buffers, as otherwise + * it would end up going through the fenced access, and we'll get + * different detiling behavior between reading and writing. + * pread/pwrite currently are reading and writing from the CPU + * perspective, requiring manual detiling by the client. + */ + if (obj_priv->tiling_mode == I915_TILING_NONE && + dev->gtt_total != 0) + ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv); + else + ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv); + #if WATCH_PWRITE if (ret) DRM_INFO("pwrite failed %d\n", ret); #endif + + drm_gem_object_unreference(obj); + return ret; } @@ -315,7 +385,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) - return -EINVAL; + return -EBADF; mutex_lock(&dev->struct_mutex); ret = i915_gem_set_domain(obj, file_priv, @@ -344,21 +414,20 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) { mutex_unlock(&dev->struct_mutex); - return -EINVAL; + return -EBADF; } #if WATCH_BUF DRM_INFO("%s: sw_finish %d (%p)\n", __func__, args->handle, obj); #endif - obj_priv = obj->driver_private; - - /** Pinned buffers may be scanout, so flush the cache - */ - if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) { - i915_gem_clflush_object(obj); - drm_agp_chipset_flush(dev); - } + obj_priv = obj->driver_private; + + /* Pinned buffers may be scanout, so flush the cache */ + if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) { + i915_gem_clflush_object(obj); + drm_agp_chipset_flush(dev); + } drm_gem_object_unreference(obj); mutex_unlock(&dev->struct_mutex); return ret; @@ -385,7 +454,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) - return -EINVAL; + return -EBADF; offset = args->offset; @@ -448,25 +517,6 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj) &dev_priv->mm.active_list); } -#if WATCH_INACTIVE -static void -i915_verify_inactive(struct drm_device *dev, char *file, int line) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; - - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { - obj = obj_priv->obj; - if (obj_priv->pin_count || obj_priv->active || (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) - DRM_ERROR("inactive %p (p %d a %d w %x) %s:%d\n", - obj, - obj_priv->pin_count, obj_priv->active, obj->write_domain, file, line); - } -} -#else -#define i915_verify_inactive(dev,file,line) -#endif static void i915_gem_object_move_to_inactive(struct drm_gem_object *obj) @@ -522,7 +572,7 @@ i915_add_request(struct drm_device *dev, uint32_t flush_domains) OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); OUT_RING(seqno); - OUT_RING(GFX_OP_USER_INTERRUPT); + OUT_RING(MI_USER_INTERRUPT); ADVANCE_LP_RING(); DRM_DEBUG("%d\n", seqno); @@ -534,7 +584,7 @@ i915_add_request(struct drm_device *dev, uint32_t flush_domains) list_add_tail(&request->list, &dev_priv->mm.request_list); if (was_empty) - schedule_delayed_work (&dev_priv->mm.retire_work, HZ); + schedule_delayed_work(&dev_priv->mm.retire_work, HZ); return seqno; } @@ -544,7 +594,6 @@ i915_add_request(struct drm_device *dev, uint32_t flush_domains) * Ensures that all commands in the ring are finished * before signalling the CPU */ - uint32_t i915_retire_commands(struct drm_device *dev) { @@ -661,7 +710,8 @@ i915_gem_retire_requests(struct drm_device *dev) list); retiring_seqno = request->seqno; - if (i915_seqno_passed(seqno, retiring_seqno) || dev_priv->mm.wedged) { + if (i915_seqno_passed(seqno, retiring_seqno) || + dev_priv->mm.wedged) { i915_gem_retire_request(dev, request); list_del(&request->list); @@ -684,7 +734,7 @@ i915_gem_retire_work_handler(struct work_struct *work) mutex_lock(&dev->struct_mutex); i915_gem_retire_requests(dev); if (!list_empty(&dev_priv->mm.request_list)) - schedule_delayed_work (&dev_priv->mm.retire_work, HZ); + schedule_delayed_work(&dev_priv->mm.retire_work, HZ); mutex_unlock(&dev->struct_mutex); } @@ -705,7 +755,8 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno) i915_user_irq_on(dev); ret = wait_event_interruptible(dev_priv->irq_queue, i915_seqno_passed(i915_get_gem_seqno(dev), - seqno) || dev_priv->mm.wedged); + seqno) || + dev_priv->mm.wedged); i915_user_irq_off(dev); dev_priv->mm.waiting_gem_seqno = 0; } @@ -744,7 +795,8 @@ i915_gem_flush(struct drm_device *dev, if (flush_domains & I915_GEM_DOMAIN_CPU) drm_agp_chipset_flush(dev); - if ((invalidate_domains|flush_domains) & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) { + if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU | + I915_GEM_DOMAIN_GTT)) { /* * read/write caches: * @@ -871,7 +923,7 @@ i915_gem_object_unbind(struct drm_gem_object *obj) */ ret = i915_gem_object_wait_rendering(obj); if (ret) { - DRM_ERROR ("wait_rendering failed: %d\n", ret); + DRM_ERROR("wait_rendering failed: %d\n", ret); return ret; } @@ -901,8 +953,8 @@ i915_gem_object_unbind(struct drm_gem_object *obj) if (obj_priv->gtt_space) { atomic_dec(&dev->gtt_count); atomic_sub(obj->size, &dev->gtt_memory); - - drm_memrange_put_block(obj_priv->gtt_space); + + drm_mm_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; } @@ -913,83 +965,6 @@ i915_gem_object_unbind(struct drm_gem_object *obj) return 0; } -#if WATCH_BUF | WATCH_EXEC | WATCH_PWRITE -static void -i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end, - uint32_t bias, uint32_t mark) -{ - uint32_t *mem = kmap_atomic(page, KM_USER0); - int i; - for (i = start; i < end; i += 4) - DRM_INFO("%08x: %08x%s\n", - (int) (bias + i), mem[i / 4], - (bias + i == mark) ? " ********" : ""); - kunmap_atomic(mem, KM_USER0); - /* give syslog time to catch up */ - msleep(1); -} - -static void -i915_gem_dump_object(struct drm_gem_object *obj, int len, - const char *where, uint32_t mark) -{ - struct drm_i915_gem_object *obj_priv = obj->driver_private; - int page; - - DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset); - for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) { - int page_len, chunk, chunk_len; - - page_len = len - page * PAGE_SIZE; - if (page_len > PAGE_SIZE) - page_len = PAGE_SIZE; - - for (chunk = 0; chunk < page_len; chunk += 128) { - chunk_len = page_len - chunk; - if (chunk_len > 128) - chunk_len = 128; - i915_gem_dump_page(obj_priv->page_list[page], - chunk, chunk + chunk_len, - obj_priv->gtt_offset + - page * PAGE_SIZE, - mark); - } - } -} -#endif - -#if WATCH_LRU -static void -i915_dump_lru(struct drm_device *dev, const char *where) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - - DRM_INFO("active list %s {\n", where); - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, - list) - { - DRM_INFO(" %p: %08x\n", obj_priv, - obj_priv->last_rendering_seqno); - } - DRM_INFO("}\n"); - DRM_INFO("flushing list %s {\n", where); - list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, - list) - { - DRM_INFO(" %p: %08x\n", obj_priv, - obj_priv->last_rendering_seqno); - } - DRM_INFO("}\n"); - DRM_INFO("inactive %s {\n", where); - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { - DRM_INFO(" %p: %08x\n", obj_priv, - obj_priv->last_rendering_seqno); - } - DRM_INFO("}\n"); -} -#endif - static int i915_gem_evict_something(struct drm_device *dev) { @@ -1063,7 +1038,8 @@ i915_gem_evict_something(struct drm_device *dev) continue; } - DRM_ERROR("inactive empty %d request empty %d flushing empty %d\n", + DRM_ERROR("inactive empty %d request empty %d " + "flushing empty %d\n", list_empty(&dev_priv->mm.inactive_list), list_empty(&dev_priv->mm.request_list), list_empty(&dev_priv->mm.flushing_list)); @@ -1084,7 +1060,7 @@ i915_gem_object_get_page_list(struct drm_gem_object *obj) struct inode *inode; struct page *page; int ret; - + if (obj_priv->page_list) return 0; @@ -1110,7 +1086,7 @@ i915_gem_object_get_page_list(struct drm_gem_object *obj) page = NULL; } ret = shmem_getpage(inode, i, &page, SGP_DIRTY, NULL); - + if (ret) { DRM_ERROR("shmem_getpage failed: %d\n", ret); i915_gem_object_free_page_list(obj); @@ -1132,7 +1108,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) struct drm_device *dev = obj->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = obj->driver_private; - struct drm_memrange_node *free_space; + struct drm_mm_node *free_space; int page_count, ret; if (alignment == 0) @@ -1143,13 +1119,11 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) } search_free: - free_space = drm_memrange_search_free(&dev_priv->mm.gtt_space, - obj->size, - alignment, 0); + free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, + obj->size, alignment, 0); if (free_space != NULL) { - obj_priv->gtt_space = - drm_memrange_get_block(free_space, obj->size, - alignment); + obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, + alignment); if (obj_priv->gtt_space != NULL) { obj_priv->gtt_space->private = obj; obj_priv->gtt_offset = obj_priv->gtt_space->start; @@ -1183,7 +1157,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) #endif ret = i915_gem_object_get_page_list(obj); if (ret) { - drm_memrange_put_block(obj_priv->gtt_space); + drm_mm_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; return ret; } @@ -1198,7 +1172,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) obj_priv->gtt_offset); if (obj_priv->agp_mem == NULL) { i915_gem_object_free_page_list(obj); - drm_memrange_put_block(obj_priv->gtt_space); + drm_mm_put_block(obj_priv->gtt_space); obj_priv->gtt_space = NULL; return -ENOMEM; } @@ -1215,7 +1189,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) return 0; } -static void +void i915_gem_clflush_object(struct drm_gem_object *obj) { struct drm_i915_gem_object *obj_priv = obj->driver_private; @@ -1354,8 +1328,8 @@ i915_gem_object_set_domain(struct drm_gem_object *obj, #if WATCH_BUF DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n", - __func__, obj, - obj->read_domains, read_domains, + __func__, obj, + obj->read_domains, read_domains, obj->write_domain, write_domain); #endif /* @@ -1393,7 +1367,8 @@ i915_gem_object_set_domain(struct drm_gem_object *obj, * flushed before the cpu cache is invalidated */ if ((invalidate_domains & I915_GEM_DOMAIN_CPU) && - (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))) { + (flush_domains & ~(I915_GEM_DOMAIN_CPU | + I915_GEM_DOMAIN_GTT))) { ret = i915_gem_object_wait_rendering(obj); if (ret) return ret; @@ -1403,7 +1378,17 @@ i915_gem_object_set_domain(struct drm_gem_object *obj, if ((write_domain | flush_domains) != 0) obj->write_domain = write_domain; + + /* If we're invalidating the CPU domain, clear the per-page CPU + * domain list as well. + */ + if (obj_priv->page_cpu_valid != NULL && + (obj->read_domains & I915_GEM_DOMAIN_CPU) && + ((read_domains & I915_GEM_DOMAIN_CPU) == 0)) { + memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); + } obj->read_domains = read_domains; + dev->invalidate_domains |= invalidate_domains; dev->flush_domains |= flush_domains; #if WATCH_BUF @@ -1416,6 +1401,57 @@ i915_gem_object_set_domain(struct drm_gem_object *obj, } /** + * Set the read/write domain on a range of the object. + * + * Currently only implemented for CPU reads, otherwise drops to normal + * i915_gem_object_set_domain(). + */ +static int +i915_gem_object_set_domain_range(struct drm_gem_object *obj, + uint64_t offset, + uint64_t size, + uint32_t read_domains, + uint32_t write_domain) +{ + struct drm_i915_gem_object *obj_priv = obj->driver_private; + int ret, i; + + if (obj->read_domains & I915_GEM_DOMAIN_CPU) + return 0; + + if (read_domains != I915_GEM_DOMAIN_CPU || + write_domain != 0) + return i915_gem_object_set_domain(obj, + read_domains, write_domain); + + /* Wait on any GPU rendering to the object to be flushed. */ + if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) { + ret = i915_gem_object_wait_rendering(obj); + if (ret) + return ret; + } + + if (obj_priv->page_cpu_valid == NULL) { + obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE, + DRM_MEM_DRIVER); + } + + /* Flush the cache on any pages that are still invalid from the CPU's + * perspective. + */ + for (i = offset / PAGE_SIZE; i < (offset + size - 1) / PAGE_SIZE; i++) { + if (obj_priv->page_cpu_valid[i]) + continue; + + drm_ttm_cache_flush(obj_priv->page_list + i, 1); + + obj_priv->page_cpu_valid[i] = 1; + } + + return 0; +} + +/** * Once all of the objects have been set in the proper domain, * perform the necessary flush and invalidate operations. * @@ -1447,76 +1483,6 @@ i915_gem_dev_set_domain(struct drm_device *dev) return flush_domains; } -#if WATCH_COHERENCY -static void -i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) -{ - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = obj->driver_private; - int page; - uint32_t *gtt_mapping; - uint32_t *backing_map = NULL; - int bad_count = 0; - - DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %dkb):\n", - __func__, obj, obj_priv->gtt_offset, handle, - obj->size / 1024); - - gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset, - obj->size); - if (gtt_mapping == NULL) { - DRM_ERROR("failed to map GTT space\n"); - return; - } - - for (page = 0; page < obj->size / PAGE_SIZE; page++) { - int i; - - backing_map = kmap_atomic(obj_priv->page_list[page], KM_USER0); - - if (backing_map == NULL) { - DRM_ERROR("failed to map backing page\n"); - goto out; - } - - for (i = 0; i < PAGE_SIZE / 4; i++) { - uint32_t cpuval = backing_map[i]; - uint32_t gttval = readl(gtt_mapping + - page * 1024 + i); - - if (cpuval != gttval) { - DRM_INFO("incoherent CPU vs GPU at 0x%08x: " - "0x%08x vs 0x%08x\n", - (int)(obj_priv->gtt_offset + - page * PAGE_SIZE + i * 4), - cpuval, gttval); - if (bad_count++ >= 8) { - DRM_INFO("...\n"); - goto out; - } - } - } - kunmap_atomic(backing_map, KM_USER0); - backing_map = NULL; - } - - out: - if (backing_map != NULL) - kunmap_atomic(backing_map, KM_USER0); - iounmap(gtt_mapping); - - /* give syslog time to catch up */ - msleep(1); - - /* Directly flush the object, since we just loaded values with the CPU - * from thebacking pages and we don't want to disturb the cache - * management that we're trying to observe. - */ - - i915_gem_clflush_object(obj); -} -#endif - /** * Pin an object to the GTT and evaluate the relocations landing in it. */ @@ -1561,7 +1527,7 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, reloc.target_handle); if (target_obj == NULL) { i915_gem_object_unpin(obj); - return -EINVAL; + return -EBADF; } target_obj_priv = target_obj->driver_private; @@ -1784,7 +1750,8 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) mutex_lock(&dev->struct_mutex); seqno = i915_file_priv->mm.last_gem_throttle_seqno; - i915_file_priv->mm.last_gem_throttle_seqno = i915_file_priv->mm.last_gem_seqno; + i915_file_priv->mm.last_gem_throttle_seqno = + i915_file_priv->mm.last_gem_seqno; if (seqno) ret = i915_wait_request(dev, seqno); mutex_unlock(&dev->struct_mutex); @@ -1841,7 +1808,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, mutex_unlock(&dev->struct_mutex); return -EIO; } - + if (dev_priv->mm.suspended) { DRM_ERROR("Execbuf while VT-switched.\n"); mutex_unlock(&dev->struct_mutex); @@ -1862,7 +1829,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, if (object_list[i] == NULL) { DRM_ERROR("Invalid object handle %d at index %d\n", exec_list[i].handle, i); - ret = -EINVAL; + ret = -EBADF; goto err; } @@ -2022,7 +1989,9 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) if (obj_priv->pin_count == 1) { atomic_inc(&dev->pin_count); atomic_add(obj->size, &dev->pin_memory); - if (!obj_priv->active && (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) == 0 && + if (!obj_priv->active && + (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | + I915_GEM_DOMAIN_GTT)) == 0 && !list_empty(&obj_priv->list)) list_del_init(&obj_priv->list); } @@ -2048,7 +2017,9 @@ i915_gem_object_unpin(struct drm_gem_object *obj) * the inactive list */ if (obj_priv->pin_count == 0) { - if (!obj_priv->active && (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) == 0) + if (!obj_priv->active && + (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | + I915_GEM_DOMAIN_GTT)) == 0) list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); atomic_dec(&dev->pin_count); @@ -2073,7 +2044,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", args->handle); mutex_unlock(&dev->struct_mutex); - return -EINVAL; + return -EBADF; } obj_priv = obj->driver_private; @@ -2084,7 +2055,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, return ret; } - /** XXX - flush the CPU caches for pinned objects + /* XXX - flush the CPU caches for pinned objects * as the X server doesn't manage domains yet */ if (obj->write_domain & I915_GEM_DOMAIN_CPU) { @@ -2113,7 +2084,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data, DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", args->handle); mutex_unlock(&dev->struct_mutex); - return -EINVAL; + return -EBADF; } i915_gem_object_unpin(obj); @@ -2137,7 +2108,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n", args->handle); mutex_unlock(&dev->struct_mutex); - return -EINVAL; + return -EBADF; } obj_priv = obj->driver_private; @@ -2187,6 +2158,7 @@ void i915_gem_free_object(struct drm_gem_object *obj) i915_gem_object_unbind(obj); + drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER); drm_free(obj->driver_private, 1, DRM_MEM_DRIVER); } @@ -2206,7 +2178,7 @@ i915_gem_set_domain(struct drm_gem_object *obj, if (ret) return ret; flush_domains = i915_gem_dev_set_domain(obj->dev); - + if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) (void) i915_add_request(dev, flush_domains); @@ -2267,7 +2239,8 @@ i915_gem_idle(struct drm_device *dev) */ i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT), ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); - seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); + seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU | + I915_GEM_DOMAIN_GTT)); if (seqno == 0) { mutex_unlock(&dev->struct_mutex); @@ -2317,6 +2290,56 @@ i915_gem_idle(struct drm_device *dev) return 0; } +static int +i915_gem_init_hws(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + int ret; + + /* If we need a physical address for the status page, it's already + * initialized at driver load time. + */ + if (!I915_NEED_GFX_HWS(dev)) + return 0; + + obj = drm_gem_object_alloc(dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate status page\n"); + return -ENOMEM; + } + obj_priv = obj->driver_private; + + ret = i915_gem_object_pin(obj, 4096); + if (ret != 0) { + drm_gem_object_unreference(obj); + return ret; + } + + dev_priv->status_gfx_addr = obj_priv->gtt_offset; + dev_priv->hws_map.offset = dev->agp->base + obj_priv->gtt_offset; + dev_priv->hws_map.size = 4096; + dev_priv->hws_map.type = 0; + dev_priv->hws_map.flags = 0; + dev_priv->hws_map.mtrr = 0; + + drm_core_ioremap(&dev_priv->hws_map, dev); + if (dev_priv->hws_map.handle == NULL) { + DRM_ERROR("Failed to map status page.\n"); + memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); + drm_gem_object_unreference(obj); + return -EINVAL; + } + dev_priv->hws_obj = obj; + dev_priv->hw_status_page = dev_priv->hws_map.handle; + memset(dev_priv->hw_status_page, 0, PAGE_SIZE); + I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr); + DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr); + + return 0; +} + int i915_gem_init_ringbuffer(struct drm_device *dev) { @@ -2325,6 +2348,10 @@ i915_gem_init_ringbuffer(struct drm_device *dev) struct drm_i915_gem_object *obj_priv; int ret; + ret = i915_gem_init_hws(dev); + if (ret != 0) + return ret; + obj = drm_gem_object_alloc(dev, 128 * 1024); if (obj == NULL) { DRM_ERROR("Failed to allocate ringbuffer\n"); @@ -2360,15 +2387,16 @@ i915_gem_init_ringbuffer(struct drm_device *dev) /* Stop the ring if it's running. */ I915_WRITE(PRB0_CTL, 0); - I915_WRITE(PRB0_HEAD, 0); - I915_WRITE(PRB0_TAIL, 0); + I915_WRITE(PRB0_HEAD, 0); + I915_WRITE(PRB0_TAIL, 0); I915_WRITE(PRB0_START, 0); /* Initialize the ring. */ I915_WRITE(PRB0_START, obj_priv->gtt_offset); - I915_WRITE(PRB0_CTL, (((obj->size - 4096) & RING_NR_PAGES) | - RING_NO_REPORT | - RING_VALID)); + I915_WRITE(PRB0_CTL, + ((obj->size - 4096) & RING_NR_PAGES) | + RING_NO_REPORT | + RING_VALID); /* Update our cache of the ring state */ i915_kernel_lost_context(dev); @@ -2388,8 +2416,18 @@ i915_gem_cleanup_ringbuffer(struct drm_device *dev) i915_gem_object_unpin(dev_priv->ring.ring_obj); drm_gem_object_unreference(dev_priv->ring.ring_obj); - + dev_priv->ring.ring_obj = NULL; memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); + + if (dev_priv->hws_obj != NULL) { + i915_gem_object_unpin(dev_priv->hws_obj); + drm_gem_object_unreference(dev_priv->hws_obj); + dev_priv->hws_obj = NULL; + memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); + + /* Write high address into HWS_PGA when disabling. */ + I915_WRITE(HWS_PGA, 0x1ffff000); + } } int @@ -2399,8 +2437,11 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_i915_private *dev_priv = dev->dev_private; int ret; + if (drm_core_check_feature(dev, DRIVER_MODESET)) + return 0; + if (dev_priv->mm.wedged) { - DRM_ERROR("Renabling wedged hardware, good luck\n"); + DRM_ERROR("Reenabling wedged hardware, good luck\n"); dev_priv->mm.wedged = 0; } @@ -2424,6 +2465,9 @@ i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, { int ret; + if (drm_core_check_feature(dev, DRIVER_MODESET)) + return 0; + mutex_lock(&dev->struct_mutex); ret = i915_gem_idle(dev); if (ret == 0) @@ -2433,263 +2477,6 @@ i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, return 0; } -static int i915_gem_active_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Active:\n"); - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, - list) - { - struct drm_gem_object *obj = obj_priv->obj; - if (obj->name) { - DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n", - obj, obj->name, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } else { - DRM_PROC_PRINT(" %p: %08x %08x %d\n", - obj, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } - } - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - -static int i915_gem_flushing_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Flushing:\n"); - list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, - list) - { - struct drm_gem_object *obj = obj_priv->obj; - if (obj->name) { - DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n", - obj, obj->name, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } else { - DRM_PROC_PRINT(" %p: %08x %08x %d\n", obj, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } - } - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - -static int i915_gem_inactive_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Inactive:\n"); - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, - list) - { - struct drm_gem_object *obj = obj_priv->obj; - if (obj->name) { - DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n", - obj, obj->name, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } else { - DRM_PROC_PRINT(" %p: %08x %08x %d\n", obj, - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - } - } - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - -static int i915_gem_request_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_request *gem_request; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Request:\n"); - list_for_each_entry(gem_request, &dev_priv->mm.request_list, - list) - { - DRM_PROC_PRINT (" %d @ %d %08x\n", - gem_request->seqno, - (int) (jiffies - gem_request->emitted_jiffies), - gem_request->flush_domains); - } - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - -static int i915_gem_seqno_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Current sequence: %d\n", i915_get_gem_seqno(dev)); - DRM_PROC_PRINT("Waiter sequence: %d\n", dev_priv->mm.waiting_gem_seqno); - DRM_PROC_PRINT("IRQ sequence: %d\n", dev_priv->mm.irq_gem_seqno); - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - - -static int i915_interrupt_info(char *buf, char **start, off_t offset, - int request, int *eof, void *data) -{ - struct drm_minor *minor = (struct drm_minor *) data; - struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int len = 0; - - if (offset > DRM_PROC_LIMIT) { - *eof = 1; - return 0; - } - - *start = &buf[offset]; - *eof = 0; - DRM_PROC_PRINT("Interrupt enable: %08x\n", - I915_READ(IER)); - DRM_PROC_PRINT("Interrupt identity: %08x\n", - I915_READ(IIR)); - DRM_PROC_PRINT("Interrupt mask: %08x\n", - I915_READ(IMR)); - DRM_PROC_PRINT("Pipe A stat: %08x\n", - I915_READ(PIPEASTAT)); - DRM_PROC_PRINT("Pipe B stat: %08x\n", - I915_READ(PIPEBSTAT)); - DRM_PROC_PRINT("Interrupts received: %d\n", - atomic_read(&dev_priv->irq_received)); - DRM_PROC_PRINT("Current sequence: %d\n", - i915_get_gem_seqno(dev)); - DRM_PROC_PRINT("Waiter sequence: %d\n", - dev_priv->mm.waiting_gem_seqno); - DRM_PROC_PRINT("IRQ sequence: %d\n", - dev_priv->mm.irq_gem_seqno); - if (len > request + offset) - return request; - *eof = 1; - return len - offset; -} - -static struct drm_proc_list { - const char *name; /**< file name */ - int (*f) (char *, char **, off_t, int, int *, void *); /**< proc callback*/ -} i915_gem_proc_list[] = { - {"i915_gem_active", i915_gem_active_info}, - {"i915_gem_flushing", i915_gem_flushing_info}, - {"i915_gem_inactive", i915_gem_inactive_info}, - {"i915_gem_request", i915_gem_request_info}, - {"i915_gem_seqno", i915_gem_seqno_info}, - {"i915_gem_interrupt", i915_interrupt_info}, -}; - -#define I915_GEM_PROC_ENTRIES ARRAY_SIZE(i915_gem_proc_list) - -int i915_gem_proc_init(struct drm_minor *minor) -{ - struct proc_dir_entry *ent; - int i, j; - - for (i = 0; i < I915_GEM_PROC_ENTRIES; i++) { - ent = create_proc_entry(i915_gem_proc_list[i].name, - S_IFREG | S_IRUGO, minor->dev_root); - if (!ent) { - DRM_ERROR("Cannot create /proc/dri/.../%s\n", - i915_gem_proc_list[i].name); - for (j = 0; j < i; j++) - remove_proc_entry(i915_gem_proc_list[i].name, - minor->dev_root); - return -1; - } - ent->read_proc = i915_gem_proc_list[i].f; - ent->data = minor; - } - return 0; -} - -void i915_gem_proc_cleanup(struct drm_minor *minor) -{ - int i; - - if (!minor->dev_root) - return; - - for (i = 0; i < I915_GEM_PROC_ENTRIES; i++) - remove_proc_entry(i915_gem_proc_list[i].name, minor->dev_root); -} - void i915_gem_lastclose(struct drm_device *dev) { @@ -2702,9 +2489,24 @@ i915_gem_lastclose(struct drm_device *dev) ret = i915_gem_idle(dev); if (ret) DRM_ERROR("failed to idle hardware: %d\n", ret); - + i915_gem_cleanup_ringbuffer(dev); } - + mutex_unlock(&dev->struct_mutex); } + +void i915_gem_load(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + INIT_LIST_HEAD(&dev_priv->mm.active_list); + INIT_LIST_HEAD(&dev_priv->mm.flushing_list); + INIT_LIST_HEAD(&dev_priv->mm.inactive_list); + INIT_LIST_HEAD(&dev_priv->mm.request_list); + INIT_DELAYED_WORK(&dev_priv->mm.retire_work, + i915_gem_retire_work_handler); + dev_priv->mm.next_gem_seqno = 1; + + i915_gem_detect_bit_6_swizzle(dev); +} |