From 609e3b037526021d20c7cc18b7fed1152206dc68 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Tue, 6 Feb 2007 14:20:33 +0100 Subject: Implement a policy for selecting memory types. --- linux-core/drmP.h | 27 +++--- linux-core/drm_bo.c | 212 +++++++++++++++++++++++++++++++++++++---------- linux-core/drm_stub.c | 1 + linux-core/i915_buffer.c | 11 +++ linux-core/i915_drv.c | 9 ++ 5 files changed, 204 insertions(+), 56 deletions(-) (limited to 'linux-core') diff --git a/linux-core/drmP.h b/linux-core/drmP.h index dd07a603..5834c9dc 100644 --- a/linux-core/drmP.h +++ b/linux-core/drmP.h @@ -673,12 +673,17 @@ typedef struct drm_mem_type_manager { */ typedef struct drm_bo_driver{ + const uint32_t *mem_type_prio; + const uint32_t *mem_busy_prio; + uint32_t num_mem_type_prio; + uint32_t num_mem_busy_prio; drm_ttm_backend_t *(*create_ttm_backend_entry) (struct drm_device *dev); int (*fence_type)(uint32_t flags, uint32_t *class, uint32_t *type); int (*invalidate_caches)(struct drm_device *dev, uint32_t flags); int (*init_mem_type)(struct drm_device *dev, uint32_t type, drm_mem_type_manager_t *man); + uint32_t (*evict_flags) (struct drm_device *dev, uint32_t type); } drm_bo_driver_t; @@ -800,19 +805,9 @@ typedef struct drm_fence_manager{ atomic_t count; } drm_fence_manager_t; - -typedef struct drm_bo_mem_region { - drm_mm_node_t *node; - uint32_t memory_type; - drm_ttm_t *ttm; - unsigned long bus_offset; - unsigned long num_pages; - uint32_t vm_flags; -} drm_bo_mem_region_t; - - typedef struct drm_buffer_manager{ struct mutex init_mutex; + struct mutex evict_mutex; int nice_mode; int initialized; drm_file_t *last_to_validate; @@ -1003,6 +998,16 @@ typedef struct drm_fence_object{ uint32_t submitted_flush; } drm_fence_object_t; +typedef struct drm_bo_mem_reg { + drm_mm_node_t *mm_node; + unsigned long size; + unsigned long num_pages; + uint32_t page_alignment; + uint32_t mem_type; + uint32_t flags; + uint32_t mask; +} drm_bo_mem_reg_t; + typedef struct drm_buffer_object{ drm_device_t *dev; diff --git a/linux-core/drm_bo.c b/linux-core/drm_bo.c index 9a27a4b5..fa659d04 100644 --- a/linux-core/drm_bo.c +++ b/linux-core/drm_bo.c @@ -56,6 +56,8 @@ static void drm_bo_destroy_locked(drm_buffer_object_t *bo); static int drm_bo_setup_vm_locked(drm_buffer_object_t *bo); static void drm_bo_takedown_vm_locked(drm_buffer_object_t *bo); static void drm_bo_unmap_virtual(drm_buffer_object_t *bo); +static int drm_bo_mem_space(drm_device_t *dev, drm_bo_mem_reg_t *mem, + int no_wait); #define DRM_FLAG_MASKED(_old, _new, _mask) {\ (_old) ^= (((_old) ^ (_new)) & (_mask)); \ @@ -497,6 +499,7 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type, int ret = 0; drm_device_t *dev = bo->dev; drm_buffer_manager_t *bm = &dev->bm; + drm_bo_mem_reg_t evict_mem; /* * Someone might have modified the buffer before we took the buffer mutex. @@ -509,22 +512,39 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type, ret = drm_bo_wait(bo, 0, 0, no_wait); - if (ret) { - if (ret != -EAGAIN) - DRM_ERROR("Failed to expire fence before " - "buffer eviction.\n"); + if (ret && ret != -EAGAIN) { + DRM_ERROR("Failed to expire fence before " + "buffer eviction.\n"); goto out; } - if (mem_type == DRM_BO_MEM_TT) { - ret = drm_move_tt_to_local(bo, 1, force_no_move); - if (ret) - goto out; - mutex_lock(&dev->struct_mutex); - list_del_init(&bo->lru); - drm_bo_add_to_lru(bo, bm); - mutex_unlock(&dev->struct_mutex); + evict_mem.num_pages = bo->num_pages; + evict_mem.page_alignment = bo->page_alignment; + evict_mem.size = evict_mem.num_pages << PAGE_SHIFT; + evict_mem.mask = dev->driver->bo_driver->evict_flags(dev, mem_type); + + ret = drm_bo_mem_space(dev, &evict_mem, no_wait); + + if (ret && ret != -EAGAIN) { + DRM_ERROR("Failed to find memory space for " + "buffer eviction.\n"); + goto out; + } + + if ((mem_type != DRM_BO_MEM_TT) && + (evict_mem.mem_type != DRM_BO_MEM_LOCAL)) { + ret = -EINVAL; + DRM_ERROR("Unsupported memory types for eviction.\n"); + goto out; } + + ret = drm_move_tt_to_local(bo, 1, force_no_move); + if (ret) + goto out; + mutex_lock(&dev->struct_mutex); + list_del_init(&bo->lru); + drm_bo_add_to_lru(bo, bm); + mutex_unlock(&dev->struct_mutex); if (ret) goto out; @@ -535,26 +555,25 @@ static int drm_bo_evict(drm_buffer_object_t * bo, unsigned mem_type, return ret; } -/* - * bo->mutex locked. - */ -int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type, - int no_wait) + +static int drm_bo_mem_force_space(drm_device_t *dev, + drm_bo_mem_reg_t *mem, + uint32_t mem_type, + int no_wait) { - drm_device_t *dev = bo->dev; drm_mm_node_t *node; drm_buffer_manager_t *bm = &dev->bm; drm_buffer_object_t *entry; drm_mem_type_manager_t *man = &bm->man[mem_type]; - drm_mm_t *mm = &man->manager; struct list_head *lru; - unsigned long size = bo->num_pages; + unsigned long num_pages = mem->num_pages; int ret; mutex_lock(&dev->struct_mutex); do { - node = drm_mm_search_free(mm, size, bo->page_alignment, 1); + node = drm_mm_search_free(&man->manager, num_pages, + mem->page_alignment, 1); if (node) break; @@ -563,11 +582,11 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type, break; entry = list_entry(lru->next, drm_buffer_object_t, lru); - atomic_inc(&entry->usage); mutex_unlock(&dev->struct_mutex); mutex_lock(&entry->mutex); - BUG_ON(bo->flags & DRM_BO_FLAG_NO_MOVE); + BUG_ON(entry->flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT)); + ret = drm_bo_evict(entry, mem_type, no_wait, 0); mutex_unlock(&entry->mutex); drm_bo_usage_deref_unlocked(entry); @@ -577,34 +596,108 @@ int drm_bo_alloc_space(drm_buffer_object_t * bo, unsigned mem_type, } while (1); if (!node) { - DRM_ERROR("Out of videoram / aperture space\n"); mutex_unlock(&dev->struct_mutex); return -ENOMEM; } - node = drm_mm_get_block(node, size, bo->page_alignment); + node = drm_mm_get_block(node, num_pages, mem->page_alignment); mutex_unlock(&dev->struct_mutex); - BUG_ON(!node); - node->private = (void *)bo; - - bo->mm_node = node; - bo->offset = node->start * PAGE_SIZE; + mem->mm_node = node; + mem->mem_type = mem_type; + mem->flags = drm_bo_type_flags(mem_type); return 0; } -static int drm_move_local_to_tt(drm_buffer_object_t * bo, int no_wait) + +static int drm_bo_mem_space(drm_device_t *dev, + drm_bo_mem_reg_t *mem, + int no_wait) { - drm_device_t *dev = bo->dev; - int ret = 0; + drm_buffer_manager_t *bm= &dev->bm; + drm_mem_type_manager_t *man; - if (!(bo->mm_node && (bo->flags & DRM_BO_FLAG_NO_MOVE))) { - BUG_ON(bo->mm_node); - ret = drm_bo_alloc_space(bo, DRM_BO_MEM_TT, no_wait); - if (ret) - return ret; + uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio; + const uint32_t *prios = dev->driver->bo_driver->mem_type_prio; + uint32_t i; + uint32_t mem_type = DRM_BO_MEM_LOCAL; + int type_found = 0; + int type_ok = 0; + int has_eagain = 0; + drm_mm_node_t *node = NULL; + int ret; + + for (i=0; imask ; + if (!type_ok) + continue; + + if (mem_type == DRM_BO_MEM_LOCAL) + break; + + man = &bm->man[mem_type]; + mutex_lock(&dev->struct_mutex); + if (man->has_type && man->use_type) { + type_found = 1; + node = drm_mm_search_free(&man->manager, mem->num_pages, + mem->page_alignment, 1); + if (node) + node = drm_mm_get_block(node, mem->num_pages, + mem->page_alignment); + } + mutex_unlock(&dev->struct_mutex); + if (node) + break; } + + if ((type_ok && (mem_type == DRM_BO_MEM_LOCAL)) || node) { + mem->mm_node = node; + mem->mem_type = mem_type; + mem->flags = drm_bo_type_flags(mem_type); + return 0; + } + + if (!type_found) { + DRM_ERROR("Requested memory types are not supported\n"); + return -EINVAL; + } + + num_prios = dev->driver->bo_driver->num_mem_busy_prio; + prios = dev->driver->bo_driver->mem_busy_prio; + + for (i=0; imask)) + continue; + + man = &bm->man[mem_type]; + ret = drm_bo_mem_force_space(dev, mem, mem_type, no_wait); + + if (ret == 0) + return 0; + + if (ret == -EAGAIN) + has_eagain = 1; + } + + ret = (has_eagain) ? -EAGAIN : -ENOMEM; + return ret; +} + - DRM_DEBUG("Flipping in to AGP 0x%08lx\n", bo->mm_node->start); + + +static int drm_move_local_to_tt(drm_buffer_object_t * bo, + drm_bo_mem_reg_t * mem, + int no_wait) +{ + drm_device_t *dev = bo->dev; + int ret = 0; + + bo->mm_node = mem->mm_node; + + DRM_DEBUG("Flipping in to AGP 0x%08lx 0x%08lx\n", + bo->mm_node->start, bo->mm_node->size); #ifdef DRM_ODD_MM_COMPAT mutex_lock(&dev->struct_mutex); @@ -631,6 +724,7 @@ static int drm_move_local_to_tt(drm_buffer_object_t * bo, int no_wait) bo->flags &= ~DRM_BO_MASK_MEM; bo->flags |= DRM_BO_FLAG_MEM_TT; bo->mem_type = DRM_BO_MEM_TT; + bo->offset = bo->mm_node->start << PAGE_SHIFT; #ifdef DRM_ODD_MM_COMPAT ret = drm_bo_remap_bound(bo); @@ -1103,14 +1197,18 @@ static void drm_buffer_user_object_unmap(drm_file_t * priv, * bo->mutex locked. */ -static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags, +static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_mem_flags, int no_wait, int force_no_move) { + drm_device_t *dev = bo->dev; + drm_buffer_manager_t *bm = &dev->bm; int ret = 0; + drm_bo_mem_reg_t mem; /* * Flush outstanding fences. */ + drm_bo_busy(bo); /* @@ -1126,16 +1224,38 @@ static int drm_bo_move_buffer(drm_buffer_object_t * bo, uint32_t new_flags, */ ret = drm_bo_wait(bo, 0, 0, no_wait); + if (ret) + return ret; - if (ret == -EINTR) - return -EAGAIN; + + mem.num_pages = bo->num_pages; + mem.size = mem.num_pages << PAGE_SHIFT; + mem.mask = new_mem_flags; + mem.page_alignment = bo->page_alignment; + + mutex_lock(&bm->evict_mutex); + mutex_lock(&dev->struct_mutex); + list_del(&bo->lru); + list_add_tail(&bo->lru,&bm->unfenced); + DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED, _DRM_BO_FLAG_UNFENCED); + mutex_unlock(&dev->struct_mutex); + + ret = drm_bo_mem_space(dev, &mem, no_wait); + mutex_unlock(&bm->evict_mutex); + if (ret) return ret; - if (new_flags & DRM_BO_FLAG_MEM_TT) { - ret = drm_move_local_to_tt(bo, no_wait); - if (ret) + if (mem.mem_type == DRM_BO_MEM_TT) { + ret = drm_move_local_to_tt(bo, &mem, no_wait); + if (ret) { + mutex_lock(&dev->struct_mutex); + list_del_init(&bo->lru); + drm_bo_add_to_lru(bo, bm); + mutex_unlock(&dev->struct_mutex); + DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED); return ret; + } } else { drm_move_tt_to_local(bo, 0, force_no_move); } @@ -1231,6 +1351,8 @@ static int drm_buffer_object_validate(drm_buffer_object_t * bo, list_add_tail(&bo->lru, &bm->unfenced); mutex_unlock(&dev->struct_mutex); } else { + DRM_FLAG_MASKED(bo->priv_flags, 0, + _DRM_BO_FLAG_UNFENCED); mutex_lock(&dev->struct_mutex); list_del_init(&bo->lru); drm_bo_add_to_lru(bo, bm); diff --git a/linux-core/drm_stub.c b/linux-core/drm_stub.c index 60123cdc..22592324 100644 --- a/linux-core/drm_stub.c +++ b/linux-core/drm_stub.c @@ -67,6 +67,7 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev, mutex_init(&dev->struct_mutex); mutex_init(&dev->ctxlist_mutex); mutex_init(&dev->bm.init_mutex); + mutex_init(&dev->bm.evict_mutex); dev->pdev = pdev; dev->pci_device = pdev->device; diff --git a/linux-core/i915_buffer.c b/linux-core/i915_buffer.c index cdbe579a..13a3e9bb 100644 --- a/linux-core/i915_buffer.c +++ b/linux-core/i915_buffer.c @@ -97,6 +97,17 @@ int i915_init_mem_type(drm_device_t *dev, uint32_t type, return 0; } +uint32_t i915_evict_flags(drm_device_t *dev, uint32_t type) +{ + switch(type) { + case DRM_BO_MEM_LOCAL: + case DRM_BO_MEM_TT: + return DRM_BO_FLAG_MEM_LOCAL; + default: + return DRM_BO_FLAG_MEM_TT; + } +} + void i915_emit_copy_blit(drm_device_t *dev, uint32_t src_offset, uint32_t dst_offset, diff --git a/linux-core/i915_drv.c b/linux-core/i915_drv.c index 64ce3c15..8c39c249 100644 --- a/linux-core/i915_drv.c +++ b/linux-core/i915_drv.c @@ -50,11 +50,20 @@ static drm_fence_driver_t i915_fence_driver = { }; #endif #ifdef I915_HAVE_BUFFER + +static uint32_t i915_mem_prios[] = {DRM_BO_MEM_PRIV0, DRM_BO_MEM_TT, DRM_BO_MEM_LOCAL}; +static uint32_t i915_busy_prios[] = {DRM_BO_MEM_TT, DRM_BO_MEM_PRIV0, DRM_BO_MEM_LOCAL}; + static drm_bo_driver_t i915_bo_driver = { + .mem_type_prio = i915_mem_prios, + .mem_busy_prio = i915_busy_prios, + .num_mem_type_prio = sizeof(i915_mem_prios)/sizeof(uint32_t), + .num_mem_busy_prio = sizeof(i915_busy_prios)/sizeof(uint32_t), .create_ttm_backend_entry = i915_create_ttm_backend_entry, .fence_type = i915_fence_types, .invalidate_caches = i915_invalidate_caches, .init_mem_type = i915_init_mem_type, + .evict_flags = i915_evict_flags, }; #endif -- cgit v1.2.3