From ab50ddaa9173ae108833db0edb209045788efc41 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Wed, 7 May 2008 13:51:29 -0700 Subject: GEM: Make dri_emit_reloc take GEM domain flags instead of TTM flags. The GEM flags are much more descriptive for what we need. Since this makes bufmgr_fake rather device-specific, move it to the intel common directory. We've wanted to do device-specific stuff to it before. --- src/mesa/drivers/dri/Makefile.template | 3 +- src/mesa/drivers/dri/common/dri_bufmgr.c | 8 +- src/mesa/drivers/dri/common/dri_bufmgr.h | 21 +- src/mesa/drivers/dri/common/dri_bufmgr_fake.c | 1174 ---------------------- src/mesa/drivers/dri/i915/Makefile | 1 + src/mesa/drivers/dri/i915/i830_vtbl.c | 6 +- src/mesa/drivers/dri/i915/i915_vtbl.c | 6 +- src/mesa/drivers/dri/i915/intel_bufmgr_fake.c | 1 + src/mesa/drivers/dri/i965/Makefile | 1 + src/mesa/drivers/dri/i965/brw_cc.c | 3 +- src/mesa/drivers/dri/i965/brw_clip_state.c | 3 +- src/mesa/drivers/dri/i965/brw_curbe.c | 3 +- src/mesa/drivers/dri/i965/brw_draw_upload.c | 9 +- src/mesa/drivers/dri/i965/brw_gs_state.c | 2 +- src/mesa/drivers/dri/i965/brw_misc_state.c | 19 +- src/mesa/drivers/dri/i965/brw_sf_state.c | 4 +- src/mesa/drivers/dri/i965/brw_vs_state.c | 2 +- src/mesa/drivers/dri/i965/brw_wm_sampler_state.c | 2 +- src/mesa/drivers/dri/i965/brw_wm_state.c | 6 +- src/mesa/drivers/dri/i965/brw_wm_surface_state.c | 16 +- src/mesa/drivers/dri/i965/intel_bufmgr_fake.c | 1 + src/mesa/drivers/dri/intel/intel_batchbuffer.c | 6 +- src/mesa/drivers/dri/intel/intel_batchbuffer.h | 9 +- src/mesa/drivers/dri/intel/intel_blit.c | 31 +- src/mesa/drivers/dri/intel/intel_bufmgr_fake.c | 1173 +++++++++++++++++++++ src/mesa/drivers/dri/intel/intel_bufmgr_fake.h | 50 + src/mesa/drivers/dri/intel/intel_bufmgr_gem.c | 30 +- src/mesa/drivers/dri/intel/intel_context.c | 1 + 28 files changed, 1331 insertions(+), 1260 deletions(-) delete mode 100644 src/mesa/drivers/dri/common/dri_bufmgr_fake.c create mode 120000 src/mesa/drivers/dri/i915/intel_bufmgr_fake.c create mode 120000 src/mesa/drivers/dri/i965/intel_bufmgr_fake.c create mode 100644 src/mesa/drivers/dri/intel/intel_bufmgr_fake.c create mode 100644 src/mesa/drivers/dri/intel/intel_bufmgr_fake.h diff --git a/src/mesa/drivers/dri/Makefile.template b/src/mesa/drivers/dri/Makefile.template index 53f9d80689..cb41662707 100644 --- a/src/mesa/drivers/dri/Makefile.template +++ b/src/mesa/drivers/dri/Makefile.template @@ -12,8 +12,7 @@ COMMON_SOURCES = \ ../common/drirenderbuffer.c COMMON_BM_SOURCES = \ - ../common/dri_bufmgr.c \ - ../common/dri_bufmgr_fake.c + ../common/dri_bufmgr.c ifeq ($(WINDOW_SYSTEM),dri) diff --git a/src/mesa/drivers/dri/common/dri_bufmgr.c b/src/mesa/drivers/dri/common/dri_bufmgr.c index 69868b6665..5967d7dafb 100644 --- a/src/mesa/drivers/dri/common/dri_bufmgr.c +++ b/src/mesa/drivers/dri/common/dri_bufmgr.c @@ -121,10 +121,12 @@ dri_bufmgr_destroy(dri_bufmgr *bufmgr) } -int dri_emit_reloc(dri_bo *reloc_buf, uint64_t flags, GLuint delta, - GLuint offset, dri_bo *target_buf) +int dri_emit_reloc(dri_bo *reloc_buf, + uint32_t read_domains, uint32_t write_domain, + uint32_t delta, uint32_t offset, dri_bo *target_buf) { - return reloc_buf->bufmgr->emit_reloc(reloc_buf, flags, delta, offset, target_buf); + return reloc_buf->bufmgr->emit_reloc(reloc_buf, read_domains, write_domain, + delta, offset, target_buf); } void *dri_process_relocs(dri_bo *batch_buf) diff --git a/src/mesa/drivers/dri/common/dri_bufmgr.h b/src/mesa/drivers/dri/common/dri_bufmgr.h index dffeb4c601..99cfb2cd05 100644 --- a/src/mesa/drivers/dri/common/dri_bufmgr.h +++ b/src/mesa/drivers/dri/common/dri_bufmgr.h @@ -135,8 +135,9 @@ struct _dri_bufmgr { * \param target Buffer whose offset should be written into the relocation * entry. */ - int (*emit_reloc)(dri_bo *reloc_buf, uint64_t flags, GLuint delta, - GLuint offset, dri_bo *target); + int (*emit_reloc)(dri_bo *reloc_buf, + uint32_t read_domains, uint32_t write_domain, + uint32_t delta, uint32_t offset, dri_bo *target); /** * Processes the relocations, either in userland or by converting the list @@ -174,22 +175,12 @@ void dri_bo_subdata(dri_bo *bo, unsigned long offset, void dri_bo_get_subdata(dri_bo *bo, unsigned long offset, unsigned long size, void *data); -void dri_bufmgr_fake_contended_lock_take(dri_bufmgr *bufmgr); -dri_bufmgr *dri_bufmgr_fake_init(unsigned long low_offset, void *low_virtual, - unsigned long size, - unsigned int (*fence_emit)(void *private), - int (*fence_wait)(void *private, - unsigned int cookie), - void *driver_priv); void dri_bufmgr_set_debug(dri_bufmgr *bufmgr, GLboolean enable_debug); -void dri_bo_fake_disable_backing_store(dri_bo *bo, - void (*invalidate_cb)(dri_bo *bo, - void *ptr), - void *ptr); void dri_bufmgr_destroy(dri_bufmgr *bufmgr); -int dri_emit_reloc(dri_bo *reloc_buf, uint64_t flags, GLuint delta, - GLuint offset, dri_bo *target_buf); +int dri_emit_reloc(dri_bo *reloc_buf, + uint32_t read_domains, uint32_t write_domain, + uint32_t delta, uint32_t offset, dri_bo *target_buf); void *dri_process_relocs(dri_bo *batch_buf); void dri_post_process_relocs(dri_bo *batch_buf); void dri_post_submit(dri_bo *batch_buf); diff --git a/src/mesa/drivers/dri/common/dri_bufmgr_fake.c b/src/mesa/drivers/dri/common/dri_bufmgr_fake.c deleted file mode 100644 index fc52674839..0000000000 --- a/src/mesa/drivers/dri/common/dri_bufmgr_fake.c +++ /dev/null @@ -1,1174 +0,0 @@ -/************************************************************************** - * - * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. - * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR - * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ - -/* Originally a fake version of the buffer manager so that we can - * prototype the changes in a driver fairly quickly, has been fleshed - * out to a fully functional interim solution. - * - * Basically wraps the old style memory management in the new - * programming interface, but is more expressive and avoids many of - * the bugs in the old texture manager. - */ -#include "mtypes.h" -#include "dri_bufmgr.h" -#include "drm.h" - -#include "simple_list.h" -#include "mm.h" -#include "imports.h" - -#define DBG(...) do { \ - if (bufmgr_fake->bufmgr.debug) \ - _mesa_printf(__VA_ARGS__); \ -} while (0) - -/* Internal flags: - */ -#define BM_NO_BACKING_STORE 0x00000001 -#define BM_NO_FENCE_SUBDATA 0x00000002 -#define BM_PINNED 0x00000004 - -/* Wrapper around mm.c's mem_block, which understands that you must - * wait for fences to expire before memory can be freed. This is - * specific to our use of memcpy for uploads - an upload that was - * processed through the command queue wouldn't need to care about - * fences. - */ -#define MAX_RELOCS 4096 - -struct fake_buffer_reloc -{ - /** Buffer object that the relocation points at. */ - dri_bo *target_buf; - /** Offset of the relocation entry within reloc_buf. */ - GLuint offset; - /** Cached value of the offset when we last performed this relocation. */ - GLuint last_target_offset; - /** Value added to target_buf's offset to get the relocation entry. */ - GLuint delta; - /** Flags to validate the target buffer under. */ - uint64_t validate_flags; -}; - -struct block { - struct block *next, *prev; - struct mem_block *mem; /* BM_MEM_AGP */ - - /** - * Marks that the block is currently in the aperture and has yet to be - * fenced. - */ - unsigned on_hardware:1; - /** - * Marks that the block is currently fenced (being used by rendering) and - * can't be freed until @fence is passed. - */ - unsigned fenced:1; - - /** Fence cookie for the block. */ - unsigned fence; /* Split to read_fence, write_fence */ - - dri_bo *bo; - void *virtual; -}; - -typedef struct _bufmgr_fake { - dri_bufmgr bufmgr; - - unsigned long low_offset; - unsigned long size; - void *virtual; - - struct mem_block *heap; - struct block lru; /* only allocated, non-fence-pending blocks here */ - - unsigned buf_nr; /* for generating ids */ - - struct block on_hardware; /* after bmValidateBuffers */ - struct block fenced; /* after bmFenceBuffers (mi_flush, emit irq, write dword) */ - /* then to bufmgr->lru or free() */ - - unsigned int last_fence; - - unsigned fail:1; - unsigned need_fence:1; - GLboolean thrashing; - - /** - * Driver callback to emit a fence, returning the cookie. - * - * Currently, this also requires that a write flush be emitted before - * emitting the fence, but this should change. - */ - unsigned int (*fence_emit)(void *private); - /** Driver callback to wait for a fence cookie to have passed. */ - int (*fence_wait)(void *private, unsigned int fence_cookie); - /** Driver-supplied argument to driver callbacks */ - void *driver_priv; - - GLboolean debug; - - GLboolean performed_rendering; - - /* keep track of the current total size of objects we have relocs for */ - unsigned long current_total_size; -} dri_bufmgr_fake; - -typedef struct _dri_bo_fake { - dri_bo bo; - - unsigned id; /* debug only */ - const char *name; - - unsigned dirty:1; - unsigned size_accounted:1; /*this buffers size has been accounted against the aperture */ - unsigned card_dirty:1; /* has the card written to this buffer - we make need to copy it back */ - unsigned int refcount; - /* Flags may consist of any of the DRM_BO flags, plus - * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the first two - * driver private flags. - */ - uint64_t flags; - unsigned int alignment; - GLboolean is_static, validated; - unsigned int map_count; - - /* Flags for the buffer to be validated with in command submission */ - uint64_t validate_flags; - - /** relocation list */ - struct fake_buffer_reloc *relocs; - GLuint nr_relocs; - - struct block *block; - void *backing_store; - void (*invalidate_cb)(dri_bo *bo, void *ptr); - void *invalidate_ptr; -} dri_bo_fake; - -static int clear_fenced(dri_bufmgr_fake *bufmgr_fake, - unsigned int fence_cookie); - -static int dri_fake_check_aperture_space(dri_bo *bo); - -#define MAXFENCE 0x7fffffff - -static GLboolean FENCE_LTE( unsigned a, unsigned b ) -{ - if (a == b) - return GL_TRUE; - - if (a < b && b - a < (1<<24)) - return GL_TRUE; - - if (a > b && MAXFENCE - a + b < (1<<24)) - return GL_TRUE; - - return GL_FALSE; -} - -static unsigned int -_fence_emit_internal(dri_bufmgr_fake *bufmgr_fake) -{ - bufmgr_fake->last_fence = bufmgr_fake->fence_emit(bufmgr_fake->driver_priv); - return bufmgr_fake->last_fence; -} - -static void -_fence_wait_internal(dri_bufmgr_fake *bufmgr_fake, unsigned int cookie) -{ - int ret; - - ret = bufmgr_fake->fence_wait(bufmgr_fake->driver_priv, cookie); - if (ret != 0) { - _mesa_printf("%s:%d: Error %d waiting for fence.\n", - __FILE__, __LINE__); - abort(); - } - clear_fenced(bufmgr_fake, cookie); -} - -static GLboolean -_fence_test(dri_bufmgr_fake *bufmgr_fake, unsigned fence) -{ - /* Slight problem with wrap-around: - */ - return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence); -} - -/** - * Allocate a memory manager block for the buffer. - */ -static GLboolean -alloc_block(dri_bo *bo) -{ - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - dri_bufmgr_fake *bufmgr_fake= (dri_bufmgr_fake *)bo->bufmgr; - struct block *block = (struct block *)calloc(sizeof *block, 1); - unsigned int align_log2 = _mesa_ffs(bo_fake->alignment) - 1; - GLuint sz; - - if (!block) - return GL_FALSE; - - sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); - - block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0); - if (!block->mem) { - free(block); - return GL_FALSE; - } - - make_empty_list(block); - - /* Insert at head or at tail??? - */ - insert_at_tail(&bufmgr_fake->lru, block); - - block->virtual = bufmgr_fake->virtual + - block->mem->ofs - bufmgr_fake->low_offset; - block->bo = bo; - - bo_fake->block = block; - - return GL_TRUE; -} - -/* Release the card storage associated with buf: - */ -static void free_block(dri_bufmgr_fake *bufmgr_fake, struct block *block) -{ - dri_bo_fake *bo_fake; - DBG("free block %p %08x %d %d\n", block, block->mem->ofs, block->on_hardware, block->fenced); - - if (!block) - return; - - bo_fake = (dri_bo_fake *)block->bo; - if (!(bo_fake->flags & BM_NO_BACKING_STORE) && (bo_fake->card_dirty == 1)) { - memcpy(bo_fake->backing_store, block->virtual, block->bo->size); - bo_fake->card_dirty = 1; - bo_fake->dirty = 1; - } - - if (block->on_hardware) { - block->bo = NULL; - } - else if (block->fenced) { - block->bo = NULL; - } - else { - DBG(" - free immediately\n"); - remove_from_list(block); - - mmFreeMem(block->mem); - free(block); - } -} - -static void -alloc_backing_store(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - assert(!bo_fake->backing_store); - assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE))); - - bo_fake->backing_store = ALIGN_MALLOC(bo->size, 64); - - DBG("alloc_backing - buf %d %p %d\n", bo_fake->id, bo_fake->backing_store, bo->size); - assert(bo_fake->backing_store); -} - -static void -free_backing_store(dri_bo *bo) -{ - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - if (bo_fake->backing_store) { - assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE))); - ALIGN_FREE(bo_fake->backing_store); - bo_fake->backing_store = NULL; - } -} - -static void -set_dirty(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - if (bo_fake->flags & BM_NO_BACKING_STORE && bo_fake->invalidate_cb != NULL) - bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr); - - assert(!(bo_fake->flags & BM_PINNED)); - - DBG("set_dirty - buf %d\n", bo_fake->id); - bo_fake->dirty = 1; -} - -static GLboolean -evict_lru(dri_bufmgr_fake *bufmgr_fake, GLuint max_fence) -{ - struct block *block, *tmp; - - DBG("%s\n", __FUNCTION__); - - foreach_s(block, tmp, &bufmgr_fake->lru) { - dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; - - if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) - continue; - - if (block->fence && max_fence && !FENCE_LTE(block->fence, max_fence)) - return 0; - - set_dirty(&bo_fake->bo); - bo_fake->block = NULL; - - free_block(bufmgr_fake, block); - return GL_TRUE; - } - - return GL_FALSE; -} - -#define foreach_s_rev(ptr, t, list) \ - for(ptr=(list)->prev,t=(ptr)->prev; list != ptr; ptr=t, t=(t)->prev) - -static GLboolean -evict_mru(dri_bufmgr_fake *bufmgr_fake) -{ - struct block *block, *tmp; - - DBG("%s\n", __FUNCTION__); - - foreach_s_rev(block, tmp, &bufmgr_fake->lru) { - dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; - - if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) - continue; - - set_dirty(&bo_fake->bo); - bo_fake->block = NULL; - - free_block(bufmgr_fake, block); - return GL_TRUE; - } - - return GL_FALSE; -} - -/** - * Removes all objects from the fenced list older than the given fence. - */ -static int clear_fenced(dri_bufmgr_fake *bufmgr_fake, - unsigned int fence_cookie) -{ - struct block *block, *tmp; - int ret = 0; - - foreach_s(block, tmp, &bufmgr_fake->fenced) { - assert(block->fenced); - - if (_fence_test(bufmgr_fake, block->fence)) { - - block->fenced = 0; - - if (!block->bo) { - DBG("delayed free: offset %x sz %x\n", - block->mem->ofs, block->mem->size); - remove_from_list(block); - mmFreeMem(block->mem); - free(block); - } - else { - DBG("return to lru: offset %x sz %x\n", - block->mem->ofs, block->mem->size); - move_to_tail(&bufmgr_fake->lru, block); - } - - ret = 1; - } - else { - /* Blocks are ordered by fence, so if one fails, all from - * here will fail also: - */ - DBG("fence not passed: offset %x sz %x %d %d \n", - block->mem->ofs, block->mem->size, block->fence, bufmgr_fake->last_fence); - break; - } - } - - DBG("%s: %d\n", __FUNCTION__, ret); - return ret; -} - -static void fence_blocks(dri_bufmgr_fake *bufmgr_fake, unsigned fence) -{ - struct block *block, *tmp; - - foreach_s (block, tmp, &bufmgr_fake->on_hardware) { - DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", block, - block->mem->size, block->mem->ofs, block->bo, fence); - block->fence = fence; - - block->on_hardware = 0; - block->fenced = 1; - - /* Move to tail of pending list here - */ - move_to_tail(&bufmgr_fake->fenced, block); - } - - assert(is_empty_list(&bufmgr_fake->on_hardware)); -} - -static GLboolean evict_and_alloc_block(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - assert(bo_fake->block == NULL); - - /* Search for already free memory: - */ - if (alloc_block(bo)) - return GL_TRUE; - - /* If we're not thrashing, allow lru eviction to dig deeper into - * recently used textures. We'll probably be thrashing soon: - */ - if (!bufmgr_fake->thrashing) { - while (evict_lru(bufmgr_fake, 0)) - if (alloc_block(bo)) - return GL_TRUE; - } - - /* Keep thrashing counter alive? - */ - if (bufmgr_fake->thrashing) - bufmgr_fake->thrashing = 20; - - /* Wait on any already pending fences - here we are waiting for any - * freed memory that has been submitted to hardware and fenced to - * become available: - */ - while (!is_empty_list(&bufmgr_fake->fenced)) { - GLuint fence = bufmgr_fake->fenced.next->fence; - _fence_wait_internal(bufmgr_fake, fence); - - if (alloc_block(bo)) - return GL_TRUE; - } - - if (!is_empty_list(&bufmgr_fake->on_hardware)) { - while (!is_empty_list(&bufmgr_fake->fenced)) { - GLuint fence = bufmgr_fake->fenced.next->fence; - _fence_wait_internal(bufmgr_fake, fence); - } - - if (!bufmgr_fake->thrashing) { - DBG("thrashing\n"); - } - bufmgr_fake->thrashing = 20; - - if (alloc_block(bo)) - return GL_TRUE; - } - - while (evict_mru(bufmgr_fake)) - if (alloc_block(bo)) - return GL_TRUE; - - DBG("%s 0x%x bytes failed\n", __FUNCTION__, bo->size); - - return GL_FALSE; -} - -/*********************************************************************** - * Public functions - */ - -/** - * Wait for hardware idle by emitting a fence and waiting for it. - */ -static void -dri_bufmgr_fake_wait_idle(dri_bufmgr_fake *bufmgr_fake) -{ - unsigned int cookie; - - cookie = bufmgr_fake->fence_emit(bufmgr_fake->driver_priv); - _fence_wait_internal(bufmgr_fake, cookie); -} - -/** - * Wait for execution pending on a buffer - */ -static void -dri_bufmgr_fake_bo_wait_idle(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - if (bo_fake->block == NULL || !bo_fake->block->fenced) - return; - - _fence_wait_internal(bufmgr_fake, bo_fake->block->fence); -} - -/* Specifically ignore texture memory sharing. - * -- just evict everything - * -- and wait for idle - */ -void -dri_bufmgr_fake_contended_lock_take(dri_bufmgr *bufmgr) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; - struct block *block, *tmp; - - bufmgr_fake->need_fence = 1; - bufmgr_fake->fail = 0; - - /* Wait for hardware idle. We don't know where acceleration has been - * happening, so we'll need to wait anyway before letting anything get - * put on the card again. - */ - dri_bufmgr_fake_wait_idle(bufmgr_fake); - - /* Check that we hadn't released the lock without having fenced the last - * set of buffers. - */ - assert(is_empty_list(&bufmgr_fake->fenced)); - assert(is_empty_list(&bufmgr_fake->on_hardware)); - - foreach_s(block, tmp, &bufmgr_fake->lru) { - assert(_fence_test(bufmgr_fake, block->fence)); - set_dirty(block->bo); - } -} - -static dri_bo * -dri_fake_bo_alloc(dri_bufmgr *bufmgr, const char *name, - unsigned long size, unsigned int alignment, - uint64_t location_mask) -{ - dri_bufmgr_fake *bufmgr_fake; - dri_bo_fake *bo_fake; - - bufmgr_fake = (dri_bufmgr_fake *)bufmgr; - - assert(size != 0); - - bo_fake = calloc(1, sizeof(*bo_fake)); - if (!bo_fake) - return NULL; - - bo_fake->bo.size = size; - bo_fake->bo.offset = -1; - bo_fake->bo.virtual = NULL; - bo_fake->bo.bufmgr = bufmgr; - bo_fake->refcount = 1; - - /* Alignment must be a power of two */ - assert((alignment & (alignment - 1)) == 0); - if (alignment == 0) - alignment = 1; - bo_fake->alignment = alignment; - bo_fake->id = ++bufmgr_fake->buf_nr; - bo_fake->name = name; - bo_fake->flags = 0; - bo_fake->is_static = GL_FALSE; - - DBG("drm_bo_alloc: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, - bo_fake->bo.size / 1024); - - return &bo_fake->bo; -} - -static dri_bo * -dri_fake_bo_alloc_static(dri_bufmgr *bufmgr, const char *name, - unsigned long offset, unsigned long size, - void *virtual, uint64_t location_mask) -{ - dri_bufmgr_fake *bufmgr_fake; - dri_bo_fake *bo_fake; - - bufmgr_fake = (dri_bufmgr_fake *)bufmgr; - - assert(size != 0); - - bo_fake = calloc(1, sizeof(*bo_fake)); - if (!bo_fake) - return NULL; - - bo_fake->bo.size = size; - bo_fake->bo.offset = offset; - bo_fake->bo.virtual = virtual; - bo_fake->bo.bufmgr = bufmgr; - bo_fake->refcount = 1; - bo_fake->id = ++bufmgr_fake->buf_nr; - bo_fake->name = name; - bo_fake->flags = BM_PINNED | DRM_BO_FLAG_NO_MOVE; - bo_fake->is_static = GL_TRUE; - - DBG("drm_bo_alloc_static: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, - bo_fake->bo.size / 1024); - - return &bo_fake->bo; -} - -static void -dri_fake_bo_reference(dri_bo *bo) -{ - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - bo_fake->refcount++; -} - -static void -dri_fake_bo_unreference(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - int i; - - if (!bo) - return; - - if (--bo_fake->refcount == 0) { - assert(bo_fake->map_count == 0); - /* No remaining references, so free it */ - if (bo_fake->block) - free_block(bufmgr_fake, bo_fake->block); - free_backing_store(bo); - - for (i = 0; i < bo_fake->nr_relocs; i++) - dri_bo_unreference(bo_fake->relocs[i].target_buf); - - DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id, bo_fake->name); - - free(bo_fake->relocs); - free(bo); - - return; - } -} - -/** - * Set the buffer as not requiring backing store, and instead get the callback - * invoked whenever it would be set dirty. - */ -void dri_bo_fake_disable_backing_store(dri_bo *bo, - void (*invalidate_cb)(dri_bo *bo, - void *ptr), - void *ptr) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - if (bo_fake->backing_store) - free_backing_store(bo); - - bo_fake->flags |= BM_NO_BACKING_STORE; - - DBG("disable_backing_store set buf %d dirty\n", bo_fake->id); - bo_fake->dirty = 1; - bo_fake->invalidate_cb = invalidate_cb; - bo_fake->invalidate_ptr = ptr; - - /* Note that it is invalid right from the start. Also note - * invalidate_cb is called with the bufmgr locked, so cannot - * itself make bufmgr calls. - */ - if (invalidate_cb != NULL) - invalidate_cb(bo, ptr); -} - -/** - * Map a buffer into bo->virtual, allocating either card memory space (If - * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary. - */ -static int -dri_fake_bo_map(dri_bo *bo, GLboolean write_enable) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - /* Static buffers are always mapped. */ - if (bo_fake->is_static) - return 0; - - /* Allow recursive mapping. Mesa may recursively map buffers with - * nested display loops, and it is used internally in bufmgr_fake - * for relocation. - */ - if (bo_fake->map_count++ != 0) - return 0; - - { - DBG("drm_bo_map: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, - bo_fake->bo.size / 1024); - - if (bo->virtual != NULL) { - _mesa_printf("%s: already mapped\n", __FUNCTION__); - abort(); - } - else if (bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED)) { - - if (!bo_fake->block && !evict_and_alloc_block(bo)) { - DBG("%s: alloc failed\n", __FUNCTION__); - bufmgr_fake->fail = 1; - return 1; - } - else { - assert(bo_fake->block); - bo_fake->dirty = 0; - - if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) && - bo_fake->block->fenced) { - dri_bufmgr_fake_bo_wait_idle(bo); - } - - bo->virtual = bo_fake->block->virtual; - } - } - else { - if (write_enable) - set_dirty(bo); - - if (bo_fake->backing_store == 0) - alloc_backing_store(bo); - - bo->virtual = bo_fake->backing_store; - } - } - - return 0; -} - -static int -dri_fake_bo_unmap(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - /* Static buffers are always mapped. */ - if (bo_fake->is_static) - return 0; - - assert(bo_fake->map_count != 0); - if (--bo_fake->map_count != 0) - return 0; - - DBG("drm_bo_unmap: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, - bo_fake->bo.size / 1024); - - bo->virtual = NULL; - - return 0; -} - -static void -dri_fake_kick_all(dri_bufmgr_fake *bufmgr_fake) -{ - struct block *block, *tmp; - - bufmgr_fake->performed_rendering = GL_FALSE; - /* okay for ever BO that is on the HW kick it off. - seriously not afraid of the POLICE right now */ - foreach_s(block, tmp, &bufmgr_fake->on_hardware) { - dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; - - block->on_hardware = 0; - free_block(bufmgr_fake, block); - bo_fake->block = NULL; - bo_fake->validated = GL_FALSE; - if (!(bo_fake->flags & BM_NO_BACKING_STORE)) - bo_fake->dirty = 1; - } -} - -static int -dri_fake_bo_validate(dri_bo *bo, uint64_t flags) -{ - dri_bufmgr_fake *bufmgr_fake; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - - /* XXX: Sanity-check whether we've already validated this one under - * different flags. See drmAddValidateItem(). - */ - bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - - DBG("drm_bo_validate: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, - bo_fake->bo.size / 1024); - - /* Sanity check: Buffers should be unmapped before being validated. - * This is not so much of a problem for bufmgr_fake, but TTM refuses, - * and the problem is harder to debug there. - */ - assert(bo_fake->map_count == 0); - - if (bo_fake->is_static) { - /* Add it to the needs-fence list */ - bufmgr_fake->need_fence = 1; - return 0; - } - - /* reset size accounted */ - bo_fake->size_accounted = 0; - - /* Allocate the card memory */ - if (!bo_fake->block && !evict_and_alloc_block(bo)) { - bufmgr_fake->fail = 1; - DBG("Failed to validate buf %d:%s\n", bo_fake->id, bo_fake->name); - return -1; - } - - assert(bo_fake->block); - assert(bo_fake->block->bo == &bo_fake->bo); - - bo->offset = bo_fake->block->mem->ofs; - - /* Upload the buffer contents if necessary */ - if (bo_fake->dirty) { - DBG("Upload dirty buf %d:%s, sz %d offset 0x%x\n", bo_fake->id, - bo_fake->name, bo->size, bo_fake->block->mem->ofs); - - assert(!(bo_fake->flags & - (BM_NO_BACKING_STORE|BM_PINNED))); - - /* Actually, should be able to just wait for a fence on the memory, - * which we would be tracking when we free it. Waiting for idle is - * a sufficiently large hammer for now. - */ - dri_bufmgr_fake_wait_idle(bufmgr_fake); - - /* we may never have mapped this BO so it might not have any backing - * store if this happens it should be rare, but 0 the card memory - * in any case */ - if (bo_fake->backing_store) - memcpy(bo_fake->block->virtual, bo_fake->backing_store, bo->size); - else - memset(bo_fake->block->virtual, 0, bo->size); - - bo_fake->dirty = 0; - } - - bo_fake->block->fenced = 0; - bo_fake->block->on_hardware = 1; - move_to_tail(&bufmgr_fake->on_hardware, bo_fake->block); - - bo_fake->validated = GL_TRUE; - bufmgr_fake->need_fence = 1; - - return 0; -} - -static void -dri_fake_fence_validated(dri_bufmgr *bufmgr) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; - unsigned int cookie; - - cookie = _fence_emit_internal(bufmgr_fake); - fence_blocks(bufmgr_fake, cookie); - - DBG("drm_fence_validated: 0x%08x cookie\n", cookie); -} - -static void -dri_fake_destroy(dri_bufmgr *bufmgr) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; - - mmDestroy(bufmgr_fake->heap); - free(bufmgr); -} - -static int -dri_fake_emit_reloc(dri_bo *reloc_buf, uint64_t flags, GLuint delta, - GLuint offset, dri_bo *target_buf) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)reloc_buf->bufmgr; - struct fake_buffer_reloc *r; - dri_bo_fake *reloc_fake = (dri_bo_fake *)reloc_buf; - dri_bo_fake *target_fake = (dri_bo_fake *)target_buf; - int i; - - assert(reloc_buf); - assert(target_buf); - - assert(target_fake->is_static || target_fake->size_accounted); - - if (reloc_fake->relocs == NULL) { - reloc_fake->relocs = malloc(sizeof(struct fake_buffer_reloc) * - MAX_RELOCS); - } - - r = &reloc_fake->relocs[reloc_fake->nr_relocs++]; - - assert(reloc_fake->nr_relocs <= MAX_RELOCS); - - dri_bo_reference(target_buf); - - r->target_buf = target_buf; - r->offset = offset; - r->last_target_offset = target_buf->offset; - r->delta = delta; - r->validate_flags = flags; - - if (bufmgr_fake->debug) { - /* Check that a conflicting relocation hasn't already been emitted. */ - for (i = 0; i < reloc_fake->nr_relocs - 1; i++) { - struct fake_buffer_reloc *r2 = &reloc_fake->relocs[i]; - - assert(r->offset != r2->offset); - } - } - - return 0; -} - -/** - * Incorporates the validation flags associated with each relocation into - * the combined validation flags for the buffer on this batchbuffer submission. - */ -static void -dri_fake_calculate_validate_flags(dri_bo *bo) -{ - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - int i; - - for (i = 0; i < bo_fake->nr_relocs; i++) { - struct fake_buffer_reloc *r = &bo_fake->relocs[i]; - dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; - - /* Do the same for the tree of buffers we depend on */ - dri_fake_calculate_validate_flags(r->target_buf); - - if (target_fake->validate_flags == 0) { - target_fake->validate_flags = r->validate_flags; - } else { - /* Mask the memory location to the intersection of all the memory - * locations the buffer is being validated to. - */ - target_fake->validate_flags = - (target_fake->validate_flags & ~DRM_BO_MASK_MEM) | - (r->validate_flags & target_fake->validate_flags & - DRM_BO_MASK_MEM); - /* All the other flags just accumulate. */ - target_fake->validate_flags |= r->validate_flags & ~DRM_BO_MASK_MEM; - } - } -} - - -static int -dri_fake_reloc_and_validate_buffer(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - int i, ret; - - assert(bo_fake->map_count == 0); - - for (i = 0; i < bo_fake->nr_relocs; i++) { - struct fake_buffer_reloc *r = &bo_fake->relocs[i]; - dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; - uint32_t reloc_data; - - /* Validate the target buffer if that hasn't been done. */ - if (!target_fake->validated) { - ret = dri_fake_reloc_and_validate_buffer(r->target_buf); - if (ret != 0) { - if (bo->virtual != NULL) - dri_bo_unmap(bo); - return ret; - } - } - - /* Calculate the value of the relocation entry. */ - if (r->target_buf->offset != r->last_target_offset) { - reloc_data = r->target_buf->offset + r->delta; - - if (bo->virtual == NULL) - dri_bo_map(bo, GL_TRUE); - - *(uint32_t *)(bo->virtual + r->offset) = reloc_data; - - r->last_target_offset = r->target_buf->offset; - } - } - - if (bo->virtual != NULL) - dri_bo_unmap(bo); - - if (bo_fake->validate_flags & DRM_BO_FLAG_WRITE) { - if (!(bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED))) { - if (bo_fake->backing_store == 0) - alloc_backing_store(bo); - - bo_fake->card_dirty = 1; - } - bufmgr_fake->performed_rendering = GL_TRUE; - } - - return dri_fake_bo_validate(bo, bo_fake->validate_flags); -} - -static void * -dri_fake_process_relocs(dri_bo *batch_buf) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)batch_buf->bufmgr; - dri_bo_fake *batch_fake = (dri_bo_fake *)batch_buf; - int ret; - int retry_count = 0; - - bufmgr_fake->performed_rendering = GL_FALSE; - - dri_fake_calculate_validate_flags(batch_buf); - - batch_fake->validate_flags = DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ; - - /* we've ran out of RAM so blow the whole lot away and retry */ - restart: - ret = dri_fake_reloc_and_validate_buffer(batch_buf); - if (bufmgr_fake->fail == 1) { - if (retry_count == 0) { - retry_count++; - dri_fake_kick_all(bufmgr_fake); - bufmgr_fake->fail = 0; - goto restart; - } else /* dump out the memory here */ - mmDumpMemInfo(bufmgr_fake->heap); - } - - assert(ret == 0); - - bufmgr_fake->current_total_size = 0; - return NULL; -} - -static void -dri_bo_fake_post_submit(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - int i; - - for (i = 0; i < bo_fake->nr_relocs; i++) { - struct fake_buffer_reloc *r = &bo_fake->relocs[i]; - dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; - - if (target_fake->validated) - dri_bo_fake_post_submit(r->target_buf); - - DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n", - bo_fake->name, (uint32_t)bo->offset, r->offset, - target_fake->name, (uint32_t)r->target_buf->offset, r->delta); - } - - assert(bo_fake->map_count == 0); - bo_fake->validated = GL_FALSE; - bo_fake->validate_flags = 0; -} - - -static void -dri_fake_post_submit(dri_bo *batch_buf) -{ - dri_fake_fence_validated(batch_buf->bufmgr); - - dri_bo_fake_post_submit(batch_buf); -} - -static int -dri_fake_check_aperture_space(dri_bo *bo) -{ - dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; - dri_bo_fake *bo_fake = (dri_bo_fake *)bo; - GLuint sz; - - sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); - - if (bo_fake->size_accounted || bo_fake->is_static) - return 0; - - if (bufmgr_fake->current_total_size + sz > bufmgr_fake->size) { - DBG("check_space: %s bo %d %d overflowed bufmgr size %d\n", bo_fake->name, bo_fake->id, sz, bufmgr_fake->size); - return -1; - } - - bufmgr_fake->current_total_size += sz; - bo_fake->size_accounted = 1; - DBG("drm_check_space: buf %d, %s %d %d\n", bo_fake->id, bo_fake->name, bo->size, bufmgr_fake->current_total_size); - return 0; -} - -dri_bufmgr * -dri_bufmgr_fake_init(unsigned long low_offset, void *low_virtual, - unsigned long size, - unsigned int (*fence_emit)(void *private), - int (*fence_wait)(void *private, unsigned int cookie), - void *driver_priv) -{ - dri_bufmgr_fake *bufmgr_fake; - - bufmgr_fake = calloc(1, sizeof(*bufmgr_fake)); - - /* Initialize allocator */ - make_empty_list(&bufmgr_fake->fenced); - make_empty_list(&bufmgr_fake->on_hardware); - make_empty_list(&bufmgr_fake->lru); - - bufmgr_fake->low_offset = low_offset; - bufmgr_fake->virtual = low_virtual; - bufmgr_fake->size = size; - bufmgr_fake->heap = mmInit(low_offset, size); - - /* Hook in methods */ - bufmgr_fake->bufmgr.bo_alloc = dri_fake_bo_alloc; - bufmgr_fake->bufmgr.bo_alloc_static = dri_fake_bo_alloc_static; - bufmgr_fake->bufmgr.bo_reference = dri_fake_bo_reference; - bufmgr_fake->bufmgr.bo_unreference = dri_fake_bo_unreference; - bufmgr_fake->bufmgr.bo_map = dri_fake_bo_map; - bufmgr_fake->bufmgr.bo_unmap = dri_fake_bo_unmap; - bufmgr_fake->bufmgr.destroy = dri_fake_destroy; - bufmgr_fake->bufmgr.emit_reloc = dri_fake_emit_reloc; - bufmgr_fake->bufmgr.process_relocs = dri_fake_process_relocs; - bufmgr_fake->bufmgr.post_submit = dri_fake_post_submit; - bufmgr_fake->bufmgr.check_aperture_space = dri_fake_check_aperture_space; - bufmgr_fake->bufmgr.debug = GL_FALSE; - - bufmgr_fake->fence_emit = fence_emit; - bufmgr_fake->fence_wait = fence_wait; - bufmgr_fake->driver_priv = driver_priv; - - return &bufmgr_fake->bufmgr; -} - diff --git a/src/mesa/drivers/dri/i915/Makefile b/src/mesa/drivers/dri/i915/Makefile index 67f251a7fa..476814c4ec 100644 --- a/src/mesa/drivers/dri/i915/Makefile +++ b/src/mesa/drivers/dri/i915/Makefile @@ -54,6 +54,7 @@ DRIVER_SOURCES = \ intel_tris.c \ intel_fbo.c \ intel_depthstencil.c \ + intel_bufmgr_fake.c \ intel_bufmgr_gem.c C_SOURCES = \ diff --git a/src/mesa/drivers/dri/i915/i830_vtbl.c b/src/mesa/drivers/dri/i915/i830_vtbl.c index c5a85fe035..86bf468a7e 100644 --- a/src/mesa/drivers/dri/i915/i830_vtbl.c +++ b/src/mesa/drivers/dri/i915/i830_vtbl.c @@ -490,14 +490,14 @@ i830_emit_state(struct intel_context *intel) OUT_BATCH(state->Buffer[I830_DESTREG_CBUFADDR0]); OUT_BATCH(state->Buffer[I830_DESTREG_CBUFADDR1]); OUT_RELOC(state->draw_region->buffer, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, state->draw_region->draw_offset); if (state->depth_region) { OUT_BATCH(state->Buffer[I830_DESTREG_DBUFADDR0]); OUT_BATCH(state->Buffer[I830_DESTREG_DBUFADDR1]); OUT_RELOC(state->depth_region->buffer, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, state->depth_region->draw_offset); } @@ -524,7 +524,7 @@ i830_emit_state(struct intel_context *intel) if (state->tex_buffer[i]) { OUT_RELOC(state->tex_buffer[i], - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_SAMPLER, 0, state->tex_offset[i] | TM0S0_USE_FENCE); } else if (state == &i830->meta) { diff --git a/src/mesa/drivers/dri/i915/i915_vtbl.c b/src/mesa/drivers/dri/i915/i915_vtbl.c index 135bfaa265..de1ec5effc 100644 --- a/src/mesa/drivers/dri/i915/i915_vtbl.c +++ b/src/mesa/drivers/dri/i915/i915_vtbl.c @@ -377,14 +377,14 @@ i915_emit_state(struct intel_context *intel) OUT_BATCH(state->Buffer[I915_DESTREG_CBUFADDR0]); OUT_BATCH(state->Buffer[I915_DESTREG_CBUFADDR1]); OUT_RELOC(state->draw_region->buffer, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, state->draw_region->draw_offset); if (state->depth_region) { OUT_BATCH(state->Buffer[I915_DESTREG_DBUFADDR0]); OUT_BATCH(state->Buffer[I915_DESTREG_DBUFADDR1]); OUT_RELOC(state->depth_region->buffer, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, state->depth_region->draw_offset); } @@ -427,7 +427,7 @@ i915_emit_state(struct intel_context *intel) if (state->tex_buffer[i]) { OUT_RELOC(state->tex_buffer[i], - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_SAMPLER, 0, state->tex_offset[i]); } else if (state == &i915->meta) { diff --git a/src/mesa/drivers/dri/i915/intel_bufmgr_fake.c b/src/mesa/drivers/dri/i915/intel_bufmgr_fake.c new file mode 120000 index 0000000000..9b840a8123 --- /dev/null +++ b/src/mesa/drivers/dri/i915/intel_bufmgr_fake.c @@ -0,0 +1 @@ +../intel/intel_bufmgr_fake.c \ No newline at end of file diff --git a/src/mesa/drivers/dri/i965/Makefile b/src/mesa/drivers/dri/i965/Makefile index ca9b7da40f..001f63ba12 100644 --- a/src/mesa/drivers/dri/i965/Makefile +++ b/src/mesa/drivers/dri/i965/Makefile @@ -9,6 +9,7 @@ DRIVER_SOURCES = \ intel_blit.c \ intel_buffer_objects.c \ intel_buffers.c \ + intel_bufmgr_fake.c \ intel_bufmgr_gem.c \ intel_context.c \ intel_decode.c \ diff --git a/src/mesa/drivers/dri/i965/brw_cc.c b/src/mesa/drivers/dri/i965/brw_cc.c index 9d8984f05c..b9338db0f5 100644 --- a/src/mesa/drivers/dri/i965/brw_cc.c +++ b/src/mesa/drivers/dri/i965/brw_cc.c @@ -257,7 +257,8 @@ cc_unit_create_from_key(struct brw_context *brw, struct brw_cc_unit_key *key) /* Emit CC viewport relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, + 0, 0, offsetof(struct brw_cc_unit_state, cc4), brw->cc.vp_bo); diff --git a/src/mesa/drivers/dri/i965/brw_clip_state.c b/src/mesa/drivers/dri/i965/brw_clip_state.c index 7cb21f894e..26c322672c 100644 --- a/src/mesa/drivers/dri/i965/brw_clip_state.c +++ b/src/mesa/drivers/dri/i965/brw_clip_state.c @@ -120,7 +120,8 @@ clip_unit_create_from_key(struct brw_context *brw, /* Emit clip program relocation */ assert(brw->clip.prog_bo); dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, + 0, clip.thread0.grf_reg_count << 1, offsetof(struct brw_clip_unit_state, thread0), brw->clip.prog_bo); diff --git a/src/mesa/drivers/dri/i965/brw_curbe.c b/src/mesa/drivers/dri/i965/brw_curbe.c index 5ff4e2964e..1b5e22f130 100644 --- a/src/mesa/drivers/dri/i965/brw_curbe.c +++ b/src/mesa/drivers/dri/i965/brw_curbe.c @@ -353,7 +353,8 @@ static void emit_constant_buffer(struct brw_context *brw) OUT_BATCH(0); } else { OUT_BATCH((CMD_CONST_BUFFER << 16) | (1 << 8) | (2 - 2)); - OUT_RELOC(brw->curbe.curbe_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + OUT_RELOC(brw->curbe.curbe_bo, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, (sz - 1) + brw->curbe.curbe_offset); } ADVANCE_BATCH(); diff --git a/src/mesa/drivers/dri/i965/brw_draw_upload.c b/src/mesa/drivers/dri/i965/brw_draw_upload.c index aa985d68b6..5222d2e450 100644 --- a/src/mesa/drivers/dri/i965/brw_draw_upload.c +++ b/src/mesa/drivers/dri/i965/brw_draw_upload.c @@ -469,7 +469,7 @@ void brw_emit_vertices( struct brw_context *brw, BRW_VB0_ACCESS_VERTEXDATA | (input->stride << BRW_VB0_PITCH_SHIFT)); OUT_RELOC(input->bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_VERTEX, 0, input->offset); OUT_BATCH(max_index); OUT_BATCH(0); /* Instance data step rate */ @@ -590,8 +590,11 @@ void brw_emit_indices(struct brw_context *brw, BEGIN_BATCH(4, IGNORE_CLIPRECTS); OUT_BATCH( ib.header.dword ); - OUT_RELOC( bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, offset); - OUT_RELOC( bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + OUT_RELOC( bo, + DRM_GEM_DOMAIN_I915_VERTEX, 0, + offset); + OUT_RELOC( bo, + DRM_GEM_DOMAIN_I915_VERTEX, 0, offset + ib_size); OUT_BATCH( 0 ); ADVANCE_BATCH(); diff --git a/src/mesa/drivers/dri/i965/brw_gs_state.c b/src/mesa/drivers/dri/i965/brw_gs_state.c index f1f9e018f1..2bf86f5573 100644 --- a/src/mesa/drivers/dri/i965/brw_gs_state.c +++ b/src/mesa/drivers/dri/i965/brw_gs_state.c @@ -107,7 +107,7 @@ gs_unit_create_from_key(struct brw_context *brw, struct brw_gs_unit_key *key) if (key->prog_active) { /* Emit GS program relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, gs.thread0.grf_reg_count << 1, offsetof(struct brw_gs_unit_state, thread0), brw->gs.prog_bo); diff --git a/src/mesa/drivers/dri/i965/brw_misc_state.c b/src/mesa/drivers/dri/i965/brw_misc_state.c index 26ec797b5f..24dfd2e24e 100644 --- a/src/mesa/drivers/dri/i965/brw_misc_state.c +++ b/src/mesa/drivers/dri/i965/brw_misc_state.c @@ -88,7 +88,9 @@ static void upload_binding_table_pointers(struct brw_context *brw) OUT_BATCH(0); /* gs */ OUT_BATCH(0); /* clip */ OUT_BATCH(0); /* sf */ - OUT_RELOC(brw->wm.bind_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); + OUT_RELOC(brw->wm.bind_bo, + DRM_GEM_DOMAIN_I915_SAMPLER, 0, + 0); ADVANCE_BATCH(); } @@ -114,18 +116,18 @@ static void upload_pipelined_state_pointers(struct brw_context *brw ) BEGIN_BATCH(7, IGNORE_CLIPRECTS); OUT_BATCH(CMD_PIPELINED_STATE_POINTERS << 16 | (7 - 2)); - OUT_RELOC(brw->vs.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); + OUT_RELOC(brw->vs.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0); if (brw->gs.prog_active) - OUT_RELOC(brw->gs.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 1); + OUT_RELOC(brw->gs.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 1); else OUT_BATCH(0); if (!brw->metaops.active) - OUT_RELOC(brw->clip.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 1); + OUT_RELOC(brw->clip.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 1); else OUT_BATCH(0); - OUT_RELOC(brw->sf.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); - OUT_RELOC(brw->wm.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); - OUT_RELOC(brw->cc.state_bo, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); + OUT_RELOC(brw->sf.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0); + OUT_RELOC(brw->wm.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0); + OUT_RELOC(brw->cc.state_bo, DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0); ADVANCE_BATCH(); brw->state.dirty.brw |= BRW_NEW_PSP; @@ -233,7 +235,8 @@ static void emit_depthbuffer(struct brw_context *brw) (region->tiled << 27) | (BRW_SURFACE_2D << 29)); OUT_RELOC(region->buffer, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE, 0); + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, + 0); OUT_BATCH((BRW_SURFACE_MIPMAPLAYOUT_BELOW << 1) | ((region->pitch - 1) << 6) | ((region->height - 1) << 19)); diff --git a/src/mesa/drivers/dri/i965/brw_sf_state.c b/src/mesa/drivers/dri/i965/brw_sf_state.c index 24388b79a5..5cf3228486 100644 --- a/src/mesa/drivers/dri/i965/brw_sf_state.c +++ b/src/mesa/drivers/dri/i965/brw_sf_state.c @@ -254,14 +254,14 @@ sf_unit_create_from_key(struct brw_context *brw, struct brw_sf_unit_key *key, /* Emit SF program relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, sf.thread0.grf_reg_count << 1, offsetof(struct brw_sf_unit_state, thread0), brw->sf.prog_bo); /* Emit SF viewport relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, sf.sf5.front_winding | (sf.sf5.viewport_transform << 1), offsetof(struct brw_sf_unit_state, sf5), brw->sf.vp_bo); diff --git a/src/mesa/drivers/dri/i965/brw_vs_state.c b/src/mesa/drivers/dri/i965/brw_vs_state.c index 2a64f3df33..73f52d7428 100644 --- a/src/mesa/drivers/dri/i965/brw_vs_state.c +++ b/src/mesa/drivers/dri/i965/brw_vs_state.c @@ -116,7 +116,7 @@ vs_unit_create_from_key(struct brw_context *brw, struct brw_vs_unit_key *key) /* Emit VS program relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, vs.thread0.grf_reg_count << 1, offsetof(struct brw_vs_unit_state, thread0), brw->vs.prog_bo); diff --git a/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c b/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c index d40332e9ae..13f7f21800 100644 --- a/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c +++ b/src/mesa/drivers/dri/i965/brw_wm_sampler_state.c @@ -306,7 +306,7 @@ static int upload_wm_samplers( struct brw_context *brw ) ret |= dri_bufmgr_check_aperture_space(brw->wm.sdc_bo[i]); dri_emit_reloc(brw->wm.sampler_bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0, i * sizeof(struct brw_sampler_state) + offsetof(struct brw_sampler_state, ss2), diff --git a/src/mesa/drivers/dri/i965/brw_wm_state.c b/src/mesa/drivers/dri/i965/brw_wm_state.c index f4da0f279e..f79b58ba7a 100644 --- a/src/mesa/drivers/dri/i965/brw_wm_state.c +++ b/src/mesa/drivers/dri/i965/brw_wm_state.c @@ -200,7 +200,7 @@ wm_unit_create_from_key(struct brw_context *brw, struct brw_wm_unit_key *key, /* Emit WM program relocation */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, wm.thread0.grf_reg_count << 1, offsetof(struct brw_wm_unit_state, thread0), brw->wm.prog_bo); @@ -208,7 +208,7 @@ wm_unit_create_from_key(struct brw_context *brw, struct brw_wm_unit_key *key, /* Emit scratch space relocation */ if (key->total_scratch != 0) { dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE, + 0, 0, wm.thread2.per_thread_scratch_space, offsetof(struct brw_wm_unit_state, thread2), brw->wm.scratch_buffer); @@ -217,7 +217,7 @@ wm_unit_create_from_key(struct brw_context *brw, struct brw_wm_unit_key *key, /* Emit sampler state relocation */ if (key->sampler_count != 0) { dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, wm.wm4.stats_enable | (wm.wm4.sampler_count << 2), offsetof(struct brw_wm_unit_state, wm4), brw->wm.sampler_bo); diff --git a/src/mesa/drivers/dri/i965/brw_wm_surface_state.c b/src/mesa/drivers/dri/i965/brw_wm_surface_state.c index 0d91391964..73f4b2b4a3 100644 --- a/src/mesa/drivers/dri/i965/brw_wm_surface_state.c +++ b/src/mesa/drivers/dri/i965/brw_wm_surface_state.c @@ -204,7 +204,7 @@ brw_create_texture_surface( struct brw_context *brw, /* Emit relocation to surface contents */ dri_emit_reloc(bo, - DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + DRM_GEM_DOMAIN_I915_SAMPLER, 0, 0, offsetof(struct brw_surface_state, ss1), key->bo); @@ -337,10 +337,14 @@ brw_update_region_surface(struct brw_context *brw, struct intel_region *region, &surf, sizeof(surf), NULL, NULL); if (region_bo != NULL) { + /* We might sample from it, and we might render to it, so flag + * them both. We might be able to figure out from other state + * a more restrictive relocation to emit. + */ dri_emit_reloc(brw->wm.surf_bo[unit], - DRM_BO_FLAG_MEM_TT | - DRM_BO_FLAG_READ | - DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_RENDER | + DRM_GEM_DOMAIN_I915_SAMPLER, + DRM_GEM_DOMAIN_I915_RENDER, 0, offsetof(struct brw_surface_state, ss1), region_bo); @@ -388,9 +392,7 @@ brw_wm_get_binding_table(struct brw_context *brw) for (i = 0; i < BRW_WM_MAX_SURF; i++) { if (brw->wm.surf_bo[i] != NULL) { dri_emit_reloc(bind_bo, - DRM_BO_FLAG_MEM_TT | - DRM_BO_FLAG_READ | - DRM_BO_FLAG_WRITE, + DRM_GEM_DOMAIN_I915_INSTRUCTION, 0, 0, i * sizeof(GLuint), brw->wm.surf_bo[i]); diff --git a/src/mesa/drivers/dri/i965/intel_bufmgr_fake.c b/src/mesa/drivers/dri/i965/intel_bufmgr_fake.c new file mode 120000 index 0000000000..9b840a8123 --- /dev/null +++ b/src/mesa/drivers/dri/i965/intel_bufmgr_fake.c @@ -0,0 +1 @@ +../intel/intel_bufmgr_fake.c \ No newline at end of file diff --git a/src/mesa/drivers/dri/intel/intel_batchbuffer.c b/src/mesa/drivers/dri/intel/intel_batchbuffer.c index bab8e645d4..f22e6c0967 100644 --- a/src/mesa/drivers/dri/intel/intel_batchbuffer.c +++ b/src/mesa/drivers/dri/intel/intel_batchbuffer.c @@ -267,11 +267,13 @@ _intel_batchbuffer_flush(struct intel_batchbuffer *batch, const char *file, GLboolean intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch, dri_bo *buffer, - GLuint flags, GLuint delta) + uint32_t read_domains, uint32_t write_domain, + uint32_t delta) { int ret; - ret = dri_emit_reloc(batch->buf, flags, delta, batch->ptr - batch->map, buffer); + ret = dri_emit_reloc(batch->buf, read_domains, write_domain, + delta, batch->ptr - batch->map, buffer); /* * Using the old buffer offset, write in what the right data would be, in case diff --git a/src/mesa/drivers/dri/intel/intel_batchbuffer.h b/src/mesa/drivers/dri/intel/intel_batchbuffer.h index feddfb46df..7268bd59da 100644 --- a/src/mesa/drivers/dri/intel/intel_batchbuffer.h +++ b/src/mesa/drivers/dri/intel/intel_batchbuffer.h @@ -79,7 +79,9 @@ void intel_batchbuffer_release_space(struct intel_batchbuffer *batch, GLboolean intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch, dri_bo *buffer, - GLuint flags, GLuint offset); + uint32_t read_domains, + uint32_t write_domain, + uint32_t offset); /* Inline functions - might actually be better off with these * non-inlined. Certainly better off switching all command packets to @@ -131,9 +133,10 @@ intel_batchbuffer_require_space(struct intel_batchbuffer *batch, #define OUT_BATCH(d) intel_batchbuffer_emit_dword(intel->batch, d) -#define OUT_RELOC(buf, cliprect_mode, delta) do { \ +#define OUT_RELOC(buf, read_domains, write_domain, delta) do { \ assert((delta) >= 0); \ - intel_batchbuffer_emit_reloc(intel->batch, buf, cliprect_mode, delta); \ + intel_batchbuffer_emit_reloc(intel->batch, buf, \ + read_domains, write_domain, delta); \ } while (0) #define ADVANCE_BATCH() do { } while(0) diff --git a/src/mesa/drivers/dri/intel/intel_blit.c b/src/mesa/drivers/dri/intel/intel_blit.c index b7d36d8cd6..3d7f64e28b 100644 --- a/src/mesa/drivers/dri/intel/intel_blit.c +++ b/src/mesa/drivers/dri/intel/intel_blit.c @@ -148,10 +148,14 @@ intelCopyBuffer(const __DRIdrawablePrivate * dPriv, OUT_BATCH((box.y1 << 16) | box.x1); OUT_BATCH((box.y2 << 16) | box.x2); - OUT_RELOC(dst->buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, 0); + OUT_RELOC(dst->buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, + 0); OUT_BATCH((src_y << 16) | src_x); OUT_BATCH(src_pitch); - OUT_RELOC(src->buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, 0); + OUT_RELOC(src->buffer, + DRM_GEM_DOMAIN_I915_RENDER, 0, + 0); ADVANCE_BATCH(); } @@ -212,7 +216,9 @@ intelEmitFillBlit(struct intel_context *intel, OUT_BATCH(BR13 | dst_pitch); OUT_BATCH((y << 16) | x); OUT_BATCH(((y + h) << 16) | (x + w)); - OUT_RELOC(dst_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, dst_offset); + OUT_RELOC(dst_buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, + dst_offset); OUT_BATCH(color); ADVANCE_BATCH(); } @@ -332,11 +338,13 @@ intelEmitCopyBlit(struct intel_context *intel, OUT_BATCH(BR13 | dst_pitch); OUT_BATCH((dst_y << 16) | dst_x); OUT_BATCH((dst_y2 << 16) | dst_x2); - OUT_RELOC(dst_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + OUT_RELOC(dst_buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, dst_offset); OUT_BATCH((src_y << 16) | src_x); OUT_BATCH(src_pitch); - OUT_RELOC(src_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + OUT_RELOC(src_buffer, + DRM_GEM_DOMAIN_I915_RENDER, 0, src_offset); ADVANCE_BATCH(); } @@ -349,11 +357,13 @@ intelEmitCopyBlit(struct intel_context *intel, OUT_BATCH(BR13 | dst_pitch); OUT_BATCH((0 << 16) | dst_x); OUT_BATCH((h << 16) | dst_x2); - OUT_RELOC(dst_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + OUT_RELOC(dst_buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, dst_offset + dst_y * dst_pitch); OUT_BATCH((0 << 16) | src_x); OUT_BATCH(src_pitch); - OUT_RELOC(src_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_READ, + OUT_RELOC(src_buffer, + DRM_GEM_DOMAIN_I915_RENDER, 0, src_offset + src_y * src_pitch); ADVANCE_BATCH(); } @@ -528,7 +538,8 @@ intelClearWithBlit(GLcontext *ctx, GLbitfield mask) OUT_BATCH(BR13); OUT_BATCH((b.y1 << 16) | b.x1); OUT_BATCH((b.y2 << 16) | b.x2); - OUT_RELOC(write_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, + OUT_RELOC(write_buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, irb_region->draw_offset); OUT_BATCH(clearVal); ADVANCE_BATCH(); @@ -600,7 +611,9 @@ intelEmitImmediateColorExpandBlit(struct intel_context *intel, OUT_BATCH(br13); OUT_BATCH((0 << 16) | 0); /* clip x1, y1 */ OUT_BATCH((100 << 16) | 100); /* clip x2, y2 */ - OUT_RELOC(dst_buffer, DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_WRITE, dst_offset); + OUT_RELOC(dst_buffer, + DRM_GEM_DOMAIN_I915_RENDER, DRM_GEM_DOMAIN_I915_RENDER, + dst_offset); OUT_BATCH(0); /* bg */ OUT_BATCH(fg_color); /* fg */ OUT_BATCH(0); /* pattern base addr */ diff --git a/src/mesa/drivers/dri/intel/intel_bufmgr_fake.c b/src/mesa/drivers/dri/intel/intel_bufmgr_fake.c new file mode 100644 index 0000000000..5d01a471c5 --- /dev/null +++ b/src/mesa/drivers/dri/intel/intel_bufmgr_fake.c @@ -0,0 +1,1173 @@ +/************************************************************************** + * + * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +/* Originally a fake version of the buffer manager so that we can + * prototype the changes in a driver fairly quickly, has been fleshed + * out to a fully functional interim solution. + * + * Basically wraps the old style memory management in the new + * programming interface, but is more expressive and avoids many of + * the bugs in the old texture manager. + */ +#include "mtypes.h" +#include "dri_bufmgr.h" +#include "intel_bufmgr_fake.h" +#include "drm.h" +#include "i915_drm.h" + +#include "simple_list.h" +#include "mm.h" +#include "imports.h" + +#define DBG(...) do { \ + if (bufmgr_fake->bufmgr.debug) \ + _mesa_printf(__VA_ARGS__); \ +} while (0) + +/* Internal flags: + */ +#define BM_NO_BACKING_STORE 0x00000001 +#define BM_NO_FENCE_SUBDATA 0x00000002 +#define BM_PINNED 0x00000004 + +/* Wrapper around mm.c's mem_block, which understands that you must + * wait for fences to expire before memory can be freed. This is + * specific to our use of memcpy for uploads - an upload that was + * processed through the command queue wouldn't need to care about + * fences. + */ +#define MAX_RELOCS 4096 + +struct fake_buffer_reloc +{ + /** Buffer object that the relocation points at. */ + dri_bo *target_buf; + /** Offset of the relocation entry within reloc_buf. */ + GLuint offset; + /** Cached value of the offset when we last performed this relocation. */ + GLuint last_target_offset; + /** Value added to target_buf's offset to get the relocation entry. */ + GLuint delta; + /** Cache domains the target buffer is read into. */ + uint32_t read_domains; + /** Cache domain the target buffer will have dirty cachelines in. */ + uint32_t write_domain; +}; + +struct block { + struct block *next, *prev; + struct mem_block *mem; /* BM_MEM_AGP */ + + /** + * Marks that the block is currently in the aperture and has yet to be + * fenced. + */ + unsigned on_hardware:1; + /** + * Marks that the block is currently fenced (being used by rendering) and + * can't be freed until @fence is passed. + */ + unsigned fenced:1; + + /** Fence cookie for the block. */ + unsigned fence; /* Split to read_fence, write_fence */ + + dri_bo *bo; + void *virtual; +}; + +typedef struct _bufmgr_fake { + dri_bufmgr bufmgr; + + unsigned long low_offset; + unsigned long size; + void *virtual; + + struct mem_block *heap; + struct block lru; /* only allocated, non-fence-pending blocks here */ + + unsigned buf_nr; /* for generating ids */ + + struct block on_hardware; /* after bmValidateBuffers */ + struct block fenced; /* after bmFenceBuffers (mi_flush, emit irq, write dword) */ + /* then to bufmgr->lru or free() */ + + unsigned int last_fence; + + unsigned fail:1; + unsigned need_fence:1; + GLboolean thrashing; + + /** + * Driver callback to emit a fence, returning the cookie. + * + * Currently, this also requires that a write flush be emitted before + * emitting the fence, but this should change. + */ + unsigned int (*fence_emit)(void *private); + /** Driver callback to wait for a fence cookie to have passed. */ + int (*fence_wait)(void *private, unsigned int fence_cookie); + /** Driver-supplied argument to driver callbacks */ + void *driver_priv; + + GLboolean debug; + + GLboolean performed_rendering; + + /* keep track of the current total size of objects we have relocs for */ + unsigned long current_total_size; +} dri_bufmgr_fake; + +typedef struct _dri_bo_fake { + dri_bo bo; + + unsigned id; /* debug only */ + const char *name; + + unsigned dirty:1; + unsigned size_accounted:1; /*this buffers size has been accounted against the aperture */ + unsigned card_dirty:1; /* has the card written to this buffer - we make need to copy it back */ + unsigned int refcount; + /* Flags may consist of any of the DRM_BO flags, plus + * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the first two + * driver private flags. + */ + uint64_t flags; + /** Cache domains the target buffer is read into. */ + uint32_t read_domains; + /** Cache domain the target buffer will have dirty cachelines in. */ + uint32_t write_domain; + + unsigned int alignment; + GLboolean is_static, validated; + unsigned int map_count; + + /** relocation list */ + struct fake_buffer_reloc *relocs; + GLuint nr_relocs; + + struct block *block; + void *backing_store; + void (*invalidate_cb)(dri_bo *bo, void *ptr); + void *invalidate_ptr; +} dri_bo_fake; + +static int clear_fenced(dri_bufmgr_fake *bufmgr_fake, + unsigned int fence_cookie); + +static int dri_fake_check_aperture_space(dri_bo *bo); + +#define MAXFENCE 0x7fffffff + +static GLboolean FENCE_LTE( unsigned a, unsigned b ) +{ + if (a == b) + return GL_TRUE; + + if (a < b && b - a < (1<<24)) + return GL_TRUE; + + if (a > b && MAXFENCE - a + b < (1<<24)) + return GL_TRUE; + + return GL_FALSE; +} + +static unsigned int +_fence_emit_internal(dri_bufmgr_fake *bufmgr_fake) +{ + bufmgr_fake->last_fence = bufmgr_fake->fence_emit(bufmgr_fake->driver_priv); + return bufmgr_fake->last_fence; +} + +static void +_fence_wait_internal(dri_bufmgr_fake *bufmgr_fake, unsigned int cookie) +{ + int ret; + + ret = bufmgr_fake->fence_wait(bufmgr_fake->driver_priv, cookie); + if (ret != 0) { + _mesa_printf("%s:%d: Error %d waiting for fence.\n", + __FILE__, __LINE__); + abort(); + } + clear_fenced(bufmgr_fake, cookie); +} + +static GLboolean +_fence_test(dri_bufmgr_fake *bufmgr_fake, unsigned fence) +{ + /* Slight problem with wrap-around: + */ + return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence); +} + +/** + * Allocate a memory manager block for the buffer. + */ +static GLboolean +alloc_block(dri_bo *bo) +{ + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + dri_bufmgr_fake *bufmgr_fake= (dri_bufmgr_fake *)bo->bufmgr; + struct block *block = (struct block *)calloc(sizeof *block, 1); + unsigned int align_log2 = _mesa_ffs(bo_fake->alignment) - 1; + GLuint sz; + + if (!block) + return GL_FALSE; + + sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); + + block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0); + if (!block->mem) { + free(block); + return GL_FALSE; + } + + make_empty_list(block); + + /* Insert at head or at tail??? + */ + insert_at_tail(&bufmgr_fake->lru, block); + + block->virtual = bufmgr_fake->virtual + + block->mem->ofs - bufmgr_fake->low_offset; + block->bo = bo; + + bo_fake->block = block; + + return GL_TRUE; +} + +/* Release the card storage associated with buf: + */ +static void free_block(dri_bufmgr_fake *bufmgr_fake, struct block *block) +{ + dri_bo_fake *bo_fake; + DBG("free block %p %08x %d %d\n", block, block->mem->ofs, block->on_hardware, block->fenced); + + if (!block) + return; + + bo_fake = (dri_bo_fake *)block->bo; + if (!(bo_fake->flags & BM_NO_BACKING_STORE) && (bo_fake->card_dirty == 1)) { + memcpy(bo_fake->backing_store, block->virtual, block->bo->size); + bo_fake->card_dirty = 1; + bo_fake->dirty = 1; + } + + if (block->on_hardware) { + block->bo = NULL; + } + else if (block->fenced) { + block->bo = NULL; + } + else { + DBG(" - free immediately\n"); + remove_from_list(block); + + mmFreeMem(block->mem); + free(block); + } +} + +static void +alloc_backing_store(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + assert(!bo_fake->backing_store); + assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE))); + + bo_fake->backing_store = ALIGN_MALLOC(bo->size, 64); + + DBG("alloc_backing - buf %d %p %d\n", bo_fake->id, bo_fake->backing_store, bo->size); + assert(bo_fake->backing_store); +} + +static void +free_backing_store(dri_bo *bo) +{ + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + if (bo_fake->backing_store) { + assert(!(bo_fake->flags & (BM_PINNED|BM_NO_BACKING_STORE))); + ALIGN_FREE(bo_fake->backing_store); + bo_fake->backing_store = NULL; + } +} + +static void +set_dirty(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + if (bo_fake->flags & BM_NO_BACKING_STORE && bo_fake->invalidate_cb != NULL) + bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr); + + assert(!(bo_fake->flags & BM_PINNED)); + + DBG("set_dirty - buf %d\n", bo_fake->id); + bo_fake->dirty = 1; +} + +static GLboolean +evict_lru(dri_bufmgr_fake *bufmgr_fake, GLuint max_fence) +{ + struct block *block, *tmp; + + DBG("%s\n", __FUNCTION__); + + foreach_s(block, tmp, &bufmgr_fake->lru) { + dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; + + if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) + continue; + + if (block->fence && max_fence && !FENCE_LTE(block->fence, max_fence)) + return 0; + + set_dirty(&bo_fake->bo); + bo_fake->block = NULL; + + free_block(bufmgr_fake, block); + return GL_TRUE; + } + + return GL_FALSE; +} + +#define foreach_s_rev(ptr, t, list) \ + for(ptr=(list)->prev,t=(ptr)->prev; list != ptr; ptr=t, t=(t)->prev) + +static GLboolean +evict_mru(dri_bufmgr_fake *bufmgr_fake) +{ + struct block *block, *tmp; + + DBG("%s\n", __FUNCTION__); + + foreach_s_rev(block, tmp, &bufmgr_fake->lru) { + dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; + + if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) + continue; + + set_dirty(&bo_fake->bo); + bo_fake->block = NULL; + + free_block(bufmgr_fake, block); + return GL_TRUE; + } + + return GL_FALSE; +} + +/** + * Removes all objects from the fenced list older than the given fence. + */ +static int clear_fenced(dri_bufmgr_fake *bufmgr_fake, + unsigned int fence_cookie) +{ + struct block *block, *tmp; + int ret = 0; + + foreach_s(block, tmp, &bufmgr_fake->fenced) { + assert(block->fenced); + + if (_fence_test(bufmgr_fake, block->fence)) { + + block->fenced = 0; + + if (!block->bo) { + DBG("delayed free: offset %x sz %x\n", + block->mem->ofs, block->mem->size); + remove_from_list(block); + mmFreeMem(block->mem); + free(block); + } + else { + DBG("return to lru: offset %x sz %x\n", + block->mem->ofs, block->mem->size); + move_to_tail(&bufmgr_fake->lru, block); + } + + ret = 1; + } + else { + /* Blocks are ordered by fence, so if one fails, all from + * here will fail also: + */ + DBG("fence not passed: offset %x sz %x %d %d \n", + block->mem->ofs, block->mem->size, block->fence, bufmgr_fake->last_fence); + break; + } + } + + DBG("%s: %d\n", __FUNCTION__, ret); + return ret; +} + +static void fence_blocks(dri_bufmgr_fake *bufmgr_fake, unsigned fence) +{ + struct block *block, *tmp; + + foreach_s (block, tmp, &bufmgr_fake->on_hardware) { + DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", block, + block->mem->size, block->mem->ofs, block->bo, fence); + block->fence = fence; + + block->on_hardware = 0; + block->fenced = 1; + + /* Move to tail of pending list here + */ + move_to_tail(&bufmgr_fake->fenced, block); + } + + assert(is_empty_list(&bufmgr_fake->on_hardware)); +} + +static GLboolean evict_and_alloc_block(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + assert(bo_fake->block == NULL); + + /* Search for already free memory: + */ + if (alloc_block(bo)) + return GL_TRUE; + + /* If we're not thrashing, allow lru eviction to dig deeper into + * recently used textures. We'll probably be thrashing soon: + */ + if (!bufmgr_fake->thrashing) { + while (evict_lru(bufmgr_fake, 0)) + if (alloc_block(bo)) + return GL_TRUE; + } + + /* Keep thrashing counter alive? + */ + if (bufmgr_fake->thrashing) + bufmgr_fake->thrashing = 20; + + /* Wait on any already pending fences - here we are waiting for any + * freed memory that has been submitted to hardware and fenced to + * become available: + */ + while (!is_empty_list(&bufmgr_fake->fenced)) { + GLuint fence = bufmgr_fake->fenced.next->fence; + _fence_wait_internal(bufmgr_fake, fence); + + if (alloc_block(bo)) + return GL_TRUE; + } + + if (!is_empty_list(&bufmgr_fake->on_hardware)) { + while (!is_empty_list(&bufmgr_fake->fenced)) { + GLuint fence = bufmgr_fake->fenced.next->fence; + _fence_wait_internal(bufmgr_fake, fence); + } + + if (!bufmgr_fake->thrashing) { + DBG("thrashing\n"); + } + bufmgr_fake->thrashing = 20; + + if (alloc_block(bo)) + return GL_TRUE; + } + + while (evict_mru(bufmgr_fake)) + if (alloc_block(bo)) + return GL_TRUE; + + DBG("%s 0x%x bytes failed\n", __FUNCTION__, bo->size); + + return GL_FALSE; +} + +/*********************************************************************** + * Public functions + */ + +/** + * Wait for hardware idle by emitting a fence and waiting for it. + */ +static void +dri_bufmgr_fake_wait_idle(dri_bufmgr_fake *bufmgr_fake) +{ + unsigned int cookie; + + cookie = bufmgr_fake->fence_emit(bufmgr_fake->driver_priv); + _fence_wait_internal(bufmgr_fake, cookie); +} + +/** + * Wait for execution pending on a buffer + */ +static void +dri_bufmgr_fake_bo_wait_idle(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + if (bo_fake->block == NULL || !bo_fake->block->fenced) + return; + + _fence_wait_internal(bufmgr_fake, bo_fake->block->fence); +} + +/* Specifically ignore texture memory sharing. + * -- just evict everything + * -- and wait for idle + */ +void +dri_bufmgr_fake_contended_lock_take(dri_bufmgr *bufmgr) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; + struct block *block, *tmp; + + bufmgr_fake->need_fence = 1; + bufmgr_fake->fail = 0; + + /* Wait for hardware idle. We don't know where acceleration has been + * happening, so we'll need to wait anyway before letting anything get + * put on the card again. + */ + dri_bufmgr_fake_wait_idle(bufmgr_fake); + + /* Check that we hadn't released the lock without having fenced the last + * set of buffers. + */ + assert(is_empty_list(&bufmgr_fake->fenced)); + assert(is_empty_list(&bufmgr_fake->on_hardware)); + + foreach_s(block, tmp, &bufmgr_fake->lru) { + assert(_fence_test(bufmgr_fake, block->fence)); + set_dirty(block->bo); + } +} + +static dri_bo * +dri_fake_bo_alloc(dri_bufmgr *bufmgr, const char *name, + unsigned long size, unsigned int alignment, + uint64_t location_mask) +{ + dri_bufmgr_fake *bufmgr_fake; + dri_bo_fake *bo_fake; + + bufmgr_fake = (dri_bufmgr_fake *)bufmgr; + + assert(size != 0); + + bo_fake = calloc(1, sizeof(*bo_fake)); + if (!bo_fake) + return NULL; + + bo_fake->bo.size = size; + bo_fake->bo.offset = -1; + bo_fake->bo.virtual = NULL; + bo_fake->bo.bufmgr = bufmgr; + bo_fake->refcount = 1; + + /* Alignment must be a power of two */ + assert((alignment & (alignment - 1)) == 0); + if (alignment == 0) + alignment = 1; + bo_fake->alignment = alignment; + bo_fake->id = ++bufmgr_fake->buf_nr; + bo_fake->name = name; + bo_fake->flags = 0; + bo_fake->is_static = GL_FALSE; + + DBG("drm_bo_alloc: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, + bo_fake->bo.size / 1024); + + return &bo_fake->bo; +} + +static dri_bo * +dri_fake_bo_alloc_static(dri_bufmgr *bufmgr, const char *name, + unsigned long offset, unsigned long size, + void *virtual, uint64_t location_mask) +{ + dri_bufmgr_fake *bufmgr_fake; + dri_bo_fake *bo_fake; + + bufmgr_fake = (dri_bufmgr_fake *)bufmgr; + + assert(size != 0); + + bo_fake = calloc(1, sizeof(*bo_fake)); + if (!bo_fake) + return NULL; + + bo_fake->bo.size = size; + bo_fake->bo.offset = offset; + bo_fake->bo.virtual = virtual; + bo_fake->bo.bufmgr = bufmgr; + bo_fake->refcount = 1; + bo_fake->id = ++bufmgr_fake->buf_nr; + bo_fake->name = name; + bo_fake->flags = BM_PINNED | DRM_BO_FLAG_NO_MOVE; + bo_fake->is_static = GL_TRUE; + + DBG("drm_bo_alloc_static: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, + bo_fake->bo.size / 1024); + + return &bo_fake->bo; +} + +static void +dri_fake_bo_reference(dri_bo *bo) +{ + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + bo_fake->refcount++; +} + +static void +dri_fake_bo_unreference(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + int i; + + if (!bo) + return; + + if (--bo_fake->refcount == 0) { + assert(bo_fake->map_count == 0); + /* No remaining references, so free it */ + if (bo_fake->block) + free_block(bufmgr_fake, bo_fake->block); + free_backing_store(bo); + + for (i = 0; i < bo_fake->nr_relocs; i++) + dri_bo_unreference(bo_fake->relocs[i].target_buf); + + DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id, bo_fake->name); + + free(bo_fake->relocs); + free(bo); + + return; + } +} + +/** + * Set the buffer as not requiring backing store, and instead get the callback + * invoked whenever it would be set dirty. + */ +void dri_bo_fake_disable_backing_store(dri_bo *bo, + void (*invalidate_cb)(dri_bo *bo, + void *ptr), + void *ptr) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + if (bo_fake->backing_store) + free_backing_store(bo); + + bo_fake->flags |= BM_NO_BACKING_STORE; + + DBG("disable_backing_store set buf %d dirty\n", bo_fake->id); + bo_fake->dirty = 1; + bo_fake->invalidate_cb = invalidate_cb; + bo_fake->invalidate_ptr = ptr; + + /* Note that it is invalid right from the start. Also note + * invalidate_cb is called with the bufmgr locked, so cannot + * itself make bufmgr calls. + */ + if (invalidate_cb != NULL) + invalidate_cb(bo, ptr); +} + +/** + * Map a buffer into bo->virtual, allocating either card memory space (If + * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary. + */ +static int +dri_fake_bo_map(dri_bo *bo, GLboolean write_enable) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + /* Static buffers are always mapped. */ + if (bo_fake->is_static) + return 0; + + /* Allow recursive mapping. Mesa may recursively map buffers with + * nested display loops, and it is used internally in bufmgr_fake + * for relocation. + */ + if (bo_fake->map_count++ != 0) + return 0; + + { + DBG("drm_bo_map: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, + bo_fake->bo.size / 1024); + + if (bo->virtual != NULL) { + _mesa_printf("%s: already mapped\n", __FUNCTION__); + abort(); + } + else if (bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED)) { + + if (!bo_fake->block && !evict_and_alloc_block(bo)) { + DBG("%s: alloc failed\n", __FUNCTION__); + bufmgr_fake->fail = 1; + return 1; + } + else { + assert(bo_fake->block); + bo_fake->dirty = 0; + + if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) && + bo_fake->block->fenced) { + dri_bufmgr_fake_bo_wait_idle(bo); + } + + bo->virtual = bo_fake->block->virtual; + } + } + else { + if (write_enable) + set_dirty(bo); + + if (bo_fake->backing_store == 0) + alloc_backing_store(bo); + + bo->virtual = bo_fake->backing_store; + } + } + + return 0; +} + +static int +dri_fake_bo_unmap(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + /* Static buffers are always mapped. */ + if (bo_fake->is_static) + return 0; + + assert(bo_fake->map_count != 0); + if (--bo_fake->map_count != 0) + return 0; + + DBG("drm_bo_unmap: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, + bo_fake->bo.size / 1024); + + bo->virtual = NULL; + + return 0; +} + +static void +dri_fake_kick_all(dri_bufmgr_fake *bufmgr_fake) +{ + struct block *block, *tmp; + + bufmgr_fake->performed_rendering = GL_FALSE; + /* okay for ever BO that is on the HW kick it off. + seriously not afraid of the POLICE right now */ + foreach_s(block, tmp, &bufmgr_fake->on_hardware) { + dri_bo_fake *bo_fake = (dri_bo_fake *)block->bo; + + block->on_hardware = 0; + free_block(bufmgr_fake, block); + bo_fake->block = NULL; + bo_fake->validated = GL_FALSE; + if (!(bo_fake->flags & BM_NO_BACKING_STORE)) + bo_fake->dirty = 1; + } +} + +static int +dri_fake_bo_validate(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + + /* XXX: Sanity-check whether we've already validated this one under + * different flags. See drmAddValidateItem(). + */ + bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + + DBG("drm_bo_validate: (buf %d: %s, %d kb)\n", bo_fake->id, bo_fake->name, + bo_fake->bo.size / 1024); + + /* Sanity check: Buffers should be unmapped before being validated. + * This is not so much of a problem for bufmgr_fake, but TTM refuses, + * and the problem is harder to debug there. + */ + assert(bo_fake->map_count == 0); + + if (bo_fake->is_static) { + /* Add it to the needs-fence list */ + bufmgr_fake->need_fence = 1; + return 0; + } + + /* reset size accounted */ + bo_fake->size_accounted = 0; + + /* Allocate the card memory */ + if (!bo_fake->block && !evict_and_alloc_block(bo)) { + bufmgr_fake->fail = 1; + DBG("Failed to validate buf %d:%s\n", bo_fake->id, bo_fake->name); + return -1; + } + + assert(bo_fake->block); + assert(bo_fake->block->bo == &bo_fake->bo); + + bo->offset = bo_fake->block->mem->ofs; + + /* Upload the buffer contents if necessary */ + if (bo_fake->dirty) { + DBG("Upload dirty buf %d:%s, sz %d offset 0x%x\n", bo_fake->id, + bo_fake->name, bo->size, bo_fake->block->mem->ofs); + + assert(!(bo_fake->flags & + (BM_NO_BACKING_STORE|BM_PINNED))); + + /* Actually, should be able to just wait for a fence on the memory, + * which we would be tracking when we free it. Waiting for idle is + * a sufficiently large hammer for now. + */ + dri_bufmgr_fake_wait_idle(bufmgr_fake); + + /* we may never have mapped this BO so it might not have any backing + * store if this happens it should be rare, but 0 the card memory + * in any case */ + if (bo_fake->backing_store) + memcpy(bo_fake->block->virtual, bo_fake->backing_store, bo->size); + else + memset(bo_fake->block->virtual, 0, bo->size); + + bo_fake->dirty = 0; + } + + bo_fake->block->fenced = 0; + bo_fake->block->on_hardware = 1; + move_to_tail(&bufmgr_fake->on_hardware, bo_fake->block); + + bo_fake->validated = GL_TRUE; + bufmgr_fake->need_fence = 1; + + return 0; +} + +static void +dri_fake_fence_validated(dri_bufmgr *bufmgr) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; + unsigned int cookie; + + cookie = _fence_emit_internal(bufmgr_fake); + fence_blocks(bufmgr_fake, cookie); + + DBG("drm_fence_validated: 0x%08x cookie\n", cookie); +} + +static void +dri_fake_destroy(dri_bufmgr *bufmgr) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bufmgr; + + mmDestroy(bufmgr_fake->heap); + free(bufmgr); +} + +static int +dri_fake_emit_reloc(dri_bo *reloc_buf, + uint32_t read_domains, uint32_t write_domain, + uint32_t delta, uint32_t offset, dri_bo *target_buf) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)reloc_buf->bufmgr; + struct fake_buffer_reloc *r; + dri_bo_fake *reloc_fake = (dri_bo_fake *)reloc_buf; + dri_bo_fake *target_fake = (dri_bo_fake *)target_buf; + int i; + + assert(reloc_buf); + assert(target_buf); + + assert(target_fake->is_static || target_fake->size_accounted); + + if (reloc_fake->relocs == NULL) { + reloc_fake->relocs = malloc(sizeof(struct fake_buffer_reloc) * + MAX_RELOCS); + } + + r = &reloc_fake->relocs[reloc_fake->nr_relocs++]; + + assert(reloc_fake->nr_relocs <= MAX_RELOCS); + + dri_bo_reference(target_buf); + + r->target_buf = target_buf; + r->offset = offset; + r->last_target_offset = target_buf->offset; + r->delta = delta; + r->read_domains = read_domains; + r->write_domain = write_domain; + + if (bufmgr_fake->debug) { + /* Check that a conflicting relocation hasn't already been emitted. */ + for (i = 0; i < reloc_fake->nr_relocs - 1; i++) { + struct fake_buffer_reloc *r2 = &reloc_fake->relocs[i]; + + assert(r->offset != r2->offset); + } + } + + return 0; +} + +/** + * Incorporates the validation flags associated with each relocation into + * the combined validation flags for the buffer on this batchbuffer submission. + */ +static void +dri_fake_calculate_domains(dri_bo *bo) +{ + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + int i; + + for (i = 0; i < bo_fake->nr_relocs; i++) { + struct fake_buffer_reloc *r = &bo_fake->relocs[i]; + dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; + + /* Do the same for the tree of buffers we depend on */ + dri_fake_calculate_domains(r->target_buf); + + target_fake->read_domains |= r->read_domains; + if (target_fake->write_domain != 0) + target_fake->write_domain = r->write_domain; + } +} + + +static int +dri_fake_reloc_and_validate_buffer(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + int i, ret; + + assert(bo_fake->map_count == 0); + + for (i = 0; i < bo_fake->nr_relocs; i++) { + struct fake_buffer_reloc *r = &bo_fake->relocs[i]; + dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; + uint32_t reloc_data; + + /* Validate the target buffer if that hasn't been done. */ + if (!target_fake->validated) { + ret = dri_fake_reloc_and_validate_buffer(r->target_buf); + if (ret != 0) { + if (bo->virtual != NULL) + dri_bo_unmap(bo); + return ret; + } + } + + /* Calculate the value of the relocation entry. */ + if (r->target_buf->offset != r->last_target_offset) { + reloc_data = r->target_buf->offset + r->delta; + + if (bo->virtual == NULL) + dri_bo_map(bo, GL_TRUE); + + *(uint32_t *)(bo->virtual + r->offset) = reloc_data; + + r->last_target_offset = r->target_buf->offset; + } + } + + if (bo->virtual != NULL) + dri_bo_unmap(bo); + + if (bo_fake->write_domain != 0) { + if (!(bo_fake->flags & (BM_NO_BACKING_STORE|BM_PINNED))) { + if (bo_fake->backing_store == 0) + alloc_backing_store(bo); + + bo_fake->card_dirty = 1; + } + bufmgr_fake->performed_rendering = GL_TRUE; + } + + return dri_fake_bo_validate(bo); +} + +static void * +dri_fake_process_relocs(dri_bo *batch_buf) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)batch_buf->bufmgr; + dri_bo_fake *batch_fake = (dri_bo_fake *)batch_buf; + int ret; + int retry_count = 0; + + bufmgr_fake->performed_rendering = GL_FALSE; + + dri_fake_calculate_domains(batch_buf); + + batch_fake->read_domains = DRM_GEM_DOMAIN_I915_COMMAND; + + /* we've ran out of RAM so blow the whole lot away and retry */ + restart: + ret = dri_fake_reloc_and_validate_buffer(batch_buf); + if (bufmgr_fake->fail == 1) { + if (retry_count == 0) { + retry_count++; + dri_fake_kick_all(bufmgr_fake); + bufmgr_fake->fail = 0; + goto restart; + } else /* dump out the memory here */ + mmDumpMemInfo(bufmgr_fake->heap); + } + + assert(ret == 0); + + bufmgr_fake->current_total_size = 0; + return NULL; +} + +static void +dri_bo_fake_post_submit(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + int i; + + for (i = 0; i < bo_fake->nr_relocs; i++) { + struct fake_buffer_reloc *r = &bo_fake->relocs[i]; + dri_bo_fake *target_fake = (dri_bo_fake *)r->target_buf; + + if (target_fake->validated) + dri_bo_fake_post_submit(r->target_buf); + + DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n", + bo_fake->name, (uint32_t)bo->offset, r->offset, + target_fake->name, (uint32_t)r->target_buf->offset, r->delta); + } + + assert(bo_fake->map_count == 0); + bo_fake->validated = GL_FALSE; + bo_fake->read_domains = 0; + bo_fake->write_domain = 0; +} + + +static void +dri_fake_post_submit(dri_bo *batch_buf) +{ + dri_fake_fence_validated(batch_buf->bufmgr); + + dri_bo_fake_post_submit(batch_buf); +} + +static int +dri_fake_check_aperture_space(dri_bo *bo) +{ + dri_bufmgr_fake *bufmgr_fake = (dri_bufmgr_fake *)bo->bufmgr; + dri_bo_fake *bo_fake = (dri_bo_fake *)bo; + GLuint sz; + + sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); + + if (bo_fake->size_accounted || bo_fake->is_static) + return 0; + + if (bufmgr_fake->current_total_size + sz > bufmgr_fake->size) { + DBG("check_space: %s bo %d %d overflowed bufmgr size %d\n", bo_fake->name, bo_fake->id, sz, bufmgr_fake->size); + return -1; + } + + bufmgr_fake->current_total_size += sz; + bo_fake->size_accounted = 1; + DBG("drm_check_space: buf %d, %s %d %d\n", bo_fake->id, bo_fake->name, bo->size, bufmgr_fake->current_total_size); + return 0; +} + +dri_bufmgr * +dri_bufmgr_fake_init(unsigned long low_offset, void *low_virtual, + unsigned long size, + unsigned int (*fence_emit)(void *private), + int (*fence_wait)(void *private, unsigned int cookie), + void *driver_priv) +{ + dri_bufmgr_fake *bufmgr_fake; + + bufmgr_fake = calloc(1, sizeof(*bufmgr_fake)); + + /* Initialize allocator */ + make_empty_list(&bufmgr_fake->fenced); + make_empty_list(&bufmgr_fake->on_hardware); + make_empty_list(&bufmgr_fake->lru); + + bufmgr_fake->low_offset = low_offset; + bufmgr_fake->virtual = low_virtual; + bufmgr_fake->size = size; + bufmgr_fake->heap = mmInit(low_offset, size); + + /* Hook in methods */ + bufmgr_fake->bufmgr.bo_alloc = dri_fake_bo_alloc; + bufmgr_fake->bufmgr.bo_alloc_static = dri_fake_bo_alloc_static; + bufmgr_fake->bufmgr.bo_reference = dri_fake_bo_reference; + bufmgr_fake->bufmgr.bo_unreference = dri_fake_bo_unreference; + bufmgr_fake->bufmgr.bo_map = dri_fake_bo_map; + bufmgr_fake->bufmgr.bo_unmap = dri_fake_bo_unmap; + bufmgr_fake->bufmgr.destroy = dri_fake_destroy; + bufmgr_fake->bufmgr.emit_reloc = dri_fake_emit_reloc; + bufmgr_fake->bufmgr.process_relocs = dri_fake_process_relocs; + bufmgr_fake->bufmgr.post_submit = dri_fake_post_submit; + bufmgr_fake->bufmgr.check_aperture_space = dri_fake_check_aperture_space; + bufmgr_fake->bufmgr.debug = GL_FALSE; + + bufmgr_fake->fence_emit = fence_emit; + bufmgr_fake->fence_wait = fence_wait; + bufmgr_fake->driver_priv = driver_priv; + + return &bufmgr_fake->bufmgr; +} + diff --git a/src/mesa/drivers/dri/intel/intel_bufmgr_fake.h b/src/mesa/drivers/dri/intel/intel_bufmgr_fake.h new file mode 100644 index 0000000000..bc7e59e61d --- /dev/null +++ b/src/mesa/drivers/dri/intel/intel_bufmgr_fake.h @@ -0,0 +1,50 @@ +/************************************************************************** + * + * Copyright © 2007 Intel Corporation + * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + * Keith Whitwell + * Eric Anholt + */ + +#ifndef _INTEL_BUFMGR_FAKE_H_ +#define _INTEL_BUFMGR_FAKE_H_ + +void dri_bufmgr_fake_contended_lock_take(dri_bufmgr *bufmgr); +dri_bufmgr *dri_bufmgr_fake_init(unsigned long low_offset, void *low_virtual, + unsigned long size, + unsigned int (*fence_emit)(void *private), + int (*fence_wait)(void *private, + unsigned int cookie), + void *driver_priv); +void dri_bo_fake_disable_backing_store(dri_bo *bo, + void (*invalidate_cb)(dri_bo *bo, + void *ptr), + void *ptr); +#endif /* _INTEL_BUFMGR_FAKE_H_ */ + diff --git a/src/mesa/drivers/dri/intel/intel_bufmgr_gem.c b/src/mesa/drivers/dri/intel/intel_bufmgr_gem.c index 69d90e19d8..5e16f9de0b 100644 --- a/src/mesa/drivers/dri/intel/intel_bufmgr_gem.c +++ b/src/mesa/drivers/dri/intel/intel_bufmgr_gem.c @@ -579,8 +579,8 @@ dri_bufmgr_gem_destroy(dri_bufmgr *bufmgr) * last known offset in target_bo. */ static int -dri_gem_emit_reloc(dri_bo *bo, uint64_t flags, GLuint delta, - GLuint offset, dri_bo *target_bo) +dri_gem_emit_reloc(dri_bo *bo, uint32_t read_domains, uint32_t write_domain, + uint32_t delta, uint32_t offset, dri_bo *target_bo) { dri_bufmgr_gem *bufmgr_gem = (dri_bufmgr_gem *)bo->bufmgr; dri_bo_gem *bo_gem = (dri_bo_gem *)bo; @@ -601,18 +601,17 @@ dri_gem_emit_reloc(dri_bo *bo, uint64_t flags, GLuint delta, bo_gem->reloc_target_bo[bo_gem->reloc_count] = target_bo; dri_bo_reference(target_bo); - /** XXX set memory domains, using existing TTM flags (which is wrong) */ - if (flags & DRM_BO_FLAG_WRITE) - { - /* assume this means the rendering buffer */ - target_bo_gem->read_domains |= DRM_GEM_DOMAIN_I915_RENDER; - target_bo_gem->write_domain = DRM_GEM_DOMAIN_I915_RENDER; - } - if (flags & DRM_BO_FLAG_READ) - { - /* assume this means the sampler buffer */ - target_bo_gem->read_domains |= DRM_GEM_DOMAIN_I915_SAMPLER; - } + /* Just accumulate the read domains into the target buffer. We don't care + * enough about minimizing the flags associated with a buffer for a + * specific set of relocations being done against it. + */ + target_bo_gem->read_domains |= read_domains; + /* XXX: this is broken if we have more than one write domain. We + * would need to be computing the write domain on the buffer based on + * order of relocs in the batchbuffer. But we only have one write buffer. + */ + target_bo_gem->write_domain = write_domain; + bo_gem->reloc_count++; return 0; } @@ -699,9 +698,6 @@ dri_gem_post_submit(dri_bo *batch_buf) dri_bo *bo = bufmgr_gem->validate_bo[i]; dri_bo_gem *bo_gem = (dri_bo_gem *)bo; - /* clear read/write domain bits */ - bo_gem->read_domains = 0; - bo_gem->write_domain = 0; /* Disconnect the buffer from the validate list */ bo_gem->validate_index = -1; dri_bo_unreference(bo); diff --git a/src/mesa/drivers/dri/intel/intel_context.c b/src/mesa/drivers/dri/intel/intel_context.c index d258e669c0..ae9e53ce6e 100644 --- a/src/mesa/drivers/dri/intel/intel_context.c +++ b/src/mesa/drivers/dri/intel/intel_context.c @@ -59,6 +59,7 @@ #include "intel_buffer_objects.h" #include "intel_fbo.h" #include "intel_decode.h" +#include "intel_bufmgr_fake.h" #include "intel_bufmgr_gem.h" #include "drirenderbuffer.h" -- cgit v1.2.3