st/nine: Retry allocations after freeing some space

Managed resources can be released whenever we need.
When we have an allocation failure, free them and retry.
Try also to flush the csmt queue in case there were some things released.

Signed-off-by: Axel Davy <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5015>

6 files changed, 68 insertions, 8 deletions

diff --git a/src/gallium/frontends/nine/basetexture9.c b/src/gallium/frontends/nine/basetexture9.c
index c59bda5d295..1f5608149eb 100644
--- a/src/gallium/frontends/nine/basetexture9.c
+++ b/src/gallium/frontends/nine/basetexture9.c
@@ -446,7 +446,7 @@ NineBaseTexture9_CreatePipeResource( struct NineBaseTexture9 *This,
             return D3D_OK;
     }
 
-    res = screen->resource_create(screen, &templ);
+    res = nine_resource_create_with_retry(This->base.base.device, screen, &templ);
     if (!res)
         return D3DERR_OUTOFVIDEOMEMORY;
     This->base.resource = res;
@@ -597,6 +597,7 @@ NineBaseTexture9_UnLoad( struct NineBaseTexture9 *This )
         This->managed.lod_resident == -1)
         return;
 
+    DBG("This=%p, releasing resource\n", This);
     pipe_resource_reference(&This->base.resource, NULL);
     This->managed.lod_resident = -1;
     This->managed.dirty = TRUE;
diff --git a/src/gallium/frontends/nine/buffer9.c b/src/gallium/frontends/nine/buffer9.c
index 629f33731ce..81baebcd8bc 100644
--- a/src/gallium/frontends/nine/buffer9.c
+++ b/src/gallium/frontends/nine/buffer9.c
@@ -385,7 +385,7 @@ NineBuffer9_Lock( struct NineBuffer9 *This,
      * with the secondary pipe, instead of waiting on the main pipe. */
     if (Flags & D3DLOCK_DISCARD && device->csmt_active) {
         struct pipe_screen *screen = NineDevice9_GetScreen(device);
-        struct pipe_resource *new_res = screen->resource_create(screen, &This->base.info);
+        struct pipe_resource *new_res = nine_resource_create_with_retry(device, screen, &This->base.info);
         if (new_res) {
             /* Use the new resource */
             pipe_resource_reference(&This->base.resource, new_res);
diff --git a/src/gallium/frontends/nine/device9.c b/src/gallium/frontends/nine/device9.c
index 2634b88a614..98f15efcbee 100644
--- a/src/gallium/frontends/nine/device9.c
+++ b/src/gallium/frontends/nine/device9.c
@@ -83,6 +83,28 @@ static void nine_setup_fpu(void)
 
 #endif
 
+struct pipe_resource *
+nine_resource_create_with_retry( struct NineDevice9 *This,
+                                 struct pipe_screen *screen,
+                                 const struct pipe_resource *templat )
+{
+    struct pipe_resource *res;
+    res = screen->resource_create(screen, templat);
+    if (res)
+        return res;
+    /* Allocation failed, retry after freeing some resources
+     * Note: Shouldn't be called from the worker thread */
+    if (!This)
+        return NULL;
+    /* Evict resources we can evict */
+    NineDevice9_EvictManagedResourcesInternal(This);
+    /* Execute anything pending, such that some
+     * deleted resources can be actually freed */
+    nine_csmt_process(This);
+    /* We could also finish the context, if needed */
+    return screen->resource_create(screen, templat);
+}
+
 void
 NineDevice9_SetDefaultState( struct NineDevice9 *This, boolean is_reset )
 {
@@ -652,6 +674,25 @@ NineDevice9_GetAvailableTextureMem( struct NineDevice9 *This )
     return This->available_texture_mem;
 }
 
+void
+NineDevice9_EvictManagedResourcesInternal( struct NineDevice9 *This )
+{
+    struct NineBaseTexture9 *tex;
+
+    DBG("This=%p\n", This);
+
+    /* This function is called internally when an allocation fails.
+     * We are supposed to release old unused managed textures/buffers,
+     * until we have enough space for the allocation.
+     * For now just release everything, except the bound textures,
+     * as this function can be called when uploading bound textures.
+     */
+    LIST_FOR_EACH_ENTRY(tex, &This->managed_textures, list2) {
+        if (!tex->bind_count)
+            NineBaseTexture9_UnLoad(tex);
+    }
+}
+
 HRESULT NINE_WINAPI
 NineDevice9_EvictManagedResources( struct NineDevice9 *This )
 {
diff --git a/src/gallium/frontends/nine/device9.h b/src/gallium/frontends/nine/device9.h
index 1116e2c5ed0..7a75a5c1a75 100644
--- a/src/gallium/frontends/nine/device9.h
+++ b/src/gallium/frontends/nine/device9.h
@@ -198,6 +198,11 @@ void
 NineDevice9_dtor( struct NineDevice9 *This );
 
 /*** Nine private ***/
+struct pipe_resource *
+nine_resource_create_with_retry( struct NineDevice9 *This,
+                                 struct pipe_screen *screen,
+                                 const struct pipe_resource *templat );
+
 void
 NineDevice9_SetDefaultState( struct NineDevice9 *This, boolean is_reset );
 
@@ -210,6 +215,9 @@ NineDevice9_GetPipe( struct NineDevice9 *This );
 const D3DCAPS9 *
 NineDevice9_GetCaps( struct NineDevice9 *This );
 
+void
+NineDevice9_EvictManagedResourcesInternal( struct NineDevice9 *This );
+
 /*** Direct3D public ***/
 
 HRESULT NINE_WINAPI
diff --git a/src/gallium/frontends/nine/resource9.c b/src/gallium/frontends/nine/resource9.c
index 852801624a9..dfab4f410c6 100644
--- a/src/gallium/frontends/nine/resource9.c
+++ b/src/gallium/frontends/nine/resource9.c
@@ -80,18 +80,28 @@ NineResource9_ctor( struct NineResource9 *This,
             This->size = util_resource_size(&This->info);
 
             p_atomic_add(&This->base.device->available_texture_mem, -This->size);
-            if (This->base.device->available_texture_mem <=
+            /* Before failing allocation, evict MANAGED memory */
+            if (This->base.device &&
+                p_atomic_read(&This->base.device->available_texture_mem) <=
+                    This->base.device->available_texture_limit)
+                NineDevice9_EvictManagedResourcesInternal(This->base.device);
+            if (p_atomic_read(&This->base.device->available_texture_mem) <=
                     This->base.device->available_texture_limit) {
+                DBG("Memory allocation failure: software limit\n");
                 return D3DERR_OUTOFVIDEOMEMORY;
             }
         }
 
         DBG("(%p) Creating pipe_resource.\n", This);
-        This->resource = screen->resource_create(screen, &This->info);
+        This->resource = nine_resource_create_with_retry(This->base.device, screen, &This->info);
         if (!This->resource)
             return D3DERR_OUTOFVIDEOMEMORY;
     }
 
+    DBG("Current texture memory count: (%d/%d)KB\n",
+        (int)(This->base.device->available_texture_mem >> 10),
+        (int)(This->base.device->available_texture_limit >> 10));
+
     This->type = Type;
     This->pool = Pool;
     This->usage = Usage;
diff --git a/src/gallium/frontends/nine/swapchain9.c b/src/gallium/frontends/nine/swapchain9.c
index 9faf83f3418..8fb20082f91 100644
--- a/src/gallium/frontends/nine/swapchain9.c
+++ b/src/gallium/frontends/nine/swapchain9.c
@@ -363,7 +363,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
                                                    tmplt.bind, FALSE, FALSE);
         if (tmplt.format == PIPE_FORMAT_NONE)
             return D3DERR_INVALIDCALL;
-        resource = This->screen->resource_create(This->screen, &tmplt);
+        resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
         if (!resource) {
             DBG("Failed to create pipe_resource.\n");
             return D3DERR_OUTOFVIDEOMEMORY;
@@ -397,7 +397,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
                 tmplt.bind |= PIPE_BIND_LINEAR;
             if (pParams->SwapEffect != D3DSWAPEFFECT_DISCARD)
                 tmplt.bind |= PIPE_BIND_RENDER_TARGET;
-            resource = This->screen->resource_create(This->screen, &tmplt);
+            resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
             pipe_resource_reference(&(This->present_buffers[i]), resource);
         }
         This->present_handles[i] = D3DWindowBuffer_create(This, resource, depth, false);
@@ -421,7 +421,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
             return D3DERR_INVALIDCALL;
 
         if (This->zsbuf) {
-            resource = This->screen->resource_create(This->screen, &tmplt);
+            resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
             if (!resource) {
                 DBG("Failed to create pipe_resource for depth buffer.\n");
                 return D3DERR_OUTOFVIDEOMEMORY;
@@ -606,7 +606,7 @@ create_present_buffer( struct NineSwapChain9 *This,
     tmplt.nr_samples = 0;
     if (This->actx->linear_framebuffer)
         tmplt.bind |= PIPE_BIND_LINEAR;
-    *resource = This->screen->resource_create(This->screen, &tmplt);
+    *resource = nine_resource_create_with_retry(This->base.device, This->screen, &tmplt);
 
     *present_handle = D3DWindowBuffer_create(This, *resource, 24, true);