virgl: Work around possible memory exhaustion

Since we don't normally flush before performing copy transfers, it's
possible in some scenarios to use too much memory for staging resources
and start failing. This can happen either because we exhaust the total
available memory (including system memory virtio-gpu swaps out to), or,
more commonly, because the total size of resources in a command buffer
doesn't fit in virtio-gpu video memory.

To reduce the chances of this happening, force a flush before a copy
transfer if the total size of queued staging resources exceeds a certain
limit. Since after a flush any queued staging resources will be
eventually released, this ensures both that each command buffer doesn't
require too much video memory, and that we don't end up consuming too
much memory for staging resources in total.

Fixes kernel errors reported when running texture_upload tests in glbench.

Signed-off-by: Alexandros Frantzis <[email protected]>
Reviewed-by: Chia-I Wu <[email protected]>

3 files changed, 22 insertions, 3 deletions

diff --git a/src/gallium/drivers/virgl/virgl_context.c b/src/gallium/drivers/virgl/virgl_context.c
index 9964520d5e7..f1b35fd1257 100644
--- a/src/gallium/drivers/virgl/virgl_context.c
+++ b/src/gallium/drivers/virgl/virgl_context.c
@@ -778,6 +778,11 @@ static void virgl_flush_eq(struct virgl_context *ctx, void *closure,
    virgl_encoder_set_sub_ctx(ctx, ctx->hw_sub_ctx_id);
 
    ctx->cbuf_initial_cdw = ctx->cbuf->cdw;
+
+   /* We have flushed the command queue, including any pending copy transfers
+    * involving staging resources.
+    */
+   ctx->queued_staging_res_size = 0;
 }
 
 static void virgl_flush_from_st(struct pipe_context *ctx,
diff --git a/src/gallium/drivers/virgl/virgl_context.h b/src/gallium/drivers/virgl/virgl_context.h
index 1449c64189a..b0fa03ca499 100644
--- a/src/gallium/drivers/virgl/virgl_context.h
+++ b/src/gallium/drivers/virgl/virgl_context.h
@@ -97,6 +97,9 @@ struct virgl_context {
 
    struct primconvert_context *primconvert;
    uint32_t hw_sub_ctx_id;
+
+   /* The total size of staging resources used in queued copy transfers. */
+   uint64_t queued_staging_res_size;
 };
 
 static inline struct virgl_sampler_view *
diff --git a/src/gallium/drivers/virgl/virgl_resource.c b/src/gallium/drivers/virgl/virgl_resource.c
index 879c3fb359a..e8baa4368e2 100644
--- a/src/gallium/drivers/virgl/virgl_resource.c
+++ b/src/gallium/drivers/virgl/virgl_resource.c
@@ -28,6 +28,12 @@
 #include "virgl_resource.h"
 #include "virgl_screen.h"
 
+/* A (soft) limit for the amount of memory we want to allow for queued staging
+ * resources. This is used to decide when we should force a flush, in order to
+ * avoid exhausting virtio-gpu memory.
+ */
+#define VIRGL_QUEUED_STAGING_RES_SIZE_LIMIT (128 * 1024 * 1024)
+
 /* We need to flush to properly sync the transfer with the current cmdbuf.
  * But there are cases where the flushing can be skipped:
  *
@@ -132,11 +138,13 @@ virgl_resource_transfer_prepare(struct virgl_context *vctx,
       copy_transfer = false;
    }
 
-   /* When performing a copy transfer there is no need to flush or wait for
-    * the target resource.
+   /* When performing a copy transfer there is no need wait for the target
+    * resource. There is normally no need to flush either, unless the amount of
+    * memory we are using for staging resources starts growing, in which case
+    * we want to flush to keep our memory consumption in check.
     */
    if (copy_transfer) {
-      flush = false;
+      flush = (vctx->queued_staging_res_size > VIRGL_QUEUED_STAGING_RES_SIZE_LIMIT);
       wait = false;
    }
 
@@ -572,6 +580,9 @@ void *virgl_transfer_uploader_map(struct virgl_context *vctx,
        */
       vtransfer->base.stride = stride;
       vtransfer->base.layer_stride = layer_stride;
+
+      /* Track the total size of active staging resources. */
+      vctx->queued_staging_res_size += size + align_offset;
    }
 
    return map_addr;