virgl: Support copy transfers

Support transfers that use a different resource as the source of data to
transfer. This will be used in upcoming commits to send data to host
buffers through a transfer upload buffer, in order to avoid waiting
when the buffer resource is busy.

Note that we don't support queueing copy transfers in the transfer
queue. Copy transfers should be emitted directly in the command queue,
allowing us to avoid flushes before them and leads to better
performance.

Signed-off-by: Alexandros Frantzis <[email protected]>
Reviewed-by: Chia-I Wu <[email protected]>

5 files changed, 70 insertions, 5 deletions

diff --git a/src/gallium/drivers/virgl/virgl_encode.c b/src/gallium/drivers/virgl/virgl_encode.c
index 9c60b99bfb6..d26c0670f72 100644
--- a/src/gallium/drivers/virgl/virgl_encode.c
+++ b/src/gallium/drivers/virgl/virgl_encode.c
@@ -511,18 +511,41 @@ int virgl_encoder_create_so_target(struct virgl_context *ctx,
    return 0;
 }
 
+enum virgl_transfer3d_encode_stride {
+   /* The stride and layer_stride are explicitly specified in the command. */
+   virgl_transfer3d_explicit_stride,
+   /* The stride and layer_stride are inferred by the host. In this case, the
+    * host will use the image stride and layer_stride for the specified level.
+    */
+   virgl_transfer3d_host_inferred_stride,
+};
+
 static void virgl_encoder_transfer3d_common(struct virgl_screen *vs,
                                             struct virgl_cmd_buf *buf,
-                                            struct virgl_transfer *xfer)
+                                            struct virgl_transfer *xfer,
+                                            enum virgl_transfer3d_encode_stride encode_stride)
+
 {
    struct pipe_transfer *transfer = &xfer->base;
    struct virgl_resource *res = virgl_resource(transfer->resource);
+   unsigned stride;
+   unsigned layer_stride;
+
+   if (encode_stride == virgl_transfer3d_explicit_stride) {
+      stride = transfer->stride;
+      layer_stride = transfer->layer_stride;
+   } else if (virgl_transfer3d_host_inferred_stride) {
+      stride = 0;
+      layer_stride = 0;
+   } else {
+      assert(!"Invalid virgl_transfer3d_encode_stride value");
+   }
 
    virgl_encoder_emit_resource(vs, buf, res);
    virgl_encoder_write_dword(buf, transfer->level);
    virgl_encoder_write_dword(buf, transfer->usage);
-   virgl_encoder_write_dword(buf, 0);
-   virgl_encoder_write_dword(buf, 0);
+   virgl_encoder_write_dword(buf, stride);
+   virgl_encoder_write_dword(buf, layer_stride);
    virgl_encoder_write_dword(buf, transfer->box.x);
    virgl_encoder_write_dword(buf, transfer->box.y);
    virgl_encoder_write_dword(buf, transfer->box.z);
@@ -567,7 +590,8 @@ int virgl_encoder_inline_write(struct virgl_context *ctx,
 
       transfer.base.box.width = length;
       virgl_encoder_write_cmd_dword(ctx, VIRGL_CMD0(VIRGL_CCMD_RESOURCE_INLINE_WRITE, 0, ((length + 3) / 4) + 11));
-      virgl_encoder_transfer3d_common(vs, ctx->cbuf, &transfer);
+      virgl_encoder_transfer3d_common(vs, ctx->cbuf, &transfer,
+                                      virgl_transfer3d_host_inferred_stride);
       virgl_encoder_write_block(ctx->cbuf, data, length);
       left_bytes -= length;
       transfer.base.box.x += length;
@@ -1121,11 +1145,31 @@ void virgl_encode_transfer(struct virgl_screen *vs, struct virgl_cmd_buf *buf,
    uint32_t command;
    command = VIRGL_CMD0(VIRGL_CCMD_TRANSFER3D, 0, VIRGL_TRANSFER3D_SIZE);
    virgl_encoder_write_dword(buf, command);
-   virgl_encoder_transfer3d_common(vs, buf, trans);
+   virgl_encoder_transfer3d_common(vs, buf, trans,
+                                   virgl_transfer3d_host_inferred_stride);
    virgl_encoder_write_dword(buf, trans->offset);
    virgl_encoder_write_dword(buf, direction);
 }
 
+void virgl_encode_copy_transfer(struct virgl_context *ctx,
+                                struct virgl_transfer *trans)
+{
+   uint32_t command;
+   struct virgl_resource *copy_src_res = virgl_resource(trans->copy_src_res);
+   struct virgl_screen *vs = virgl_screen(ctx->base.screen);
+
+   command = VIRGL_CMD0(VIRGL_CCMD_COPY_TRANSFER3D, 0, VIRGL_COPY_TRANSFER3D_SIZE);
+   virgl_encoder_write_cmd_dword(ctx, command);
+   /* Copy transfers need to explicitly specify the stride, since it may differ
+    * from the image stride.
+    */
+   virgl_encoder_transfer3d_common(vs, ctx->cbuf, trans, virgl_transfer3d_explicit_stride);
+   virgl_encoder_emit_resource(vs, ctx->cbuf, copy_src_res);
+   virgl_encoder_write_dword(ctx->cbuf, trans->copy_src_offset);
+   /* At the moment all copy transfers are synchronized. */
+   virgl_encoder_write_dword(ctx->cbuf, 1);
+}
+
 void virgl_encode_end_transfers(struct virgl_cmd_buf *buf)
 {
    uint32_t command, diff;
diff --git a/src/gallium/drivers/virgl/virgl_encode.h b/src/gallium/drivers/virgl/virgl_encode.h
index 150ed17f994..3cbec0f021c 100644
--- a/src/gallium/drivers/virgl/virgl_encode.h
+++ b/src/gallium/drivers/virgl/virgl_encode.h
@@ -292,5 +292,8 @@ int virgl_encode_get_query_result_qbo(struct virgl_context *ctx,
 void virgl_encode_transfer(struct virgl_screen *vs, struct virgl_cmd_buf *buf,
                            struct virgl_transfer *trans, uint32_t direction);
 
+void virgl_encode_copy_transfer(struct virgl_context *ctx,
+                                struct virgl_transfer *trans);
+
 void virgl_encode_end_transfers(struct virgl_cmd_buf *buf);
 #endif
diff --git a/src/gallium/drivers/virgl/virgl_resource.c b/src/gallium/drivers/virgl/virgl_resource.c
index e334d55fa79..fd01df1c10a 100644
--- a/src/gallium/drivers/virgl/virgl_resource.c
+++ b/src/gallium/drivers/virgl/virgl_resource.c
@@ -401,6 +401,8 @@ virgl_resource_create_transfer(struct slab_child_pool *pool,
    trans->base.layer_stride = metadata->layer_stride[level];
    trans->offset = offset;
    util_range_init(&trans->range);
+   trans->copy_src_res = NULL;
+   trans->copy_src_offset = 0;
 
    if (trans->base.resource->target != PIPE_TEXTURE_3D &&
        trans->base.resource->target != PIPE_TEXTURE_CUBE &&
@@ -417,6 +419,7 @@ virgl_resource_create_transfer(struct slab_child_pool *pool,
 void virgl_resource_destroy_transfer(struct slab_child_pool *pool,
                                      struct virgl_transfer *trans)
 {
+   pipe_resource_reference(&trans->copy_src_res, NULL);
    util_range_destroy(&trans->range);
    slab_free(pool, trans);
 }
diff --git a/src/gallium/drivers/virgl/virgl_resource.h b/src/gallium/drivers/virgl/virgl_resource.h
index 87b3ea71673..358ce3d926f 100644
--- a/src/gallium/drivers/virgl/virgl_resource.h
+++ b/src/gallium/drivers/virgl/virgl_resource.h
@@ -73,6 +73,13 @@ struct virgl_transfer {
    struct list_head queue_link;
    struct pipe_transfer *resolve_transfer;
    void *hw_res_map;
+   /* If not NULL, denotes that this is a copy transfer, i.e.,
+    * that the transfer source data should be taken from this
+    * resource instead of the original transfer resource.
+    */
+   struct pipe_resource *copy_src_res;
+   /* The offset in the copy source resource to copy data from. */
+   uint32_t copy_src_offset;
 };
 
 void virgl_resource_destroy(struct pipe_screen *screen,
diff --git a/src/gallium/drivers/virgl/virgl_transfer_queue.c b/src/gallium/drivers/virgl/virgl_transfer_queue.c
index 4831cbd235c..6144dfe4861 100644
--- a/src/gallium/drivers/virgl/virgl_transfer_queue.c
+++ b/src/gallium/drivers/virgl/virgl_transfer_queue.c
@@ -223,6 +223,7 @@ static void add_internal(struct virgl_transfer_queue *queue,
                          struct virgl_transfer *transfer)
 {
    uint32_t dwords = VIRGL_TRANSFER3D_SIZE + 1;
+
    if (queue->tbuf) {
       if (queue->num_dwords + dwords >= VIRGL_MAX_TBUF_DWORDS) {
          struct list_iteration_args iter;
@@ -296,6 +297,10 @@ int virgl_transfer_queue_unmap(struct virgl_transfer_queue *queue,
    res = transfer->base.resource;
    pipe_resource_reference(&pres, res);
 
+   /* We don't support copy transfers in the transfer queue. */
+   assert(!transfer->copy_src_res);
+
+   /* Attempt to merge multiple intersecting transfers into a single one. */
    if (res->target == PIPE_BUFFER) {
       memset(&iter, 0, sizeof(iter));
       iter.current = transfer;
@@ -367,6 +372,9 @@ virgl_transfer_queue_extend(struct virgl_transfer_queue *queue,
    struct virgl_transfer *queued = NULL;
    struct list_iteration_args iter;
 
+   /* We don't support extending from copy transfers. */
+   assert(!transfer->copy_src_res);
+
    if (transfer->base.resource->target == PIPE_BUFFER) {
       memset(&iter, 0, sizeof(iter));
       iter.current = transfer;