panfrost: Add a panfrost_flush_all_batches() helper

And use it in panfrost_flush() to flush all batches, and not only the
one currently bound to the context.

We also replace all internal calls to panfrost_flush() by
panfrost_flush_all_batches() ones.

Signed-off-by: Boris Brezillon <[email protected]>
Reviewed-by: Alyssa Rosenzweig <[email protected]>

5 files changed, 64 insertions, 15 deletions

diff --git a/src/gallium/drivers/panfrost/pan_compute.c b/src/gallium/drivers/panfrost/pan_compute.c
index 4639c1b03c3..036dffbb17b 100644
--- a/src/gallium/drivers/panfrost/pan_compute.c
+++ b/src/gallium/drivers/panfrost/pan_compute.c
@@ -133,7 +133,7 @@ panfrost_launch_grid(struct pipe_context *pipe,
         /* Queue the job */
         panfrost_scoreboard_queue_compute_job(batch, transfer);
 
-        panfrost_flush(pipe, NULL, PIPE_FLUSH_END_OF_FRAME);
+        panfrost_flush_all_batches(ctx, true);
 }
 
 void
diff --git a/src/gallium/drivers/panfrost/pan_context.c b/src/gallium/drivers/panfrost/pan_context.c
index 5317cfe6f7c..a956221b049 100644
--- a/src/gallium/drivers/panfrost/pan_context.c
+++ b/src/gallium/drivers/panfrost/pan_context.c
@@ -1342,7 +1342,6 @@ panfrost_flush(
         unsigned flags)
 {
         struct panfrost_context *ctx = pan_context(pipe);
-        struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx);
         struct util_dynarray fences;
 
         /* We must collect the fences before the flush is done, otherwise we'll
@@ -1350,13 +1349,18 @@ panfrost_flush(
          */
         if (fence) {
                 util_dynarray_init(&fences, NULL);
-                panfrost_batch_fence_reference(batch->out_sync);
-                util_dynarray_append(&fences, struct panfrost_batch_fence *,
-                                     batch->out_sync);
+                hash_table_foreach(ctx->batches, hentry) {
+                        struct panfrost_batch *batch = hentry->data;
+
+                        panfrost_batch_fence_reference(batch->out_sync);
+                        util_dynarray_append(&fences,
+                                             struct panfrost_batch_fence *,
+                                             batch->out_sync);
+                }
         }
 
-        /* Submit the frame itself */
-        panfrost_batch_submit(batch);
+        /* Submit all pending jobs */
+        panfrost_flush_all_batches(ctx, false);
 
         if (fence) {
                 struct panfrost_fence *f = panfrost_fence_create(ctx, &fences);
@@ -2315,7 +2319,7 @@ panfrost_set_framebuffer_state(struct pipe_context *pctx,
         }
 
         if (!is_scanout || has_draws)
-                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
         else
                 assert(!ctx->payloads[PIPE_SHADER_VERTEX].postfix.framebuffer &&
                        !ctx->payloads[PIPE_SHADER_FRAGMENT].postfix.framebuffer);
@@ -2547,6 +2551,7 @@ panfrost_get_query_result(struct pipe_context *pipe,
                           union pipe_query_result *vresult)
 {
         struct panfrost_query *query = (struct panfrost_query *) q;
+        struct panfrost_context *ctx = pan_context(pipe);
 
 
         switch (query->type) {
@@ -2554,7 +2559,7 @@ panfrost_get_query_result(struct pipe_context *pipe,
         case PIPE_QUERY_OCCLUSION_PREDICATE:
         case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
                 /* Flush first */
-                panfrost_flush(pipe, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
 
                 /* Read back the query results */
                 unsigned *result = (unsigned *) query->transfer.cpu;
@@ -2570,7 +2575,7 @@ panfrost_get_query_result(struct pipe_context *pipe,
 
         case PIPE_QUERY_PRIMITIVES_GENERATED:
         case PIPE_QUERY_PRIMITIVES_EMITTED:
-                panfrost_flush(pipe, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
                 vresult->u64 = query->end - query->start;
                 break;
 
diff --git a/src/gallium/drivers/panfrost/pan_job.c b/src/gallium/drivers/panfrost/pan_job.c
index 211e48bafd4..3ccf4bb6b3e 100644
--- a/src/gallium/drivers/panfrost/pan_job.c
+++ b/src/gallium/drivers/panfrost/pan_job.c
@@ -856,7 +856,7 @@ panfrost_batch_submit_jobs(struct panfrost_batch *batch)
         return ret;
 }
 
-void
+static void
 panfrost_batch_submit(struct panfrost_batch *batch)
 {
         assert(batch);
@@ -904,8 +904,52 @@ out:
 
         panfrost_free_batch(batch);
 
+}
+
+void
+panfrost_flush_all_batches(struct panfrost_context *ctx, bool wait)
+{
+        struct util_dynarray fences, syncobjs;
+
+        if (wait) {
+                util_dynarray_init(&fences, NULL);
+                util_dynarray_init(&syncobjs, NULL);
+        }
+
+        hash_table_foreach(ctx->batches, hentry) {
+                struct panfrost_batch *batch = hentry->data;
+
+                assert(batch);
+
+                if (wait) {
+                        panfrost_batch_fence_reference(batch->out_sync);
+                        util_dynarray_append(&fences, struct panfrost_batch_fence *,
+                                             batch->out_sync);
+                        util_dynarray_append(&syncobjs, uint32_t,
+                                             batch->out_sync->syncobj);
+                }
+
+                panfrost_batch_submit(batch);
+        }
+
+        assert(!ctx->batches->entries);
+
         /* Collect batch fences before returning */
         panfrost_gc_fences(ctx);
+
+        if (!wait)
+                return;
+
+        drmSyncobjWait(pan_screen(ctx->base.screen)->fd,
+                       util_dynarray_begin(&syncobjs),
+                       util_dynarray_num_elements(&syncobjs, uint32_t),
+                       INT64_MAX, DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL, NULL);
+
+        util_dynarray_foreach(&fences, struct panfrost_batch_fence *, fence)
+                panfrost_batch_fence_unreference(*fence);
+
+        util_dynarray_fini(&fences);
+        util_dynarray_fini(&syncobjs);
 }
 
 void
diff --git a/src/gallium/drivers/panfrost/pan_job.h b/src/gallium/drivers/panfrost/pan_job.h
index 63813dff652..e95e156a40f 100644
--- a/src/gallium/drivers/panfrost/pan_job.h
+++ b/src/gallium/drivers/panfrost/pan_job.h
@@ -183,7 +183,7 @@ panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size,
                          uint32_t create_flags, uint32_t access_flags);
 
 void
-panfrost_batch_submit(struct panfrost_batch *batch);
+panfrost_flush_all_batches(struct panfrost_context *ctx, bool wait);
 
 void
 panfrost_batch_set_requirements(struct panfrost_batch *batch);
diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c
index 363a330c4fd..1f7605adcd5 100644
--- a/src/gallium/drivers/panfrost/pan_resource.c
+++ b/src/gallium/drivers/panfrost/pan_resource.c
@@ -580,7 +580,7 @@ panfrost_transfer_map(struct pipe_context *pctx,
 
         if (is_bound && (usage & PIPE_TRANSFER_READ)) {
                 assert(level == 0);
-                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
         }
 
         /* TODO: Respect usage flags */
@@ -752,7 +752,7 @@ panfrost_generate_mipmap(
         bool has_draws = batch->last_job.gpu;
 
         if (has_draws)
-                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
 
         /* We've flushed the original buffer if needed, now trigger a blit */
 
@@ -766,7 +766,7 @@ panfrost_generate_mipmap(
          * the state tracker deal with it. */
 
         if (blit_res)
-                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+                panfrost_flush_all_batches(ctx, true);
 
         return blit_res;
 }