ilo: rework query support

This fixes some corner cases, but more importantly, the new code should be
easier to reason about.

Signed-off-by: Chia-I Wu <[email protected]>

8 files changed, 357 insertions, 402 deletions

diff --git a/src/gallium/drivers/ilo/ilo_3d.c b/src/gallium/drivers/ilo/ilo_3d.c
index 30686d6075f..0a746a1887e 100644
--- a/src/gallium/drivers/ilo/ilo_3d.c
+++ b/src/gallium/drivers/ilo/ilo_3d.c
@@ -38,246 +38,142 @@
 #include "ilo_3d.h"
 
 static void
-process_query_for_occlusion_counter(struct ilo_3d *hw3d,
-                                    struct ilo_query *q)
+ilo_3d_own_render_ring(struct ilo_3d *hw3d)
 {
-   uint64_t *vals, depth_count = 0;
-   int i;
-
-   /* in pairs */
-   assert(q->reg_read % 2 == 0);
-
-   vals = intel_bo_map(q->bo, false);
-   for (i = 1; i < q->reg_read; i += 2)
-      depth_count += vals[i] - vals[i - 1];
-   intel_bo_unmap(q->bo);
-
-   /* accumulate so that the query can be resumed if wanted */
-   q->data.u64 += depth_count;
-   q->reg_read = 0;
+   ilo_cp_set_owner(hw3d->cp, INTEL_RING_RENDER, &hw3d->owner);
 }
 
 static uint64_t
-timestamp_to_ns(uint64_t timestamp)
+query_timestamp_to_ns(const struct ilo_3d *hw3d, uint64_t timestamp)
 {
    /* see ilo_get_timestamp() */
    return (timestamp & 0xffffffff) * 80;
 }
 
+/**
+ * Process the bo and accumulate the result.  The bo is emptied.
+ */
 static void
-process_query_for_timestamp(struct ilo_3d *hw3d, struct ilo_query *q)
-{
-   uint64_t *vals, timestamp;
-
-   assert(q->reg_read == 1);
-
-   vals = intel_bo_map(q->bo, false);
-   timestamp = vals[0];
-   intel_bo_unmap(q->bo);
-
-   q->data.u64 = timestamp_to_ns(timestamp);
-   q->reg_read = 0;
-}
-
-static void
-process_query_for_time_elapsed(struct ilo_3d *hw3d, struct ilo_query *q)
-{
-   uint64_t *vals, elapsed = 0;
-   int i;
-
-   /* in pairs */
-   assert(q->reg_read % 2 == 0);
-
-   vals = intel_bo_map(q->bo, false);
-
-   for (i = 1; i < q->reg_read; i += 2)
-      elapsed += vals[i] - vals[i - 1];
-
-   intel_bo_unmap(q->bo);
-
-   /* accumulate so that the query can be resumed if wanted */
-   q->data.u64 += timestamp_to_ns(elapsed);
-   q->reg_read = 0;
-}
-
-static void
-process_query_for_pipeline_statistics(struct ilo_3d *hw3d,
-                                      struct ilo_query *q)
+query_process_bo(const struct ilo_3d *hw3d, struct ilo_query *q)
 {
    const uint64_t *vals;
+   uint64_t tmp;
    int i;
 
-   assert(q->reg_read % 22 == 0);
+   if (!q->used)
+      return;
 
    vals = intel_bo_map(q->bo, false);
-
-   for (i = 0; i < q->reg_read; i += 22) {
-      struct pipe_query_data_pipeline_statistics *stats =
-         &q->data.pipeline_statistics;
-      const uint64_t *begin = vals + i;
-      const uint64_t *end = begin + 11;
-
-      stats->ia_vertices    += end[0] - begin[0];
-      stats->ia_primitives  += end[1] - begin[1];
-      stats->vs_invocations += end[2] - begin[2];
-      stats->gs_invocations += end[3] - begin[3];
-      stats->gs_primitives  += end[4] - begin[4];
-      stats->c_invocations  += end[5] - begin[5];
-      stats->c_primitives   += end[6] - begin[6];
-      stats->ps_invocations += end[7] - begin[7];
-      stats->hs_invocations += end[8] - begin[8];
-      stats->ds_invocations += end[9] - begin[9];
-      stats->cs_invocations += end[10] - begin[10];
+   if (!vals) {
+      q->used = 0;
+      return;
    }
 
-   intel_bo_unmap(q->bo);
-
-   q->reg_read = 0;
-}
-
-static void
-ilo_3d_resume_queries(struct ilo_3d *hw3d)
-{
-   struct ilo_query *q;
+   switch (q->type) {
+   case PIPE_QUERY_OCCLUSION_COUNTER:
+   case PIPE_QUERY_TIME_ELAPSED:
+      assert(q->stride == sizeof(*vals) * 2);
 
-   /* resume occlusion queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->occlusion_queries, list) {
-      /* accumulate the result if the bo is alreay full */
-      if (q->reg_read >= q->reg_total)
-         process_query_for_occlusion_counter(hw3d, q);
+      tmp = 0;
+      for (i = 0; i < q->used; i++)
+         tmp += vals[2 * i + 1] - vals[2 * i];
 
-      ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline,
-            q->bo, q->reg_read++);
-   }
+      if (q->type == PIPE_QUERY_TIME_ELAPSED)
+         tmp = query_timestamp_to_ns(hw3d, tmp);
 
-   /* resume timer queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->time_elapsed_queries, list) {
-      /* accumulate the result if the bo is alreay full */
-      if (q->reg_read >= q->reg_total)
-         process_query_for_time_elapsed(hw3d, q);
+      q->result.u64 += tmp;
+      break;
+   case PIPE_QUERY_TIMESTAMP:
+      assert(q->stride == sizeof(*vals));
 
-      ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline,
-            q->bo, q->reg_read++);
+      q->result.u64 = query_timestamp_to_ns(hw3d, vals[q->used - 1]);
+      break;
+   case PIPE_QUERY_PIPELINE_STATISTICS:
+      assert(q->stride == sizeof(*vals) * 22);
+
+      for (i = 0; i < q->used; i++) {
+         struct pipe_query_data_pipeline_statistics *stats =
+            &q->result.pipeline_statistics;
+         const uint64_t *begin = vals + 22 * i;
+         const uint64_t *end = begin + 11;
+
+         stats->ia_vertices    += end[0] - begin[0];
+         stats->ia_primitives  += end[1] - begin[1];
+         stats->vs_invocations += end[2] - begin[2];
+         stats->gs_invocations += end[3] - begin[3];
+         stats->gs_primitives  += end[4] - begin[4];
+         stats->c_invocations  += end[5] - begin[5];
+         stats->c_primitives   += end[6] - begin[6];
+         stats->ps_invocations += end[7] - begin[7];
+         stats->hs_invocations += end[8] - begin[8];
+         stats->ds_invocations += end[9] - begin[9];
+         stats->cs_invocations += end[10] - begin[10];
+      }
+      break;
+   default:
+      break;
    }
 
-   /* resume pipeline statistics queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->pipeline_statistics_queries, list) {
-      /* accumulate the result if the bo is alreay full */
-      if (q->reg_read >= q->reg_total)
-         process_query_for_pipeline_statistics(hw3d, q);
+   intel_bo_unmap(q->bo);
 
-      ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline,
-            q->bo, q->reg_read);
-      q->reg_read += 11;
-   }
+   q->used = 0;
 }
 
 static void
-ilo_3d_pause_queries(struct ilo_3d *hw3d)
+query_begin_bo(struct ilo_3d *hw3d, struct ilo_query *q)
 {
-   struct ilo_query *q;
+   uint32_t offset;
 
-   /* pause occlusion queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->occlusion_queries, list) {
-      assert(q->reg_read < q->reg_total);
-      ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline,
-            q->bo, q->reg_read++);
-   }
+   /* bo is full */
+   if (q->used >= q->count)
+      query_process_bo(hw3d, q);
 
-   /* pause timer queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->time_elapsed_queries, list) {
-      assert(q->reg_read < q->reg_total);
-      ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline,
-            q->bo, q->reg_read++);
-   }
+   offset = q->stride * q->used;
 
-   /* pause pipeline statistics queries */
-   LIST_FOR_EACH_ENTRY(q, &hw3d->pipeline_statistics_queries, list) {
-      assert(q->reg_read < q->reg_total);
-      ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline,
-            q->bo, q->reg_read);
-      q->reg_read += 11;
+   /* write the beginning value to the bo */
+   switch (q->type) {
+   case PIPE_QUERY_OCCLUSION_COUNTER:
+      ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline, q->bo, offset);
+      break;
+   case PIPE_QUERY_TIMESTAMP:
+      /* no-op */
+      break;
+   case PIPE_QUERY_TIME_ELAPSED:
+      ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline, q->bo, offset);
+      break;
+   case PIPE_QUERY_PIPELINE_STATISTICS:
+      ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline, q->bo, offset);
+      break;
+   default:
+      assert(!"unknown query type");
+      break;
    }
 }
 
 static void
-ilo_3d_own_render_ring(struct ilo_3d *hw3d)
-{
-   ilo_cp_set_owner(hw3d->cp, INTEL_RING_RENDER, &hw3d->owner);
-}
-
-static void
-ilo_3d_reserve_for_query(struct ilo_3d *hw3d, struct ilo_query *q,
-                         enum ilo_3d_pipeline_action act)
+query_end_bo(struct ilo_3d *hw3d, struct ilo_query *q)
 {
-   q->reg_cmd_size = ilo_3d_pipeline_estimate_size(hw3d->pipeline, act, NULL);
+   uint32_t offset;
 
-   /* XXX we should check the aperture size */
-   if (ilo_cp_space(hw3d->cp) < q->reg_cmd_size * 2) {
-      ilo_cp_submit(hw3d->cp, "out of space");
-      assert(ilo_cp_space(hw3d->cp) >= q->reg_cmd_size * 2);
-   }
+   assert(q->used < q->count);
 
-   /* reserve space for pausing the query */
-   hw3d->owner.reserve += q->reg_cmd_size;
-}
+   offset = q->stride * q->used;
+   if (q->in_pairs)
+      offset += q->stride >> 1;
 
-/**
- * Begin a query.
- */
-void
-ilo_3d_begin_query(struct ilo_context *ilo, struct ilo_query *q)
-{
-   struct ilo_3d *hw3d = ilo->hw3d;
-
-   ilo_3d_own_render_ring(hw3d);
+   q->used++;
 
+   /* write the ending value to the bo */
    switch (q->type) {
    case PIPE_QUERY_OCCLUSION_COUNTER:
-      ilo_3d_reserve_for_query(hw3d, q, ILO_3D_PIPELINE_WRITE_DEPTH_COUNT);
-      q->data.u64 = 0;
-
-      if (ilo_query_alloc_bo(q, 2, -1, hw3d->cp->winsys)) {
-         ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline,
-               q->bo, q->reg_read++);
-
-         list_add(&q->list, &hw3d->occlusion_queries);
-      }
+      ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline, q->bo, offset);
       break;
    case PIPE_QUERY_TIMESTAMP:
-      /* nop */
-      break;
    case PIPE_QUERY_TIME_ELAPSED:
-      ilo_3d_reserve_for_query(hw3d, q, ILO_3D_PIPELINE_WRITE_TIMESTAMP);
-      q->data.u64 = 0;
-
-      if (ilo_query_alloc_bo(q, 2, -1, hw3d->cp->winsys)) {
-         ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline,
-               q->bo, q->reg_read++);
-
-         list_add(&q->list, &hw3d->time_elapsed_queries);
-      }
-      break;
-   case PIPE_QUERY_PRIMITIVES_GENERATED:
-      q->data.u64 = 0;
-      list_add(&q->list, &hw3d->prim_generated_queries);
-      break;
-   case PIPE_QUERY_PRIMITIVES_EMITTED:
-      q->data.u64 = 0;
-      list_add(&q->list, &hw3d->prim_emitted_queries);
+      ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline, q->bo, offset);
       break;
    case PIPE_QUERY_PIPELINE_STATISTICS:
-      ilo_3d_reserve_for_query(hw3d, q, ILO_3D_PIPELINE_WRITE_STATISTICS);
-      memset(&q->data.pipeline_statistics, 0,
-            sizeof(q->data.pipeline_statistics));
-
-      if (ilo_query_alloc_bo(q, 11 * 2, -1, hw3d->cp->winsys)) {
-         ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline,
-               q->bo, q->reg_read);
-         q->reg_read += 11;
-
-         list_add(&q->list, &hw3d->pipeline_statistics_queries);
-      }
+      ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline, q->bo, offset);
       break;
    default:
       assert(!"unknown query type");
@@ -285,93 +181,103 @@ ilo_3d_begin_query(struct ilo_context *ilo, struct ilo_query *q)
    }
 }
 
-/**
- * End a query.
- */
-void
-ilo_3d_end_query(struct ilo_context *ilo, struct ilo_query *q)
+bool
+ilo_3d_init_query(struct pipe_context *pipe, struct ilo_query *q)
 {
-   struct ilo_3d *hw3d = ilo->hw3d;
-
-   ilo_3d_own_render_ring(hw3d);
+   struct ilo_context *ilo = ilo_context(pipe);
+   unsigned bo_size;
 
    switch (q->type) {
    case PIPE_QUERY_OCCLUSION_COUNTER:
-      list_del(&q->list);
-
-      assert(q->reg_read < q->reg_total);
-      assert(hw3d->owner.reserve >= q->reg_cmd_size);
-      hw3d->owner.reserve -= q->reg_cmd_size;
-
-      ilo_3d_pipeline_emit_write_depth_count(hw3d->pipeline,
-            q->bo, q->reg_read++);
+      q->cmd_len = ilo_3d_pipeline_estimate_size(ilo->hw3d->pipeline,
+            ILO_3D_PIPELINE_WRITE_DEPTH_COUNT, q);
+      q->stride = sizeof(uint64_t);
+      q->in_pairs = true;
       break;
    case PIPE_QUERY_TIMESTAMP:
-      q->data.u64 = 0;
-
-      if (ilo_query_alloc_bo(q, 1, 1, hw3d->cp->winsys)) {
-         ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline,
-               q->bo, q->reg_read++);
-      }
+      q->cmd_len = ilo_3d_pipeline_estimate_size(ilo->hw3d->pipeline,
+            ILO_3D_PIPELINE_WRITE_TIMESTAMP, q);
+      q->stride = sizeof(uint64_t);
+      q->in_pairs = false;
       break;
    case PIPE_QUERY_TIME_ELAPSED:
-      list_del(&q->list);
-
-      assert(q->reg_read < q->reg_total);
-      assert(hw3d->owner.reserve >= q->reg_cmd_size);
-      hw3d->owner.reserve -= q->reg_cmd_size;
-
-      ilo_3d_pipeline_emit_write_timestamp(hw3d->pipeline,
-            q->bo, q->reg_read++);
+      q->cmd_len = ilo_3d_pipeline_estimate_size(ilo->hw3d->pipeline,
+            ILO_3D_PIPELINE_WRITE_TIMESTAMP, q);
+      q->stride = sizeof(uint64_t);
+      q->in_pairs = true;
+      break;
+   case PIPE_QUERY_PIPELINE_STATISTICS:
+      q->cmd_len = ilo_3d_pipeline_estimate_size(ilo->hw3d->pipeline,
+            ILO_3D_PIPELINE_WRITE_STATISTICS, q);
+      q->stride = sizeof(uint64_t) * 11;
+      q->in_pairs = true;
       break;
    case PIPE_QUERY_PRIMITIVES_GENERATED:
    case PIPE_QUERY_PRIMITIVES_EMITTED:
-      list_del(&q->list);
-      break;
-   case PIPE_QUERY_PIPELINE_STATISTICS:
-      list_del(&q->list);
-
-      assert(q->reg_read + 11 <= q->reg_total);
-      assert(hw3d->owner.reserve >= q->reg_cmd_size);
-      hw3d->owner.reserve -= q->reg_cmd_size;
-
-      ilo_3d_pipeline_emit_write_statistics(hw3d->pipeline,
-            q->bo, q->reg_read);
-      q->reg_read += 11;
+      return true;
       break;
    default:
-      assert(!"unknown query type");
+      return false;
       break;
    }
+
+   /* double cmd_len and stride if in pairs */
+   q->cmd_len <<= q->in_pairs;
+   q->stride <<= q->in_pairs;
+
+   bo_size = (q->stride > 4096) ? q->stride : 4096;
+   q->bo = intel_winsys_alloc_buffer(ilo->winsys, "query", bo_size, false);
+   if (!q->bo)
+      return false;
+
+   q->count = bo_size / q->stride;
+
+   return true;
 }
 
-/**
- * Process the raw query data.
- */
 void
-ilo_3d_process_query(struct ilo_context *ilo, struct ilo_query *q)
+ilo_3d_begin_query(struct pipe_context *pipe, struct ilo_query *q)
 {
-   struct ilo_3d *hw3d = ilo->hw3d;
+   struct ilo_3d *hw3d = ilo_context(pipe)->hw3d;
+
+   ilo_3d_own_render_ring(hw3d);
+
+   if (q->bo) {
+      /* need to submit first */
+      if (!ilo_builder_validate(&hw3d->cp->builder, 1, &q->bo) ||
+          ilo_cp_space(hw3d->cp) < q->cmd_len) {
+         ilo_cp_submit(hw3d->cp, "out of aperture or space");
+
+         assert(ilo_builder_validate(&hw3d->cp->builder, 1, &q->bo));
+         assert(ilo_cp_space(hw3d->cp) >= q->cmd_len);
+
+         ilo_3d_own_render_ring(hw3d);
+      }
+
+      /* reserve the space for ending/pausing the query */
+      hw3d->owner.reserve += q->cmd_len >> q->in_pairs;
+
+      query_begin_bo(hw3d, q);
+   }
 
    switch (q->type) {
    case PIPE_QUERY_OCCLUSION_COUNTER:
-      if (q->bo)
-         process_query_for_occlusion_counter(hw3d, q);
+      list_add(&q->list, &hw3d->occlusion_queries);
       break;
    case PIPE_QUERY_TIMESTAMP:
-      if (q->bo)
-         process_query_for_timestamp(hw3d, q);
+      /* no-op */
       break;
    case PIPE_QUERY_TIME_ELAPSED:
-      if (q->bo)
-         process_query_for_time_elapsed(hw3d, q);
+      list_add(&q->list, &hw3d->time_elapsed_queries);
       break;
    case PIPE_QUERY_PRIMITIVES_GENERATED:
+      list_add(&q->list, &hw3d->prim_generated_queries);
+      break;
    case PIPE_QUERY_PRIMITIVES_EMITTED:
+      list_add(&q->list, &hw3d->prim_emitted_queries);
       break;
    case PIPE_QUERY_PIPELINE_STATISTICS:
-      if (q->bo)
-         process_query_for_pipeline_statistics(hw3d, q);
+      list_add(&q->list, &hw3d->pipeline_statistics_queries);
       break;
    default:
       assert(!"unknown query type");
@@ -379,19 +285,34 @@ ilo_3d_process_query(struct ilo_context *ilo, struct ilo_query *q)
    }
 }
 
+void
+ilo_3d_end_query(struct pipe_context *pipe, struct ilo_query *q)
+{
+   struct ilo_3d *hw3d = ilo_context(pipe)->hw3d;
+
+   ilo_3d_own_render_ring(hw3d);
+
+   if (q->bo) {
+      /* reclaim the reserved space */
+      hw3d->owner.reserve -= q->cmd_len >> q->in_pairs;
+      assert(hw3d->owner.reserve >= 0);
+
+      query_end_bo(hw3d, q);
+   }
+
+   list_delinit(&q->list);
+}
+
 /**
- * Hook for CP new-batch.
+ * Process the raw query data.
  */
 void
-ilo_3d_cp_submitted(struct ilo_3d *hw3d)
+ilo_3d_process_query(struct pipe_context *pipe, struct ilo_query *q)
 {
-   /* invalidate the pipeline */
-   ilo_3d_pipeline_invalidate(hw3d->pipeline,
-         ILO_3D_PIPELINE_INVALIDATE_BATCH_BO |
-         ILO_3D_PIPELINE_INVALIDATE_STATE_BO |
-         ILO_3D_PIPELINE_INVALIDATE_KERNEL_BO);
+   struct ilo_3d *hw3d = ilo_context(pipe)->hw3d;
 
-   hw3d->new_batch = true;
+   if (q->bo)
+      query_process_bo(hw3d, q);
 }
 
 static void
@@ -405,7 +326,35 @@ ilo_3d_own_cp(struct ilo_cp *cp, void *data)
       assert(ilo_cp_space(hw3d->cp) >= hw3d->owner.reserve * 2);
    }
 
-   ilo_3d_resume_queries(hw3d);
+   while (true) {
+      struct ilo_builder_snapshot snapshot;
+      struct ilo_query *q;
+
+      ilo_builder_batch_snapshot(&hw3d->cp->builder, &snapshot);
+
+      /* resume occlusion queries */
+      LIST_FOR_EACH_ENTRY(q, &hw3d->occlusion_queries, list)
+         query_begin_bo(hw3d, q);
+
+      /* resume timer queries */
+      LIST_FOR_EACH_ENTRY(q, &hw3d->time_elapsed_queries, list)
+         query_begin_bo(hw3d, q);
+
+      /* resume pipeline statistics queries */
+      LIST_FOR_EACH_ENTRY(q, &hw3d->pipeline_statistics_queries, list)
+         query_begin_bo(hw3d, q);
+
+      if (!ilo_builder_validate(&hw3d->cp->builder, 0, NULL)) {
+         ilo_builder_batch_restore(&hw3d->cp->builder, &snapshot);
+
+         if (ilo_builder_batch_used(&hw3d->cp->builder)) {
+            ilo_cp_submit(hw3d->cp, "out of aperture");
+            continue;
+         }
+      }
+
+      break;
+   }
 
    assert(ilo_cp_space(hw3d->cp) >= hw3d->owner.reserve);
 }
@@ -414,8 +363,36 @@ static void
 ilo_3d_release_cp(struct ilo_cp *cp, void *data)
 {
    struct ilo_3d *hw3d = data;
+   struct ilo_query *q;
+
+   assert(ilo_cp_space(hw3d->cp) >= hw3d->owner.reserve);
+
+   /* pause occlusion queries */
+   LIST_FOR_EACH_ENTRY(q, &hw3d->occlusion_queries, list)
+      query_end_bo(hw3d, q);
+
+   /* pause timer queries */
+   LIST_FOR_EACH_ENTRY(q, &hw3d->time_elapsed_queries, list)
+      query_end_bo(hw3d, q);
+
+   /* pause pipeline statistics queries */
+   LIST_FOR_EACH_ENTRY(q, &hw3d->pipeline_statistics_queries, list)
+      query_end_bo(hw3d, q);
+}
+
+/**
+ * Hook for CP new-batch.
+ */
+void
+ilo_3d_cp_submitted(struct ilo_3d *hw3d)
+{
+   /* invalidate the pipeline */
+   ilo_3d_pipeline_invalidate(hw3d->pipeline,
+         ILO_3D_PIPELINE_INVALIDATE_BATCH_BO |
+         ILO_3D_PIPELINE_INVALIDATE_STATE_BO |
+         ILO_3D_PIPELINE_INVALIDATE_KERNEL_BO);
 
-   ilo_3d_pause_queries(hw3d);
+   hw3d->new_batch = true;
 }
 
 /**
@@ -512,10 +489,10 @@ update_prim_count(struct ilo_3d *hw3d, int generated, int emitted)
    struct ilo_query *q;
 
    LIST_FOR_EACH_ENTRY(q, &hw3d->prim_generated_queries, list)
-      q->data.u64 += generated;
+      q->result.u64 += generated;
 
    LIST_FOR_EACH_ENTRY(q, &hw3d->prim_emitted_queries, list)
-      q->data.u64 += emitted;
+      q->result.u64 += emitted;
 }
 
 bool
diff --git a/src/gallium/drivers/ilo/ilo_3d.h b/src/gallium/drivers/ilo/ilo_3d.h
index ff3a59e3584..9053dcdcc99 100644
--- a/src/gallium/drivers/ilo/ilo_3d.h
+++ b/src/gallium/drivers/ilo/ilo_3d.h
@@ -72,14 +72,17 @@ ilo_3d_cp_submitted(struct ilo_3d *hw3d);
 void
 ilo_3d_draw_rectlist(struct ilo_3d *hw3d, const struct ilo_blitter *blitter);
 
+bool
+ilo_3d_init_query(struct pipe_context *pipe, struct ilo_query *q);
+
 void
-ilo_3d_begin_query(struct ilo_context *ilo, struct ilo_query *q);
+ilo_3d_begin_query(struct pipe_context *pipe, struct ilo_query *q);
 
 void
-ilo_3d_end_query(struct ilo_context *ilo, struct ilo_query *q);
+ilo_3d_end_query(struct pipe_context *pipe, struct ilo_query *q);
 
 void
-ilo_3d_process_query(struct ilo_context *ilo, struct ilo_query *q);
+ilo_3d_process_query(struct pipe_context *pipe, struct ilo_query *q);
 
 bool
 ilo_3d_pass_render_condition(struct ilo_context *ilo);
diff --git a/src/gallium/drivers/ilo/ilo_3d_pipeline.c b/src/gallium/drivers/ilo/ilo_3d_pipeline.c
index 37e33323192..87c69349019 100644
--- a/src/gallium/drivers/ilo/ilo_3d_pipeline.c
+++ b/src/gallium/drivers/ilo/ilo_3d_pipeline.c
@@ -237,10 +237,10 @@ ilo_3d_pipeline_emit_flush(struct ilo_3d_pipeline *p)
  */
 void
 ilo_3d_pipeline_emit_write_timestamp(struct ilo_3d_pipeline *p,
-                                     struct intel_bo *bo, int index)
+                                     struct intel_bo *bo, uint32_t offset)
 {
    handle_invalid_batch_bo(p, true);
-   p->emit_write_timestamp(p, bo, index);
+   p->emit_write_timestamp(p, bo, offset);
 }
 
 /**
@@ -248,10 +248,10 @@ ilo_3d_pipeline_emit_write_timestamp(struct ilo_3d_pipeline *p,
  */
 void
 ilo_3d_pipeline_emit_write_depth_count(struct ilo_3d_pipeline *p,
-                                       struct intel_bo *bo, int index)
+                                       struct intel_bo *bo, uint32_t offset)
 {
    handle_invalid_batch_bo(p, true);
-   p->emit_write_depth_count(p, bo, index);
+   p->emit_write_depth_count(p, bo, offset);
 }
 
 /**
@@ -259,10 +259,10 @@ ilo_3d_pipeline_emit_write_depth_count(struct ilo_3d_pipeline *p,
  */
 void
 ilo_3d_pipeline_emit_write_statistics(struct ilo_3d_pipeline *p,
-                                      struct intel_bo *bo, int index)
+                                      struct intel_bo *bo, uint32_t offset)
 {
    handle_invalid_batch_bo(p, true);
-   p->emit_write_statistics(p, bo, index);
+   p->emit_write_statistics(p, bo, offset);
 }
 
 void
diff --git a/src/gallium/drivers/ilo/ilo_3d_pipeline.h b/src/gallium/drivers/ilo/ilo_3d_pipeline.h
index 0bf4ee2092c..5342b4bdc57 100644
--- a/src/gallium/drivers/ilo/ilo_3d_pipeline.h
+++ b/src/gallium/drivers/ilo/ilo_3d_pipeline.h
@@ -81,13 +81,13 @@ struct ilo_3d_pipeline {
    void (*emit_flush)(struct ilo_3d_pipeline *pipeline);
 
    void (*emit_write_timestamp)(struct ilo_3d_pipeline *pipeline,
-                                struct intel_bo *bo, int index);
+                                struct intel_bo *bo, uint32_t offset);
 
    void (*emit_write_depth_count)(struct ilo_3d_pipeline *pipeline,
-                                  struct intel_bo *bo, int index);
+                                  struct intel_bo *bo, uint32_t offset);
 
    void (*emit_write_statistics)(struct ilo_3d_pipeline *pipeline,
-                                 struct intel_bo *bo, int index);
+                                 struct intel_bo *bo, uint32_t offset);
 
    void (*emit_rectlist)(struct ilo_3d_pipeline *pipeline,
                          const struct ilo_blitter *blitter);
@@ -176,15 +176,15 @@ ilo_3d_pipeline_emit_flush(struct ilo_3d_pipeline *p);
 
 void
 ilo_3d_pipeline_emit_write_timestamp(struct ilo_3d_pipeline *p,
-                                     struct intel_bo *bo, int index);
+                                     struct intel_bo *bo, uint32_t offset);
 
 void
 ilo_3d_pipeline_emit_write_depth_count(struct ilo_3d_pipeline *p,
-                                       struct intel_bo *bo, int index);
+                                       struct intel_bo *bo, uint32_t offset);
 
 void
 ilo_3d_pipeline_emit_write_statistics(struct ilo_3d_pipeline *p,
-                                      struct intel_bo *bo, int index);
+                                      struct intel_bo *bo, uint32_t offset);
 
 void
 ilo_3d_pipeline_emit_rectlist(struct ilo_3d_pipeline *p,
diff --git a/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.c b/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.c
index 5deb77228a3..383e9fe46bb 100644
--- a/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.c
+++ b/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.c
@@ -1466,20 +1466,21 @@ ilo_3d_pipeline_emit_flush_gen6(struct ilo_3d_pipeline *p)
 
 void
 ilo_3d_pipeline_emit_write_timestamp_gen6(struct ilo_3d_pipeline *p,
-                                          struct intel_bo *bo, int index)
+                                          struct intel_bo *bo,
+                                          uint32_t offset)
 {
    if (ilo_dev_gen(p->dev) == ILO_GEN(6))
       gen6_wa_pipe_control_post_sync(p, true);
 
    gen6_PIPE_CONTROL(p->builder,
          GEN6_PIPE_CONTROL_WRITE_TIMESTAMP,
-         bo, index * sizeof(uint64_t),
-         true);
+         bo, offset, true);
 }
 
 void
 ilo_3d_pipeline_emit_write_depth_count_gen6(struct ilo_3d_pipeline *p,
-                                            struct intel_bo *bo, int index)
+                                            struct intel_bo *bo,
+                                            uint32_t offset)
 {
    if (ilo_dev_gen(p->dev) == ILO_GEN(6))
       gen6_wa_pipe_control_post_sync(p, false);
@@ -1487,15 +1488,15 @@ ilo_3d_pipeline_emit_write_depth_count_gen6(struct ilo_3d_pipeline *p,
    gen6_PIPE_CONTROL(p->builder,
          GEN6_PIPE_CONTROL_DEPTH_STALL |
          GEN6_PIPE_CONTROL_WRITE_PS_DEPTH_COUNT,
-         bo, index * sizeof(uint64_t),
-         true);
+         bo, offset, true);
 }
 
 void
 ilo_3d_pipeline_emit_write_statistics_gen6(struct ilo_3d_pipeline *p,
-                                           struct intel_bo *bo, int index)
+                                           struct intel_bo *bo,
+                                           uint32_t offset)
 {
-   uint32_t regs[] = {
+   const uint32_t regs[] = {
       GEN6_REG_IA_VERTICES_COUNT,
       GEN6_REG_IA_PRIMITIVES_COUNT,
       GEN6_REG_VS_INVOCATION_COUNT,
@@ -1513,20 +1514,17 @@ ilo_3d_pipeline_emit_write_statistics_gen6(struct ilo_3d_pipeline *p,
    p->emit_flush(p);
 
    for (i = 0; i < Elements(regs); i++) {
-      const uint32_t bo_offset = (index + i) * sizeof(uint64_t);
-
       if (regs[i]) {
          /* store lower 32 bits */
-         gen6_MI_STORE_REGISTER_MEM(p->builder,
-               bo, bo_offset, regs[i]);
+         gen6_MI_STORE_REGISTER_MEM(p->builder, bo, offset, regs[i]);
          /* store higher 32 bits */
-         gen6_MI_STORE_REGISTER_MEM(p->builder,
-               bo, bo_offset + 4, regs[i] + 4);
+         gen6_MI_STORE_REGISTER_MEM(p->builder, bo, offset + 4, regs[i] + 4);
       }
       else {
-         gen6_MI_STORE_DATA_IMM(p->builder,
-               bo, bo_offset, 0, true);
+         gen6_MI_STORE_DATA_IMM(p->builder, bo, offset, 0, true);
       }
+
+      offset += 8;
    }
 }
 
diff --git a/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.h b/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.h
index 59892cbfa7d..6121a06458a 100644
--- a/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.h
+++ b/src/gallium/drivers/ilo/ilo_3d_pipeline_gen6.h
@@ -158,15 +158,18 @@ ilo_3d_pipeline_emit_flush_gen6(struct ilo_3d_pipeline *p);
 
 void
 ilo_3d_pipeline_emit_write_timestamp_gen6(struct ilo_3d_pipeline *p,
-                                          struct intel_bo *bo, int index);
+                                          struct intel_bo *bo,
+                                          uint32_t offset);
 
 void
 ilo_3d_pipeline_emit_write_depth_count_gen6(struct ilo_3d_pipeline *p,
-                                            struct intel_bo *bo, int index);
+                                            struct intel_bo *bo,
+                                            uint32_t offset);
 
 void
 ilo_3d_pipeline_emit_write_statistics_gen6(struct ilo_3d_pipeline *p,
-                                           struct intel_bo *bo, int index);
+                                           struct intel_bo *bo,
+                                           uint32_t offset);
 
 void
 ilo_3d_pipeline_init_gen6(struct ilo_3d_pipeline *p);
diff --git a/src/gallium/drivers/ilo/ilo_query.c b/src/gallium/drivers/ilo/ilo_query.c
index 812e056996e..e0cd4ee7554 100644
--- a/src/gallium/drivers/ilo/ilo_query.c
+++ b/src/gallium/drivers/ilo/ilo_query.c
@@ -33,31 +33,30 @@
 #include "ilo_query.h"
 
 static const struct {
-   const char *name;
-
-   void (*begin)(struct ilo_context *ilo, struct ilo_query *q);
-   void (*end)(struct ilo_context *ilo, struct ilo_query *q);
-   void (*process)(struct ilo_context *ilo, struct ilo_query *q);
-} query_info[PIPE_QUERY_TYPES] = {
-#define INFO(prefix, desc) {        \
-   .name = desc,                    \
-   .begin = prefix ## _begin_query, \
-   .end = prefix ## _end_query,     \
-   .process = prefix ## _process_query, \
+   bool (*init)(struct pipe_context *pipe, struct ilo_query *q);
+   void (*begin)(struct pipe_context *pipe, struct ilo_query *q);
+   void (*end)(struct pipe_context *pipe, struct ilo_query *q);
+   void (*process)(struct pipe_context *pipe, struct ilo_query *q);
+} ilo_query_table[PIPE_QUERY_TYPES] = {
+#define INFO(prefix) {                    \
+   .init = prefix ## _init_query,         \
+   .begin = prefix ## _begin_query,       \
+   .end = prefix ## _end_query,           \
+   .process = prefix ## _process_query,   \
 }
-#define INFOX(prefix, desc) { desc, NULL, NULL, NULL, }
-
-   [PIPE_QUERY_OCCLUSION_COUNTER]      = INFO(ilo_3d, "occlusion counter"),
-   [PIPE_QUERY_OCCLUSION_PREDICATE]    = INFOX(ilo_3d, "occlusion pred."),
-   [PIPE_QUERY_TIMESTAMP]              = INFO(ilo_3d, "timestamp"),
-   [PIPE_QUERY_TIMESTAMP_DISJOINT]     = INFOX(ilo_3d, "timestamp disjoint"),
-   [PIPE_QUERY_TIME_ELAPSED]           = INFO(ilo_3d, "time elapsed"),
-   [PIPE_QUERY_PRIMITIVES_GENERATED]   = INFO(ilo_3d, "primitives generated"),
-   [PIPE_QUERY_PRIMITIVES_EMITTED]     = INFO(ilo_3d, "primitives emitted"),
-   [PIPE_QUERY_SO_STATISTICS]          = INFOX(ilo_3d, "so statistics"),
-   [PIPE_QUERY_SO_OVERFLOW_PREDICATE]  = INFOX(ilo_3d, "so overflow pred."),
-   [PIPE_QUERY_GPU_FINISHED]           = INFOX(ilo_3d, "gpu finished"),
-   [PIPE_QUERY_PIPELINE_STATISTICS]    = INFO(ilo_3d, "pipeline statistics"),
+#define INFOX(prefix) { NULL, NULL, NULL, NULL, }
+
+   [PIPE_QUERY_OCCLUSION_COUNTER]      = INFO(ilo_3d),
+   [PIPE_QUERY_OCCLUSION_PREDICATE]    = INFOX(ilo_3d),
+   [PIPE_QUERY_TIMESTAMP]              = INFO(ilo_3d),
+   [PIPE_QUERY_TIMESTAMP_DISJOINT]     = INFOX(ilo_3d),
+   [PIPE_QUERY_TIME_ELAPSED]           = INFO(ilo_3d),
+   [PIPE_QUERY_PRIMITIVES_GENERATED]   = INFO(ilo_3d),
+   [PIPE_QUERY_PRIMITIVES_EMITTED]     = INFO(ilo_3d),
+   [PIPE_QUERY_SO_STATISTICS]          = INFOX(ilo_3d),
+   [PIPE_QUERY_SO_OVERFLOW_PREDICATE]  = INFOX(ilo_3d),
+   [PIPE_QUERY_GPU_FINISHED]           = INFOX(ilo_3d),
+   [PIPE_QUERY_PIPELINE_STATISTICS]    = INFO(ilo_3d),
 
 #undef INFO
 #undef INFOX
@@ -91,8 +90,15 @@ ilo_create_query(struct pipe_context *pipe, unsigned query_type, unsigned index)
       return NULL;
 
    q->type = query_type;
+   q->index = index;
+
    list_inithead(&q->list);
 
+   if (!ilo_query_table[q->type].init(pipe, q)) {
+      FREE(q);
+      return NULL;
+   }
+
    return (struct pipe_query *) q;
 }
 
@@ -110,62 +116,71 @@ ilo_destroy_query(struct pipe_context *pipe, struct pipe_query *query)
 static void
 ilo_begin_query(struct pipe_context *pipe, struct pipe_query *query)
 {
-   struct ilo_context *ilo = ilo_context(pipe);
    struct ilo_query *q = ilo_query(query);
 
+   if (q->active)
+      return;
+
+   util_query_clear_result(&q->result, q->type);
+   q->used = 0;
    q->active = true;
 
-   query_info[q->type].begin(ilo, q);
+   ilo_query_table[q->type].begin(pipe, q);
 }
 
 static void
 ilo_end_query(struct pipe_context *pipe, struct pipe_query *query)
 {
-   struct ilo_context *ilo = ilo_context(pipe);
    struct ilo_query *q = ilo_query(query);
 
-   query_info[q->type].end(ilo, q);
+   if (!q->active) {
+      /* require ilo_begin_query() first */
+      if (q->in_pairs)
+         return;
+
+      ilo_begin_query(pipe, query);
+   }
 
-   /*
-    * some queries such as timestamp query does not require a call to
-    * begin_query() so q->active is always false
-    */
    q->active = false;
+
+   ilo_query_table[q->type].end(pipe, q);
 }
 
 /**
- * The type (union pipe_query_result) indicates only the size of the buffer.
- * Callers expect the result to be "serialized".
+ * Serialize the result.  The size of \p buf is
+ * sizeof(union pipe_query_result).
  */
 static void
-serialize_query_data(unsigned type, const union pipe_query_result *data,
-                     void *buf)
+query_serialize(const struct ilo_query *q, void *buf)
 {
-   switch (type) {
+   switch (q->type) {
    case PIPE_QUERY_OCCLUSION_COUNTER:
    case PIPE_QUERY_TIMESTAMP:
    case PIPE_QUERY_TIME_ELAPSED:
    case PIPE_QUERY_PRIMITIVES_GENERATED:
    case PIPE_QUERY_PRIMITIVES_EMITTED:
       {
-         uint64_t *r = buf;
-         r[0] = data->u64;
+         uint64_t *dst = buf;
+         dst[0] = q->result.u64;
       }
       break;
    case PIPE_QUERY_PIPELINE_STATISTICS:
       {
-         uint64_t *r = buf;
-         r[0] = data->pipeline_statistics.ia_vertices;
-         r[1] = data->pipeline_statistics.ia_primitives;
-         r[2] = data->pipeline_statistics.vs_invocations;
-         r[3] = data->pipeline_statistics.gs_invocations;
-         r[4] = data->pipeline_statistics.gs_primitives;
-         r[5] = data->pipeline_statistics.c_invocations;
-         r[6] = data->pipeline_statistics.c_primitives;
-         r[7] = data->pipeline_statistics.ps_invocations;
-         r[8] = data->pipeline_statistics.hs_invocations;
-         r[9] = data->pipeline_statistics.ds_invocations;
-         r[10] = data->pipeline_statistics.cs_invocations;
+         const struct pipe_query_data_pipeline_statistics *stats =
+            &q->result.pipeline_statistics;
+         uint64_t *dst = buf;
+
+         dst[0] = stats->ia_vertices;
+         dst[1] = stats->ia_primitives;
+         dst[2] = stats->vs_invocations;
+         dst[3] = stats->gs_invocations;
+         dst[4] = stats->gs_primitives;
+         dst[5] = stats->c_invocations;
+         dst[6] = stats->c_primitives;
+         dst[7] = stats->ps_invocations;
+         dst[8] = stats->hs_invocations;
+         dst[9] = stats->ds_invocations;
+         dst[10] = stats->cs_invocations;
       }
       break;
    default:
@@ -178,71 +193,30 @@ static boolean
 ilo_get_query_result(struct pipe_context *pipe, struct pipe_query *query,
                      boolean wait, union pipe_query_result *result)
 {
-   struct ilo_context *ilo = ilo_context(pipe);
    struct ilo_query *q = ilo_query(query);
 
    if (q->active)
       return false;
 
    if (q->bo) {
-      if (ilo_builder_has_reloc(&ilo->cp->builder, q->bo))
-         ilo_cp_submit(ilo->cp, "syncing for queries");
+      struct ilo_cp *cp = ilo_context(pipe)->cp;
+
+      if (ilo_builder_has_reloc(&cp->builder, q->bo))
+         ilo_cp_submit(cp, "syncing for queries");
 
       if (!wait && intel_bo_is_busy(q->bo))
          return false;
-
-      query_info[q->type].process(ilo, q);
    }
 
+   ilo_query_table[q->type].process(pipe, q);
+
    if (result)
-      serialize_query_data(q->type, &q->data, (void *) result);
+      query_serialize(q, (void *) result);
 
    return true;
 }
 
 /**
- * Allocate a query bo for reading hardware statistics.
- *
- * \param reg_count specifies how many registers need to be read.
- * \param repeat_count specifies how many times the registers are read.  If
- *        zero or negative, a 4KB bo is allocated.
- */
-bool
-ilo_query_alloc_bo(struct ilo_query *q, int reg_count, int repeat_count,
-                   struct intel_winsys *winsys)
-{
-   const char *name;
-   int reg_total;
-
-   name = query_info[q->type].name;
-
-   reg_total = reg_count * repeat_count;
-   if (reg_total <= 0)
-      reg_total = 4096 / sizeof(uint64_t);
-
-   /* (re-)allocate the bo */
-   if (q->reg_total < reg_total) {
-      /* registers are 64-bit */
-      const int size = reg_total * sizeof(uint64_t);
-
-      if (q->bo)
-         intel_bo_unreference(q->bo);
-
-      q->bo = intel_winsys_alloc_buffer(winsys,
-            name, size, false);
-      q->reg_total = (q->bo) ? reg_total : 0;
-   }
-
-   /* avoid partial reads */
-   if (reg_count)
-      q->reg_total -= q->reg_total % reg_count;
-
-   q->reg_read = 0;
-
-   return (q->bo != NULL);
-}
-
-/**
  * Initialize query-related functions.
  */
 void
diff --git a/src/gallium/drivers/ilo/ilo_query.h b/src/gallium/drivers/ilo/ilo_query.h
index 7fedb2fbe03..e451e4e13af 100644
--- a/src/gallium/drivers/ilo/ilo_query.h
+++ b/src/gallium/drivers/ilo/ilo_query.h
@@ -39,24 +39,24 @@ struct ilo_context;
  */
 struct ilo_query {
    unsigned type;
-   bool active;
+   unsigned index;
 
    struct list_head list;
 
-   /* storage for the collected data */
-   union pipe_query_result data;
+   bool active;
 
    /* for queries that need to read hardware statistics */
+   int cmd_len; /* in dwords, as expected by ilo_cp */
+   bool in_pairs;
    struct intel_bo *bo;
-   int reg_read, reg_total;
-   int reg_cmd_size; /* in dwords, as expected by ilo_cp */
+   int stride, count;
+   int used;
+
+   /* storage for the collected data */
+   union pipe_query_result result;
 };
 
 void
 ilo_init_query_functions(struct ilo_context *ilo);
 
-bool
-ilo_query_alloc_bo(struct ilo_query *q, int reg_count, int repeat_count,
-                   struct intel_winsys *winsys);
-
 #endif /* ILO_QUERY_H */