i965: Expose OA counters via INTEL_performance_query

This adds support for exposing basic Observation Architecture
performance counters on Haswell.

This support is based on the i915 perf kernel interface which is used
to configure the OA unit, allowing Mesa to emit MI_REPORT_PERF_COUNT
commands around queries to collect counter snapshots.

To take into account the small chance that some of the 32bit counters
could wrap around for long queries (~50 milliseconds for a GT3 Haswell @
1.1GHz) the implementation also collects periodic metrics.

Signed-off-by: Robert Bragg <[email protected]>
Reviewed-by: Lionel Landwerlin <[email protected]>
Acked-by: Kenneth Graunke <[email protected]>

2 files changed, 1158 insertions, 13 deletions

diff --git a/src/mesa/drivers/dri/i965/brw_context.h b/src/mesa/drivers/dri/i965/brw_context.h
index 0aa27995ab9..c67dc7ceafb 100644
--- a/src/mesa/drivers/dri/i965/brw_context.h
+++ b/src/mesa/drivers/dri/i965/brw_context.h
@@ -1114,7 +1114,59 @@ struct brw_context
       struct brw_perf_query_info *queries;
       int n_queries;
 
+      /* The i915 perf stream we open to setup + enable the OA counters */
+      int oa_stream_fd;
+
+      /* An i915 perf stream fd gives exclusive access to the OA unit that will
+       * report counter snapshots for a specific counter set/profile in a
+       * specific layout/format so we can only start OA queries that are
+       * compatible with the currently open fd...
+       */
+      int current_oa_metrics_set_id;
+      int current_oa_format;
+
+      /* List of buffers containing OA reports */
+      struct exec_list sample_buffers;
+
+      /* Cached list of empty sample buffers */
+      struct exec_list free_sample_buffers;
+
+      int n_active_oa_queries;
       int n_active_pipeline_stats_queries;
+
+      /* The number of queries depending on running OA counters which
+       * extends beyond brw_end_perf_query() since we need to wait until
+       * the last MI_RPC command has parsed by the GPU.
+       *
+       * Accurate accounting is important here as emitting an
+       * MI_REPORT_PERF_COUNT command while the OA unit is disabled will
+       * effectively hang the gpu.
+       */
+      int n_oa_users;
+
+      /* To help catch an spurious problem with the hardware or perf
+       * forwarding samples, we emit each MI_REPORT_PERF_COUNT command
+       * with a unique ID that we can explicitly check for...
+       */
+      int next_query_start_report_id;
+
+      /**
+       * An array of queries whose results haven't yet been assembled
+       * based on the data in buffer objects.
+       *
+       * These may be active, or have already ended.  However, the
+       * results have not been requested.
+       */
+      struct brw_perf_query_object **unaccumulated;
+      int unaccumulated_elements;
+      int unaccumulated_array_size;
+
+      /* The total number of query objects so we can relinquish
+       * our exclusive access to perf if the application deletes
+       * all of its objects. (NB: We only disable perf while
+       * there are no active queries)
+       */
+      int n_query_instances;
    } perfquery;
 
    int num_atoms[BRW_NUM_PIPELINES];
diff --git a/src/mesa/drivers/dri/i965/brw_performance_query.c b/src/mesa/drivers/dri/i965/brw_performance_query.c
index 6cd5bf4caa5..64b3beeeae0 100644
--- a/src/mesa/drivers/dri/i965/brw_performance_query.c
+++ b/src/mesa/drivers/dri/i965/brw_performance_query.c
@@ -26,12 +26,21 @@
  *
  * Implementation of the GL_INTEL_performance_query extension.
  *
- * Currently this driver only exposes the 64bit Pipeline Statistics
- * Registers for Gen6+, with support for Observability Counters to be
- * added later for Gen7.5+
+ * Currently there are two possible counter sources exposed here:
+ *
+ * On Gen6+ hardware we have numerous 64bit Pipeline Statistics Registers
+ * that we can snapshot at the beginning and end of a query.
+ *
+ * On Gen7.5+ we have Observability Architecture counters which are
+ * covered in separate document from the rest of the PRMs.  It is available at:
+ * https://01.org/linuxgraphics/documentation/driver-documentation-prms
+ * => 2013 Intel Core Processor Family => Observability Performance Counters
+ * (This one volume covers Sandybridge, Ivybridge, Baytrail, and Haswell,
+ * though notably we currently only support OA counters for Haswell+)
  */
 
 #include <limits.h>
+#include <dirent.h>
 
 #include <asm/unistd.h>
 #include <sys/types.h>
@@ -40,6 +49,9 @@
 #include <sys/mman.h>
 #include <sys/ioctl.h>
 
+#include <xf86drm.h>
+#include <i915_drm.h>
+
 #include "main/hash.h"
 #include "main/macros.h"
 #include "main/mtypes.h"
@@ -47,14 +59,145 @@
 
 #include "util/bitset.h"
 #include "util/ralloc.h"
+#include "util/hash_table.h"
+#include "util/list.h"
 
 #include "brw_context.h"
 #include "brw_defines.h"
 #include "brw_performance_query.h"
+#include "brw_oa_hsw.h"
 #include "intel_batchbuffer.h"
 
 #define FILE_DEBUG_FLAG DEBUG_PERFMON
 
+/*
+ * The largest OA format we can use on Haswell includes:
+ * 1 timestamp, 45 A counters, 8 B counters and 8 C counters.
+ */
+#define MAX_OA_REPORT_COUNTERS 62
+
+#define I915_PERF_OA_SAMPLE_SIZE (8 +   /* drm_i915_perf_record_header */ \
+                                  256)  /* OA counter report */
+
+/**
+ * Periodic OA samples are read() into these buffer structures via the
+ * i915 perf kernel interface and appended to the
+ * brw->perfquery.sample_buffers linked list. When we process the
+ * results of an OA metrics query we need to consider all the periodic
+ * samples between the Begin and End MI_REPORT_PERF_COUNT command
+ * markers.
+ *
+ * 'Periodic' is a simplification as there are other automatic reports
+ * written by the hardware also buffered here.
+ *
+ * Considering three queries, A, B and C:
+ *
+ *  Time ---->
+ *                ________________A_________________
+ *                |                                |
+ *                | ________B_________ _____C___________
+ *                | |                | |           |   |
+ *
+ * And an illustration of sample buffers read over this time frame:
+ * [HEAD ][     ][     ][     ][     ][     ][     ][     ][TAIL ]
+ *
+ * These nodes may hold samples for query A:
+ * [     ][     ][  A  ][  A  ][  A  ][  A  ][  A  ][     ][     ]
+ *
+ * These nodes may hold samples for query B:
+ * [     ][     ][  B  ][  B  ][  B  ][     ][     ][     ][     ]
+ *
+ * These nodes may hold samples for query C:
+ * [     ][     ][     ][     ][     ][  C  ][  C  ][  C  ][     ]
+ *
+ * The illustration assumes we have an even distribution of periodic
+ * samples so all nodes have the same size plotted against time:
+ *
+ * Note, to simplify code, the list is never empty.
+ *
+ * With overlapping queries we can see that periodic OA reports may
+ * relate to multiple queries and care needs to be take to keep
+ * track of sample buffers until there are no queries that might
+ * depend on their contents.
+ *
+ * We use a node ref counting system where a reference ensures that a
+ * node and all following nodes can't be freed/recycled until the
+ * reference drops to zero.
+ *
+ * E.g. with a ref of one here:
+ * [  0  ][  0  ][  1  ][  0  ][  0  ][  0  ][  0  ][  0  ][  0  ]
+ *
+ * These nodes could be freed or recycled ("reaped"):
+ * [  0  ][  0  ]
+ *
+ * These must be preserved until the leading ref drops to zero:
+ *               [  1  ][  0  ][  0  ][  0  ][  0  ][  0  ][  0  ]
+ *
+ * When a query starts we take a reference on the current tail of
+ * the list, knowing that no already-buffered samples can possibly
+ * relate to the newly-started query. A pointer to this node is
+ * also saved in the query object's ->oa.samples_head.
+ *
+ * E.g. starting query A while there are two nodes in .sample_buffers:
+ *                ________________A________
+ *                |
+ *
+ * [  0  ][  1  ]
+ *           ^_______ Add a reference and store pointer to node in
+ *                    A->oa.samples_head
+ *
+ * Moving forward to when the B query starts with no new buffer nodes:
+ * (for reference, i915 perf reads() are only done when queries finish)
+ *                ________________A_______
+ *                | ________B___
+ *                | |
+ *
+ * [  0  ][  2  ]
+ *           ^_______ Add a reference and store pointer to
+ *                    node in B->oa.samples_head
+ *
+ * Once a query is finished, after an OA query has become 'Ready',
+ * once the End OA report has landed and after we we have processed
+ * all the intermediate periodic samples then we drop the
+ * ->oa.samples_head reference we took at the start.
+ *
+ * So when the B query has finished we have:
+ *                ________________A________
+ *                | ______B___________
+ *                | |                |
+ * [  0  ][  1  ][  0  ][  0  ][  0  ]
+ *           ^_______ Drop B->oa.samples_head reference
+ *
+ * We still can't free these due to the A->oa.samples_head ref:
+ *        [  1  ][  0  ][  0  ][  0  ]
+ *
+ * When the A query finishes: (note there's a new ref for C's samples_head)
+ *                ________________A_________________
+ *                |                                |
+ *                |                    _____C_________
+ *                |                    |           |
+ * [  0  ][  0  ][  0  ][  0  ][  1  ][  0  ][  0  ]
+ *           ^_______ Drop A->oa.samples_head reference
+ *
+ * And we can now reap these nodes up to the C->oa.samples_head:
+ * [  X  ][  X  ][  X  ][  X  ]
+ *                  keeping -> [  1  ][  0  ][  0  ]
+ *
+ * We reap old sample buffers each time we finish processing an OA
+ * query by iterating the sample_buffers list from the head until we
+ * find a referenced node and stop.
+ *
+ * Reaped buffers move to a perfquery.free_sample_buffers list and
+ * when we come to read() we first look to recycle a buffer from the
+ * free_sample_buffers list before allocating a new buffer.
+ */
+struct brw_oa_sample_buf {
+   struct exec_node link;
+   int refcount;
+   int len;
+   uint8_t buf[I915_PERF_OA_SAMPLE_SIZE * 10];
+};
+
 /**
  * i965 representation of a performance query object.
  *
@@ -68,13 +211,56 @@ struct brw_perf_query_object
 
    const struct brw_perf_query_info *query;
 
-   struct {
-      /**
-       * BO containing starting and ending snapshots for the
-       * statistics counters.
-       */
-      drm_intel_bo *bo;
-   } pipeline_stats;
+   /* See query->kind to know which state below is in use... */
+   union {
+      struct {
+
+         /**
+          * BO containing OA counter snapshots at query Begin/End time.
+          */
+         drm_intel_bo *bo;
+
+         /**
+          * The MI_REPORT_PERF_COUNT command lets us specify a unique
+          * ID that will be reflected in the resulting OA report
+          * that's written by the GPU. This is the ID we're expecting
+          * in the begin report and the the end report should be
+          * @begin_report_id + 1.
+          */
+         int begin_report_id;
+
+         /**
+          * Reference the head of the brw->perfquery.sample_buffers
+          * list at the time that the query started (so we only need
+          * to look at nodes after this point when looking for samples
+          * related to this query)
+          *
+          * (See struct brw_oa_sample_buf description for more details)
+          */
+         struct exec_node *samples_head;
+
+         /**
+          * Storage for the final accumulated OA counters.
+          */
+         uint64_t accumulator[MAX_OA_REPORT_COUNTERS];
+
+         /**
+          * false while in the unaccumulated_elements list, and set to
+          * true when the final, end MI_RPC snapshot has been
+          * accumulated.
+          */
+         bool results_accumulated;
+
+      } oa;
+
+      struct {
+         /**
+          * BO containing starting and ending snapshots for the
+          * statistics counters.
+          */
+         drm_intel_bo *bo;
+      } pipeline_stats;
+   };
 };
 
 /** Downcasting convenience macro. */
@@ -88,15 +274,32 @@ brw_perf_query(struct gl_perf_query_object *o)
 #define STATS_BO_END_OFFSET_BYTES   (STATS_BO_SIZE / 2)
 #define MAX_STAT_COUNTERS           (STATS_BO_END_OFFSET_BYTES / 8)
 
+#define MI_RPC_BO_SIZE              4096
+#define MI_RPC_BO_END_OFFSET_BYTES  (MI_RPC_BO_SIZE / 2)
+
 /******************************************************************************/
 
+static bool
+brw_is_perf_query_ready(struct gl_context *ctx,
+                        struct gl_perf_query_object *o);
+
 static void
 dump_perf_query_callback(GLuint id, void *query_void, void *brw_void)
 {
+   struct gl_context *ctx = brw_void;
    struct gl_perf_query_object *o = query_void;
    struct brw_perf_query_object *obj = query_void;
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+      DBG("%4d: %-6s %-8s BO: %-4s OA data: %-10s %-15s\n",
+          id,
+          o->Used ? "Dirty," : "New,",
+          o->Active ? "Active," : (o->Ready ? "Ready," : "Pending,"),
+          obj->oa.bo ? "yes," : "no,",
+          brw_is_perf_query_ready(ctx, o) ? "ready," : "not ready,",
+          obj->oa.results_accumulated ? "accumulated" : "not accumulated");
+      break;
    case PIPELINE_STATS:
       DBG("%4d: %-6s %-8s BO: %-4s\n",
           id,
@@ -111,13 +314,68 @@ static void
 dump_perf_queries(struct brw_context *brw)
 {
    struct gl_context *ctx = &brw->ctx;
-   DBG("Queries: (Open queries = %d)\n",
-       brw->perfquery.n_active_pipeline_stats_queries);
+   DBG("Queries: (Open queries = %d, OA users = %d)\n",
+       brw->perfquery.n_active_oa_queries, brw->perfquery.n_oa_users);
    _mesa_HashWalk(ctx->PerfQuery.Objects, dump_perf_query_callback, brw);
 }
 
 /******************************************************************************/
 
+static struct brw_oa_sample_buf *
+get_free_sample_buf(struct brw_context *brw)
+{
+   struct exec_node *node = exec_list_pop_head(&brw->perfquery.free_sample_buffers);
+   struct brw_oa_sample_buf *buf;
+
+   if (node)
+      buf = exec_node_data(struct brw_oa_sample_buf, node, link);
+   else {
+      buf = ralloc_size(brw, sizeof(*buf));
+
+      exec_node_init(&buf->link);
+      buf->refcount = 0;
+      buf->len = 0;
+   }
+
+   return buf;
+}
+
+static void
+reap_old_sample_buffers(struct brw_context *brw)
+{
+   struct exec_node *tail_node =
+      exec_list_get_tail(&brw->perfquery.sample_buffers);
+   struct brw_oa_sample_buf *tail_buf =
+      exec_node_data(struct brw_oa_sample_buf, tail_node, link);
+
+   /* Remove all old, unreferenced sample buffers walking forward from
+    * the head of the list, except always leave at least one node in
+    * the list so we always have a node to reference when we Begin
+    * a new query.
+    */
+   foreach_list_typed_safe(struct brw_oa_sample_buf, buf, link,
+                           &brw->perfquery.sample_buffers)
+   {
+      if (buf->refcount == 0 && buf != tail_buf) {
+         exec_node_remove(&buf->link);
+         exec_list_push_head(&brw->perfquery.free_sample_buffers, &buf->link);
+      } else
+         return;
+   }
+}
+
+static void
+free_sample_bufs(struct brw_context *brw)
+{
+   foreach_list_typed_safe(struct brw_oa_sample_buf, buf, link,
+                           &brw->perfquery.free_sample_buffers)
+      ralloc_free(buf);
+
+   exec_list_make_empty(&brw->perfquery.free_sample_buffers);
+}
+
+/******************************************************************************/
+
 /**
  * Driver hook for glGetPerfQueryInfoINTEL().
  */
@@ -138,6 +396,10 @@ brw_get_perf_query_info(struct gl_context *ctx,
    *n_counters = query->n_counters;
 
    switch (query->kind) {
+   case OA_COUNTERS:
+      *n_active = brw->perfquery.n_active_oa_queries;
+      break;
+
    case PIPELINE_STATS:
       *n_active = brw->perfquery.n_active_pipeline_stats_queries;
       break;
@@ -200,6 +462,397 @@ snapshot_statistics_registers(struct brw_context *brw,
 }
 
 /**
+ * Emit an MI_REPORT_PERF_COUNT command packet.
+ *
+ * This asks the GPU to write a report of the current OA counter
+ * values into @bo at the given offset and containing the given
+ * @report_id which we can cross-reference when parsing the report.
+ */
+static void
+emit_mi_report_perf_count(struct brw_context *brw,
+                          drm_intel_bo *bo,
+                          uint32_t offset_in_bytes,
+                          uint32_t report_id)
+{
+   assert(offset_in_bytes % 64 == 0);
+
+   BEGIN_BATCH(3);
+   OUT_BATCH(GEN6_MI_REPORT_PERF_COUNT);
+   OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
+             offset_in_bytes);
+   OUT_BATCH(report_id);
+   ADVANCE_BATCH();
+}
+
+/**
+ * Add a query to the global list of "unaccumulated queries."
+ *
+ * Queries are tracked here until all the associated OA reports have
+ * been accumulated via accumulate_oa_reports() after the end
+ * MI_REPORT_PERF_COUNT has landed in query->oa.bo.
+ */
+static void
+add_to_unaccumulated_query_list(struct brw_context *brw,
+                                struct brw_perf_query_object *obj)
+{
+   if (brw->perfquery.unaccumulated_elements >=
+       brw->perfquery.unaccumulated_array_size)
+   {
+      brw->perfquery.unaccumulated_array_size *= 1.5;
+      brw->perfquery.unaccumulated =
+         reralloc(brw, brw->perfquery.unaccumulated,
+                  struct brw_perf_query_object *,
+                  brw->perfquery.unaccumulated_array_size);
+   }
+
+   brw->perfquery.unaccumulated[brw->perfquery.unaccumulated_elements++] = obj;
+}
+
+/**
+ * Remove a query from the global list of unaccumulated queries once
+ * after successfully accumulating the OA reports associated with the
+ * query in accumulate_oa_reports() or when discarding unwanted query
+ * results.
+ */
+static void
+drop_from_unaccumulated_query_list(struct brw_context *brw,
+                                   struct brw_perf_query_object *obj)
+{
+   for (int i = 0; i < brw->perfquery.unaccumulated_elements; i++) {
+      if (brw->perfquery.unaccumulated[i] == obj) {
+         int last_elt = --brw->perfquery.unaccumulated_elements;
+
+         if (i == last_elt)
+            brw->perfquery.unaccumulated[i] = NULL;
+         else {
+            brw->perfquery.unaccumulated[i] =
+               brw->perfquery.unaccumulated[last_elt];
+         }
+
+         break;
+      }
+   }
+
+   /* Drop our samples_head reference so that associated periodic
+    * sample data buffers can potentially be reaped if they aren't
+    * referenced by any other queries...
+    */
+
+   struct brw_oa_sample_buf *buf =
+      exec_node_data(struct brw_oa_sample_buf, obj->oa.samples_head, link);
+
+   assert(buf->refcount > 0);
+   buf->refcount--;
+
+   obj->oa.samples_head = NULL;
+
+   reap_old_sample_buffers(brw);
+}
+
+static uint64_t
+timebase_scale(struct brw_context *brw, uint32_t u32_time_delta)
+{
+   uint64_t tmp = ((uint64_t)u32_time_delta) * 1000000000ull;
+
+   return tmp ? tmp / brw->perfquery.sys_vars.timestamp_frequency : 0;
+}
+
+static void
+accumulate_uint32(const uint32_t *report0,
+                  const uint32_t *report1,
+                  uint64_t *accumulator)
+{
+   *accumulator += (uint32_t)(*report1 - *report0);
+}
+
+/**
+ * Given pointers to starting and ending OA snapshots, add the deltas for each
+ * counter to the results.
+ */
+static void
+add_deltas(struct brw_context *brw,
+           struct brw_perf_query_object *obj,
+           const uint32_t *start,
+           const uint32_t *end)
+{
+   const struct brw_perf_query_info *query = obj->query;
+   uint64_t *accumulator = obj->oa.accumulator;
+   int i;
+
+   switch (query->oa_format) {
+   case I915_OA_FORMAT_A45_B8_C8:
+      accumulate_uint32(start + 1, end + 1, accumulator); /* timestamp */
+
+      for (i = 0; i < 61; i++)
+         accumulate_uint32(start + 3 + i, end + 3 + i, accumulator + 1 + i);
+
+      break;
+   default:
+      unreachable("Can't accumulate OA counters in unknown format");
+   }
+}
+
+static bool
+inc_n_oa_users(struct brw_context *brw)
+{
+   if (brw->perfquery.n_oa_users == 0 &&
+       drmIoctl(brw->perfquery.oa_stream_fd,
+                I915_PERF_IOCTL_ENABLE, 0) < 0)
+   {
+      return false;
+   }
+   ++brw->perfquery.n_oa_users;
+
+   return true;
+}
+
+static void
+dec_n_oa_users(struct brw_context *brw)
+{
+   /* Disabling the i915 perf stream will effectively disable the OA
+    * counters.  Note it's important to be sure there are no outstanding
+    * MI_RPC commands at this point since they could stall the CS
+    * indefinitely once OACONTROL is disabled.
+    */
+   --brw->perfquery.n_oa_users;
+   if (brw->perfquery.n_oa_users == 0 &&
+       drmIoctl(brw->perfquery.oa_stream_fd, I915_PERF_IOCTL_DISABLE, 0) < 0)
+   {
+      DBG("WARNING: Error disabling i915 perf stream: %m\n");
+   }
+}
+
+/* In general if we see anything spurious while accumulating results,
+ * we don't try and continue accumulating the current query, hoping
+ * for the best, we scrap anything outstanding, and then hope for the
+ * best with new queries.
+ */
+static void
+discard_all_queries(struct brw_context *brw)
+{
+   while (brw->perfquery.unaccumulated_elements) {
+      struct brw_perf_query_object *obj = brw->perfquery.unaccumulated[0];
+
+      obj->oa.results_accumulated = true;
+      drop_from_unaccumulated_query_list(brw, brw->perfquery.unaccumulated[0]);
+
+      dec_n_oa_users(brw);
+   }
+}
+
+static bool
+read_oa_samples(struct brw_context *brw)
+{
+   while (1) {
+      struct brw_oa_sample_buf *buf = get_free_sample_buf(brw);
+      int len;
+
+      while ((len = read(brw->perfquery.oa_stream_fd, buf->buf,
+                         sizeof(buf->buf))) < 0 && errno == EINTR)
+         ;
+
+      if (len <= 0) {
+         exec_list_push_tail(&brw->perfquery.free_sample_buffers, &buf->link);
+
+         if (len < 0) {
+            if (errno == EAGAIN)
+               return true;
+            else {
+               DBG("Error reading i915 perf samples: %m\n");
+               return false;
+            }
+         } else {
+            DBG("Spurious EOF reading i915 perf samples\n");
+            return false;
+         }
+      }
+
+      buf->len = len;
+      exec_list_push_tail(&brw->perfquery.sample_buffers, &buf->link);
+   }
+
+   unreachable("not reached");
+   return false;
+}
+
+/**
+ * Accumulate raw OA counter values based on deltas between pairs
+ * of OA reports.
+ *
+ * Accumulation starts from the first report captured via
+ * MI_REPORT_PERF_COUNT (MI_RPC) by brw_begin_perf_query() until the
+ * last MI_RPC report requested by brw_end_perf_query(). Between these
+ * two reports there may also some number of periodically sampled OA
+ * reports collected via the i915 perf interface - depending on the
+ * duration of the query.
+ *
+ * These periodic snapshots help to ensure we handle counter overflow
+ * correctly by being frequent enough to ensure we don't miss multiple
+ * overflows of a counter between snapshots.
+ */
+static void
+accumulate_oa_reports(struct brw_context *brw,
+                      struct brw_perf_query_object *obj)
+{
+   struct gl_perf_query_object *o = &obj->base;
+   uint32_t *query_buffer;
+   uint32_t *start;
+   uint32_t *last;
+   uint32_t *end;
+   struct exec_node *first_samples_node;
+
+   assert(o->Ready);
+
+   /* Collect the latest periodic OA reports from i915 perf */
+   if (!read_oa_samples(brw))
+      goto error;
+
+   drm_intel_bo_map(obj->oa.bo, false);
+   query_buffer = obj->oa.bo->virtual;
+
+   start = last = query_buffer;
+   end = query_buffer + (MI_RPC_BO_END_OFFSET_BYTES / sizeof(uint32_t));
+
+   if (start[0] != obj->oa.begin_report_id) {
+      DBG("Spurious start report id=%"PRIu32"\n", start[0]);
+      goto error;
+   }
+   if (end[0] != (obj->oa.begin_report_id + 1)) {
+      DBG("Spurious end report id=%"PRIu32"\n", end[0]);
+      goto error;
+   }
+
+   /* See if we have any periodic reports to accumulate too... */
+
+   /* N.B. The oa.samples_head was set when the query began and
+    * pointed to the tail of the brw->perfquery.sample_buffers list at
+    * the time the query started. Since the buffer existed before the
+    * first MI_REPORT_PERF_COUNT command was emitted we therefore know
+    * that no data in this particular node's buffer can possibly be
+    * associated with the query - so skip ahead one...
+    */
+   first_samples_node = obj->oa.samples_head->next;
+
+   foreach_list_typed_from(struct brw_oa_sample_buf, buf, link,
+                           &brw->perfquery.sample_buffers,
+                           first_samples_node)
+   {
+      int offset = 0;
+
+      while (offset < buf->len) {
+         const struct drm_i915_perf_record_header *header =
+            (const struct drm_i915_perf_record_header *)(buf->buf + offset);
+
+         assert(header->size != 0);
+         assert(header->size <= buf->len);
+
+         offset += header->size;
+
+         switch (header->type) {
+         case DRM_I915_PERF_RECORD_SAMPLE: {
+            uint32_t *report = (uint32_t *)(header + 1);
+
+            /* Ignore reports that come before the start marker.
+             * (Note: takes care to allow overflow of 32bit timestamps)
+             */
+            if (timebase_scale(brw, report[1] - start[1]) > 5000000000)
+               continue;
+
+            /* Ignore reports that come after the end marker.
+             * (Note: takes care to allow overflow of 32bit timestamps)
+             */
+            if (timebase_scale(brw, report[1] - end[1]) <= 5000000000)
+               goto end;
+
+            add_deltas(brw, obj, last, report);
+
+            last = report;
+
+            break;
+         }
+
+         case DRM_I915_PERF_RECORD_OA_BUFFER_LOST:
+             DBG("i915 perf: OA error: all reports lost\n");
+             goto error;
+         case DRM_I915_PERF_RECORD_OA_REPORT_LOST:
+             DBG("i915 perf: OA report lost\n");
+             break;
+         }
+      }
+   }
+
+end:
+
+   add_deltas(brw, obj, last, end);
+
+   DBG("Marking %d accumulated - results gathered\n", o->Id);
+
+   drm_intel_bo_unmap(obj->oa.bo);
+   obj->oa.results_accumulated = true;
+   drop_from_unaccumulated_query_list(brw, obj);
+   dec_n_oa_users(brw);
+
+   return;
+
+error:
+
+   drm_intel_bo_unmap(obj->oa.bo);
+   discard_all_queries(brw);
+}
+
+/******************************************************************************/
+
+static bool
+open_i915_perf_oa_stream(struct brw_context *brw,
+                         int metrics_set_id,
+                         int report_format,
+                         int period_exponent,
+                         int drm_fd,
+                         uint32_t ctx_id)
+{
+   uint64_t properties[] = {
+      /* Single context sampling */
+      DRM_I915_PERF_PROP_CTX_HANDLE, ctx_id,
+
+      /* Include OA reports in samples */
+      DRM_I915_PERF_PROP_SAMPLE_OA, true,
+
+      /* OA unit configuration */
+      DRM_I915_PERF_PROP_OA_METRICS_SET, metrics_set_id,
+      DRM_I915_PERF_PROP_OA_FORMAT, report_format,
+      DRM_I915_PERF_PROP_OA_EXPONENT, period_exponent,
+   };
+   struct drm_i915_perf_open_param param = {
+      .flags = I915_PERF_FLAG_FD_CLOEXEC |
+               I915_PERF_FLAG_FD_NONBLOCK |
+               I915_PERF_FLAG_DISABLED,
+      .num_properties = ARRAY_SIZE(properties) / 2,
+      .properties_ptr = (uint64_t)properties
+   };
+   int fd = drmIoctl(drm_fd, DRM_IOCTL_I915_PERF_OPEN, &param);
+   if (fd == -1) {
+      DBG("Error opening i915 perf OA stream: %m\n");
+      return false;
+   }
+
+   brw->perfquery.oa_stream_fd = fd;
+
+   brw->perfquery.current_oa_metrics_set_id = metrics_set_id;
+   brw->perfquery.current_oa_format = report_format;
+
+   return true;
+}
+
+static void
+close_perf(struct brw_context *brw)
+{
+   if (brw->perfquery.oa_stream_fd != -1) {
+      close(brw->perfquery.oa_stream_fd);
+      brw->perfquery.oa_stream_fd = -1;
+   }
+}
+
+/**
  * Driver hook for glBeginPerfQueryINTEL().
  */
 static bool
@@ -268,6 +921,112 @@ brw_begin_perf_query(struct gl_context *ctx,
    brw_emit_mi_flush(brw);
 
    switch (query->kind) {
+   case OA_COUNTERS:
+
+      /* Opening an i915 perf stream implies exclusive access to the OA unit
+       * which will generate counter reports for a specific counter set with a
+       * specific layout/format so we can't begin any OA based queries that
+       * require a different counter set or format unless we get an opportunity
+       * to close the stream and open a new one...
+       */
+      if (brw->perfquery.oa_stream_fd != -1 &&
+          brw->perfquery.current_oa_metrics_set_id !=
+          query->oa_metrics_set_id) {
+
+         if (brw->perfquery.n_oa_users != 0)
+            return false;
+         else
+            close_perf(brw);
+      }
+
+      /* If the OA counters aren't already on, enable them. */
+      if (brw->perfquery.oa_stream_fd == -1) {
+         __DRIscreen *screen = brw->screen->driScrnPriv;
+         uint32_t ctx_id;
+         int period_exponent;
+
+         if (drm_intel_gem_context_get_id(brw->hw_ctx, &ctx_id) != 0)
+            return false;
+
+         /* The timestamp for HSW+ increments every 80ns
+          *
+          * The period_exponent gives a sampling period as follows:
+          *   sample_period = 80ns * 2^(period_exponent + 1)
+          *
+          * The overflow period for Haswell can be calculated as:
+          *
+          * 2^32 / (n_eus * max_gen_freq * 2)
+          * (E.g. 40 EUs @ 1GHz = ~53ms)
+          *
+          * We currently sample every 42 milliseconds...
+          */
+         period_exponent = 18;
+
+         if (!open_i915_perf_oa_stream(brw,
+                                       query->oa_metrics_set_id,
+                                       query->oa_format,
+                                       period_exponent,
+                                       screen->fd, /* drm fd */
+                                       ctx_id))
+            return false;
+      } else {
+         assert(brw->perfquery.current_oa_metrics_set_id ==
+                query->oa_metrics_set_id &&
+                brw->perfquery.current_oa_format ==
+                query->oa_format);
+      }
+
+      if (!inc_n_oa_users(brw)) {
+         DBG("WARNING: Error enabling i915 perf stream: %m\n");
+         return false;
+      }
+
+      if (obj->oa.bo) {
+         drm_intel_bo_unreference(obj->oa.bo);
+         obj->oa.bo = NULL;
+      }
+
+      obj->oa.bo =
+         drm_intel_bo_alloc(brw->bufmgr, "perf. query OA MI_RPC bo",
+                            MI_RPC_BO_SIZE, 64);
+#ifdef DEBUG
+      /* Pre-filling the BO helps debug whether writes landed. */
+      drm_intel_bo_map(obj->oa.bo, true);
+      memset((char *) obj->oa.bo->virtual, 0x80, MI_RPC_BO_SIZE);
+      drm_intel_bo_unmap(obj->oa.bo);
+#endif
+
+      obj->oa.begin_report_id = brw->perfquery.next_query_start_report_id;
+      brw->perfquery.next_query_start_report_id += 2;
+
+      /* Take a starting OA counter snapshot. */
+      emit_mi_report_perf_count(brw, obj->oa.bo, 0,
+                                obj->oa.begin_report_id);
+      ++brw->perfquery.n_active_oa_queries;
+
+      /* No already-buffered samples can possibly be associated with this query
+       * so create a marker within the list of sample buffers enabling us to
+       * easily ignore earlier samples when processing this query after
+       * completion.
+       */
+      assert(!exec_list_is_empty(&brw->perfquery.sample_buffers));
+      obj->oa.samples_head = exec_list_get_tail(&brw->perfquery.sample_buffers);
+
+      struct brw_oa_sample_buf *buf =
+         exec_node_data(struct brw_oa_sample_buf, obj->oa.samples_head, link);
+
+      /* This reference will ensure that future/following sample
+       * buffers (that may relate to this query) can't be freed until
+       * this drops to zero.
+       */
+      buf->refcount++;
+
+      memset(obj->oa.accumulator, 0, sizeof(obj->oa.accumulator));
+      obj->oa.results_accumulated = false;
+
+      add_to_unaccumulated_query_list(brw, obj);
+      break;
+
    case PIPELINE_STATS:
       if (obj->pipeline_stats.bo) {
          drm_intel_bo_unreference(obj->pipeline_stats.bo);
@@ -312,6 +1071,28 @@ brw_end_perf_query(struct gl_context *ctx,
    brw_emit_mi_flush(brw);
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+
+      /* NB: It's possible that the query will have already been marked
+       * as 'accumulated' if an error was seen while reading samples
+       * from perf. In this case we mustn't try and emit a closing
+       * MI_RPC command in case the OA unit has already been disabled
+       */
+      if (!obj->oa.results_accumulated) {
+         /* Take an ending OA counter snapshot. */
+         emit_mi_report_perf_count(brw, obj->oa.bo,
+                                   MI_RPC_BO_END_OFFSET_BYTES,
+                                   obj->oa.begin_report_id + 1);
+      }
+
+      --brw->perfquery.n_active_oa_queries;
+
+      /* NB: even though the query has now ended, it can't be accumulated
+       * until the end MI_REPORT_PERF_COUNT snapshot has been written
+       * to query->oa.bo
+       */
+      break;
+
    case PIPELINE_STATS:
       snapshot_statistics_registers(brw, obj,
                                     STATS_BO_END_OFFSET_BYTES);
@@ -330,6 +1111,10 @@ brw_wait_perf_query(struct gl_context *ctx, struct gl_perf_query_object *o)
    assert(!o->Ready);
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+      bo = obj->oa.bo;
+      break;
+
    case PIPELINE_STATS:
       bo = obj->pipeline_stats.bo;
       break;
@@ -339,7 +1124,8 @@ brw_wait_perf_query(struct gl_context *ctx, struct gl_perf_query_object *o)
       return;
 
    /* If the current batch references our results bo then we need to
-    * flush first... */
+    * flush first...
+    */
    if (drm_intel_bo_references(brw->batch.bo, bo))
       intel_batchbuffer_flush(brw);
 
@@ -362,6 +1148,12 @@ brw_is_perf_query_ready(struct gl_context *ctx,
       return true;
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+      return (obj->oa.results_accumulated ||
+              (obj->oa.bo &&
+               !drm_intel_bo_references(brw->batch.bo, obj->oa.bo) &&
+               !drm_intel_bo_busy(obj->oa.bo)));
+
    case PIPELINE_STATS:
       return (obj->pipeline_stats.bo &&
               !drm_intel_bo_references(brw->batch.bo, obj->pipeline_stats.bo) &&
@@ -373,6 +1165,49 @@ brw_is_perf_query_ready(struct gl_context *ctx,
 }
 
 static int
+get_oa_counter_data(struct brw_context *brw,
+                    struct brw_perf_query_object *obj,
+                    size_t data_size,
+                    uint8_t *data)
+{
+   const struct brw_perf_query_info *query = obj->query;
+   int n_counters = query->n_counters;
+   int written = 0;
+
+   if (!obj->oa.results_accumulated) {
+      accumulate_oa_reports(brw, obj);
+      assert(obj->oa.results_accumulated);
+   }
+
+   for (int i = 0; i < n_counters; i++) {
+      const struct brw_perf_query_counter *counter = &query->counters[i];
+      uint64_t *out_uint64;
+      float *out_float;
+
+      if (counter->size) {
+         switch (counter->data_type) {
+         case GL_PERFQUERY_COUNTER_DATA_UINT64_INTEL:
+            out_uint64 = (uint64_t *)(data + counter->offset);
+            *out_uint64 = counter->oa_counter_read_uint64(brw, query,
+                                                          obj->oa.accumulator);
+            break;
+         case GL_PERFQUERY_COUNTER_DATA_FLOAT_INTEL:
+            out_float = (float *)(data + counter->offset);
+            *out_float = counter->oa_counter_read_float(brw, query,
+                                                        obj->oa.accumulator);
+            break;
+         default:
+            /* So far we aren't using uint32, double or bool32... */
+            unreachable("unexpected counter data type");
+         }
+         written = counter->offset + counter->size;
+      }
+   }
+
+   return written;
+}
+
+static int
 get_pipeline_stats_data(struct brw_context *brw,
                         struct brw_perf_query_object *obj,
                         size_t data_size,
@@ -433,6 +1268,10 @@ brw_get_perf_query_data(struct gl_context *ctx,
    assert(o->Ready);
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+      written = get_oa_counter_data(brw, obj, data_size, (uint8_t *)data);
+      break;
+
    case PIPELINE_STATS:
       written = get_pipeline_stats_data(brw, obj, data_size, (uint8_t *)data);
       break;
@@ -456,6 +1295,8 @@ brw_new_perf_query_object(struct gl_context *ctx, unsigned query_index)
 
    obj->query = query;
 
+   brw->perfquery.n_query_instances++;
+
    return &obj->base;
 }
 
@@ -466,6 +1307,7 @@ static void
 brw_delete_perf_query(struct gl_context *ctx,
                       struct gl_perf_query_object *o)
 {
+   struct brw_context *brw = brw_context(ctx);
    struct brw_perf_query_object *obj = brw_perf_query(o);
 
    /* We can assume that the frontend waits for a query to complete
@@ -478,6 +1320,20 @@ brw_delete_perf_query(struct gl_context *ctx,
    DBG("Delete(%d)\n", o->Id);
 
    switch (obj->query->kind) {
+   case OA_COUNTERS:
+      if (obj->oa.bo) {
+         if (!obj->oa.results_accumulated) {
+            drop_from_unaccumulated_query_list(brw, obj);
+            dec_n_oa_users(brw);
+         }
+
+         drm_intel_bo_unreference(obj->oa.bo);
+         obj->oa.bo = NULL;
+      }
+
+      obj->oa.results_accumulated = false;
+      break;
+
    case PIPELINE_STATS:
       if (obj->pipeline_stats.bo) {
          drm_intel_bo_unreference(obj->pipeline_stats.bo);
@@ -487,6 +1343,15 @@ brw_delete_perf_query(struct gl_context *ctx,
    }
 
    free(obj);
+
+   /* As an indication that the INTEL_performance_query extension is no
+    * longer in use, it's a good time to free our cache of sample
+    * buffers and close any current i915-perf stream.
+    */
+   if (--brw->perfquery.n_query_instances == 0) {
+      free_sample_bufs(brw);
+      close_perf(brw);
+   }
 }
 
 /******************************************************************************/
@@ -618,16 +1483,244 @@ init_pipeline_statistic_query_registers(struct brw_context *brw)
    query->data_size = sizeof(uint64_t) * query->n_counters;
 }
 
+static bool
+read_file_uint64(const char *file, uint64_t *val)
+{
+    char buf[32];
+    int fd, n;
+
+    fd = open(file, 0);
+    if (fd < 0)
+	return false;
+    n = read(fd, buf, sizeof (buf) - 1);
+    close(fd);
+    if (n < 0)
+	return false;
+
+    buf[n] = '\0';
+    *val = strtoull(buf, NULL, 0);
+
+    return true;
+}
+
+static void
+enumerate_sysfs_metrics(struct brw_context *brw, const char *sysfs_dev_dir)
+{
+   char buf[256];
+   DIR *metricsdir = NULL;
+   struct dirent *metric_entry;
+
+   strncpy(buf, sysfs_dev_dir, sizeof(buf));
+   strncat(buf, "/metrics", sizeof(buf));
+
+   metricsdir = opendir(buf);
+   if (!metricsdir) {
+      DBG("Failed to open %s: %m\n", buf);
+      return;
+   }
+
+   while ((metric_entry = readdir(metricsdir))) {
+      struct hash_entry *entry;
+
+      if ((metric_entry->d_type != DT_DIR &&
+           metric_entry->d_type != DT_LNK) ||
+          metric_entry->d_name[0] == '.')
+         continue;
+
+      DBG("metric set: %s\n", metric_entry->d_name);
+      entry = _mesa_hash_table_search(brw->perfquery.oa_metrics_table,
+                                      metric_entry->d_name);
+      if (entry) {
+         struct brw_perf_query_info *query;
+         uint64_t id;
+
+         snprintf(buf, sizeof(buf), "%s/metrics/%s/id",
+                  sysfs_dev_dir, metric_entry->d_name);
+
+         if (!read_file_uint64(buf, &id)) {
+            DBG("Failed to read metric set id from %s: %m", buf);
+            continue;
+         }
+
+         query = append_query_info(brw);
+         *query = *(struct brw_perf_query_info *)entry->data;
+         query->oa_metrics_set_id = id;
+
+         DBG("metric set known by mesa: id = %" PRIu64"\n",
+             query->oa_metrics_set_id);
+      } else
+         DBG("metric set not known by mesa (skipping)\n");
+   }
+
+   closedir(metricsdir);
+}
+
+static bool
+read_sysfs_drm_device_file_uint64(struct brw_context *brw,
+                                  const char *sysfs_dev_dir,
+                                  const char *file,
+                                  uint64_t *value)
+{
+   char buf[512];
+
+   snprintf(buf, sizeof(buf), "%s/%s", sysfs_dev_dir, file);
+
+   return read_file_uint64(buf, value);
+}
+
+static bool
+init_oa_sys_vars(struct brw_context *brw, const char *sysfs_dev_dir)
+{
+   uint64_t min_freq_mhz = 0, max_freq_mhz = 0;
+
+   if (!read_sysfs_drm_device_file_uint64(brw, sysfs_dev_dir,
+                                          "gt_min_freq_mhz",
+                                          &min_freq_mhz))
+      return false;
+
+   if (!read_sysfs_drm_device_file_uint64(brw, sysfs_dev_dir,
+                                          "gt_max_freq_mhz",
+                                          &max_freq_mhz))
+      return false;
+
+   brw->perfquery.sys_vars.gt_min_freq = min_freq_mhz * 1000000;
+   brw->perfquery.sys_vars.gt_max_freq = max_freq_mhz * 1000000;
+
+   if (brw->is_haswell) {
+      const struct gen_device_info *info = &brw->screen->devinfo;
+
+      brw->perfquery.sys_vars.timestamp_frequency = 12500000;
+
+      if (info->gt == 1) {
+         brw->perfquery.sys_vars.n_eus = 10;
+         brw->perfquery.sys_vars.n_eu_slices = 1;
+         brw->perfquery.sys_vars.subslice_mask = 0x1;
+      } else if (info->gt == 2) {
+         brw->perfquery.sys_vars.n_eus = 20;
+         brw->perfquery.sys_vars.n_eu_slices = 1;
+         brw->perfquery.sys_vars.subslice_mask = 0x3;
+      } else if (info->gt == 3) {
+         brw->perfquery.sys_vars.n_eus = 40;
+         brw->perfquery.sys_vars.n_eu_slices = 2;
+         brw->perfquery.sys_vars.subslice_mask = 0xf;
+      } else
+         unreachable("not reached");
+
+      return true;
+   } else
+      return false;
+}
+
+static bool
+get_sysfs_dev_dir(struct brw_context *brw,
+                  char *path_buf,
+                  int path_buf_len)
+{
+   __DRIscreen *screen = brw->screen->driScrnPriv;
+   struct stat sb;
+   int min, maj;
+   DIR *drmdir;
+   struct dirent *drm_entry;
+
+   assert(path_buf);
+   assert(path_buf_len);
+   path_buf[0] = '\0';
+
+   if (fstat(screen->fd, &sb)) {
+      DBG("Failed to stat DRM fd\n");
+      return false;
+   }
+
+   maj = major(sb.st_rdev);
+   min = minor(sb.st_rdev);
+
+   if (!S_ISCHR(sb.st_mode)) {
+      DBG("DRM fd is not a character device as expected\n");
+      return false;
+   }
+
+   snprintf(path_buf, path_buf_len,
+            "/sys/dev/char/%d:%d/device/drm", maj, min);
+
+   drmdir = opendir(path_buf);
+   if (!drmdir) {
+      DBG("Failed to open %s: %m\n", path_buf);
+      return false;
+   }
+
+   while ((drm_entry = readdir(drmdir))) {
+      if ((drm_entry->d_type == DT_DIR ||
+           drm_entry->d_type == DT_LNK) &&
+          strncmp(drm_entry->d_name, "card", 4) == 0)
+      {
+         snprintf(path_buf, path_buf_len,
+                  "/sys/dev/char/%d:%d/device/drm/%s",
+                  maj, min, drm_entry->d_name);
+         closedir(drmdir);
+         return true;
+      }
+   }
+
+   closedir(drmdir);
+
+   DBG("Failed to find cardX directory under /sys/dev/char/%d:%d/device/drm\n",
+       maj, min);
+
+   return false;
+}
+
 static unsigned
 brw_init_perf_query_info(struct gl_context *ctx)
 {
    struct brw_context *brw = brw_context(ctx);
+   struct stat sb;
+   char sysfs_dev_dir[128];
 
    if (brw->perfquery.n_queries)
       return brw->perfquery.n_queries;
 
    init_pipeline_statistic_query_registers(brw);
 
+   /* The existence of this sysctl parameter implies the kernel supports
+    * the i915 perf interface.
+    */
+   if (brw->is_haswell &&
+       stat("/proc/sys/dev/i915/perf_stream_paranoid", &sb) == 0 &&
+       get_sysfs_dev_dir(brw, sysfs_dev_dir, sizeof(sysfs_dev_dir)) &&
+       init_oa_sys_vars(brw, sysfs_dev_dir))
+   {
+      brw->perfquery.oa_metrics_table =
+         _mesa_hash_table_create(NULL, _mesa_key_hash_string,
+                                 _mesa_key_string_equal);
+
+      /* Index all the metric sets mesa knows about before looking to
+       * see what the kernel is advertising.
+       */
+      brw_oa_register_queries_hsw(brw);
+
+      enumerate_sysfs_metrics(brw, sysfs_dev_dir);
+   }
+
+   brw->perfquery.unaccumulated =
+      ralloc_array(brw, struct brw_perf_query_object *, 2);
+   brw->perfquery.unaccumulated_elements = 0;
+   brw->perfquery.unaccumulated_array_size = 2;
+
+   exec_list_make_empty(&brw->perfquery.sample_buffers);
+   exec_list_make_empty(&brw->perfquery.free_sample_buffers);
+
+   /* It's convenient to guarantee that this linked list of sample
+    * buffers is never empty so we add an empty head so when we
+    * Begin an OA query we can always take a reference on a buffer
+    * in this list.
+    */
+   struct brw_oa_sample_buf *buf = get_free_sample_buf(brw);
+   exec_list_push_head(&brw->perfquery.sample_buffers, &buf->link);
+
+   brw->perfquery.oa_stream_fd = -1;
+
+   brw->perfquery.next_query_start_report_id = 1000;
+
    return brw->perfquery.n_queries;
 }