tgsi: add BALLOT/READ_* opcodes

v2 (Nicolai):
- BALLOT isn't per-channel
- expand the documentation (also for VOTE_*)

v3:
- only BALLOT returns a 64-bit lanemask (Boyan)
- relax the requirement on READ_INVOC: the invocation number to read
  from must be uniform within a sub-group. This matches the
  GL_ARB_shader_ballot spect (and the v_readlane instruction of AMD
  GCN)

v4:
- hopefully really fix the doc of VOTE_* returns (Ilia)

Signed-off-by: Ilia Mirkin <[email protected]>
Signed-off-by: Nicolai Hähnle <[email protected]>
Reviewed-by: Marek Olšák <[email protected]> (v2)

3 files changed, 63 insertions, 17 deletions

diff --git a/src/gallium/auxiliary/tgsi/tgsi_info.c b/src/gallium/auxiliary/tgsi/tgsi_info.c
index 5a6a9bcdcd1..30bad6d6f7a 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_info.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_info.c
@@ -113,7 +113,7 @@ static const struct tgsi_opcode_info opcode_info[TGSI_OPCODE_LAST] =
    { 0, 0, 0, 0, 0, 0, 0, NONE, "BRK", TGSI_OPCODE_BRK },
    { 0, 1, 0, 0, 1, 0, 1, NONE, "IF", TGSI_OPCODE_IF },
    { 0, 1, 0, 0, 1, 0, 1, NONE, "UIF", TGSI_OPCODE_UIF },
-   { 0, 1, 0, 0, 0, 0, 1, NONE, "", 76 },      /* removed */
+   { 1, 2, 0, 0, 0, 0, 0, COMP, "READ_INVOC", TGSI_OPCODE_READ_INVOC },
    { 0, 0, 0, 0, 1, 1, 1, NONE, "ELSE", TGSI_OPCODE_ELSE },
    { 0, 0, 0, 0, 0, 1, 0, NONE, "ENDIF", TGSI_OPCODE_ENDIF },
    { 1, 1, 0, 0, 0, 0, 0, COMP, "DDX_FINE", TGSI_OPCODE_DDX_FINE },
@@ -125,7 +125,7 @@ static const struct tgsi_opcode_info opcode_info[TGSI_OPCODE_LAST] =
    { 1, 1, 0, 0, 0, 0, 0, COMP, "NOT", TGSI_OPCODE_NOT },
    { 1, 1, 0, 0, 0, 0, 0, COMP, "TRUNC", TGSI_OPCODE_TRUNC },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "SHL", TGSI_OPCODE_SHL },
-   { 0, 0, 0, 0, 0, 0, 0, NONE, "", 88 },      /* removed */
+   { 1, 1, 0, 0, 0, 0, 0, OTHR, "BALLOT", TGSI_OPCODE_BALLOT },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "AND", TGSI_OPCODE_AND },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "OR", TGSI_OPCODE_OR },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "MOD", TGSI_OPCODE_MOD },
@@ -143,7 +143,7 @@ static const struct tgsi_opcode_info opcode_info[TGSI_OPCODE_LAST] =
    { 1, 1, 1, 0, 0, 0, 0, OTHR, "TXQ_LZ", TGSI_OPCODE_TXQ_LZ },
    { 1, 1, 1, 0, 0, 0, 0, OTHR, "TXQS", TGSI_OPCODE_TXQS },
    { 1, 1, 0, 0, 0, 0, 0, OTHR, "RESQ", TGSI_OPCODE_RESQ },
-   { 0, 0, 0, 0, 0, 0, 0, NONE, "", 106 },     /* removed */
+   { 1, 1, 0, 0, 0, 0, 0, COMP, "READ_FIRST", TGSI_OPCODE_READ_FIRST },
    { 0, 0, 0, 0, 0, 0, 0, NONE, "NOP", TGSI_OPCODE_NOP },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "FSEQ", TGSI_OPCODE_FSEQ },
    { 1, 2, 0, 0, 0, 0, 0, COMP, "FSGE", TGSI_OPCODE_FSGE },
diff --git a/src/gallium/docs/source/tgsi.rst b/src/gallium/docs/source/tgsi.rst
index 05b06ce6f1e..9362d430f07 100644
--- a/src/gallium/docs/source/tgsi.rst
+++ b/src/gallium/docs/source/tgsi.rst
@@ -2859,22 +2859,68 @@ only be used with 32-bit integer image formats.
   resource[offset] = (dst_x > src_x ? dst_x : src_x)
 
 
-.. _voteopcodes:
+.. _interlaneopcodes:
+
+Inter-lane opcodes
+^^^^^^^^^^^^^^^^^^
+
+These opcodes reduce the given value across the shader invocations
+running in the current SIMD group. Every thread in the subgroup will receive
+the same result. The BALLOT operations accept a single-channel argument that
+is treated as a boolean and produce a 64-bit value.
+
+.. opcode:: VOTE_ANY - Value is set in any of the active invocations
+
+  Syntax: ``VOTE_ANY dst, value``
+
+  Example: ``VOTE_ANY TEMP[0].x, TEMP[1].x``
+
+
+.. opcode:: VOTE_ALL - Value is set in all of the active invocations
+
+  Syntax: ``VOTE_ALL dst, value``
+
+  Example: ``VOTE_ALL TEMP[0].x, TEMP[1].x``
+
+
+.. opcode:: VOTE_EQ - Value is the same in all of the active invocations
+
+  Syntax: ``VOTE_EQ dst, value``
+
+  Example: ``VOTE_EQ TEMP[0].x, TEMP[1].x``
+
+
+.. opcode:: BALLOT - Lanemask of whether the value is set in each active
+            invocation
+
+  Syntax: ``BALLOT dst, value``
+
+  Example: ``BALLOT TEMP[0].xy, TEMP[1].x``
+
+  When the argument is a constant true, this produces a bitmask of active
+  invocations. In fragment shaders, this can include helper invocations
+  (invocations whose outputs and writes to memory are discarded, but which
+  are used to compute derivatives).
+
+
+.. opcode:: READ_FIRST - Broadcast the value from the first active
+            invocation to all active lanes
+
+  Syntax: ``READ_FIRST dst, value``
+
+  Example: ``READ_FIRST TEMP[0], TEMP[1]``
 
-Vote opcodes
-^^^^^^^^^^^^
 
-These opcodes compare the given value across the shader invocations
-running in the current SIMD group. The details of exactly which
-invocations get compared are implementation-defined, and it would be a
-correct implementation to only ever consider the current thread's
-value. (i.e. SIMD group of 1). The argument is treated as a boolean.
+.. opcode:: READ_INVOC - Retrieve the value from the given invocation
+            (need not be uniform)
 
-.. opcode:: VOTE_ANY - Value is set in any of the current invocations
+  Syntax: ``READ_INVOC dst, value, invocation``
 
-.. opcode:: VOTE_ALL - Value is set in all of the current invocations
+  Example: ``READ_INVOC TEMP[0].xy, TEMP[1].xy, TEMP[2].x``
 
-.. opcode:: VOTE_EQ - Value is the same in all of the current invocations
+  invocation.x controls the invocation number to read from for all channels.
+  The invocation number must be the same across all active invocations in a
+  sub-group; otherwise, the results are undefined.
 
 
 Explanation of symbols used
diff --git a/src/gallium/include/pipe/p_shader_tokens.h b/src/gallium/include/pipe/p_shader_tokens.h
index 3d99a5f88bc..7e8b3caf23b 100644
--- a/src/gallium/include/pipe/p_shader_tokens.h
+++ b/src/gallium/include/pipe/p_shader_tokens.h
@@ -404,7 +404,7 @@ struct tgsi_property_data {
 #define TGSI_OPCODE_BRK                 73
 #define TGSI_OPCODE_IF                  74
 #define TGSI_OPCODE_UIF                 75
-                                /* gap */
+#define TGSI_OPCODE_READ_INVOC          76
 #define TGSI_OPCODE_ELSE                77
 #define TGSI_OPCODE_ENDIF               78
 
@@ -418,7 +418,7 @@ struct tgsi_property_data {
 #define TGSI_OPCODE_NOT                 85
 #define TGSI_OPCODE_TRUNC               86
 #define TGSI_OPCODE_SHL                 87
-                                /* gap */
+#define TGSI_OPCODE_BALLOT              88
 #define TGSI_OPCODE_AND                 89
 #define TGSI_OPCODE_OR                  90
 #define TGSI_OPCODE_MOD                 91
@@ -436,7 +436,7 @@ struct tgsi_property_data {
 #define TGSI_OPCODE_TXQ_LZ              103 /* TXQ for mipmap level 0 */
 #define TGSI_OPCODE_TXQS                104
 #define TGSI_OPCODE_RESQ                105
-                                /* gap */
+#define TGSI_OPCODE_READ_FIRST          106
 #define TGSI_OPCODE_NOP                 107
 
 #define TGSI_OPCODE_FSEQ                108