vc4: Make scheduling of NOPs a separate step from QIR -> QPU translation.

This should also be used as a way to pair QIR instructions into QPU
instructions later.

3 files changed, 212 insertions, 90 deletions

diff --git a/src/gallium/drivers/vc4/vc4_program.c b/src/gallium/drivers/vc4/vc4_program.c
index 12d188185cd..0e415a8baf6 100644
--- a/src/gallium/drivers/vc4/vc4_program.c
+++ b/src/gallium/drivers/vc4/vc4_program.c
@@ -600,7 +600,7 @@ vc4_shader_tgsi_to_qir(struct vc4_compiled_shader *shader, enum qstage stage,
 
         if (vc4_debug & VC4_DEBUG_SHADERDB) {
                 fprintf(stderr, "SHADER-DB: %s: %d instructions\n",
-                        qir_get_stage_name(c->stage), c->num_qpu_insts);
+                        qir_get_stage_name(c->stage), c->qpu_inst_count);
                 fprintf(stderr, "SHADER-DB: %s: %d uniforms\n",
                         qir_get_stage_name(c->stage), trans->num_uniforms);
         }
@@ -647,7 +647,7 @@ vc4_fs_compile(struct vc4_context *vc4, struct vc4_compiled_shader *shader,
         shader->num_inputs = trans->num_inputs;
         copy_uniform_state_to_shader(shader, 0, trans);
         shader->bo = vc4_bo_alloc_mem(vc4->screen, trans->c->qpu_insts,
-                                      trans->c->num_qpu_insts * sizeof(uint64_t),
+                                      trans->c->qpu_inst_count * sizeof(uint64_t),
                                       "fs_code");
 
         qir_compile_destroy(trans->c);
@@ -668,8 +668,8 @@ vc4_vs_compile(struct vc4_context *vc4, struct vc4_compiled_shader *shader,
                                                               &key->base);
         copy_uniform_state_to_shader(shader, 1, cs_trans);
 
-        uint32_t vs_size = vs_trans->c->num_qpu_insts * sizeof(uint64_t);
-        uint32_t cs_size = cs_trans->c->num_qpu_insts * sizeof(uint64_t);
+        uint32_t vs_size = vs_trans->c->qpu_inst_count * sizeof(uint64_t);
+        uint32_t cs_size = cs_trans->c->qpu_inst_count * sizeof(uint64_t);
         shader->coord_shader_offset = vs_size; /* XXX: alignment? */
         shader->bo = vc4_bo_alloc(vc4->screen,
                                   shader->coord_shader_offset + cs_size,
diff --git a/src/gallium/drivers/vc4/vc4_qir.h b/src/gallium/drivers/vc4/vc4_qir.h
index 75b1933e6a6..cd7133cdc88 100644
--- a/src/gallium/drivers/vc4/vc4_qir.h
+++ b/src/gallium/drivers/vc4/vc4_qir.h
@@ -126,8 +126,10 @@ struct qcompile {
         struct simple_node instructions;
         uint32_t immediates[1024];
 
+        struct simple_node qpu_inst_list;
         uint64_t *qpu_insts;
-        uint32_t num_qpu_insts;
+        uint32_t qpu_inst_count;
+        uint32_t qpu_inst_size;
 };
 
 struct qcompile *qir_compile_init(void);
diff --git a/src/gallium/drivers/vc4/vc4_qpu_emit.c b/src/gallium/drivers/vc4/vc4_qpu_emit.c
index f02fe822b07..b8a506eb174 100644
--- a/src/gallium/drivers/vc4/vc4_qpu_emit.c
+++ b/src/gallium/drivers/vc4/vc4_qpu_emit.c
@@ -33,13 +33,34 @@ vc4_dump_program(struct qcompile *c)
 {
         fprintf(stderr, "%s:\n", qir_get_stage_name(c->stage));
 
-        for (int i = 0; i < c->num_qpu_insts; i++) {
+        for (int i = 0; i < c->qpu_inst_count; i++) {
                 fprintf(stderr, "0x%016"PRIx64" ", c->qpu_insts[i]);
                 vc4_qpu_disasm(&c->qpu_insts[i], 1);
                 fprintf(stderr, "\n");
         }
 }
 
+struct queued_qpu_inst {
+        struct simple_node link;
+        uint64_t inst;
+};
+
+static void
+queue(struct qcompile *c, uint64_t inst)
+{
+        struct queued_qpu_inst *q = calloc(1, sizeof(*q));
+        q->inst = inst;
+        insert_at_tail(&c->qpu_inst_list, &q->link);
+}
+
+static uint64_t *
+last_inst(struct qcompile *c)
+{
+        struct queued_qpu_inst *q =
+                (struct queued_qpu_inst *)last_elem(&c->qpu_inst_list);
+        return &q->inst;
+}
+
 /**
  * This is used to resolve the fact that we might register-allocate two
  * different operands of an instruction to the same physical register file
@@ -50,23 +71,119 @@ vc4_dump_program(struct qcompile *c)
  * instruction, instead.
  */
 static void
-fixup_raddr_conflict(uint64_t *insts, uint32_t *ni,
+fixup_raddr_conflict(struct qcompile *c,
                struct qpu_reg src0, struct qpu_reg *src1)
 {
         if ((src0.mux == QPU_MUX_A || src0.mux == QPU_MUX_B) &&
             (src1->mux == QPU_MUX_A || src1->mux == QPU_MUX_B) &&
             src0.addr != src1->addr) {
-                insts[(*ni)++] = qpu_inst(qpu_a_MOV(qpu_r3(), *src1),
-                                          qpu_m_NOP());
+                queue(c, qpu_inst(qpu_a_MOV(qpu_r3(), *src1),
+                                  qpu_m_NOP()));
                 *src1 = qpu_r3();
         }
 }
 
+static void
+serialize_one_inst(struct qcompile *c, uint64_t inst)
+{
+        if (c->qpu_inst_count >= c->qpu_inst_size) {
+                c->qpu_inst_size = MAX2(16, c->qpu_inst_size * 2);
+                c->qpu_insts = realloc(c->qpu_insts,
+                                       c->qpu_inst_size * sizeof(uint64_t));
+        }
+        c->qpu_insts[c->qpu_inst_count++] = inst;
+}
+
+static void
+serialize_insts(struct qcompile *c)
+{
+        int last_sfu_write = -10;
+
+        while (!is_empty_list(&c->qpu_inst_list)) {
+                struct queued_qpu_inst *q =
+                        (struct queued_qpu_inst *)first_elem(&c->qpu_inst_list);
+                uint32_t last_waddr_a = QPU_W_NOP, last_waddr_b = QPU_W_NOP;
+                uint32_t raddr_a = QPU_GET_FIELD(q->inst, QPU_RADDR_A);
+                uint32_t raddr_b = QPU_GET_FIELD(q->inst, QPU_RADDR_B);
+
+                if (c->qpu_inst_count > 0) {
+                        uint64_t last_inst = c->qpu_insts[c->qpu_inst_count -
+                                                          1];
+                        uint32_t last_waddr_add = QPU_GET_FIELD(last_inst,
+                                                                QPU_WADDR_ADD);
+                        uint32_t last_waddr_mul = QPU_GET_FIELD(last_inst,
+                                                                QPU_WADDR_MUL);
+
+                        if (last_inst & QPU_WS) {
+                                last_waddr_a = last_waddr_mul;
+                                last_waddr_b = last_waddr_add;
+                        } else {
+                                last_waddr_a = last_waddr_add;
+                                last_waddr_b = last_waddr_mul;
+                        }
+                }
+
+                uint32_t src_muxes[] = {
+                        QPU_GET_FIELD(q->inst, QPU_ADD_A),
+                        QPU_GET_FIELD(q->inst, QPU_ADD_B),
+                        QPU_GET_FIELD(q->inst, QPU_MUL_A),
+                        QPU_GET_FIELD(q->inst, QPU_MUL_B),
+                };
+
+                /* "An instruction must not read from a location in physical
+                 *  regfile A or B that was written to by the previous
+                 *  instruction."
+                 */
+                bool needs_raddr_vs_waddr_nop = false;
+                bool reads_r4 = false;
+                for (int i = 0; i < ARRAY_SIZE(src_muxes); i++) {
+                        if ((raddr_a < 32 &&
+                             src_muxes[i] == QPU_MUX_A &&
+                             last_waddr_a == raddr_a) ||
+                            (raddr_b < 32 &&
+                             src_muxes[i] == QPU_MUX_B &&
+                             last_waddr_b == raddr_b)) {
+                                needs_raddr_vs_waddr_nop = true;
+                        }
+                        if (src_muxes[i] == QPU_MUX_R4)
+                                reads_r4 = true;
+                }
+
+                if (needs_raddr_vs_waddr_nop) {
+                        serialize_one_inst(c, qpu_inst(qpu_a_NOP(),
+                                                       qpu_m_NOP()));
+                }
+
+                /* "After an SFU lookup instruction, accumulator r4 must not
+                 *  be read in the following two instructions. Any other
+                 *  instruction that results in r4 being written (that is, TMU
+                 *  read, TLB read, SFU lookup) cannot occur in the two
+                 *  instructions following an SFU lookup."
+                 */
+                if (reads_r4) {
+                        while (c->qpu_inst_count - last_sfu_write < 3) {
+                                serialize_one_inst(c, qpu_inst(qpu_a_NOP(),
+                                                               qpu_m_NOP()));
+                        }
+                }
+
+                uint32_t waddr_a = QPU_GET_FIELD(q->inst, QPU_WADDR_ADD);
+                uint32_t waddr_m = QPU_GET_FIELD(q->inst, QPU_WADDR_MUL);
+                if ((waddr_a >= QPU_W_SFU_RECIP && waddr_a <= QPU_W_SFU_LOG) ||
+                    (waddr_m >= QPU_W_SFU_RECIP && waddr_m <= QPU_W_SFU_LOG)) {
+                        last_sfu_write = c->qpu_inst_count;
+                }
+
+                serialize_one_inst(c, q->inst);
+
+                remove_from_list(&q->link);
+                free(q);
+        }
+}
+
 void
 vc4_generate_code(struct qcompile *c)
 {
-        uint64_t *insts = malloc(sizeof(uint64_t) * 1024); /* XXX: sizing */
-        uint32_t ni = 0;
         struct qpu_reg allocate_to_qpu_reg[3 + 32 + 32];
         bool reg_in_use[ARRAY_SIZE(allocate_to_qpu_reg)];
         int *reg_allocated = calloc(c->num_temps, sizeof(*reg_allocated));
@@ -84,6 +201,8 @@ vc4_generate_code(struct qcompile *c)
         for (int i = 0; i < 32; i++)
                 allocate_to_qpu_reg[i + 3 + 32] = qpu_rb(i);
 
+        make_empty_list(&c->qpu_inst_list);
+
         struct simple_node *node;
         foreach(node, &c->instructions) {
                 struct qinst *qinst = (struct qinst *)node;
@@ -99,8 +218,8 @@ vc4_generate_code(struct qcompile *c)
         switch (c->stage) {
         case QSTAGE_VERT:
         case QSTAGE_COORD:
-                insts[ni++] = qpu_load_imm_ui(qpu_vrsetup(), 0x00401a00);
-                insts[ni++] = qpu_load_imm_ui(qpu_vwsetup(), 0x00001a00);
+                queue(c, qpu_load_imm_ui(qpu_vrsetup(), 0x00401a00));
+                queue(c, qpu_load_imm_ui(qpu_vwsetup(), 0x00001a00));
                 break;
         case QSTAGE_FRAG:
                 break;
@@ -208,8 +327,8 @@ vc4_generate_code(struct qcompile *c)
                         /* Skip emitting the MOV if it's a no-op. */
                         if (dst.mux == QPU_MUX_A || dst.mux == QPU_MUX_B ||
                             dst.mux != src[0].mux || dst.addr != src[0].addr) {
-                                insts[ni++] = qpu_inst(qpu_a_MOV(dst, src[0]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_MOV(dst, src[0]),
+                                                  qpu_m_NOP()));
                         }
                         break;
 
@@ -217,30 +336,28 @@ vc4_generate_code(struct qcompile *c)
                 case QOP_SNE:
                 case QOP_SGE:
                 case QOP_SLT:
-                        fixup_raddr_conflict(insts, &ni, src[0], &src[1]);
-                        insts[ni++] = qpu_inst(qpu_a_SUB(qpu_ra(QPU_W_NOP),
-                                                         src[0], src[1]),
-                                               qpu_m_NOP());
-                        insts[ni - 1] |= QPU_SF;
-
-                        insts[ni++] = qpu_load_imm_f(dst, 0.0);
-                        insts[ni++] = qpu_load_imm_f(dst, 1.0);
-                        insts[ni - 1] = ((insts[ni - 1] & ~QPU_COND_ADD_MASK)
+                        fixup_raddr_conflict(c, src[0], &src[1]);
+                        queue(c, qpu_inst(qpu_a_SUB(qpu_ra(QPU_W_NOP),
+                                                    src[0], src[1]),
+                                          qpu_m_NOP()));
+                        *last_inst(c) |= QPU_SF;
+
+                        queue(c, qpu_load_imm_f(dst, 0.0));
+                        queue(c, qpu_load_imm_f(dst, 1.0));
+                        *last_inst(c) = ((*last_inst(c) & ~QPU_COND_ADD_MASK)
                                          | QPU_SET_FIELD(compareflags[qinst->op - QOP_SEQ],
                                                          QPU_COND_ADD));
 
                         break;
 
                 case QOP_VPM_WRITE:
-                        insts[ni++] = qpu_inst(qpu_a_MOV(qpu_ra(QPU_W_VPM),
-                                                         src[0]),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_MOV(qpu_ra(QPU_W_VPM), src[0]),
+                                          qpu_m_NOP()));
                         break;
 
                 case QOP_VPM_READ:
-                        insts[ni++] = qpu_inst(qpu_a_MOV(dst,
-                                                         qpu_ra(QPU_R_VPM)),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_MOV(dst, qpu_ra(QPU_R_VPM)),
+                                          qpu_m_NOP()));
                         break;
 
                 case QOP_RCP:
@@ -249,70 +366,70 @@ vc4_generate_code(struct qcompile *c)
                 case QOP_LOG2:
                         switch (qinst->op) {
                         case QOP_RCP:
-                                insts[ni++] = qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIP),
-                                                                 src[0]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIP),
+                                                            src[0]),
+                                                  qpu_m_NOP()));
                                 break;
                         case QOP_RSQ:
-                                insts[ni++] = qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIPSQRT),
-                                                                 src[0]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIPSQRT),
+                                                            src[0]),
+                                                  qpu_m_NOP()));
                                 break;
                         case QOP_EXP2:
-                                insts[ni++] = qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_EXP),
-                                                                 src[0]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_EXP),
+                                                            src[0]),
+                                                  qpu_m_NOP()));
                                 break;
                         case QOP_LOG2:
-                                insts[ni++] = qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_LOG),
-                                                                 src[0]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_MOV(qpu_rb(QPU_W_SFU_LOG),
+                                                            src[0]),
+                                                  qpu_m_NOP()));
                                 break;
                         default:
                                 abort();
                         }
 
-                        insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
-                        insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
-
-                        insts[ni++] = qpu_inst(qpu_a_MOV(dst, qpu_r4()),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_MOV(dst, qpu_r4()),
+                                          qpu_m_NOP()));
 
                         break;
 
                 case QOP_PACK_COLORS:
                         for (int i = 0; i < 4; i++) {
-                                insts[ni++] = qpu_inst(qpu_a_NOP(),
-                                                       qpu_m_MOV(qpu_r3(), src[i]));
-                                insts[ni - 1] |= QPU_PM;
-                                insts[ni - 1] |= QPU_SET_FIELD(QPU_PACK_MUL_8A + i,
+                                queue(c, qpu_inst(qpu_a_NOP(),
+                                                  qpu_m_MOV(qpu_r3(), src[i])));
+                                *last_inst(c) |= QPU_PM;
+                                *last_inst(c) |= QPU_SET_FIELD(QPU_PACK_MUL_8A + i,
                                                                QPU_PACK);
                         }
 
-                        insts[ni++] = qpu_inst(qpu_a_MOV(dst, qpu_r3()),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_MOV(dst, qpu_r3()),
+                                          qpu_m_NOP()));
+
                         break;
 
                 case QOP_TLB_COLOR_WRITE:
-                        insts[ni++] = qpu_inst(qpu_a_MOV(qpu_tlbc(),
-                                                         src[0]),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_MOV(qpu_tlbc(),
+                                                    src[0]),
+                                          qpu_m_NOP()));
                         break;
 
                 case QOP_VARY_ADD_C:
-                        insts[ni++] = qpu_inst(qpu_a_FADD(dst,
-                                                          src[0], qpu_r5()),
-                                               qpu_m_NOP());
+                        queue(c, qpu_inst(qpu_a_FADD(dst,
+                                                     src[0], qpu_r5()),
+                                          qpu_m_NOP()));
                         break;
 
                 case QOP_PACK_SCALED:
-                        insts[ni++] = qpu_inst(qpu_a_MOV(dst, src[0]),
-                                               qpu_m_NOP());
-                        insts[ni - 1] |= QPU_SET_FIELD(QPU_PACK_A_16A, QPU_PACK);
+                        queue(c, qpu_inst(qpu_a_MOV(dst, src[0]),
+                                          qpu_m_NOP()));
+                        *last_inst(c) |= QPU_SET_FIELD(QPU_PACK_A_16A,
+                                                       QPU_PACK);
 
-                        insts[ni++] = qpu_inst(qpu_a_MOV(dst, src[1]),
-                                               qpu_m_NOP());
-                        insts[ni - 1] |= QPU_SET_FIELD(QPU_PACK_A_16B, QPU_PACK);
+                        queue(c, qpu_inst(qpu_a_MOV(dst, src[1]),
+                                          qpu_m_NOP()));
+                        *last_inst(c) |= QPU_SET_FIELD(QPU_PACK_A_16B,
+                                                       QPU_PACK);
 
                         break;
 
@@ -327,51 +444,54 @@ vc4_generate_code(struct qcompile *c)
                         if (qir_get_op_nsrc(qinst->op) == 1)
                                 src[1] = src[0];
 
-                        fixup_raddr_conflict(insts, &ni, src[0], &src[1]);
+                        fixup_raddr_conflict(c, src[0], &src[1]);
 
                         if (translate[qinst->op].is_mul) {
-                                insts[ni++] = qpu_inst(qpu_a_NOP(),
-                                                       qpu_m_alu2(translate[qinst->op].op,
-                                                                  dst, src[0], src[1]));
+                                queue(c, qpu_inst(qpu_a_NOP(),
+                                                  qpu_m_alu2(translate[qinst->op].op,
+                                                             dst,
+                                                             src[0], src[1])));
                         } else {
-                                insts[ni++] = qpu_inst(qpu_a_alu2(translate[qinst->op].op,
-                                                                  dst, src[0], src[1]),
-                                                       qpu_m_NOP());
+                                queue(c, qpu_inst(qpu_a_alu2(translate[qinst->op].op,
+                                                             dst,
+                                                             src[0], src[1]),
+                                                  qpu_m_NOP()));
                         }
                         break;
                 }
-
-                if ((dst.mux == QPU_MUX_A || dst.mux == QPU_MUX_B) &&
-                    dst.addr < 32)
-                        insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
         }
 
+        serialize_insts(c);
+
         /* thread end can't have VPM write */
-        if (QPU_GET_FIELD(insts[ni - 1], QPU_WADDR_ADD) == QPU_W_VPM ||
-            QPU_GET_FIELD(insts[ni - 1], QPU_WADDR_MUL) == QPU_W_VPM)
-                insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
+        if (QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
+                          QPU_WADDR_ADD) == QPU_W_VPM ||
+            QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
+                          QPU_WADDR_MUL) == QPU_W_VPM) {
+                serialize_one_inst(c, qpu_inst(qpu_a_NOP(), qpu_m_NOP()));
+        }
 
-        insts[ni - 1] = qpu_set_sig(insts[ni - 1], QPU_SIG_PROG_END);
-        insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
-        insts[ni++] = qpu_inst(qpu_a_NOP(), qpu_m_NOP());
+        c->qpu_insts[c->qpu_inst_count - 1] =
+                qpu_set_sig(c->qpu_insts[c->qpu_inst_count - 1],
+                            QPU_SIG_PROG_END);
+        serialize_one_inst(c, qpu_inst(qpu_a_NOP(), qpu_m_NOP()));
+        serialize_one_inst(c, qpu_inst(qpu_a_NOP(), qpu_m_NOP()));
 
         switch (c->stage) {
         case QSTAGE_VERT:
         case QSTAGE_COORD:
                 break;
         case QSTAGE_FRAG:
-                insts[2] = qpu_set_sig(insts[2], QPU_SIG_WAIT_FOR_SCOREBOARD);
-                insts[ni - 1] = qpu_set_sig(insts[ni - 1],
-                                            QPU_SIG_SCOREBOARD_UNLOCK);
+                c->qpu_insts[2] = qpu_set_sig(c->qpu_insts[2],
+                                              QPU_SIG_WAIT_FOR_SCOREBOARD);
+                c->qpu_insts[c->qpu_inst_count - 1] =
+                        qpu_set_sig(c->qpu_insts[c->qpu_inst_count - 1],
+                                    QPU_SIG_SCOREBOARD_UNLOCK);
                 break;
         }
 
-        c->qpu_insts = insts;
-        c->num_qpu_insts = ni;
-
         if (vc4_debug & VC4_DEBUG_QPU)
                 vc4_dump_program(c);
 
-        vc4_qpu_validate(insts, ni);
+        vc4_qpu_validate(c->qpu_insts, c->qpu_inst_count);
 }
-