diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/freedreno/Makefile.sources | 1 | ||||
| -rw-r--r-- | src/freedreno/ir3/ir3.h | 3 | ||||
| -rw-r--r-- | src/freedreno/ir3/ir3_compiler_nir.c | 3 | ||||
| -rw-r--r-- | src/freedreno/ir3/ir3_postsched.c | 669 | ||||
| -rw-r--r-- | src/freedreno/ir3/ir3_ra.c | 7 | ||||
| -rw-r--r-- | src/freedreno/ir3/meson.build | 1 |
6 files changed, 679 insertions, 5 deletions
diff --git a/src/freedreno/Makefile.sources b/src/freedreno/Makefile.sources index fb0b9f2159f..c67fa375e03 100644 --- a/src/freedreno/Makefile.sources +++ b/src/freedreno/Makefile.sources @@ -44,6 +44,7 @@ ir3_SOURCES := \ ir3/ir3_nir_lower_tex_prefetch.c \ ir3/ir3_nir_lower_tg4_to_tex.c \ ir3/ir3_nir_move_varying_inputs.c \ + ir3/ir3_postsched.c \ ir3/ir3_print.c \ ir3/ir3_ra.c \ ir3/ir3_sched.c \ diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index b10f2f0da79..f78a7d6f364 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -1138,6 +1138,9 @@ void ir3_sun(struct ir3 *ir); void ir3_sched_add_deps(struct ir3 *ir); int ir3_sched(struct ir3 *ir); +struct ir3_context; +int ir3_postsched(struct ir3_context *ctx); + bool ir3_a6xx_fixup_atomic_dests(struct ir3 *ir, struct ir3_shader_variant *so); /* register assignment: */ diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index 13052648814..52a049a4123 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -3481,6 +3481,9 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, ir3_debug_print(ir, "AFTER RA"); + ir3_postsched(ctx); + ir3_debug_print(ir, "AFTER POSTSCHED"); + if (compiler->gpu_id >= 600) { if (ir3_a6xx_fixup_atomic_dests(ir, so)) { ir3_debug_print(ir, "AFTER ATOMIC FIXUP"); diff --git a/src/freedreno/ir3/ir3_postsched.c b/src/freedreno/ir3/ir3_postsched.c new file mode 100644 index 00000000000..4290e882249 --- /dev/null +++ b/src/freedreno/ir3/ir3_postsched.c @@ -0,0 +1,669 @@ +/* + * Copyright (C) 2019 Google, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Rob Clark <[email protected]> + */ + + +#include "util/dag.h" +#include "util/u_math.h" + +#include "ir3.h" +#include "ir3_compiler.h" +#include "ir3_context.h" + +#ifdef DEBUG +#define SCHED_DEBUG (ir3_shader_debug & IR3_DBG_SCHEDMSGS) +#else +#define SCHED_DEBUG 0 +#endif +#define d(fmt, ...) do { if (SCHED_DEBUG) { \ + printf("PSCHED: "fmt"\n", ##__VA_ARGS__); \ +} } while (0) + +#define di(instr, fmt, ...) do { if (SCHED_DEBUG) { \ + printf("PSCHED: "fmt": ", ##__VA_ARGS__); \ + ir3_print_instr(instr); \ +} } while (0) + +/* + * Post RA Instruction Scheduling + */ + +struct ir3_postsched_ctx { + struct ir3_context *ctx; + + void *mem_ctx; + struct ir3_block *block; /* the current block */ + struct dag *dag; + + struct list_head unscheduled_list; /* unscheduled instructions */ + struct ir3_instruction *scheduled; /* last scheduled instr */ + struct ir3_instruction *pred; /* current p0.x user, if any */ + + bool error; +}; + +struct ir3_postsched_node { + struct dag_node dag; /* must be first for util_dynarray_foreach */ + struct ir3_instruction *instr; + bool partially_evaluated_path; + + unsigned delay; + unsigned max_delay; +}; + +#define foreach_sched_node(__n, __list) \ + list_for_each_entry(struct ir3_postsched_node, __n, __list, dag.link) + +#define foreach_bit(b, mask) \ + for (uint32_t _m = ({debug_assert((mask) >= 1); (mask);}); _m && ({(b) = u_bit_scan(&_m); 1;});) + +// TODO deduplicate +static bool is_sfu_or_mem(struct ir3_instruction *instr) +{ + return is_sfu(instr) || is_mem(instr); +} + +static void +schedule(struct ir3_postsched_ctx *ctx, struct ir3_instruction *instr) +{ + debug_assert(ctx->block == instr->block); + + /* maybe there is a better way to handle this than just stuffing + * a nop.. ideally we'd know about this constraint in the + * scheduling and depth calculation.. + */ + if (ctx->scheduled && is_sfu_or_mem(ctx->scheduled) && is_sfu_or_mem(instr)) + ir3_NOP(ctx->block); + + /* remove from unscheduled_list: + */ + list_delinit(&instr->node); + + if (writes_pred(instr)) { + ctx->pred = instr; + } + + di(instr, "schedule"); + + list_addtail(&instr->node, &instr->block->instr_list); + ctx->scheduled = instr; + + struct ir3_postsched_node *n = instr->data; + dag_prune_head(ctx->dag, &n->dag); +} + +static void +dump_state(struct ir3_postsched_ctx *ctx) +{ + if (!SCHED_DEBUG) + return; + + foreach_sched_node (n, &ctx->dag->heads) { + di(n->instr, "maxdel=%3d ", n->max_delay); + + util_dynarray_foreach(&n->dag.edges, struct dag_edge, edge) { + struct ir3_postsched_node *child = + (struct ir3_postsched_node *)edge->child; + + di(child->instr, " -> (%d parents) ", child->dag.parent_count); + } + } +} + +/* find instruction to schedule: */ +static struct ir3_instruction * +choose_instr(struct ir3_postsched_ctx *ctx) +{ + struct ir3_postsched_node *chosen = NULL; + + dump_state(ctx); + + foreach_sched_node (n, &ctx->dag->heads) { + if (!is_meta(n->instr)) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "prio: chose (meta)"); + return chosen->instr; + } + + /* Try to schedule inputs with a higher priority, if possible, as + * the last bary.f unlocks varying storage to unblock more VS + * warps. + */ + foreach_sched_node (n, &ctx->dag->heads) { + if (!is_input(n->instr)) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "prio: chose (input)"); + return chosen->instr; + } + + /* Next prioritize discards: */ + foreach_sched_node (n, &ctx->dag->heads) { + unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); + + if (d > 0) + continue; + + if (!is_kill(n->instr)) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "csp: chose (kill, hard ready)"); + return chosen->instr; + } + + /* Next prioritize expensive instructions: */ + foreach_sched_node (n, &ctx->dag->heads) { + unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); + + if (d > 0) + continue; + + if (!(is_sfu(n->instr) || is_tex(n->instr))) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "csp: chose (sfu/tex, hard ready)"); + return chosen->instr; + } + + /* First try to find a ready leader w/ soft delay (ie. including extra + * delay for things like tex fetch which can be synchronized w/ sync + * bit (but we probably do want to schedule some other instructions + * while we wait) + */ + foreach_sched_node (n, &ctx->dag->heads) { + unsigned d = ir3_delay_calc(ctx->block, n->instr, true, false); + + if (d > 0) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "csp: chose (soft ready)"); + return chosen->instr; + } + + /* Next try to find a ready leader that can be scheduled without nop's, + * which in the case of things that need (sy)/(ss) could result in + * stalls.. but we've already decided there is not a better option. + */ + foreach_sched_node (n, &ctx->dag->heads) { + unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); + + if (d > 0) + continue; + + if (!chosen || (chosen->max_delay < n->max_delay)) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "csp: chose (hard ready)"); + return chosen->instr; + } + + /* Otherwise choose leader with maximum cost: + * + * TODO should we try to balance cost and delays? I guess it is + * a balance between now-nop's and future-nop's? + */ + foreach_sched_node (n, &ctx->dag->heads) { + if (!chosen || chosen->max_delay < n->max_delay) + chosen = n; + } + + if (chosen) { + di(chosen->instr, "csp: chose (leader)"); + return chosen->instr; + } + + return NULL; +} + +struct ir3_postsched_deps_state { + struct ir3_context *ctx; + + enum { F, R } direction; + + bool merged; + + /* Track the mapping between sched node (instruction) that last + * wrote a given register (in whichever direction we are iterating + * the block) + * + * Note, this table is twice as big as the # of regs, to deal with + * half-precision regs. The approach differs depending on whether + * the half and full precision register files are "merged" (conflict, + * ie. a6xx+) in which case we consider each full precision dep + * as two half-precision dependencies, vs older separate (non- + * conflicting) in which case the first half of the table is used + * for full precision and 2nd half for half-precision. + */ + struct ir3_postsched_node *regs[2 * 256]; +}; + +/* bounds checking read/write accessors, since OoB access to stuff on + * the stack is gonna cause a bad day. + */ +#define dep_reg(state, idx) *({ \ + assert((idx) < ARRAY_SIZE((state)->regs)); \ + &(state)->regs[(idx)]; \ + }) + +static void +add_dep(struct ir3_postsched_deps_state *state, + struct ir3_postsched_node *before, + struct ir3_postsched_node *after) +{ + if (!before || !after) + return; + + assert(before != after); + + if (state->direction == F) { + dag_add_edge(&before->dag, &after->dag, NULL); + } else { + dag_add_edge(&after->dag, &before->dag, NULL); + } +} + +static void +add_single_reg_dep(struct ir3_postsched_deps_state *state, + struct ir3_postsched_node *node, unsigned num, bool write) +{ + add_dep(state, dep_reg(state, num), node); + if (write) { + dep_reg(state, num) = node; + } +} + +/* This is where we handled full vs half-precision, and potential conflicts + * between half and full precision that result in additional dependencies. + * The 'reg' arg is really just to know half vs full precision. + */ +static void +add_reg_dep(struct ir3_postsched_deps_state *state, + struct ir3_postsched_node *node, const struct ir3_register *reg, + unsigned num, bool write) +{ + if (state->merged) { + if (reg->flags & IR3_REG_HALF) { + /* single conflict in half-reg space: */ + add_single_reg_dep(state, node, num, write); + } else { + /* two conflicts in half-reg space: */ + add_single_reg_dep(state, node, 2 * num + 0, write); + add_single_reg_dep(state, node, 2 * num + 1, write); + } + } else { + if (reg->flags & IR3_REG_HALF) + num += ARRAY_SIZE(state->regs) / 2; + add_single_reg_dep(state, node, num, write); + } +} + +static void +calculate_deps(struct ir3_postsched_deps_state *state, + struct ir3_postsched_node *node) +{ + static const struct ir3_register half_reg = { .flags = IR3_REG_HALF }; + struct ir3_register *reg; + int b; + + /* Add dependencies on instructions that previously (or next, + * in the reverse direction) wrote any of our src registers: + */ + foreach_src_n (reg, i, node->instr) { + /* NOTE: relative access for a src can be either const or gpr: */ + if (reg->flags & IR3_REG_RELATIV) { + /* also reads a0.x: */ + add_reg_dep(state, node, &half_reg, regid(REG_A0, 0), false); + } + + if (reg->flags & (IR3_REG_CONST | IR3_REG_IMMED)) + continue; + + if (reg->flags & IR3_REG_RELATIV) { + /* mark entire array as read: */ + struct ir3_array *arr = ir3_lookup_array(state->ctx->ir, reg->array.id); + for (unsigned i = 0; i < arr->length; i++) { + add_reg_dep(state, node, reg, arr->reg + i, false); + } + } else { + foreach_bit (b, reg->wrmask) { + add_reg_dep(state, node, reg, reg->num + b, false); + + struct ir3_postsched_node *dep = dep_reg(state, reg->num + b); + if (dep && (state->direction == F)) { + unsigned d = ir3_delayslots(dep->instr, node->instr, i); + node->delay = MAX2(node->delay, d); + } + } + } + } + + if (dest_regs(node->instr) == 0) + return; + + /* And then after we update the state for what this instruction + * wrote: + */ + reg = node->instr->regs[0]; + if (reg->flags & IR3_REG_RELATIV) { + /* mark the entire array as written: */ + struct ir3_array *arr = ir3_lookup_array(state->ctx->ir, reg->array.id); + for (unsigned i = 0; i < arr->length; i++) { + add_reg_dep(state, node, reg, arr->reg + i, true); + } + + /* also reads a0.x: */ + add_reg_dep(state, node, &half_reg, regid(REG_A0, 0), false); + } else { + foreach_bit (b, reg->wrmask) { + add_reg_dep(state, node, reg, reg->num + b, true); + } + } +} + +static void +calculate_forward_deps(struct ir3_postsched_ctx *ctx) +{ + struct ir3_postsched_deps_state state = { + .ctx = ctx->ctx, + .direction = F, + .merged = ctx->ctx->compiler->gpu_id >= 600, + }; + + foreach_instr (instr, &ctx->unscheduled_list) { + calculate_deps(&state, instr->data); + } +} + +static void +calculate_reverse_deps(struct ir3_postsched_ctx *ctx) +{ + struct ir3_postsched_deps_state state = { + .ctx = ctx->ctx, + .direction = R, + .merged = ctx->ctx->compiler->gpu_id >= 600, + }; + + foreach_instr_rev (instr, &ctx->unscheduled_list) { + calculate_deps(&state, instr->data); + } +} + +static void +sched_node_init(struct ir3_postsched_ctx *ctx, struct ir3_instruction *instr) +{ + struct ir3_postsched_node *n = rzalloc(ctx->mem_ctx, struct ir3_postsched_node); + + dag_init_node(ctx->dag, &n->dag); + + n->instr = instr; + instr->data = n; +} + +static void +sched_dag_max_delay_cb(struct dag_node *node, void *state) +{ + struct ir3_postsched_node *n = (struct ir3_postsched_node *)node; + uint32_t max_delay = 0; + + util_dynarray_foreach(&n->dag.edges, struct dag_edge, edge) { + struct ir3_postsched_node *child = (struct ir3_postsched_node *)edge->child; + max_delay = MAX2(child->max_delay, max_delay); + } + + n->max_delay = MAX2(n->max_delay, max_delay + n->delay); +} + +static void +sched_dag_init(struct ir3_postsched_ctx *ctx) +{ + ctx->mem_ctx = ralloc_context(NULL); + + ctx->dag = dag_create(ctx->mem_ctx); + + foreach_instr (instr, &ctx->unscheduled_list) + sched_node_init(ctx, instr); + + calculate_forward_deps(ctx); + calculate_reverse_deps(ctx); + + /* + * Normal srcs won't be in SSA at this point, those are dealt with in + * calculate_forward_deps() and calculate_reverse_deps(). But we still + * have the false-dep information in SSA form, so go ahead and add + * dependencies for that here: + */ + foreach_instr (instr, &ctx->unscheduled_list) { + struct ir3_postsched_node *n = instr->data; + struct ir3_instruction *src; + + foreach_ssa_src_n (src, i, instr) { + if (src->block != instr->block) + continue; + + /* we can end up with unused false-deps.. just skip them: */ + if (src->flags & IR3_INSTR_UNUSED) + continue; + + struct ir3_postsched_node *sn = src->data; + + /* don't consider dependencies in other blocks: */ + if (src->block != instr->block) + continue; + + dag_add_edge(&sn->dag, &n->dag, NULL); + } + } + + // TODO do we want to do this after reverse-dependencies? + dag_traverse_bottom_up(ctx->dag, sched_dag_max_delay_cb, NULL); +} + +static void +sched_dag_destroy(struct ir3_postsched_ctx *ctx) +{ + ralloc_free(ctx->mem_ctx); + ctx->mem_ctx = NULL; + ctx->dag = NULL; +} + +static void +sched_block(struct ir3_postsched_ctx *ctx, struct ir3_block *block) +{ + ctx->block = block; + ctx->scheduled = NULL; + ctx->pred = NULL; + + /* move all instructions to the unscheduled list, and + * empty the block's instruction list (to which we will + * be inserting). + */ + list_replace(&block->instr_list, &ctx->unscheduled_list); + list_inithead(&block->instr_list); + + // TODO once we are using post-sched for everything we can + // just not stick in NOP's prior to post-sched, and drop this. + // for now keep this, since it makes post-sched optional: + foreach_instr_safe (instr, &ctx->unscheduled_list) { + switch (instr->opc) { + case OPC_NOP: + case OPC_BR: + case OPC_JUMP: + list_delinit(&instr->node); + break; + default: + break; + } + } + + sched_dag_init(ctx); + + /* First schedule all meta:input instructions, followed by + * tex-prefetch. We want all of the instructions that load + * values into registers before the shader starts to go + * before any other instructions. But in particular we + * want inputs to come before prefetches. This is because + * a FS's bary_ij input may not actually be live in the + * shader, but it should not be scheduled on top of any + * other input (but can be overwritten by a tex prefetch) + */ + foreach_instr_safe (instr, &ctx->unscheduled_list) + if (instr->opc == OPC_META_INPUT) + schedule(ctx, instr); + + foreach_instr_safe (instr, &ctx->unscheduled_list) + if (instr->opc == OPC_META_TEX_PREFETCH) + schedule(ctx, instr); + + while (!list_is_empty(&ctx->unscheduled_list)) { + struct ir3_instruction *instr; + + instr = choose_instr(ctx); + + /* this shouldn't happen: */ + if (!instr) { + ctx->error = true; + break; + } + + unsigned delay = ir3_delay_calc(ctx->block, instr, false, false); + d("delay=%u", delay); + + /* and if we run out of instructions that can be scheduled, + * then it is time for nop's: + */ + debug_assert(delay <= 6); + while (delay > 0) { + ir3_NOP(block); + delay--; + } + + schedule(ctx, instr); + } + + sched_dag_destroy(ctx); +} + + +static bool +is_self_mov(struct ir3_instruction *instr) +{ + if (!is_same_type_mov(instr)) + return false; + + if (instr->regs[0]->num != instr->regs[1]->num) + return false; + + if (instr->regs[0]->flags & IR3_REG_RELATIV) + return false; + + if (instr->regs[1]->flags & (IR3_REG_CONST | IR3_REG_IMMED | + IR3_REG_RELATIV | IR3_REG_FNEG | IR3_REG_FABS | + IR3_REG_SNEG | IR3_REG_SABS | IR3_REG_BNOT | + IR3_REG_EVEN | IR3_REG_POS_INF)) + return false; + + return true; +} + +/* sometimes we end up w/ in-place mov's, ie. mov.u32u32 r1.y, r1.y + * as a result of places were before RA we are not sure that it is + * safe to eliminate. We could eliminate these earlier, but sometimes + * they are tangled up in false-dep's, etc, so it is easier just to + * let them exist until after RA + */ +static void +cleanup_self_movs(struct ir3 *ir) +{ + foreach_block (block, &ir->block_list) { + foreach_instr_safe (instr, &block->instr_list) { + struct ir3_register *reg; + + foreach_src (reg, instr) { + if (!reg->instr) + continue; + + if (is_self_mov(reg->instr)) { + list_delinit(®->instr->node); + reg->instr = reg->instr->regs[1]->instr; + } + } + + for (unsigned i = 0; i < instr->deps_count; i++) { + if (is_self_mov(instr->deps[i])) { + list_delinit(&instr->deps[i]->node); + instr->deps[i] = instr->deps[i]->regs[1]->instr; + } + } + } + } +} + +int +ir3_postsched(struct ir3_context *cctx) +{ + struct ir3_postsched_ctx ctx = { + .ctx = cctx, + }; + + ir3_remove_nops(cctx->ir); + cleanup_self_movs(cctx->ir); + + foreach_block (block, &cctx->ir->block_list) { + sched_block(&ctx, block); + } + + if (ctx.error) + return -1; + + return 0; +} diff --git a/src/freedreno/ir3/ir3_ra.c b/src/freedreno/ir3/ir3_ra.c index 823968a5cec..0d1028a76d7 100644 --- a/src/freedreno/ir3/ir3_ra.c +++ b/src/freedreno/ir3/ir3_ra.c @@ -385,14 +385,11 @@ size_to_class(unsigned sz, bool half, bool high) static bool writes_gpr(struct ir3_instruction *instr) { - if (is_store(instr)) - return false; - if (instr->regs_count == 0) + if (dest_regs(instr) == 0) return false; /* is dest a normal temp register: */ struct ir3_register *reg = instr->regs[0]; - if (reg->flags & (IR3_REG_CONST | IR3_REG_IMMED)) - return false; + debug_assert(!(reg->flags & (IR3_REG_CONST | IR3_REG_IMMED))); if ((reg->num == regid(REG_A0, 0)) || (reg->num == regid(REG_P0, 0))) return false; diff --git a/src/freedreno/ir3/meson.build b/src/freedreno/ir3/meson.build index 0c2cb21930a..89ec273034e 100644 --- a/src/freedreno/ir3/meson.build +++ b/src/freedreno/ir3/meson.build @@ -71,6 +71,7 @@ libfreedreno_ir3_files = files( 'ir3_nir_lower_tex_prefetch.c', 'ir3_nir_lower_tg4_to_tex.c', 'ir3_nir_move_varying_inputs.c', + 'ir3_postsched.c', 'ir3_print.c', 'ir3_ra.c', 'ir3_sched.c', |