summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/intel/compiler/brw_eu.h11
-rw-r--r--src/intel/compiler/brw_eu_emit.c141
-rw-r--r--src/intel/compiler/brw_eu_validate.c65
-rw-r--r--src/intel/compiler/brw_inst.h57
4 files changed, 255 insertions, 19 deletions
diff --git a/src/intel/compiler/brw_eu.h b/src/intel/compiler/brw_eu.h
index 2c4ccaecb94..a7041ea4a34 100644
--- a/src/intel/compiler/brw_eu.h
+++ b/src/intel/compiler/brw_eu.h
@@ -811,6 +811,17 @@ brw_send_indirect_message(struct brw_codegen *p,
struct brw_reg desc,
unsigned desc_imm);
+void
+brw_send_indirect_split_message(struct brw_codegen *p,
+ unsigned sfid,
+ struct brw_reg dst,
+ struct brw_reg payload0,
+ struct brw_reg payload1,
+ struct brw_reg desc,
+ unsigned desc_imm,
+ struct brw_reg ex_desc,
+ unsigned ex_desc_imm);
+
void brw_ff_sync(struct brw_codegen *p,
struct brw_reg dest,
unsigned msg_reg_nr,
diff --git a/src/intel/compiler/brw_eu_emit.c b/src/intel/compiler/brw_eu_emit.c
index 06084dcdc50..9be82d1b87c 100644
--- a/src/intel/compiler/brw_eu_emit.c
+++ b/src/intel/compiler/brw_eu_emit.c
@@ -96,7 +96,19 @@ brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest)
gen7_convert_mrf_to_grf(p, &dest);
- {
+ if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC) {
+ assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
+ dest.file == BRW_ARCHITECTURE_REGISTER_FILE);
+ assert(dest.address_mode == BRW_ADDRESS_DIRECT);
+ assert(dest.subnr % 16 == 0);
+ assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1 &&
+ dest.vstride == dest.width + 1);
+ assert(!dest.negate && !dest.abs);
+ brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr);
+ brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16);
+ brw_inst_set_send_dst_reg_file(devinfo, inst, dest.file);
+ } else {
brw_inst_set_dst_file_type(devinfo, inst, dest.file, dest.type);
brw_inst_set_dst_address_mode(devinfo, inst, dest.address_mode);
@@ -177,8 +189,11 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
gen7_convert_mrf_to_grf(p, &reg);
- if (devinfo->gen >= 6 && (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
- brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC)) {
+ if (devinfo->gen >= 6 &&
+ (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC)) {
/* Any source modifiers or regions will be ignored, since this just
* identifies the MRF/GRF to start reading the message contents from.
* Check for some likely failures.
@@ -188,7 +203,17 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
assert(reg.address_mode == BRW_ADDRESS_DIRECT);
}
- {
+ if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC) {
+ assert(reg.file == BRW_GENERAL_REGISTER_FILE);
+ assert(reg.address_mode == BRW_ADDRESS_DIRECT);
+ assert(reg.subnr % 16 == 0);
+ assert(reg.hstride == BRW_HORIZONTAL_STRIDE_1 &&
+ reg.vstride == reg.width + 1);
+ assert(!reg.negate && !reg.abs);
+ brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr);
+ brw_inst_set_src0_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
+ } else {
brw_inst_set_src0_file_type(devinfo, inst, reg.file, reg.type);
brw_inst_set_src0_abs(devinfo, inst, reg.abs);
brw_inst_set_src0_negate(devinfo, inst, reg.negate);
@@ -282,7 +307,18 @@ brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
if (reg.file == BRW_GENERAL_REGISTER_FILE)
assert(reg.nr < 128);
- {
+ if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC) {
+ assert(reg.file == BRW_GENERAL_REGISTER_FILE ||
+ reg.file == BRW_ARCHITECTURE_REGISTER_FILE);
+ assert(reg.address_mode == BRW_ADDRESS_DIRECT);
+ assert(reg.subnr == 0);
+ assert(reg.hstride == BRW_HORIZONTAL_STRIDE_1 &&
+ reg.vstride == reg.width + 1);
+ assert(!reg.negate && !reg.abs);
+ brw_inst_set_send_src1_reg_nr(devinfo, inst, reg.nr);
+ brw_inst_set_send_src1_reg_file(devinfo, inst, reg.file);
+ } else {
/* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
*
* "Accumulator registers may be accessed explicitly as src0
@@ -2484,6 +2520,101 @@ brw_send_indirect_message(struct brw_codegen *p,
brw_inst_set_sfid(devinfo, send, sfid);
}
+void
+brw_send_indirect_split_message(struct brw_codegen *p,
+ unsigned sfid,
+ struct brw_reg dst,
+ struct brw_reg payload0,
+ struct brw_reg payload1,
+ struct brw_reg desc,
+ unsigned desc_imm,
+ struct brw_reg ex_desc,
+ unsigned ex_desc_imm)
+{
+ const struct gen_device_info *devinfo = p->devinfo;
+ struct brw_inst *send;
+
+ dst = retype(dst, BRW_REGISTER_TYPE_UW);
+
+ assert(desc.type == BRW_REGISTER_TYPE_UD);
+
+ if (desc.file == BRW_IMMEDIATE_VALUE) {
+ desc.ud |= desc_imm;
+ } else {
+ struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
+
+ brw_push_insn_state(p);
+ brw_set_default_access_mode(p, BRW_ALIGN_1);
+ brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+ brw_set_default_exec_size(p, BRW_EXECUTE_1);
+ brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
+
+ /* Load the indirect descriptor to an address register using OR so the
+ * caller can specify additional descriptor bits with the desc_imm
+ * immediate.
+ */
+ brw_OR(p, addr, desc, brw_imm_ud(desc_imm));
+
+ brw_pop_insn_state(p);
+ desc = addr;
+ }
+
+ if (ex_desc.file == BRW_IMMEDIATE_VALUE) {
+ ex_desc.ud |= ex_desc_imm;
+ } else {
+ struct brw_reg addr = retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD);
+
+ brw_push_insn_state(p);
+ brw_set_default_access_mode(p, BRW_ALIGN_1);
+ brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+ brw_set_default_exec_size(p, BRW_EXECUTE_1);
+ brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
+
+ /* Load the indirect extended descriptor to an address register using OR
+ * so the caller can specify additional descriptor bits with the
+ * desc_imm immediate.
+ *
+ * Even though the instruction dispatcher always pulls the SFID from the
+ * instruction itself, the extended descriptor sent to the actual unit
+ * gets the SFID from the extended descriptor which comes from the
+ * address register. If we don't OR it in, the external unit gets
+ * confused and hangs the GPU.
+ */
+ brw_OR(p, addr, ex_desc, brw_imm_ud(ex_desc_imm | sfid));
+
+ brw_pop_insn_state(p);
+ ex_desc = addr;
+ }
+
+ send = next_insn(p, BRW_OPCODE_SENDS);
+ brw_set_dest(p, send, dst);
+ brw_set_src0(p, send, retype(payload0, BRW_REGISTER_TYPE_UD));
+ brw_set_src1(p, send, retype(payload1, BRW_REGISTER_TYPE_UD));
+
+ if (desc.file == BRW_IMMEDIATE_VALUE) {
+ brw_inst_set_send_sel_reg32_desc(devinfo, send, 0);
+ brw_inst_set_send_desc(devinfo, send, desc.ud);
+ } else {
+ assert(desc.file == BRW_ARCHITECTURE_REGISTER_FILE);
+ assert(desc.nr == BRW_ARF_ADDRESS);
+ assert(desc.subnr == 0);
+ brw_inst_set_send_sel_reg32_desc(devinfo, send, 1);
+ }
+
+ if (ex_desc.file == BRW_IMMEDIATE_VALUE) {
+ brw_inst_set_send_sel_reg32_ex_desc(devinfo, send, 0);
+ brw_inst_set_send_ex_desc(devinfo, send, ex_desc.ud);
+ } else {
+ assert(ex_desc.file == BRW_ARCHITECTURE_REGISTER_FILE);
+ assert(ex_desc.nr == BRW_ARF_ADDRESS);
+ assert((ex_desc.subnr & 0x3) == 0);
+ brw_inst_set_send_sel_reg32_ex_desc(devinfo, send, 1);
+ brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo, send, ex_desc.subnr >> 2);
+ }
+
+ brw_inst_set_sfid(devinfo, send, sfid);
+}
+
static void
brw_send_indirect_surface_message(struct brw_codegen *p,
unsigned sfid,
diff --git a/src/intel/compiler/brw_eu_validate.c b/src/intel/compiler/brw_eu_validate.c
index 5e505992842..358a0347a93 100644
--- a/src/intel/compiler/brw_eu_validate.c
+++ b/src/intel/compiler/brw_eu_validate.c
@@ -102,6 +102,18 @@ inst_is_send(const struct gen_device_info *devinfo, const brw_inst *inst)
}
}
+static bool
+inst_is_split_send(const struct gen_device_info *devinfo, const brw_inst *inst)
+{
+ switch (brw_inst_opcode(devinfo, inst)) {
+ case BRW_OPCODE_SENDS:
+ case BRW_OPCODE_SENDSC:
+ return true;
+ default:
+ return false;
+ }
+}
+
static unsigned
signed_type(unsigned type)
{
@@ -248,6 +260,12 @@ sources_not_null(const struct gen_device_info *devinfo,
if (num_sources == 3)
return (struct string){};
+ /* Nothing to test. Split sends can only encode a file in sources that are
+ * allowed to be NULL.
+ */
+ if (inst_is_split_send(devinfo, inst))
+ return (struct string){};
+
if (num_sources >= 1)
ERROR_IF(src0_is_null(devinfo, inst), "src0 is null");
@@ -263,8 +281,41 @@ send_restrictions(const struct gen_device_info *devinfo,
{
struct string error_msg = { .str = NULL, .len = 0 };
- if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
- brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC) {
+ if (inst_is_split_send(devinfo, inst)) {
+ ERROR_IF(brw_inst_send_src1_reg_file(devinfo, inst) == BRW_ARCHITECTURE_REGISTER_FILE &&
+ brw_inst_send_src1_reg_nr(devinfo, inst) != BRW_ARF_NULL,
+ "src1 of split send must be a GRF or NULL");
+
+ ERROR_IF(brw_inst_eot(devinfo, inst) &&
+ brw_inst_src0_da_reg_nr(devinfo, inst) < 112,
+ "send with EOT must use g112-g127");
+ ERROR_IF(brw_inst_eot(devinfo, inst) &&
+ brw_inst_send_src1_reg_file(devinfo, inst) == BRW_GENERAL_REGISTER_FILE &&
+ brw_inst_send_src1_reg_nr(devinfo, inst) < 112,
+ "send with EOT must use g112-g127");
+
+ if (brw_inst_send_src1_reg_file(devinfo, inst) == BRW_GENERAL_REGISTER_FILE) {
+ /* Assume minimums if we don't know */
+ unsigned mlen = 1;
+ if (!brw_inst_send_sel_reg32_desc(devinfo, inst)) {
+ const uint32_t desc = brw_inst_send_desc(devinfo, inst);
+ mlen = brw_message_desc_mlen(devinfo, desc);
+ }
+
+ unsigned ex_mlen = 1;
+ if (!brw_inst_send_sel_reg32_ex_desc(devinfo, inst)) {
+ const uint32_t ex_desc = brw_inst_send_ex_desc(devinfo, inst);
+ ex_mlen = brw_message_ex_desc_ex_mlen(devinfo, ex_desc);
+ }
+ const unsigned src0_reg_nr = brw_inst_src0_da_reg_nr(devinfo, inst);
+ const unsigned src1_reg_nr = brw_inst_send_src1_reg_nr(devinfo, inst);
+ ERROR_IF((src0_reg_nr <= src1_reg_nr &&
+ src1_reg_nr < src0_reg_nr + mlen) ||
+ (src1_reg_nr <= src0_reg_nr &&
+ src0_reg_nr < src1_reg_nr + ex_mlen),
+ "split send payloads must not overlap");
+ }
+ } else if (inst_is_send(devinfo, inst)) {
ERROR_IF(brw_inst_src0_address_mode(devinfo, inst) != BRW_ADDRESS_DIRECT,
"send must use direct addressing");
@@ -534,6 +585,12 @@ general_restrictions_on_region_parameters(const struct gen_device_info *devinfo,
if (num_sources == 3)
return (struct string){};
+ /* Split sends don't have the bits in the instruction to encode regions so
+ * there's nothing to check.
+ */
+ if (inst_is_split_send(devinfo, inst))
+ return (struct string){};
+
if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_16) {
if (desc->ndst != 0 && !dst_is_null(devinfo, inst))
ERROR_IF(brw_inst_dst_hstride(devinfo, inst) != BRW_HORIZONTAL_STRIDE_1,
@@ -1124,6 +1181,10 @@ special_requirements_for_handling_double_precision_data_types(
if (num_sources == 3 || num_sources == 0)
return (struct string){};
+ /* Split sends don't have types so there's no doubles there. */
+ if (inst_is_split_send(devinfo, inst))
+ return (struct string){};
+
enum brw_reg_type exec_type = execution_type(devinfo, inst);
unsigned exec_type_size = brw_reg_type_to_size(exec_type);
diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h
index f60325c1a6b..71316f12215 100644
--- a/src/intel/compiler/brw_inst.h
+++ b/src/intel/compiler/brw_inst.h
@@ -455,6 +455,19 @@ FJ(gen4_jump_count, 111, 96, devinfo->gen < 6)
FC(gen4_pop_count, 115, 112, devinfo->gen < 6)
/** @} */
+/**
+ * SEND instructions:
+ * @{
+ */
+FC(send_ex_desc_ia_subreg_nr, 82, 80, devinfo->gen >= 9)
+FC(send_src0_address_mode, 79, 79, devinfo->gen >= 9)
+FC(send_sel_reg32_desc, 77, 77, devinfo->gen >= 9)
+FC(send_sel_reg32_ex_desc, 61, 61, devinfo->gen >= 9)
+FC(send_src1_reg_nr, 51, 44, devinfo->gen >= 9)
+FC(send_src1_reg_file, 36, 36, devinfo->gen >= 9)
+FC(send_dst_reg_file, 35, 35, devinfo->gen >= 9)
+/** @} */
+
/* Message descriptor bits */
#define MD(x) ((x) + 96)
@@ -513,11 +526,21 @@ brw_inst_set_send_ex_desc(const struct gen_device_info *devinfo,
brw_inst *inst, uint32_t value)
{
assert(devinfo->gen >= 9);
- brw_inst_set_bits(inst, 94, 91, GET_BITS(value, 31, 28));
- brw_inst_set_bits(inst, 88, 85, GET_BITS(value, 27, 24));
- brw_inst_set_bits(inst, 83, 80, GET_BITS(value, 23, 20));
- brw_inst_set_bits(inst, 67, 64, GET_BITS(value, 19, 16));
- assert(GET_BITS(value, 15, 0) == 0);
+ if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC) {
+ brw_inst_set_bits(inst, 94, 91, GET_BITS(value, 31, 28));
+ brw_inst_set_bits(inst, 88, 85, GET_BITS(value, 27, 24));
+ brw_inst_set_bits(inst, 83, 80, GET_BITS(value, 23, 20));
+ brw_inst_set_bits(inst, 67, 64, GET_BITS(value, 19, 16));
+ assert(GET_BITS(value, 15, 0) == 0);
+ } else {
+ assert(brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC);
+ brw_inst_set_bits(inst, 95, 80, GET_BITS(value, 31, 16));
+ assert(GET_BITS(value, 15, 10) == 0);
+ brw_inst_set_bits(inst, 67, 64, GET_BITS(value, 9, 6));
+ assert(GET_BITS(value, 5, 0) == 0);
+ }
}
/**
@@ -530,10 +553,18 @@ brw_inst_send_ex_desc(const struct gen_device_info *devinfo,
const brw_inst *inst)
{
assert(devinfo->gen >= 9);
- return (brw_inst_bits(inst, 94, 91) << 28 |
- brw_inst_bits(inst, 88, 85) << 24 |
- brw_inst_bits(inst, 83, 80) << 20 |
- brw_inst_bits(inst, 67, 64) << 16);
+ if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC) {
+ return (brw_inst_bits(inst, 94, 91) << 28 |
+ brw_inst_bits(inst, 88, 85) << 24 |
+ brw_inst_bits(inst, 83, 80) << 20 |
+ brw_inst_bits(inst, 67, 64) << 16);
+ } else {
+ assert(brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC);
+ return (brw_inst_bits(inst, 95, 80) << 16 |
+ brw_inst_bits(inst, 67, 64) << 6);
+ }
}
/**
@@ -956,9 +987,11 @@ brw_inst_##reg##_ia16_addr_imm(const struct gen_device_info *devinfo, \
* Compared to Align1, these are missing the low 4 bits.
* -Gen 4- ----Gen8----
*/
-BRW_IA16_ADDR_IMM(src1, 105, 96, 121, 104, 100)
-BRW_IA16_ADDR_IMM(src0, 73, 64, 95, 72, 68)
-BRW_IA16_ADDR_IMM(dst, 57, 52, 47, 56, 52)
+BRW_IA16_ADDR_IMM(src1, 105, 96, 121, 104, 100)
+BRW_IA16_ADDR_IMM(src0, 73, 64, 95, 72, 68)
+BRW_IA16_ADDR_IMM(dst, 57, 52, 47, 56, 52)
+BRW_IA16_ADDR_IMM(send_src0, -1, -1, 78, 72, 68)
+BRW_IA16_ADDR_IMM(send_dst, -1, -1, 62, 56, 52)
/**
* Fetch a set of contiguous bits from the instruction.
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-29 14:40:10 +0000