diff options
Diffstat (limited to 'src/intel')
| -rw-r--r-- | src/intel/compiler/brw_eu_compact.c | 1040 | ||||
| -rw-r--r-- | src/intel/compiler/brw_inst.h | 12 |
2 files changed, 868 insertions, 184 deletions
diff --git a/src/intel/compiler/brw_eu_compact.c b/src/intel/compiler/brw_eu_compact.c index dcc8f2841d5..9155da04c77 100644 --- a/src/intel/compiler/brw_eu_compact.c +++ b/src/intel/compiler/brw_eu_compact.c @@ -1,5 +1,5 @@ /* - * Copyright © 2012 Intel Corporation + * Copyright © 2012-2018 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -70,6 +70,12 @@ * while JIP (must be negative) * * Gen 8 adds support for compacting 3-src instructions. + * + * Gen12 reduces the number of bits that available to compacted immediates from + * 13 to 12, but improves the compaction of floating-point immediates by + * allowing the high bits to be encoded (the sign, 8-bit exponent, and the + * three most significant bits of the mantissa), rather than the lowest bits of + * the mantissa. */ #include "brw_eu.h" @@ -672,6 +678,149 @@ static const uint32_t gen11_datatype_table[32] = { 0b001001011001001001000, }; +static const uint32_t gen12_control_index_table[32] = { + 0b000000000000000000100, /* (16|M0) */ + 0b000000000000000000011, /* (8|M0) */ + 0b000000010000000000000, /* (W) (1|M0) */ + 0b000000010000000000100, /* (W) (16|M0) */ + 0b000000010000000000011, /* (W) (8|M0) */ + 0b010000000000000000100, /* (16|M0) (ge)f0.0 */ + 0b000000000000000100100, /* (16|M16) */ + 0b010100000000000000100, /* (16|M0) (lt)f0.0 */ + 0b000000000000000000000, /* (1|M0) */ + 0b000010000000000000100, /* (16|M0) (sat) */ + 0b000000000000000010011, /* (8|M8) */ + 0b001100000000000000100, /* (16|M0) (gt)f0.0 */ + 0b000100000000000000100, /* (16|M0) (eq)f0.0 */ + 0b000100010000000000100, /* (W) (16|M0) (eq)f0.0 */ + 0b001000000000000000100, /* (16|M0) (ne)f0.0 */ + 0b000000000000100000100, /* (f0.0) (16|M0) */ + 0b010100000000000000011, /* (8|M0) (lt)f0.0 */ + 0b000000000000110000100, /* (f1.0) (16|M0) */ + 0b000000010000000000001, /* (W) (2|M0) */ + 0b000000000000101000100, /* (f0.1) (16|M0) */ + 0b000000000000111000100, /* (f1.1) (16|M0) */ + 0b010000010000000000100, /* (W) (16|M0) (ge)f0.0 */ + 0b000000000000000100011, /* (8|M16) */ + 0b000000000000000110011, /* (8|M24) */ + 0b010100010000000000100, /* (W) (16|M0) (lt)f0.0 */ + 0b010000000000000000011, /* (8|M0) (ge)f0.0 */ + 0b000100010000000000000, /* (W) (1|M0) (eq)f0.0 */ + 0b000010000000000000011, /* (8|M0) (sat) */ + 0b010100000000010000100, /* (16|M0) (lt)f1.0 */ + 0b000100000000000000011, /* (8|M0) (eq)f0.0 */ + 0b000001000000000000011, /* (8|M0) {AccWrEn} */ + 0b000000010000000100100, /* (W) (16|M16) */ +}; + +static const uint32_t gen12_datatype_table[32] = { + 0b11010110100101010100, /* grf<1>:f grf:f grf:f */ + 0b00000110100101010100, /* grf<1>:f grf:f arf:ub */ + 0b00000010101101010100, /* grf<1>:f imm:f arf:ub */ + 0b01010110110101010100, /* grf<1>:f grf:f imm:f */ + 0b11010100100101010100, /* arf<1>:f grf:f grf:f */ + 0b11010010100101010100, /* grf<1>:f arf:f grf:f */ + 0b01010100110101010100, /* arf<1>:f grf:f imm:f */ + 0b00000000100000000000, /* arf<1>:ub arf:ub arf:ub */ + 0b11010000100101010100, /* arf<1>:f arf:f grf:f */ + 0b00101110110011001100, /* grf<1>:d grf:d imm:w */ + 0b10110110100011001100, /* grf<1>:d grf:d grf:d */ + 0b01010010110101010100, /* grf<1>:f arf:f imm:f */ + 0b10010110100001000100, /* grf<1>:ud grf:ud grf:ud */ + 0b01010000110101010100, /* arf<1>:f arf:f imm:f */ + 0b00110110110011001100, /* grf<1>:d grf:d imm:d */ + 0b00010110110001000100, /* grf<1>:ud grf:ud imm:ud */ + 0b00000111000101010100, /* grf<2>:f grf:f arf:ub */ + 0b00101100110011001100, /* arf<1>:d grf:d imm:w */ + 0b00000000100000100010, /* arf<1>:uw arf:uw arf:ub */ + 0b00000010100001000100, /* grf<1>:ud arf:ud arf:ub */ + 0b00100110110000101010, /* grf<1>:w grf:uw imm:uv */ + 0b00001110110000100010, /* grf<1>:uw grf:uw imm:uw */ + 0b10010111000001000100, /* grf<2>:ud grf:ud grf:ud */ + 0b00000110100101001100, /* grf<1>:d grf:f arf:ub */ + 0b10001100100011001100, /* arf<1>:d grf:d grf:uw */ + 0b00000110100001010100, /* grf<1>:f grf:ud arf:ub */ + 0b00101110110001001100, /* grf<1>:d grf:ud imm:w */ + 0b00000010100000100010, /* grf<1>:uw arf:uw arf:ub */ + 0b00000110100000110100, /* grf<1>:f grf:uw arf:ub */ + 0b00000110100000010100, /* grf<1>:f grf:ub arf:ub */ + 0b00000110100011010100, /* grf<1>:f grf:d arf:ub */ + 0b00000010100101010100, /* grf<1>:f arf:f arf:ub */ +}; + +static const uint16_t gen12_subreg_table[32] = { + 0b000000000000000, /* .0 .0 .0 */ + 0b100000000000000, /* .0 .0 .16 */ + 0b001000000000000, /* .0 .0 .4 */ + 0b011000000000000, /* .0 .0 .12 */ + 0b000000010000000, /* .0 .4 .0 */ + 0b010000000000000, /* .0 .0 .8 */ + 0b101000000000000, /* .0 .0 .20 */ + 0b000000000001000, /* .8 .0 .0 */ + 0b000000100000000, /* .0 .8 .0 */ + 0b110000000000000, /* .0 .0 .24 */ + 0b111000000000000, /* .0 .0 .28 */ + 0b000001000000000, /* .0 .16 .0 */ + 0b000000000000100, /* .4 .0 .0 */ + 0b000001100000000, /* .0 .24 .0 */ + 0b000001010000000, /* .0 .20 .0 */ + 0b000000110000000, /* .0 .12 .0 */ + 0b000001110000000, /* .0 .28 .0 */ + 0b000000000011100, /* .28 .0 .0 */ + 0b000000000010000, /* .16 .0 .0 */ + 0b000000000001100, /* .12 .0 .0 */ + 0b000000000011000, /* .24 .0 .0 */ + 0b000000000010100, /* .20 .0 .0 */ + 0b000000000000010, /* .2 .0 .0 */ + 0b000000101000000, /* .0 .10 .0 */ + 0b000000001000000, /* .0 .2 .0 */ + 0b000000010000100, /* .4 .4 .0 */ + 0b000000001011100, /* .28 .2 .0 */ + 0b000000001000010, /* .2 .2 .0 */ + 0b000000110001100, /* .12 .12 .0 */ + 0b000000000100000, /* .0 .1 .0 */ + 0b000000001100000, /* .0 .3 .0 */ + 0b110001100000000, /* .0 .24 .24 */ +}; + +static const uint16_t gen12_src0_index_table[16] = { + 0b010001100100, /* r<8;8,1> */ + 0b000000000000, /* r<0;1,0> */ + 0b010001100110, /* -r<8;8,1> */ + 0b010001100101, /* (abs)r<8;8,1> */ + 0b000000000010, /* -r<0;1,0> */ + 0b001000000000, /* r<2;1,0> */ + 0b001001000000, /* r<2;4,0> */ + 0b001101000000, /* r<4;4,0> */ + 0b001000100100, /* r<2;2,1> */ + 0b001100000000, /* r<4;1,0> */ + 0b001000100110, /* -r<2;2,1> */ + 0b001101000100, /* r<4;4,1> */ + 0b010001100111, /* -(abs)r<8;8,1> */ + 0b000100000000, /* r<1;1,0> */ + 0b000000000001, /* (abs)r<0;1,0> */ + 0b111100010000, /* r[a]<1,0> */ +}; + +static const uint16_t gen12_src1_index_table[16] = { + 0b000100011001, /* r<8;8,1> */ + 0b000000000000, /* r<0;1,0> */ + 0b100100011001, /* -r<8;8,1> */ + 0b100000000000, /* -r<0;1,0> */ + 0b010100011001, /* (abs)r<8;8,1> */ + 0b100011010000, /* -r<4;4,0> */ + 0b000010000000, /* r<2;1,0> */ + 0b000010001001, /* r<2;2,1> */ + 0b100010001001, /* -r<2;2,1> */ + 0b000011010000, /* r<4;4,0> */ + 0b000011010001, /* r<4;4,1> */ + 0b000011000000, /* r<4;1,0> */ + 0b110100011001, /* -(abs)r<8;8,1> */ + 0b010000000000, /* (abs)r<0;1,0> */ + 0b110000000000, /* -(abs)r<0;1,0> */ + 0b100011010001, /* -r<4;4,1> */ +}; + /* This is actually the control index table for Cherryview (26 bits), but the * only difference from Broadwell (24 bits) is that it has two extra 0-bits at * the start. @@ -699,6 +848,111 @@ static const uint64_t gen8_3src_source_index_table[4] = { 0b0000001110010011100100111001000001111000000100000, }; +static const uint64_t gen12_3src_control_index_table[32] = { + 0b000001001010010101000000000000000100, /* (16|M0) grf<1>:f :f :f :f */ + 0b000001001010010101000000000000000011, /* (8|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000000011, /* (8|M0) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000000011, /* (W) (8|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000000011, /* (W) (8|M0) arf<1>:f :f :f :f */ + 0b000001001000010101000000000000010011, /* (8|M8) arf<1>:f :f :f :f */ + 0b000001001010010101000000000000010011, /* (8|M8) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000010011, /* (W) (8|M8) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000010011, /* (W) (8|M8) grf<1>:f :f :f :f */ + 0b000001001010010101000010000000000100, /* (W) (16|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000000100, /* (16|M0) arf<1>:f :f :f :f */ + 0b000001001010010101010000000000000100, /* (16|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000000000000100100, /* (16|M16) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000000100, /* (W) (16|M0) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000000000, /* (W) (1|M0) grf<1>:f :f :f :f */ + 0b000001001010010101010000000000000011, /* (8|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001000010101000010000000110011, /* (W) (8|M24) arf<1>:f :f :f :f */ + 0b000001001000010101000010000000100011, /* (W) (8|M16) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000110011, /* (W) (8|M24) grf<1>:f :f :f :f */ + 0b000001001010010101000010000000100011, /* (W) (8|M16) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000100011, /* (8|M16) arf<1>:f :f :f :f */ + 0b000001001000010101000000000000110011, /* (8|M24) arf<1>:f :f :f :f */ + 0b000001001010010101000000000000100011, /* (8|M16) grf<1>:f :f :f :f */ + 0b000001001010010101000000000000110011, /* (8|M24) grf<1>:f :f :f :f */ + 0b000001001000010101010000000000000100, /* (16|M0) (sat)arf<1>:f :f :f :f */ + 0b000001001010010101010010000000000100, /* (W) (16|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000010000000100100, /* (W) (16|M16) grf<1>:f :f :f :f */ + 0b000001001010010001000010000000000000, /* (W) (1|M0) grf<1>:ud :ud :ud :ud */ + 0b000001001000010101000000000000100100, /* (16|M16) arf<1>:f :f :f :f */ + 0b000001001010010101010000000000100100, /* (16|M16) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000010000000000010, /* (W) (4|M0) grf<1>:f :f :f :f */ + 0b000001001000010101010000000000000011, /* (8|M0) (sat)arf<1>:f :f :f :f */ +}; + +static const uint32_t gen12_3src_source_index_table[32] = { + 0b100101100001100000000, /* grf<0;0> grf<8;1> grf<0> */ + 0b100101100001001000010, /* arf<4;1> grf<8;1> grf<0> */ + 0b101101100001101000011, /* grf<8;1> grf<8;1> grf<1> */ + 0b100101100001101000011, /* grf<8;1> grf<8;1> grf<0> */ + 0b101100000000101000011, /* grf<8;1> grf<0;0> grf<1> */ + 0b101101100001101001011, /* -grf<8;1> grf<8;1> grf<1> */ + 0b101001100001101000011, /* grf<8;1> arf<8;1> grf<1> */ + 0b100001100001100000000, /* grf<0;0> arf<8;1> grf<0> */ + 0b101101100001100000000, /* grf<0;0> grf<8;1> grf<1> */ + 0b101101100101101000011, /* grf<8;1> grf<8;1> -grf<1> */ + 0b101101110001101000011, /* grf<8;1> -grf<8;1> grf<1> */ + 0b101100000000100000000, /* grf<0;0> grf<0;0> grf<1> */ + 0b100001100001101000011, /* grf<8;1> arf<8;1> grf<0> */ + 0b100101110001100000000, /* grf<0;0> -grf<8;1> grf<0> */ + 0b100101110001101000011, /* grf<8;1> -grf<8;1> grf<0> */ + 0b100101100001101001011, /* -grf<8;1> grf<8;1> grf<0> */ + 0b100100000000101000011, /* grf<8;1> grf<0;0> grf<0> */ + 0b100101100001100001000, /* -grf<0;0> grf<8;1> grf<0> */ + 0b100100000000100000000, /* grf<0;0> grf<0;0> grf<0> */ + 0b101101110001100000000, /* grf<0;0> -grf<8;1> grf<1> */ + 0b100101100101100000000, /* grf<0;0> grf<8;1> -grf<0> */ + 0b101001100001100000000, /* grf<0;0> arf<8;1> grf<1> */ + 0b100101100101101000011, /* grf<8;1> grf<8;1> -grf<0> */ + 0b101101100101101001011, /* -grf<8;1> grf<8;1> -grf<1> */ + 0b101001100001101001011, /* -grf<8;1> arf<8;1> grf<1> */ + 0b101101110001101001011, /* -grf<8;1> -grf<8;1> grf<1> */ + 0b101100010000101000011, /* grf<8;1> -grf<0;0> grf<1> */ + 0b101100000100101000011, /* grf<8;1> grf<0;0> -grf<1> */ + 0b101101100001100001000, /* -grf<0;0> grf<8;1> grf<1> */ + 0b101101100101100000000, /* grf<0;0> grf<8;1> -grf<1> */ + 0b100100000100101000011, /* grf<8;1> grf<0;0> -grf<0> */ + 0b101001100101101000011, /* grf<8;1> arf<8;1> -grf<1> */ +}; + +static const uint32_t gen12_3src_subreg_table[32] = { + 0b00000000000000000000, /* .0 .0 .0 .0 */ + 0b00100000000000000000, /* .0 .0 .0 .4 */ + 0b00000000000110000000, /* .0 .12 .0 .0 */ + 0b10100000000000000000, /* .0 .0 .0 .20 */ + 0b10000000001110000000, /* .0 .28 .0 .16 */ + 0b01100000000000000000, /* .0 .0 .0 .12 */ + 0b01000000000000000000, /* .0 .0 .0 .8 */ + 0b00000010000000000000, /* .0 .0 .8 .0 */ + 0b00000001000000000000, /* .0 .0 .4 .0 */ + 0b11000000000000000000, /* .0 .0 .0 .24 */ + 0b10000000000000000000, /* .0 .0 .0 .16 */ + 0b11100000000000000000, /* .0 .0 .0 .28 */ + 0b00000110000000000000, /* .0 .0 .24 .0 */ + 0b00000000000010000000, /* .0 .4 .0 .0 */ + 0b00000100000000000000, /* .0 .0 .16 .0 */ + 0b00000011000000000000, /* .0 .0 .12 .0 */ + 0b00000101000000000000, /* .0 .0 .20 .0 */ + 0b00000111000000000000, /* .0 .0 .28 .0 */ + 0b00000000000100000000, /* .0 .8 .0 .0 */ + 0b00000000001000000000, /* .0 .16 .0 .0 */ + 0b00000000001100000000, /* .0 .24 .0 .0 */ + 0b00000000001010000000, /* .0 .20 .0 .0 */ + 0b00000000001110000000, /* .0 .28 .0 .0 */ + 0b11000000001110000000, /* .0 .28 .0 .24 */ + 0b00100000000100000000, /* .0 .8 .0 .4 */ + 0b00100000000110000000, /* .0 .12 .0 .4 */ + 0b01000000000110000000, /* .0 .12 .0 .8 */ + 0b10000000001100000000, /* .0 .24 .0 .16 */ + 0b10000000001010000000, /* .0 .20 .0 .16 */ + 0b01100000000010000000, /* .0 .4 .0 .12 */ + 0b10100000001110000000, /* .0 .28 .0 .20 */ + 0b01000000000010000000, /* .0 .4 .0 .8 */ +}; + static const uint32_t *control_index_table; static const uint32_t *datatype_table; static const uint16_t *subreg_table; @@ -709,9 +963,20 @@ static bool set_control_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, const brw_inst *src) { - uint32_t uncompacted; /* 17b/G45; 19b/IVB+ */ - - if (devinfo->gen >= 8) { + uint32_t uncompacted; /* 17b/G45; 19b/IVB+; 21b/TGL+ */ + + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 95, 92) << 17) | /* 4b */ + (brw_inst_bits(src, 34, 34) << 16) | /* 1b */ + (brw_inst_bits(src, 33, 33) << 15) | /* 1b */ + (brw_inst_bits(src, 32, 32) << 14) | /* 1b */ + (brw_inst_bits(src, 31, 31) << 13) | /* 1b */ + (brw_inst_bits(src, 28, 28) << 12) | /* 1b */ + (brw_inst_bits(src, 27, 24) << 8) | /* 4b */ + (brw_inst_bits(src, 23, 22) << 6) | /* 2b */ + (brw_inst_bits(src, 21, 19) << 3) | /* 3b */ + (brw_inst_bits(src, 18, 16)); /* 3b */ + } else if (devinfo->gen >= 8) { uncompacted = (brw_inst_bits(src, 33, 31) << 16) | /* 3b */ (brw_inst_bits(src, 23, 12) << 4) | /* 12b */ (brw_inst_bits(src, 10, 9) << 2) | /* 2b */ @@ -740,11 +1005,28 @@ set_control_index(const struct gen_device_info *devinfo, static bool set_datatype_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, - const brw_inst *src) + const brw_inst *src, bool is_immediate) { - uint32_t uncompacted; /* 18b/G45+; 21b/BDW+ */ - - if (devinfo->gen >= 8) { + uint32_t uncompacted; /* 18b/G45+; 21b/BDW+; 20b/TGL+ */ + + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 91, 88) << 15) | /* 4b */ + (brw_inst_bits(src, 66, 66) << 14) | /* 1b */ + (brw_inst_bits(src, 50, 50) << 13) | /* 1b */ + (brw_inst_bits(src, 49, 48) << 11) | /* 2b */ + (brw_inst_bits(src, 47, 47) << 10) | /* 1b */ + (brw_inst_bits(src, 46, 46) << 9) | /* 1b */ + (brw_inst_bits(src, 43, 40) << 5) | /* 4b */ + (brw_inst_bits(src, 39, 36) << 1) | /* 4b */ + (brw_inst_bits(src, 35, 35)); /* 1b */ + + /* Src1.RegFile overlaps with the immediate, so ignore it if an immediate + * is present + */ + if (!is_immediate) { + uncompacted |= brw_inst_bits(src, 98, 98) << 19; /* 1b */ + } + } else if (devinfo->gen >= 8) { uncompacted = (brw_inst_bits(src, 63, 61) << 18) | /* 3b */ (brw_inst_bits(src, 94, 89) << 12) | /* 6b */ (brw_inst_bits(src, 46, 35)); /* 12b */ @@ -767,12 +1049,21 @@ static bool set_subreg_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, const brw_inst *src, bool is_immediate) { - uint16_t uncompacted = /* 15b */ - (brw_inst_bits(src, 52, 48) << 0) | /* 5b */ - (brw_inst_bits(src, 68, 64) << 5); /* 5b */ + uint16_t uncompacted; /* 15b */ + + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 55, 51) << 0) | /* 5b */ + (brw_inst_bits(src, 71, 67) << 5); /* 5b */ - if (!is_immediate) - uncompacted |= brw_inst_bits(src, 100, 96) << 10; /* 5b */ + if (!is_immediate) + uncompacted |= brw_inst_bits(src, 103, 99) << 10; /* 5b */ + } else { + uncompacted = (brw_inst_bits(src, 52, 48) << 0) | /* 5b */ + (brw_inst_bits(src, 68, 64) << 5); /* 5b */ + + if (!is_immediate) + uncompacted |= brw_inst_bits(src, 100, 96) << 10; /* 5b */ + } for (int i = 0; i < 32; i++) { if (subreg_table[i] == uncompacted) { @@ -788,9 +1079,22 @@ static bool set_src0_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, const brw_inst *src) { - uint16_t uncompacted = brw_inst_bits(src, 88, 77); /* 12b */ + uint16_t uncompacted; /* 12b */ + int table_len; + + if (devinfo->gen >= 12) { + table_len = ARRAY_SIZE(gen12_src0_index_table); + uncompacted = (brw_inst_bits(src, 87, 84) << 8) | /* 4b */ + (brw_inst_bits(src, 83, 81) << 5) | /* 3b */ + (brw_inst_bits(src, 80, 80) << 4) | /* 1b */ + (brw_inst_bits(src, 65, 64) << 2) | /* 2b */ + (brw_inst_bits(src, 45, 44)); /* 2b */ + } else { + table_len = ARRAY_SIZE(gen8_src_index_table); + uncompacted = brw_inst_bits(src, 88, 77); /* 12b */ + } - for (int i = 0; i < 32; i++) { + for (int i = 0; i < table_len; i++) { if (src0_index_table[i] == uncompacted) { brw_compact_inst_set_src0_index(devinfo, dst, i); return true; @@ -802,16 +1106,34 @@ set_src0_index(const struct gen_device_info *devinfo, static bool set_src1_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, - const brw_inst *src, bool is_immediate) + const brw_inst *src, bool is_immediate, unsigned imm) { if (is_immediate) { - uint16_t imm = (brw_inst_imm_ud(devinfo, src) >> 8) & 0x1f; - brw_compact_inst_set_src1_index(devinfo, dst, imm); + if (devinfo->gen >= 12) { + /* src1 index takes the low 4 bits of the 12-bit compacted value */ + brw_compact_inst_set_src1_index(devinfo, dst, imm & 0xf); + } else { + /* src1 index takes the high 5 bits of the 13-bit compacted value */ + brw_compact_inst_set_src1_index(devinfo, dst, imm >> 8); + } return true; } else { - uint16_t uncompacted = brw_inst_bits(src, 120, 109); /* 12b */ + uint16_t uncompacted; /* 12b */ + int table_len; + + if (devinfo->gen >= 12) { + table_len = ARRAY_SIZE(gen12_src0_index_table); + uncompacted = (brw_inst_bits(src, 121, 120) << 10) | /* 2b */ + (brw_inst_bits(src, 119, 116) << 6) | /* 4b */ + (brw_inst_bits(src, 115, 113) << 3) | /* 3b */ + (brw_inst_bits(src, 112, 112) << 2) | /* 1b */ + (brw_inst_bits(src, 97, 96)); /* 2b */ + } else { + table_len = ARRAY_SIZE(gen8_src_index_table); + uncompacted = brw_inst_bits(src, 120, 109); /* 12b */ + } - for (int i = 0; i < 32; i++) { + for (int i = 0; i < table_len; i++) { if (src1_index_table[i] == uncompacted) { brw_compact_inst_set_src1_index(devinfo, dst, i); return true; @@ -828,17 +1150,48 @@ set_3src_control_index(const struct gen_device_info *devinfo, { assert(devinfo->gen >= 8); - uint32_t uncompacted = /* 24b/BDW; 26b/CHV */ - (brw_inst_bits(src, 34, 32) << 21) | /* 3b */ - (brw_inst_bits(src, 28, 8)); /* 21b */ + if (devinfo->gen >= 12) { + uint64_t uncompacted = /* 36b/TGL+ */ + (brw_inst_bits(src, 95, 92) << 32) | /* 4b */ + (brw_inst_bits(src, 90, 88) << 29) | /* 3b */ + (brw_inst_bits(src, 82, 80) << 26) | /* 3b */ + (brw_inst_bits(src, 50, 50) << 25) | /* 1b */ + (brw_inst_bits(src, 48, 48) << 24) | /* 1b */ + (brw_inst_bits(src, 42, 40) << 21) | /* 3b */ + (brw_inst_bits(src, 39, 39) << 20) | /* 1b */ + (brw_inst_bits(src, 38, 36) << 17) | /* 3b */ + (brw_inst_bits(src, 34, 34) << 16) | /* 1b */ + (brw_inst_bits(src, 33, 33) << 15) | /* 1b */ + (brw_inst_bits(src, 32, 32) << 14) | /* 1b */ + (brw_inst_bits(src, 31, 31) << 13) | /* 1b */ + (brw_inst_bits(src, 28, 28) << 12) | /* 1b */ + (brw_inst_bits(src, 27, 24) << 8) | /* 4b */ + (brw_inst_bits(src, 23, 23) << 7) | /* 1b */ + (brw_inst_bits(src, 22, 22) << 6) | /* 1b */ + (brw_inst_bits(src, 21, 19) << 3) | /* 3b */ + (brw_inst_bits(src, 18, 16)); /* 3b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_control_index_table); i++) { + if (gen12_3src_control_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_control_index(devinfo, dst, i); + return true; + } + } + } else { + uint32_t uncompacted = /* 24b/BDW; 26b/CHV/SKL+ */ + (brw_inst_bits(src, 34, 32) << 21) | /* 3b */ + (brw_inst_bits(src, 28, 8)); /* 21b */ - if (devinfo->gen >= 9 || devinfo->is_cherryview) - uncompacted |= brw_inst_bits(src, 36, 35) << 24; /* 2b */ + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + uncompacted |= + brw_inst_bits(src, 36, 35) << 24; /* 2b */ + } - for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) { - if (gen8_3src_control_index_table[i] == uncompacted) { - brw_compact_inst_set_3src_control_index(devinfo, dst, i); - return true; + for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) { + if (gen8_3src_control_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_control_index(devinfo, dst, i); + return true; + } } } @@ -851,27 +1204,75 @@ set_3src_source_index(const struct gen_device_info *devinfo, { assert(devinfo->gen >= 8); - uint64_t uncompacted = /* 46b/BDW; 49b/CHV */ - (brw_inst_bits(src, 83, 83) << 43) | /* 1b */ - (brw_inst_bits(src, 114, 107) << 35) | /* 8b */ - (brw_inst_bits(src, 93, 86) << 27) | /* 8b */ - (brw_inst_bits(src, 72, 65) << 19) | /* 8b */ - (brw_inst_bits(src, 55, 37)); /* 19b */ - - if (devinfo->gen >= 9 || devinfo->is_cherryview) { - uncompacted |= - (brw_inst_bits(src, 126, 125) << 47) | /* 2b */ - (brw_inst_bits(src, 105, 104) << 45) | /* 2b */ - (brw_inst_bits(src, 84, 84) << 44); /* 1b */ + if (devinfo->gen >= 12) { + uint32_t uncompacted = /* 21b/TGL+ */ + (brw_inst_bits(src, 114, 114) << 20) | /* 1b */ + (brw_inst_bits(src, 113, 112) << 18) | /* 2b */ + (brw_inst_bits(src, 98, 98) << 17) | /* 1b */ + (brw_inst_bits(src, 97, 96) << 15) | /* 2b */ + (brw_inst_bits(src, 91, 91) << 14) | /* 1b */ + (brw_inst_bits(src, 87, 86) << 12) | /* 2b */ + (brw_inst_bits(src, 85, 84) << 10) | /* 2b */ + (brw_inst_bits(src, 83, 83) << 9) | /* 1b */ + (brw_inst_bits(src, 66, 66) << 8) | /* 1b */ + (brw_inst_bits(src, 65, 64) << 6) | /* 2b */ + (brw_inst_bits(src, 47, 47) << 5) | /* 1b */ + (brw_inst_bits(src, 46, 46) << 4) | /* 1b */ + (brw_inst_bits(src, 45, 44) << 2) | /* 2b */ + (brw_inst_bits(src, 43, 43) << 1) | /* 1b */ + (brw_inst_bits(src, 35, 35)); /* 1b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_source_index_table); i++) { + if (gen12_3src_source_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return true; + } + } } else { - uncompacted |= - (brw_inst_bits(src, 125, 125) << 45) | /* 1b */ - (brw_inst_bits(src, 104, 104) << 44); /* 1b */ + uint64_t uncompacted = /* 46b/BDW; 49b/CHV/SKL+ */ + (brw_inst_bits(src, 83, 83) << 43) | /* 1b */ + (brw_inst_bits(src, 114, 107) << 35) | /* 8b */ + (brw_inst_bits(src, 93, 86) << 27) | /* 8b */ + (brw_inst_bits(src, 72, 65) << 19) | /* 8b */ + (brw_inst_bits(src, 55, 37)); /* 19b */ + + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + uncompacted |= + (brw_inst_bits(src, 126, 125) << 47) | /* 2b */ + (brw_inst_bits(src, 105, 104) << 45) | /* 2b */ + (brw_inst_bits(src, 84, 84) << 44); /* 1b */ + } else { + uncompacted |= + (brw_inst_bits(src, 125, 125) << 45) | /* 1b */ + (brw_inst_bits(src, 104, 104) << 44); /* 1b */ + } + + for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_source_index_table); i++) { + if (gen8_3src_source_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return true; + } + } } - for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_source_index_table); i++) { - if (gen8_3src_source_index_table[i] == uncompacted) { - brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return false; +} + +static bool +set_3src_subreg_index(const struct gen_device_info *devinfo, + brw_compact_inst *dst, const brw_inst *src) +{ + assert(devinfo->gen >= 12); + + uint32_t uncompacted = /* 20b/TGL+ */ + (brw_inst_bits(src, 119, 115) << 15) | /* 5b */ + (brw_inst_bits(src, 103, 99) << 10) | /* 5b */ + (brw_inst_bits(src, 71, 67) << 5) | /* 5b */ + (brw_inst_bits(src, 55, 51)); /* 5b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_subreg_table); i++) { + if (gen12_3src_subreg_table[i] == uncompacted) { + brw_compact_inst_set_3src_subreg_index(devinfo, dst, i); return true; } } @@ -897,7 +1298,10 @@ has_unmapped_bits(const struct gen_device_info *devinfo, const brw_inst *src) * - Imm64[27:31] (bits 91-95 on Gen7, bit 95 on Gen8) * - UIP[31] (bit 95 on Gen8) */ - if (devinfo->gen >= 8) { + if (devinfo->gen >= 12) { + assert(!brw_inst_bits(src, 7, 7)); + return false; + } else if (devinfo->gen >= 8) { assert(!brw_inst_bits(src, 7, 7)); return brw_inst_bits(src, 95, 95) || brw_inst_bits(src, 47, 47) || @@ -918,7 +1322,9 @@ has_3src_unmapped_bits(const struct gen_device_info *devinfo, * fields of the compacted instruction. All of them seem to be reserved * bits currently. */ - if (devinfo->gen >= 9 || devinfo->is_cherryview) { + if (devinfo->gen >= 12) { + assert(!brw_inst_bits(src, 7, 7)); + } else if (devinfo->gen >= 9 || devinfo->is_cherryview) { assert(!brw_inst_bits(src, 127, 127) && !brw_inst_bits(src, 7, 7)); } else { @@ -958,19 +1364,31 @@ brw_try_compact_3src_instruction(const struct gen_device_info *devinfo, if (!set_3src_source_index(devinfo, dst, src)) return false; - compact(dst_reg_nr); - compact_a16(src0_rep_ctrl); + if (devinfo->gen >= 12) { + if (!set_3src_subreg_index(devinfo, dst, src)) + return false; + + compact(swsb); + compact(debug_control); + compact(dst_reg_nr); + compact(src0_reg_nr); + compact(src1_reg_nr); + compact(src2_reg_nr); + } else { + compact(dst_reg_nr); + compact_a16(src0_rep_ctrl); + compact(debug_control); + compact(saturate); + compact_a16(src1_rep_ctrl); + compact_a16(src2_rep_ctrl); + compact(src0_reg_nr); + compact(src1_reg_nr); + compact(src2_reg_nr); + compact_a16(src0_subreg_nr); + compact_a16(src1_subreg_nr); + compact_a16(src2_subreg_nr); + } brw_compact_inst_set_3src_cmpt_control(devinfo, dst, true); - compact(debug_control); - compact(saturate); - compact_a16(src1_rep_ctrl); - compact_a16(src2_rep_ctrl); - compact(src0_reg_nr); - compact(src1_reg_nr); - compact(src2_reg_nr); - compact_a16(src0_subreg_nr); - compact_a16(src1_subreg_nr); - compact_a16(src2_subreg_nr); #undef compact #undef compact_a16 @@ -978,20 +1396,129 @@ brw_try_compact_3src_instruction(const struct gen_device_info *devinfo, return true; } -/* Compacted instructions have 12-bits for immediate sources, and a 13th bit - * that's replicated through the high 20 bits. +/* On SNB through ICL, compacted instructions have 12-bits for immediate + * sources, and a 13th bit that's replicated through the high 20 bits. * * Effectively this means we get 12-bit integers, 0.0f, and some limited uses * of packed vectors as compactable immediates. + * + * On TGL+, the high 12-bits of floating-point values (:f and :hf) are encoded + * rather than the low 12-bits. For signed integer the 12th bit is replicated, + * while for unsigned integers it is not. + * + * Returns the compacted immediate, or -1 if immediate cannot be compacted */ +static int +compact_immediate(const struct gen_device_info *devinfo, + enum brw_reg_type type, unsigned imm) +{ + if (devinfo->gen >= 12) { + switch (type) { + case BRW_REGISTER_TYPE_F: + /* We get the high 12-bits as-is; rest must be zero */ + if ((imm & 0xfffff) == 0) + return (imm >> 20) & 0xfff; + break; + case BRW_REGISTER_TYPE_HF: + /* We get the high 12-bits as-is; rest must be zero */ + if ((imm & 0xf) == 0) + return (imm >> 4) & 0xfff; + break; + case BRW_REGISTER_TYPE_UD: + case BRW_REGISTER_TYPE_VF: + case BRW_REGISTER_TYPE_UV: + case BRW_REGISTER_TYPE_V: + /* We get the low 12-bits as-is; rest must be zero */ + if ((imm & 0xfffff000) == 0) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_UW: + /* We get the low 12-bits as-is; rest must be zero */ + if ((imm & 0xf000) == 0) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_D: + /* We get the low 11-bits as-is; 12th is replicated */ + if (((int)imm >> 11) == 0 || ((int)imm >> 11) == -1) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_W: + /* We get the low 11-bits as-is; 12th is replicated */ + if (((short)imm >> 11) == 0 || ((short)imm >> 11) == -1) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_NF: + case BRW_REGISTER_TYPE_DF: + case BRW_REGISTER_TYPE_Q: + case BRW_REGISTER_TYPE_UQ: + case BRW_REGISTER_TYPE_B: + case BRW_REGISTER_TYPE_UB: + unreachable("not reached"); + } + } else { + /* We get the low 12 bits as-is; 13th is replicated */ + if (((int)imm >> 12) == 0 || ((int)imm >> 12 == -1)) { + return imm & 0x1fff; + } + } + + return -1; +} + +static int +uncompact_immediate(const struct gen_device_info *devinfo, + enum brw_reg_type type, unsigned compact_imm) +{ + if (devinfo->gen >= 12) { + switch (type) { + case BRW_REGISTER_TYPE_F: + return compact_imm << 20; + case BRW_REGISTER_TYPE_HF: + return (compact_imm << 20) | (compact_imm << 4); + case BRW_REGISTER_TYPE_UD: + case BRW_REGISTER_TYPE_VF: + case BRW_REGISTER_TYPE_UV: + case BRW_REGISTER_TYPE_V: + return compact_imm; + case BRW_REGISTER_TYPE_UW: + /* Replicate */ + return compact_imm << 16 | compact_imm; + case BRW_REGISTER_TYPE_D: + /* Extend the 12th bit into the high 20 bits */ + return (int)(compact_imm << 20) >> 20; + case BRW_REGISTER_TYPE_W: + /* Extend the 12th bit into the high 4 bits and replicate */ + return ( (int)(compact_imm << 20) >> 4) | + ((short)(compact_imm << 4) >> 4); + case BRW_REGISTER_TYPE_NF: + case BRW_REGISTER_TYPE_DF: + case BRW_REGISTER_TYPE_Q: + case BRW_REGISTER_TYPE_UQ: + case BRW_REGISTER_TYPE_B: + case BRW_REGISTER_TYPE_UB: + unreachable("not reached"); + } + } else { + /* Replicate the 13th bit into the high 19 bits */ + return (int)(compact_imm << 19) >> 19; + } + + unreachable("not reached"); +} + static bool -is_compactable_immediate(unsigned imm) +has_immediate(const struct gen_device_info *devinfo, const brw_inst *inst, + enum brw_reg_type *type) { - /* We get the low 12 bits as-is. */ - imm &= ~0xfff; + if (brw_inst_src0_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) { + *type = brw_inst_src0_type(devinfo, inst); + return *type != -1; + } else if (brw_inst_src1_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) { + *type = brw_inst_src1_type(devinfo, inst); + return *type != -1; + } - /* We get one bit replicated through the top 20 bits. */ - return imm == 0 || imm == 0xfffff000; + return false; } /** @@ -1037,8 +1564,7 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) (brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_DF || brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_UQ || brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_Q))) { - enum brw_reg_file file = brw_inst_src1_reg_file(devinfo, &inst); - brw_inst_set_src1_file_type(devinfo, &inst, file, BRW_REGISTER_TYPE_UD); + brw_inst_set_src1_reg_hw_type(devinfo, &inst, 0); } /* Compacted instructions only have 12-bits (plus 1 for the other 20) @@ -1051,8 +1577,12 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) * to do this there. * * If we see a 0.0:F, change the type to VF so that it can be compacted. + * + * Compaction of floating-point immediates is improved on Gen12, thus + * removing the need for this. */ - if (brw_inst_imm_ud(devinfo, &inst) == 0x0 && + if (devinfo->gen < 12 && + brw_inst_imm_ud(devinfo, &inst) == 0x0 && brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_F && brw_inst_dst_type(devinfo, &inst) == BRW_REGISTER_TYPE_F && brw_inst_dst_hstride(devinfo, &inst) == BRW_HORIZONTAL_STRIDE_1) { @@ -1062,8 +1592,12 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) /* There are no mappings for dst:d | i:d, so if the immediate is suitable * set the types to :UD so the instruction can be compacted. + * + * FINISHME: Use dst:f | imm:f on Gen12 */ - if (is_compactable_immediate(brw_inst_imm_ud(devinfo, &inst)) && + if (devinfo->gen < 12 && + compact_immediate(devinfo, BRW_REGISTER_TYPE_D, + brw_inst_imm_ud(devinfo, &inst)) != -1 && brw_inst_cond_modifier(devinfo, &inst) == BRW_CONDITIONAL_NONE && brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_D && brw_inst_dst_type(devinfo, &inst) == BRW_REGISTER_TYPE_D) { @@ -1105,13 +1639,20 @@ brw_try_compact_instruction(const struct gen_device_info *devinfo, } } - bool is_immediate = - brw_inst_src0_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE || - brw_inst_src1_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE; - if (is_immediate && - (devinfo->gen < 6 || - !is_compactable_immediate(brw_inst_imm_ud(devinfo, src)))) { - return false; + enum brw_reg_type type; + bool is_immediate = has_immediate(devinfo, src, &type); + + unsigned compacted_imm = 0; + + if (is_immediate) { + /* Instructions with immediates cannot be compacted on Gen < 6 */ + if (devinfo->gen < 6) + return false; + + compacted_imm = compact_immediate(devinfo, type, + brw_inst_imm_ud(devinfo, src)); + if (compacted_imm == -1) + return false; } if (has_unmapped_bits(devinfo, src)) @@ -1121,49 +1662,61 @@ brw_try_compact_instruction(const struct gen_device_info *devinfo, #define compact(field) \ brw_compact_inst_set_##field(devinfo, &temp, brw_inst_##field(devinfo, src)) +#define compact_reg(field) \ + brw_compact_inst_set_##field##_reg_nr(devinfo, &temp, \ + brw_inst_##field##_da_reg_nr(devinfo, src)) compact(hw_opcode); compact(debug_control); if (!set_control_index(devinfo, &temp, src)) return false; - if (!set_datatype_index(devinfo, &temp, src)) + if (!set_datatype_index(devinfo, &temp, src, is_immediate)) return false; if (!set_subreg_index(devinfo, &temp, src, is_immediate)) return false; - - if (devinfo->gen >= 6) { - compact(acc_wr_control); - } else { - compact(mask_control_ex); - } - - compact(cond_modifier); - - if (devinfo->gen <= 6) - compact(flag_subreg_nr); - - brw_compact_inst_set_cmpt_control(devinfo, &temp, true); - if (!set_src0_index(devinfo, &temp, src)) return false; - if (!set_src1_index(devinfo, &temp, src, is_immediate)) + if (!set_src1_index(devinfo, &temp, src, is_immediate, compacted_imm)) return false; - brw_compact_inst_set_dst_reg_nr(devinfo, &temp, - brw_inst_dst_da_reg_nr(devinfo, src)); - brw_compact_inst_set_src0_reg_nr(devinfo, &temp, - brw_inst_src0_da_reg_nr(devinfo, src)); + if (devinfo->gen >= 12) { + compact(swsb); + compact_reg(dst); + compact_reg(src0); - if (is_immediate) { - brw_compact_inst_set_src1_reg_nr(devinfo, &temp, - brw_inst_imm_ud(devinfo, src) & 0xff); + if (is_immediate) { + /* src1 reg takes the high 8 bits (of the 12-bit compacted value) */ + brw_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm >> 4); + } else { + compact_reg(src1); + } } else { - brw_compact_inst_set_src1_reg_nr(devinfo, &temp, - brw_inst_src1_da_reg_nr(devinfo, src)); + if (devinfo->gen >= 6) { + compact(acc_wr_control); + } else { + compact(mask_control_ex); + } + + if (devinfo->gen <= 6) + compact(flag_subreg_nr); + + compact(cond_modifier); + + compact_reg(dst); + compact_reg(src0); + + if (is_immediate) { + /* src1 reg takes the low 8 bits (of the 13-bit compacted value) */ + brw_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm & 0xff); + } else { + compact_reg(src1); + } } + brw_compact_inst_set_cmpt_control(devinfo, &temp, true); #undef compact +#undef compact_reg *dst = temp; @@ -1177,7 +1730,18 @@ set_uncompacted_control(const struct gen_device_info *devinfo, brw_inst *dst, uint32_t uncompacted = control_index_table[brw_compact_inst_control_index(devinfo, src)]; - if (devinfo->gen >= 8) { + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 95, 92, (uncompacted >> 17)); + brw_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); + brw_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); + brw_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); + brw_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); + brw_inst_set_bits(dst, 23, 22, (uncompacted >> 6) & 0x3); + brw_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); + } else if (devinfo->gen >= 8) { brw_inst_set_bits(dst, 33, 31, (uncompacted >> 16)); brw_inst_set_bits(dst, 23, 12, (uncompacted >> 4) & 0xfff); brw_inst_set_bits(dst, 10, 9, (uncompacted >> 2) & 0x3); @@ -1199,7 +1763,18 @@ set_uncompacted_datatype(const struct gen_device_info *devinfo, brw_inst *dst, uint32_t uncompacted = datatype_table[brw_compact_inst_datatype_index(devinfo, src)]; - if (devinfo->gen >= 8) { + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 98, 98, (uncompacted >> 19)); + brw_inst_set_bits(dst, 91, 88, (uncompacted >> 15) & 0xf); + brw_inst_set_bits(dst, 66, 66, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 50, 50, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 49, 48, (uncompacted >> 11) & 0x3); + brw_inst_set_bits(dst, 47, 47, (uncompacted >> 10) & 0x1); + brw_inst_set_bits(dst, 46, 46, (uncompacted >> 9) & 0x1); + brw_inst_set_bits(dst, 43, 40, (uncompacted >> 5) & 0xf); + brw_inst_set_bits(dst, 39, 36, (uncompacted >> 1) & 0xf); + brw_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); + } else if (devinfo->gen >= 8) { brw_inst_set_bits(dst, 63, 61, (uncompacted >> 18)); brw_inst_set_bits(dst, 94, 89, (uncompacted >> 12) & 0x3f); brw_inst_set_bits(dst, 46, 35, (uncompacted >> 0) & 0xfff); @@ -1216,9 +1791,15 @@ set_uncompacted_subreg(const struct gen_device_info *devinfo, brw_inst *dst, uint16_t uncompacted = subreg_table[brw_compact_inst_subreg_index(devinfo, src)]; - brw_inst_set_bits(dst, 100, 96, (uncompacted >> 10)); - brw_inst_set_bits(dst, 68, 64, (uncompacted >> 5) & 0x1f); - brw_inst_set_bits(dst, 52, 48, (uncompacted >> 0) & 0x1f); + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 103, 99, (uncompacted >> 10)); + brw_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); + } else { + brw_inst_set_bits(dst, 100, 96, (uncompacted >> 10)); + brw_inst_set_bits(dst, 68, 64, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 52, 48, (uncompacted >> 0) & 0x1f); + } } static void @@ -1228,21 +1809,31 @@ set_uncompacted_src0(const struct gen_device_info *devinfo, brw_inst *dst, uint32_t compacted = brw_compact_inst_src0_index(devinfo, src); uint16_t uncompacted = src0_index_table[compacted]; - brw_inst_set_bits(dst, 88, 77, uncompacted); + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 87, 84, (uncompacted >> 8)); + brw_inst_set_bits(dst, 83, 81, (uncompacted >> 5) & 0x7); + brw_inst_set_bits(dst, 80, 80, (uncompacted >> 4) & 0x1); + brw_inst_set_bits(dst, 65, 64, (uncompacted >> 2) & 0x3); + brw_inst_set_bits(dst, 45, 44, (uncompacted >> 0) & 0x3); + } else { + brw_inst_set_bits(dst, 88, 77, uncompacted); + } } static void set_uncompacted_src1(const struct gen_device_info *devinfo, brw_inst *dst, - brw_compact_inst *src, bool is_immediate) + brw_compact_inst *src) { - if (is_immediate) { - signed high5 = brw_compact_inst_src1_index(devinfo, src); - /* Replicate top bit of src1_index into high 20 bits of the immediate. */ - brw_inst_set_imm_ud(devinfo, dst, (high5 << 27) >> 19); + uint16_t uncompacted = + src1_index_table[brw_compact_inst_src1_index(devinfo, src)]; + + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 121, 120, (uncompacted >> 10)); + brw_inst_set_bits(dst, 119, 116, (uncompacted >> 6) & 0xf); + brw_inst_set_bits(dst, 115, 113, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 112, 112, (uncompacted >> 2) & 0x1); + brw_inst_set_bits(dst, 97, 96, (uncompacted >> 0) & 0x3); } else { - uint16_t uncompacted = - src1_index_table[brw_compact_inst_src1_index(devinfo, src)]; - brw_inst_set_bits(dst, 120, 109, uncompacted); } } @@ -1253,14 +1844,38 @@ set_uncompacted_3src_control_index(const struct gen_device_info *devinfo, { assert(devinfo->gen >= 8); - uint32_t compacted = brw_compact_inst_3src_control_index(devinfo, src); - uint32_t uncompacted = gen8_3src_control_index_table[compacted]; + if (devinfo->gen >= 12) { + uint64_t compacted = brw_compact_inst_3src_control_index(devinfo, src); + uint64_t uncompacted = gen12_3src_control_index_table[compacted]; + + brw_inst_set_bits(dst, 95, 92, (uncompacted >> 32)); + brw_inst_set_bits(dst, 90, 88, (uncompacted >> 29) & 0x7); + brw_inst_set_bits(dst, 82, 80, (uncompacted >> 26) & 0x7); + brw_inst_set_bits(dst, 50, 50, (uncompacted >> 25) & 0x1); + brw_inst_set_bits(dst, 48, 48, (uncompacted >> 24) & 0x1); + brw_inst_set_bits(dst, 42, 40, (uncompacted >> 21) & 0x7); + brw_inst_set_bits(dst, 39, 39, (uncompacted >> 20) & 0x1); + brw_inst_set_bits(dst, 38, 36, (uncompacted >> 17) & 0x7); + brw_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); + brw_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); + brw_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); + brw_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); + brw_inst_set_bits(dst, 23, 23, (uncompacted >> 7) & 0x1); + brw_inst_set_bits(dst, 22, 22, (uncompacted >> 6) & 0x1); + brw_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); + } else { + uint32_t compacted = brw_compact_inst_3src_control_index(devinfo, src); + uint32_t uncompacted = gen8_3src_control_index_table[compacted]; - brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7); - brw_inst_set_bits(dst, 28, 8, (uncompacted >> 0) & 0x1fffff); + brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7); + brw_inst_set_bits(dst, 28, 8, (uncompacted >> 0) & 0x1fffff); - if (devinfo->gen >= 9 || devinfo->is_cherryview) - brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3); + if (devinfo->gen >= 9 || devinfo->is_cherryview) + brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3); + } } static void @@ -1270,25 +1885,61 @@ set_uncompacted_3src_source_index(const struct gen_device_info *devinfo, assert(devinfo->gen >= 8); uint32_t compacted = brw_compact_inst_3src_source_index(devinfo, src); - uint64_t uncompacted = gen8_3src_source_index_table[compacted]; - - brw_inst_set_bits(dst, 83, 83, (uncompacted >> 43) & 0x1); - brw_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff); - brw_inst_set_bits(dst, 93, 86, (uncompacted >> 27) & 0xff); - brw_inst_set_bits(dst, 72, 65, (uncompacted >> 19) & 0xff); - brw_inst_set_bits(dst, 55, 37, (uncompacted >> 0) & 0x7ffff); - - if (devinfo->gen >= 9 || devinfo->is_cherryview) { - brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3); - brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3); - brw_inst_set_bits(dst, 84, 84, (uncompacted >> 44) & 0x1); + + if (devinfo->gen >= 12) { + uint32_t uncompacted = gen12_3src_source_index_table[compacted]; + + brw_inst_set_bits(dst, 114, 114, (uncompacted >> 20)); + brw_inst_set_bits(dst, 113, 112, (uncompacted >> 18) & 0x3); + brw_inst_set_bits(dst, 98, 98, (uncompacted >> 17) & 0x1); + brw_inst_set_bits(dst, 97, 96, (uncompacted >> 15) & 0x3); + brw_inst_set_bits(dst, 91, 91, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 87, 86, (uncompacted >> 12) & 0x3); + brw_inst_set_bits(dst, 85, 84, (uncompacted >> 10) & 0x3); + brw_inst_set_bits(dst, 83, 83, (uncompacted >> 9) & 0x1); + brw_inst_set_bits(dst, 66, 66, (uncompacted >> 8) & 0x1); + brw_inst_set_bits(dst, 65, 64, (uncompacted >> 6) & 0x3); + brw_inst_set_bits(dst, 47, 47, (uncompacted >> 5) & 0x1); + brw_inst_set_bits(dst, 46, 46, (uncompacted >> 4) & 0x1); + brw_inst_set_bits(dst, 45, 44, (uncompacted >> 2) & 0x3); + brw_inst_set_bits(dst, 43, 43, (uncompacted >> 1) & 0x1); + brw_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); } else { - brw_inst_set_bits(dst, 125, 125, (uncompacted >> 45) & 0x1); - brw_inst_set_bits(dst, 104, 104, (uncompacted >> 44) & 0x1); + uint64_t uncompacted = gen8_3src_source_index_table[compacted]; + + brw_inst_set_bits(dst, 83, 83, (uncompacted >> 43) & 0x1); + brw_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff); + brw_inst_set_bits(dst, 93, 86, (uncompacted >> 27) & 0xff); + brw_inst_set_bits(dst, 72, 65, (uncompacted >> 19) & 0xff); + brw_inst_set_bits(dst, 55, 37, (uncompacted >> 0) & 0x7ffff); + + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3); + brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3); + brw_inst_set_bits(dst, 84, 84, (uncompacted >> 44) & 0x1); + } else { + brw_inst_set_bits(dst, 125, 125, (uncompacted >> 45) & 0x1); + brw_inst_set_bits(dst, 104, 104, (uncompacted >> 44) & 0x1); + } } } static void +set_uncompacted_3src_subreg_index(const struct gen_device_info *devinfo, + brw_inst *dst, brw_compact_inst *src) +{ + assert(devinfo->gen >= 12); + + uint32_t compacted = brw_compact_inst_3src_subreg_index(devinfo, src); + uint32_t uncompacted = gen12_3src_subreg_table[compacted]; + + brw_inst_set_bits(dst, 119, 115, (uncompacted >> 15)); + brw_inst_set_bits(dst, 103, 99, (uncompacted >> 10) & 0x1f); + brw_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); +} + +static void brw_uncompact_3src_instruction(const struct gen_device_info *devinfo, brw_inst *dst, brw_compact_inst *src) { @@ -1301,22 +1952,35 @@ brw_uncompact_3src_instruction(const struct gen_device_info *devinfo, uncompact(hw_opcode); - set_uncompacted_3src_control_index(devinfo, dst, src); - set_uncompacted_3src_source_index(devinfo, dst, src); - - uncompact(dst_reg_nr); - uncompact_a16(src0_rep_ctrl); + if (devinfo->gen >= 12) { + set_uncompacted_3src_control_index(devinfo, dst, src); + set_uncompacted_3src_source_index(devinfo, dst, src); + set_uncompacted_3src_subreg_index(devinfo, dst, src); + + uncompact(debug_control); + uncompact(swsb); + uncompact(dst_reg_nr); + uncompact(src0_reg_nr); + uncompact(src1_reg_nr); + uncompact(src2_reg_nr); + } else { + set_uncompacted_3src_control_index(devinfo, dst, src); + set_uncompacted_3src_source_index(devinfo, dst, src); + + uncompact(dst_reg_nr); + uncompact_a16(src0_rep_ctrl); + uncompact(debug_control); + uncompact(saturate); + uncompact_a16(src1_rep_ctrl); + uncompact_a16(src2_rep_ctrl); + uncompact(src0_reg_nr); + uncompact(src1_reg_nr); + uncompact(src2_reg_nr); + uncompact_a16(src0_subreg_nr); + uncompact_a16(src1_subreg_nr); + uncompact_a16(src2_subreg_nr); + } brw_inst_set_3src_cmpt_control(devinfo, dst, false); - uncompact(debug_control); - uncompact(saturate); - uncompact_a16(src1_rep_ctrl); - uncompact_a16(src2_rep_ctrl); - uncompact(src0_reg_nr); - uncompact(src1_reg_nr); - uncompact(src2_reg_nr); - uncompact_a16(src0_subreg_nr); - uncompact_a16(src1_subreg_nr); - uncompact_a16(src2_subreg_nr); #undef uncompact #undef uncompact_a16 @@ -1337,48 +2001,51 @@ brw_uncompact_instruction(const struct gen_device_info *devinfo, brw_inst *dst, #define uncompact(field) \ brw_inst_set_##field(devinfo, dst, brw_compact_inst_##field(devinfo, src)) +#define uncompact_reg(field) \ + brw_inst_set_##field##_da_reg_nr(devinfo, dst, \ + brw_compact_inst_##field##_reg_nr(devinfo, src)) uncompact(hw_opcode); uncompact(debug_control); set_uncompacted_control(devinfo, dst, src); set_uncompacted_datatype(devinfo, dst, src); - - /* src0/1 register file fields are in the datatype table. */ - bool is_immediate = brw_inst_src0_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE || - brw_inst_src1_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE; - set_uncompacted_subreg(devinfo, dst, src); + set_uncompacted_src0(devinfo, dst, src); - if (devinfo->gen >= 6) { - uncompact(acc_wr_control); + enum brw_reg_type type; + if (has_immediate(devinfo, dst, &type)) { + unsigned imm = uncompact_immediate(devinfo, type, + brw_compact_inst_imm(devinfo, src)); + brw_inst_set_imm_ud(devinfo, dst, imm); } else { - uncompact(mask_control_ex); + set_uncompacted_src1(devinfo, dst, src); + uncompact_reg(src1); } - uncompact(cond_modifier); - - if (devinfo->gen <= 6) - uncompact(flag_subreg_nr); + if (devinfo->gen >= 12) { + uncompact(swsb); + uncompact_reg(dst); + uncompact_reg(src0); + } else { + if (devinfo->gen >= 6) { + uncompact(acc_wr_control); + } else { + uncompact(mask_control_ex); + } - set_uncompacted_src0(devinfo, dst, src); - set_uncompacted_src1(devinfo, dst, src, is_immediate); + uncompact(cond_modifier); - brw_inst_set_dst_da_reg_nr(devinfo, dst, - brw_compact_inst_dst_reg_nr(devinfo, src)); - brw_inst_set_src0_da_reg_nr(devinfo, dst, - brw_compact_inst_src0_reg_nr(devinfo, src)); + if (devinfo->gen <= 6) + uncompact(flag_subreg_nr); - if (is_immediate) { - brw_inst_set_imm_ud(devinfo, dst, - brw_inst_imm_ud(devinfo, dst) | - brw_compact_inst_src1_reg_nr(devinfo, src)); - } else { - brw_inst_set_src1_da_reg_nr(devinfo, dst, - brw_compact_inst_src1_reg_nr(devinfo, src)); + uncompact_reg(dst); + uncompact_reg(src0); } + brw_inst_set_cmpt_control(devinfo, dst, false); #undef uncompact +#undef uncompact_reg } void brw_debug_compact_uncompact(const struct gen_device_info *devinfo, @@ -1488,14 +2155,19 @@ brw_init_compaction_tables(const struct gen_device_info *devinfo) assert(gen8_subreg_table[ARRAY_SIZE(gen8_subreg_table) - 1] != 0); assert(gen8_src_index_table[ARRAY_SIZE(gen8_src_index_table) - 1] != 0); assert(gen11_datatype_table[ARRAY_SIZE(gen11_datatype_table) - 1] != 0); + assert(gen12_control_index_table[ARRAY_SIZE(gen12_control_index_table) - 1] != 0); + assert(gen12_datatype_table[ARRAY_SIZE(gen12_datatype_table) - 1] != 0); + assert(gen12_subreg_table[ARRAY_SIZE(gen12_subreg_table) - 1] != 0); + assert(gen12_src0_index_table[ARRAY_SIZE(gen12_src0_index_table) - 1] != 0); + assert(gen12_src1_index_table[ARRAY_SIZE(gen12_src1_index_table) - 1] != 0); switch (devinfo->gen) { case 12: - control_index_table = NULL; - datatype_table = NULL; - subreg_table = NULL; - src0_index_table = NULL; - src1_index_table = NULL; + control_index_table = gen12_control_index_table;; + datatype_table = gen12_datatype_table; + subreg_table = gen12_subreg_table; + src0_index_table = gen12_src0_index_table; + src1_index_table = gen12_src1_index_table; break; case 11: control_index_table = gen8_control_index_table; @@ -1544,7 +2216,7 @@ void brw_compact_instructions(struct brw_codegen *p, int start_offset, struct disasm_info *disasm) { - if (unlikely(INTEL_DEBUG & DEBUG_NO_COMPACTION) || p->devinfo->gen > 11) + if (unlikely(INTEL_DEBUG & DEBUG_NO_COMPACTION)) return; const struct gen_device_info *devinfo = p->devinfo; diff --git a/src/intel/compiler/brw_inst.h b/src/intel/compiler/brw_inst.h index 3927190f776..9daf8f21849 100644 --- a/src/intel/compiler/brw_inst.h +++ b/src/intel/compiler/brw_inst.h @@ -1397,6 +1397,18 @@ FC(swsb, /* 4+ */ -1, -1, /* 12+ */ 15, 8, devinfo->gen >= 12) F(debug_control, /* 4+ */ 7, 7, /* 12+ */ 7, 7) F(hw_opcode, /* 4+ */ 6, 0, /* 12+ */ 6, 0) /* Same location as brw_inst */ +static inline unsigned +brw_compact_inst_imm(const struct gen_device_info *devinfo, + const brw_compact_inst *inst) +{ + if (devinfo->gen >= 12) { + return brw_compact_inst_bits(inst, 63, 52); + } else { + return (brw_compact_inst_bits(inst, 39, 35) << 8) | + (brw_compact_inst_bits(inst, 63, 56)); + } +} + /** * (Gen8+) Compacted three-source instructions: * @{ |