gk104/ir: Use the new rcp/rsq in library

[imirkin: add a few more "long" prefixes to safen things up]
Acked-by: Ilia Mirkin <[email protected]>
Cc: 19.0 <[email protected]>

2 files changed, 333 insertions, 14 deletions

diff --git a/src/gallium/drivers/nouveau/codegen/lib/gk104.asm b/src/gallium/drivers/nouveau/codegen/lib/gk104.asm
index cd65b547279..576da1bab60 100644
--- a/src/gallium/drivers/nouveau/codegen/lib/gk104.asm
+++ b/src/gallium/drivers/nouveau/codegen/lib/gk104.asm
@@ -543,6 +543,8 @@ $p2 suldgb b32 $r3 cg zero u8 g[$r4d] $r2 $p0
 $p1 suldgb b32 $r3 cv zero u8 g[$r4d] $r2 $p0
 long mov b32 $r3 0x3f800000
 long nop
+sched 0x00 0x00 0x00 0x00 0x00 0x00 0x00
+long nop
 long ret
 
 
@@ -554,7 +556,144 @@ long ret
 // SIZE:    9 * 8 bytes
 //
 gk104_rcp_f64:
-   long nop
+   // Step 1: classify input according to exponent and value, and calculate
+   // result for 0/inf/nan. $r2 holds the exponent value, which starts at
+   // bit 52 (bit 20 of the upper half) and is 11 bits in length
+   ext u32 $r2 $r1 0xb14
+   add b32 $r3 $r2 0xffffffff
+   joinat #rcp_rejoin
+   // We want to check whether the exponent is 0 or 0x7ff (i.e. NaN, inf,
+   // denorm, or 0). Do this by substracting 1 from the exponent, which will
+   // mean that it's > 0x7fd in those cases when doing unsigned comparison
+   set $p0 0x1 gt u32 $r3 0x7fd
+   // $r3: 0 for norms, 0x36 for denorms, -1 for others
+   long mov b32 $r3 0x0
+   sched 0x2f 0x04 0x2d 0x2b 0x2f 0x28 0x28
+   join (not $p0) nop
+   // Process all special values: NaN, inf, denorm, 0
+   mov b32 $r3 0xffffffff
+   // A number is NaN if its abs value is greater than or unordered with inf
+   set $p0 0x1 gtu f64 abs $r0d 0x7ff0000000000000
+   (not $p0) bra #rcp_inf_or_denorm_or_zero
+   // NaN -> NaN, the next line sets the "quiet" bit of the result. This
+   // behavior is both seen on the CPU and the blob
+   join or b32 $r1 $r1 0x80000
+rcp_inf_or_denorm_or_zero:
+   and b32 $r4 $r1 0x7ff00000
+   // Other values with nonzero in exponent field should be inf
+   set $p0 0x1 eq s32 $r4 0x0
+   sched 0x2b 0x04 0x2f 0x2d 0x2b 0x2f 0x20
+   $p0 bra #rcp_denorm_or_zero
+   // +/-Inf -> +/-0
+   xor b32 $r1 $r1 0x7ff00000
+   join mov b32 $r0 0x0
+rcp_denorm_or_zero:
+   set $p0 0x1 gtu f64 abs $r0d 0x0
+   $p0 bra #rcp_denorm
+   // +/-0 -> +/-Inf
+   join or b32 $r1 $r1 0x7ff00000
+rcp_denorm:
+   // non-0 denorms: multiply with 2^54 (the 0x36 in $r3), join with norms
+   mul rn f64 $r0d $r0d 0x4350000000000000
+   sched 0x2f 0x28 0x2b 0x28 0x28 0x04 0x28
+   join mov b32 $r3 0x36
+rcp_rejoin:
+   // All numbers with -1 in $r3 have their result ready in $r0d, return them
+   // others need further calculation
+   set $p0 0x1 lt s32 $r3 0x0
+   $p0 bra #rcp_end
+   // Step 2: Before the real calculation goes on, renormalize the values to
+   // range [1, 2) by setting exponent field to 0x3ff (the exponent of 1)
+   // result in $r6d. The exponent will be recovered later.
+   ext u32 $r2 $r1 0xb14
+   and b32 $r7 $r1 0x800fffff
+   add b32 $r7 $r7 0x3ff00000
+   long mov b32 $r6 $r0
+   sched 0x2b 0x04 0x28 0x28 0x2a 0x2b 0x2e
+   // Step 3: Convert new value to float (no overflow will occur due to step
+   // 2), calculate rcp and do newton-raphson step once
+   cvt rz f32 $r5 f64 $r6d
+   long rcp f32 $r4 $r5
+   mov b32 $r0 0xbf800000
+   fma rn f32 $r5 $r4 $r5 $r0
+   fma rn f32 $r0 neg $r4 $r5 $r4
+   // Step 4: convert result $r0 back to double, do newton-raphson steps
+   cvt f64 $r0d f32 $r0
+   cvt f64 $r6d neg f64 $r6d
+   sched 0x2e 0x29 0x29 0x29 0x29 0x29 0x29
+   cvt f64 $r8d f32 0x3f800000
+   // 4 Newton-Raphson Steps, tmp in $r4d, result in $r0d
+   // The formula used here (and above) is:
+   //     RCP_{n + 1} = 2 * RCP_{n} - x * RCP_{n} * RCP_{n}
+   // The following code uses 2 FMAs for each step, and it will basically
+   // looks like:
+   //     tmp = -src * RCP_{n} + 1
+   //     RCP_{n + 1} = RCP_{n} * tmp + RCP_{n}
+   fma rn f64 $r4d $r6d $r0d $r8d
+   fma rn f64 $r0d $r0d $r4d $r0d
+   fma rn f64 $r4d $r6d $r0d $r8d
+   fma rn f64 $r0d $r0d $r4d $r0d
+   fma rn f64 $r4d $r6d $r0d $r8d
+   fma rn f64 $r0d $r0d $r4d $r0d
+   sched 0x29 0x20 0x28 0x28 0x28 0x28 0x28
+   fma rn f64 $r4d $r6d $r0d $r8d
+   fma rn f64 $r0d $r0d $r4d $r0d
+   // Step 5: Exponent recovery and final processing
+   // The exponent is recovered by adding what we added to the exponent.
+   // Suppose we want to calculate rcp(x), but we have rcp(cx), then
+   //     rcp(x) = c * rcp(cx)
+   // The delta in exponent comes from two sources:
+   //   1) The renormalization in step 2. The delta is:
+   //      0x3ff - $r2
+   //   2) (For the denorm input) The 2^54 we multiplied at rcp_denorm, stored
+   //      in $r3
+   // These 2 sources are calculated in the first two lines below, and then
+   // added to the exponent extracted from the result above.
+   // Note that after processing, the new exponent may >= 0x7ff (inf)
+   // or <= 0 (denorm). Those cases will be handled respectively below
+   subr b32 $r2 $r2 0x3ff
+   long add b32 $r4 $r2 $r3
+   ext u32 $r3 $r1 0xb14
+   // New exponent in $r3
+   long add b32 $r3 $r3 $r4
+   add b32 $r2 $r3 0xffffffff
+   sched 0x28 0x2b 0x28 0x2b 0x28 0x28 0x2b
+   // (exponent-1) < 0x7fe (unsigned) means the result is in norm range
+   // (same logic as in step 1)
+   set $p0 0x1 lt u32 $r2 0x7fe
+   (not $p0) bra #rcp_result_inf_or_denorm
+   // Norms: convert exponents back and return
+   shl b32 $r4 $r4 clamp 0x14
+   long add b32 $r1 $r4 $r1
+   bra #rcp_end
+rcp_result_inf_or_denorm:
+   // New exponent >= 0x7ff means that result is inf
+   set $p0 0x1 ge s32 $r3 0x7ff
+   (not $p0) bra #rcp_result_denorm
+   sched 0x20 0x25 0x28 0x2b 0x23 0x25 0x2f
+   // Infinity
+   and b32 $r1 $r1 0x80000000
+   long mov b32 $r0 0x0
+   add b32 $r1 $r1 0x7ff00000
+   bra #rcp_end
+rcp_result_denorm:
+   // Denorm result comes from huge input. The greatest possible fp64, i.e.
+   // 0x7fefffffffffffff's rcp is 0x0004000000000000, 1/4 of the smallest
+   // normal value. Other rcp result should be greater than that. If we
+   // set the exponent field to 1, we can recover the result by multiplying
+   // it with 1/2 or 1/4. 1/2 is used if the "exponent" $r3 is 0, otherwise
+   // 1/4 ($r3 should be -1 then). This is quite tricky but greatly simplifies
+   // the logic here.
+   set $p0 0x1 ne u32 $r3 0x0
+   and b32 $r1 $r1 0x800fffff
+   // 0x3e800000: 1/4
+   $p0 cvt f64 $r6d f32 0x3e800000
+   sched 0x2f 0x28 0x2c 0x2e 0x2a 0x20 0x27
+   // 0x3f000000: 1/2
+   (not $p0) cvt f64 $r6d f32 0x3f000000
+   add b32 $r1 $r1 0x00100000
+   mul rn f64 $r0d $r0d $r6d
+rcp_end:
    long ret
 
 // RSQ F64: Newton Raphson rsqrt(x): r_{i+1} = r_i * (1.5 - 0.5 * x * r_i * r_i)
@@ -565,7 +704,67 @@ gk104_rcp_f64:
 // SIZE:    14 * 8 bytes
 //
 gk104_rsq_f64:
-   long nop
+   // Before getting initial result rsqrt64h, two special cases should be
+   // handled first.
+   // 1. NaN: set the highest bit in mantissa so it'll be surely recognized
+   //    as NaN in rsqrt64h
+   set $p0 0x1 gtu f64 abs $r0d 0x7ff0000000000000
+   $p0 or b32 $r1 $r1 0x00080000
+   and b32 $r2 $r1 0x7fffffff
+   sched 0x27 0x20 0x28 0x2c 0x25 0x28 0x28
+   // 2. denorms and small normal values: using their original value will
+   //    lose precision either at rsqrt64h or the first step in newton-raphson
+   //    steps below. Take 2 as a threshold in exponent field, and multiply
+   //    with 2^54 if the exponent is smaller or equal. (will multiply 2^27
+   //    to recover in the end)
+   ext u32 $r3 $r1 0xb14
+   set $p1 0x1 le u32 $r3 0x2
+   long or b32 $r2 $r0 $r2
+   $p1 mul rn f64 $r0d $r0d 0x4350000000000000
+   rsqrt64h $r5 $r1
+   // rsqrt64h will give correct result for 0/inf/nan, the following logic
+   // checks whether the input is one of those (exponent is 0x7ff or all 0
+   // except for the sign bit)
+   set b32 $r6 ne u32 $r3 0x7ff
+   long and b32 $r2 $r2 $r6
+   sched 0x28 0x2b 0x20 0x27 0x28 0x2e 0x28
+   set $p0 0x1 ne u32 $r2 0x0
+   $p0 bra #rsq_norm
+   // For 0/inf/nan, make sure the sign bit agrees with input and return
+   and b32 $r1 $r1 0x80000000
+   long mov b32 $r0 0x0
+   long or b32 $r1 $r1 $r5
+   long ret
+rsq_norm:
+   // For others, do 4 Newton-Raphson steps with the formula:
+   //     RSQ_{n + 1} = RSQ_{n} * (1.5 - 0.5 * x * RSQ_{n} * RSQ_{n})
+   // In the code below, each step is written as:
+   //     tmp1 = 0.5 * x * RSQ_{n}
+   //     tmp2 = -RSQ_{n} * tmp1 + 0.5
+   //     RSQ_{n + 1} = RSQ_{n} * tmp2 + RSQ_{n}
+   long mov b32 $r4 0x0
+   sched 0x2f 0x29 0x29 0x29 0x29 0x29 0x29
+   // 0x3f000000: 1/2
+   cvt f64 $r8d f32 0x3f000000
+   mul rn f64 $r2d $r0d $r8d
+   mul rn f64 $r0d $r2d $r4d
+   fma rn f64 $r6d neg $r4d $r0d $r8d
+   fma rn f64 $r4d $r4d $r6d $r4d
+   mul rn f64 $r0d $r2d $r4d
+   fma rn f64 $r6d neg $r4d $r0d $r8d
+   sched 0x29 0x29 0x29 0x29 0x29 0x29 0x29
+   fma rn f64 $r4d $r4d $r6d $r4d
+   mul rn f64 $r0d $r2d $r4d
+   fma rn f64 $r6d neg $r4d $r0d $r8d
+   fma rn f64 $r4d $r4d $r6d $r4d
+   mul rn f64 $r0d $r2d $r4d
+   fma rn f64 $r6d neg $r4d $r0d $r8d
+   fma rn f64 $r4d $r4d $r6d $r4d
+   sched 0x29 0x20 0x28 0x2e 0x00 0x00 0x00
+   // Multiply 2^27 to result for small inputs to recover
+   $p1 mul rn f64 $r4d $r4d 0x41a0000000000000
+   long mov b32 $r1 $r5
+   long mov b32 $r0 $r4
    long ret
 
 //
diff --git a/src/gallium/drivers/nouveau/codegen/lib/gk104.asm.h b/src/gallium/drivers/nouveau/codegen/lib/gk104.asm.h
index 37998768efe..ed948dee471 100644
--- a/src/gallium/drivers/nouveau/codegen/lib/gk104.asm.h
+++ b/src/gallium/drivers/nouveau/codegen/lib/gk104.asm.h
@@ -481,12 +481,132 @@ uint64_t gk104_builtin_code[] = {
 	0xd40040000840c785,
 	0x18fe00000000dde2,
 	0x4000000000001de4,
-	0x9000000000001de7,
-/* 0x0f08: gk104_rcp_f64 */
+	0x2000000000000007,
 	0x4000000000001de4,
 	0x9000000000001de7,
-/* 0x0f18: gk104_rsq_f64 */
-	0x4000000000001de4,
+/* 0x0f18: gk104_rcp_f64 */
+	0x7000c02c50109c03,
+	0x0bfffffffc20dc02,
+	0x6000000280000007,
+	0x1a0ec01ff431dc03,
+	0x180000000000dde2,
+	0x228282f2b2d042f7,
+	0x40000000000021f4,
+	0x1bfffffffc00dde2,
+	0x1e0edffc0001dc81,
+	0x40000000200021e7,
+	0x3800200000105c52,
+/* 0x0f70: rcp_inf_or_denorm_or_zero */
+	0x39ffc00000111c02,
+	0x190e0000fc41dc23,
+	0x2202f2b2d2f042b7,
+	0x40000000400001e7,
+	0x39ffc00000105c82,
+	0x1800000000001df2,
+/* 0x0fa0: rcp_denorm_or_zero */
+	0x1e0ec0000001dc81,
+	0x40000000200001e7,
+	0x39ffc00000105c52,
+/* 0x0fb8: rcp_denorm */
+	0x5000d0d400001c01,
+	0x2280428282b282f7,
+	0x18000000d800ddf2,
+/* 0x0fd0: rcp_rejoin */
+	0x188e0000fc31dc23,
+	0x40000006000001e7,
+	0x7000c02c50109c03,
+	0x3a003ffffc11dc02,
+	0x08ffc0000071dc02,
+	0x2800000000019de4,
+	0x22e2b2a2828042b7,
+	0x1006000019a15c04,
+	0xc800000010511c00,
+	0x1afe000000001de2,
+	0x3000000014415c00,
+	0x3008000014401e00,
+	0x1000000001301c04,
+	0x1000000019b19d04,
+	0x22929292929292e7,
+	0x1000cfe001321c04,
+	0x2010000000611c01,
+	0x2000000010001c01,
+	0x2010000000611c01,
+	0x2000000010001c01,
+	0x2010000000611c01,
+	0x2000000010001c01,
+	0x2282828282820297,
+	0x2010000000611c01,
+	0x2000000010001c01,
+	0x0800000ffc209e02,
+	0x480000000c211c03,
+	0x7000c02c5010dc03,
+	0x480000001030dc03,
+	0x0bfffffffc309c02,
+	0x22b28282b282b287,
+	0x188ec01ff821dc03,
+	0x40000000600021e7,
+	0x6000c00050411c03,
+	0x4800000004405c03,
+	0x40000001c0001de7,
+/* 0x10f0: rcp_result_inf_or_denorm */
+	0x1b0ec01ffc31dc23,
+	0x40000000a00021e7,
+	0x22f25232b2825207,
+	0x3a00000000105c02,
+	0x1800000000001de2,
+	0x09ffc00000105c02,
+	0x40000000e0001de7,
+/* 0x1128: rcp_result_denorm */
+	0x1a8e0000fc31dc03,
+	0x3a003ffffc105c02,
+	0x1000cfa001318004,
+	0x227202a2e2c282f7,
+	0x1000cfc00131a004,
+	0x0800400000105c02,
+	0x5000000018001c01,
+/* 0x1160: rcp_end */
+	0x9000000000001de7,
+/* 0x1168: gk104_rsq_f64 */
+	0x1e0edffc0001dc81,
+	0x3800200000104042,
+	0x39fffffffc109c02,
+	0x22828252c2820277,
+	0x7000c02c5010dc03,
+	0x198ec0000833dc03,
+	0x6800000008009c43,
+	0x5000d0d400000401,
+	0xc80000001c115c00,
+	0x128ec01ffc319c03,
+	0x6800000018209c03,
+	0x2282e2827202b287,
+	0x1a8e0000fc21dc03,
+	0x40000000800001e7,
+	0x3a00000000105c02,
+	0x1800000000001de2,
+	0x6800000014105c43,
+	0x9000000000001de7,
+/* 0x11f8: rsq_norm */
+	0x1800000000011de2,
+	0x22929292929292f7,
+	0x1000cfc001321c04,
+	0x5000000020009c01,
+	0x5000000010201c01,
+	0x2010000000419e01,
+	0x2008000018411c01,
+	0x5000000010201c01,
+	0x2010000000419e01,
+	0x2292929292929297,
+	0x2008000018411c01,
+	0x5000000010201c01,
+	0x2010000000419e01,
+	0x2008000018411c01,
+	0x5000000010201c01,
+	0x2010000000419e01,
+	0x2008000018411c01,
+	0x20000002e2820297,
+	0x5000d06800410401,
+	0x2800000014005de4,
+	0x2800000010001de4,
 	0x9000000000001de7,
 	0xc800000003f01cc5,
 	0x2c00000100005c04,
@@ -495,7 +615,7 @@ uint64_t gk104_builtin_code[] = {
 	0x680100000c1fdc03,
 	0x4000000a60001c47,
 	0x180000004000dde2,
-/* 0x0f60: spill_cfstack */
+/* 0x12e0: spill_cfstack */
 	0x78000009c0000007,
 	0x0c0000000430dd02,
 	0x4003ffffa0001ca7,
@@ -543,14 +663,14 @@ uint64_t gk104_builtin_code[] = {
 	0x4000000100001ea7,
 	0x480100000c001c03,
 	0x0800000000105c42,
-/* 0x10d8: shared_loop */
+/* 0x1458: shared_loop */
 	0xc100000000309c85,
 	0x9400000500009c85,
 	0x0c00000010001d02,
 	0x0800000000105d42,
 	0x0c0000001030dd02,
 	0x4003ffff40001ca7,
-/* 0x1108: shared_done */
+/* 0x1488: shared_done */
 	0x2800406420001de4,
 	0x2800406430005de4,
 	0xe000000000001c45,
@@ -564,7 +684,7 @@ uint64_t gk104_builtin_code[] = {
 	0x480000000c209c03,
 	0x4801000008001c03,
 	0x0800000000105c42,
-/* 0x1170: search_cstack */
+/* 0x14f0: search_cstack */
 	0x280040646000dde4,
 	0x8400000020009f05,
 	0x190ec0002821dc03,
@@ -573,17 +693,17 @@ uint64_t gk104_builtin_code[] = {
 	0x0800000000105c42,
 	0x0c0000004030dd02,
 	0x00029dff0ffc5cbf,
-/* 0x11b0: entry_found */
+/* 0x1530: entry_found */
 	0x8400000000009f85,
 	0x2800406400001de4,
 	0x2800406410005de4,
 	0x9400000010009c85,
 	0x4000000000001df4,
-/* 0x11d8: end_exit */
+/* 0x1558: end_exit */
 	0x9800000003ffdcc5,
 	0xd000000000008007,
 	0xa000000000004007,
-/* 0x11f0: end_cont */
+/* 0x1570: end_cont */
 	0xd000000000008007,
 	0x3400c3fffc201c04,
 	0xc000000003f01ec5,
@@ -593,6 +713,6 @@ uint64_t gk104_builtin_code[] = {
 uint64_t gk104_builtin_offsets[] = {
 	0x0000000000000000,
 	0x00000000000000f0,
-	0x0000000000000f08,
 	0x0000000000000f18,
+	0x0000000000001168,
 };