nvc0: add MP trap handler for nve4

4 files changed, 314 insertions, 15 deletions

diff --git a/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm b/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm
index 2ed8c0af871..5adc9ff38a5 100644
--- a/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm
+++ b/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm
@@ -561,3 +561,138 @@ long ret
 //
 long nop
 long ret
+//
+// Trap handler.
+// Requires at least 4 GPRs and 32 bytes of l[] memory to temporarily save GPRs.
+// Low 32 bytes of l[] memory shouldn't be used if resumeability is required.
+//
+// Trap info:
+// 0x000: mutex
+// 0x004: PC
+// 0x008: trapstat
+// 0x00c: warperr
+// 0x010: tidx
+// 0x014: tidy
+// 0x018: tidz
+// 0x01c: ctaidx
+// 0x020: ctaidy
+// 0x024: ctaidz
+// 0x030: $r0q
+// 0x130: $flags
+// 0x140: s[]
+//
+st b128 wb l[0x00] $r0q
+// check state of the warp and continue if it didn't cause the trap
+long mov b32 $r1 $trapstat
+long mov b32 $r3 $warperr
+mov $r2 $flags mask 0xffff
+and b32 0 $c $r1 $r3
+e $c bra #end_cont
+// spill control flow stack to l[]
+long mov b32 $r3 16
+spill_cfstack:
+preret #end_exit
+sub b32 $r3 $c $r3 0x1
+lg $c bra #spill_cfstack
+// retrieve pointer to trap info
+mov b32 $r0 c0[0x1900]
+mov b32 $r1 c0[0x1904]
+// we only let a single faulting thread store its state
+mov b32 $r3 0x1
+exch b32 $r3 g[$r0d] $r3
+joinat #end_exit
+set $p0 0x1 eq u32 $r3 0x1
+join $p0 nop
+// store $c and $p registers
+st b32 wb g[$r0d+0x130] $r2
+// store $trapstat and $warperr
+long mov b32 $r2 $trapstat
+long mov b32 $r3 $warperr
+st b64 wb g[$r0d+0x8] $r2d
+// store registers
+st b128 wb g[$r0d+0x40] $r4q
+st b128 wb g[$r0d+0x50] $r8q
+st b128 wb g[$r0d+0x60] $r12q
+st b128 wb g[$r0d+0x70] $r16q
+st b128 wb g[$r0d+0x80] $r20q
+st b128 wb g[$r0d+0x90] $r24q
+st b128 wb g[$r0d+0xa0] $r28q
+st b128 wb g[$r0d+0xb0] $r32q
+st b128 wb g[$r0d+0xc0] $r36q
+st b128 wb g[$r0d+0xd0] $r40q
+st b128 wb g[$r0d+0xe0] $r44q
+st b128 wb g[$r0d+0xf0] $r48q
+st b128 wb g[$r0d+0x100] $r52q
+st b128 wb g[$r0d+0x110] $r56q
+st b128 wb g[$r0d+0x120] $r60q
+ld b64 $r2d cs l[0x0]
+st b64 wb g[$r0d+0x30] $r2d
+ld b64 $r2d cs l[0x8]
+st b64 wb g[$r0d+0x38] $r2d
+// store thread id
+long mov b32 $r2 $tidx
+long mov b32 $r3 $tidy
+st b64 wb g[$r0d+0x10] $r2d
+long mov b32 $r2 $tidz
+long mov b32 $r3 $ctaidx
+st b64 wb g[$r0d+0x18] $r2d
+long mov b32 $r2 $ctaidy
+long mov b32 $r3 $ctaidz
+st b64 wb g[$r0d+0x20] $r2d
+// store shared memory (in reverse order so $r0d is base again at the end)
+long mov b32 $r3 $smemsz
+sub b32 $r3 $c $r3 0x4
+s $c bra #shared_done
+add b32 $r0 $c $r0 $r3
+add b32 $r1 $r1 0x0 $c
+shared_loop:
+long ld b32 $r2 s[$r3]
+long st b32 wb g[$r0d+0x140] $r2
+sub b32 $r0 $c $r0 0x4
+sub b32 $r1 $r1 0x0 $c
+sub b32 $r3 $c $r3 0x4
+lg $c bra #shared_loop
+shared_done:
+// search the stack for trap entry to retrieve PC
+mov b32 $r0 c0[0x1908]
+mov b32 $r1 c0[0x190c]
+membar sys
+// invalidate caches so we can read stack entries via g[]
+cctl ivall 0 l[0]
+cctl ivall 0 g[$r0d]
+// get offsets
+mov b32 $r2 $physid
+ext u32 $r3 $r2 0x0814 // MP id
+ext u32 $r2 $r2 0x0608 // warp id
+mul $r2 u32 $r2 u32 c0[0x1914] // warp offset
+mul $r3 u32 $r3 u32 c0[0x1910] // MP offset
+add b32 $r2 $r2 $r3 // MP + warp offset
+add b32 $r0 $c $r0 $r2
+add b32 $r1 $r1 0x0 $c
+search_cstack:
+mov b32 $r3 c0[0x1918] // cstack size
+ld u8 $r2 cv g[$r0d+0x8]
+set $p0 0x1 eq u32 $r2 0xa
+$p0 bra #entry_found
+add b32 $r0 $c $r0 0x10
+add b32 $r1 $r1 0x0 $c
+sub b32 $r3 $c $r3 0x10
+lg $c bra #search_cstack
+bra #end_exit
+entry_found:
+// load PC (may be unaligned and spread out)
+ld b32 $r2 cv g[$r0d]
+mov b32 $r0 c0[0x1900]
+mov b32 $r1 c0[0x1904]
+st b32 wb g[$r0d+0x4] $r2
+join nop
+// invalidate caches and exit
+end_exit:
+cctl ivall 0 g[0]
+bpt pause 0x0
+rtt terminate
+end_cont:
+bpt pause 0x0
+mov $flags $r2 mask 0xffff
+ld b128 $r0q cs l[0x00]
+rtt
diff --git a/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm.h b/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm.h
index 526e75989d5..ad386f8771a 100644
--- a/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm.h
+++ b/src/gallium/drivers/nvc0/codegen/target_lib_nve4.asm.h
@@ -488,6 +488,99 @@ static const uint64_t nve4_builtin_code[] =
    0x9000000000001de7ULL,
    0x4000000000001de4ULL,
    0x9000000000001de7ULL,
+   0xc800000003f01cc5ULL,
+   0x2c00000100005c04ULL,
+   0x2c0000010800dc04ULL,
+   0x3000c3fffff09c04ULL,
+   0x680100000c1fdc03ULL,
+   0x4000000a60001c47ULL,
+   0x180000004000dde2ULL,
+   0x78000009c0000007ULL,
+   0x0c0000000430dd02ULL,
+   0x4003ffffa0001ca7ULL,
+   0x2800406400001de4ULL,
+   0x2800406410005de4ULL,
+   0x180000000400dde2ULL,
+   0x547e18000000dd05ULL,
+   0x60000008e0000007ULL,
+   0x190ec0000431dc03ULL,
+   0x40000000000001f4ULL,
+   0x94000004c0009c85ULL,
+   0x2c00000100009c04ULL,
+   0x2c0000010800dc04ULL,
+   0x9400000020009ca5ULL,
+   0x9400000100011cc5ULL,
+   0x9400000140021cc5ULL,
+   0x9400000180031cc5ULL,
+   0x94000001c0041cc5ULL,
+   0x9400000200051cc5ULL,
+   0x9400000240061cc5ULL,
+   0x9400000280071cc5ULL,
+   0x94000002c0081cc5ULL,
+   0x9400000300091cc5ULL,
+   0x94000003400a1cc5ULL,
+   0x94000003800b1cc5ULL,
+   0x94000003c00c1cc5ULL,
+   0x94000004000d1cc5ULL,
+   0x94000004400e1cc5ULL,
+   0x94000004800f1cc5ULL,
+   0xc000000003f09ea5ULL,
+   0x94000000c0009ca5ULL,
+   0xc000000023f09ea5ULL,
+   0x94000000e0009ca5ULL,
+   0x2c00000084009c04ULL,
+   0x2c0000008800dc04ULL,
+   0x9400000040009ca5ULL,
+   0x2c0000008c009c04ULL,
+   0x2c0000009400dc04ULL,
+   0x9400000060009ca5ULL,
+   0x2c00000098009c04ULL,
+   0x2c0000009c00dc04ULL,
+   0x9400000080009ca5ULL,
+   0x2c000000c800dc04ULL,
+   0x0c0000001030dd02ULL,
+   0x4000000100001ea7ULL,
+   0x480100000c001c03ULL,
+   0x0800000000105c42ULL,
+   0xc100000000309c85ULL,
+   0x9400000500009c85ULL,
+   0x0c00000010001d02ULL,
+   0x0800000000105d42ULL,
+   0x0c0000001030dd02ULL,
+   0x4003ffff40001ca7ULL,
+   0x2800406420001de4ULL,
+   0x2800406430005de4ULL,
+   0xe000000000001c45ULL,
+   0xd000000003ffdcc5ULL,
+   0x9c000000000fdcc5ULL,
+   0x2c0000000c009c04ULL,
+   0x7000c0205020dc03ULL,
+   0x7000c01820209c03ULL,
+   0x5000406450209c03ULL,
+   0x500040644030dc03ULL,
+   0x480000000c209c03ULL,
+   0x4801000008001c03ULL,
+   0x0800000000105c42ULL,
+   0x280040646000dde4ULL,
+   0x8400000020009f05ULL,
+   0x190ec0002821dc03ULL,
+   0x40000000800001e7ULL,
+   0x0c00000040001c02ULL,
+   0x0800000000105c42ULL,
+   0x0c0000004030dd02ULL,
+   0x00029dff0ffc5cbfULL,
+   0x8400000000009f85ULL,
+   0x2800406400001de4ULL,
+   0x2800406410005de4ULL,
+   0x9400000010009c85ULL,
+   0x4000000000001df4ULL,
+   0x9800000003ffdcc5ULL,
+   0xd000000000008007ULL,
+   0xa000000000004007ULL,
+   0xd000000000008007ULL,
+   0x3400c3fffc201c04ULL,
+   0xc000000003f01ec5ULL,
+   0xa000000000000007ULL
 };
 
 static const uint16_t nve4_builtin_offsets[NVC0_BUILTIN_COUNT] =
diff --git a/src/gallium/drivers/nvc0/nve4_compute.c b/src/gallium/drivers/nvc0/nve4_compute.c
index b5344e4eed6..c61d90cf0d1 100644
--- a/src/gallium/drivers/nvc0/nve4_compute.c
+++ b/src/gallium/drivers/nvc0/nve4_compute.c
@@ -151,6 +151,25 @@ nve4_screen_compute_setup(struct nvc0_screen *screen,
    PUSH_DATA (push, 1);
    PUSH_DATA (push, 3); /* 7 */
    PUSH_DATA (push, 1);
+
+#ifdef DEBUG
+   BEGIN_NVC0(push, NVE4_COMPUTE(UPLOAD_ADDRESS_HIGH), 2);
+   PUSH_DATAh(push, screen->parm->offset + NVE4_CP_INPUT_TRAP_INFO_PTR);
+   PUSH_DATA (push, screen->parm->offset + NVE4_CP_INPUT_TRAP_INFO_PTR);
+   BEGIN_NVC0(push, NVE4_COMPUTE(UPLOAD_SIZE), 2);
+   PUSH_DATA (push, 28);
+   PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_UNK0184_UNKVAL);
+   BEGIN_1IC0(push, NVE4_COMPUTE(UPLOAD_EXEC), 8);
+   PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_UNKVAL_DATA);
+   PUSH_DATA (push, screen->parm->offset + NVE4_CP_PARAM_TRAP_INFO);
+   PUSH_DATAh(push, screen->parm->offset + NVE4_CP_PARAM_TRAP_INFO);
+   PUSH_DATA (push, screen->tls->offset);
+   PUSH_DATAh(push, screen->tls->offset);
+   PUSH_DATA (push, screen->tls->size / 2); /* MP TEMP block size */
+   PUSH_DATA (push, screen->tls->size / 2 / 64); /* warp TEMP block size */
+   PUSH_DATA (push, 0); /* warp cfstack size */
+#endif
+
    BEGIN_NVC0(push, NVE4_COMPUTE(FLUSH), 1);
    PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
 
@@ -617,3 +636,34 @@ nve4_compute_dump_launch_desc(const struct nve4_cp_launch_desc *desc)
    }
 }
 
+static void
+nve4_compute_trap_info(struct nvc0_context *nvc0)
+{
+   struct nvc0_screen *screen = nvc0->screen;
+   struct nouveau_bo *bo = screen->parm;
+   int ret, i;
+   volatile struct nve4_mp_trap_info *info;
+   uint8_t *map;
+
+   ret = nouveau_bo_map(bo, NOUVEAU_BO_RDWR, nvc0->base.client);
+   if (ret)
+      return;
+   map = (uint8_t *)bo->map;
+   info = (volatile struct nve4_mp_trap_info *)(map + NVE4_CP_PARAM_TRAP_INFO);
+
+   if (info->lock) {
+      debug_printf("trapstat = %08x\n", info->trapstat);
+      debug_printf("warperr = %08x\n", info->warperr);
+      debug_printf("PC = %x\n", info->pc);
+      debug_printf("tid = %u %u %u\n",
+                   info->tid[0], info->tid[1], info->tid[2]);
+      debug_printf("ctaid = %u %u %u\n",
+                   info->ctaid[0], info->ctaid[1], info->ctaid[2]);
+      for (i = 0; i <= 63; ++i)
+         debug_printf("$r%i = %08x\n", i, info->r[i]);
+      for (i = 0; i <= 6; ++i)
+         debug_printf("$p%i = %i\n", i, (info->flags >> i) & 1);
+      debug_printf("$c = %x\n", info->flags >> 12);
+   }
+   info->lock = 0;
+}
diff --git a/src/gallium/drivers/nvc0/nve4_compute.h b/src/gallium/drivers/nvc0/nve4_compute.h
index 11d89283cc0..91d64fff695 100644
--- a/src/gallium/drivers/nvc0/nve4_compute.h
+++ b/src/gallium/drivers/nvc0/nve4_compute.h
@@ -7,21 +7,28 @@
 
 /* Input space is implemented as c0[], to which we bind the screen->parm bo.
  */
-#define NVE4_CP_INPUT_USER          0x0000
-#define NVE4_CP_INPUT_USER_LIMIT    0x1000
-#define NVE4_CP_INPUT_GRID_INFO(i) (0x1000 + (i) * 4)
-#define NVE4_CP_INPUT_NTID(i)      (0x1000 + (i) * 4)
-#define NVE4_CP_INPUT_NCTAID(i)    (0x100c + (i) * 4)
-#define NVE4_CP_INPUT_GRIDID        0x1018
-#define NVE4_CP_INPUT_TEX(i)       (0x1040 + (i) * 4)
-#define NVE4_CP_INPUT_TEX_STRIDE    4
-#define NVE4_CP_INPUT_TEX_MAX       32
-#define NVE4_CP_INPUT_MS_OFFSETS    0x10c0
-#define NVE4_CP_INPUT_SUF_STRIDE    64
-#define NVE4_CP_INPUT_SUF(i)       (0x1100 + (i) * NVE4_CP_INPUT_SUF_STRIDE)
-#define NVE4_CP_INPUT_SUF_MAX       32
-#define NVE4_CP_INPUT_SIZE          0x1900
-#define NVE4_CP_PARAM_SIZE          0x2000
+#define NVE4_CP_INPUT_USER           0x0000
+#define NVE4_CP_INPUT_USER_LIMIT     0x1000
+#define NVE4_CP_INPUT_GRID_INFO(i)  (0x1000 + (i) * 4)
+#define NVE4_CP_INPUT_NTID(i)       (0x1000 + (i) * 4)
+#define NVE4_CP_INPUT_NCTAID(i)     (0x100c + (i) * 4)
+#define NVE4_CP_INPUT_GRIDID         0x1018
+#define NVE4_CP_INPUT_TEX(i)        (0x1040 + (i) * 4)
+#define NVE4_CP_INPUT_TEX_STRIDE     4
+#define NVE4_CP_INPUT_TEX_MAX        32
+#define NVE4_CP_INPUT_MS_OFFSETS     0x10c0
+#define NVE4_CP_INPUT_SUF_STRIDE     64
+#define NVE4_CP_INPUT_SUF(i)        (0x1100 + (i) * NVE4_CP_INPUT_SUF_STRIDE)
+#define NVE4_CP_INPUT_SUF_MAX        32
+#define NVE4_CP_INPUT_TRAP_INFO_PTR  0x1900
+#define NVE4_CP_INPUT_TEMP_PTR       0x1908
+#define NVE4_CP_INPUT_MP_TEMP_SIZE   0x1910
+#define NVE4_CP_INPUT_WARP_TEMP_SIZE 0x1914
+#define NVE4_CP_INPUT_CSTACK_SIZE    0x1918
+#define NVE4_CP_INPUT_SIZE           0x1a00
+#define NVE4_CP_PARAM_TRAP_INFO      0x2000
+#define NVE4_CP_PARAM_TRAP_INFO_SZ  (1 << 16)
+#define NVE4_CP_PARAM_SIZE          (NVE4_CP_PARAM_TRAP_INFO + (1 << 16))
 
 struct nve4_cp_launch_desc
 {
@@ -111,4 +118,18 @@ nve4_cp_launch_desc_set_ctx_cb(struct nve4_cp_launch_desc *desc,
    }
 }
 
+struct nve4_mp_trap_info {
+   u32 lock;
+   u32 pc;
+   u32 trapstat;
+   u32 warperr;
+   u32 tid[3];
+   u32 ctaid[3];
+   u32 pad028[2];
+   u32 r[64];
+   u32 flags;
+   u32 pad134[3];
+   u32 s[0x3000];
+};
+
 #endif /* NVE4_COMPUTE_H */