icp: fix all !ENDBR objtool warnings in x86 Asm code

Currently, only Blake3 x86 Asm code has signs of being ENDBR-aware.
At least, under certain conditions it includes some header file and
uses some custom macro from there.
Linux has its own NOENDBR since several releases ago. It's defined
in the same <asm/linkage.h>, so currently <sys/asm_linkage.h>
already is provided with it.

Let's unify those two into one %ENDBR macro. At first, check if it's
present already. If so -- use Linux kernel version. Otherwise, try
to go that second way and use %_CET_ENDBR from <cet.h> if available.
If no, fall back to just empty definition.
This fixes a couple more 'relocations to !ENDBR' across the module.
And now that we always have the latest/actual ENDBR definition, use
it at the entrance of the few corresponding functions that objtool
still complains about. This matches the way how it's used in the
upstream x86 core Asm code.

Reviewed-by: Attila Fülöp <[email protected]>
Reviewed-by: Tino Reichardt <[email protected]>
Reviewed-by: Richard Yao <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Alexander Lobakin <[email protected]>
Closes #14035

9 files changed, 23 insertions, 50 deletions

diff --git a/module/icp/asm-x86_64/aes/aes_amd64.S b/module/icp/asm-x86_64/aes/aes_amd64.S
index f546e8933..a0525dd46 100644
--- a/module/icp/asm-x86_64/aes/aes_amd64.S
+++ b/module/icp/asm-x86_64/aes/aes_amd64.S
@@ -704,6 +704,7 @@ enc_tab:
 
 
 ENTRY_NP(aes_encrypt_amd64)
+	ENDBR
 #ifdef	GLADMAN_INTERFACE
 	// Original interface
 	sub	$[4*8], %rsp	// gnu/linux/opensolaris binary interface
@@ -809,6 +810,7 @@ dec_tab:
 
 
 ENTRY_NP(aes_decrypt_amd64)
+	ENDBR
 #ifdef	GLADMAN_INTERFACE
 	// Original interface
 	sub	$[4*8], %rsp	// gnu/linux/opensolaris binary interface
diff --git a/module/icp/asm-x86_64/blake3/blake3_avx2.S b/module/icp/asm-x86_64/blake3/blake3_avx2.S
index f4d9cb766..cb08430b8 100644
--- a/module/icp/asm-x86_64/blake3/blake3_avx2.S
+++ b/module/icp/asm-x86_64/blake3/blake3_avx2.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_avx2
 .text
@@ -47,7 +37,7 @@
 .type zfs_blake3_hash_many_avx2,@function
 .p2align  6
 zfs_blake3_hash_many_avx2:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
diff --git a/module/icp/asm-x86_64/blake3/blake3_avx512.S b/module/icp/asm-x86_64/blake3/blake3_avx512.S
index 71b5715c8..960406ea2 100644
--- a/module/icp/asm-x86_64/blake3/blake3_avx512.S
+++ b/module/icp/asm-x86_64/blake3/blake3_avx512.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_avx512
 .global zfs_blake3_compress_in_place_avx512
@@ -52,7 +42,7 @@
 
 .p2align  6
 zfs_blake3_hash_many_avx512:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -2409,7 +2399,7 @@ zfs_blake3_hash_many_avx512:
         jmp     4b
 .p2align 6
 zfs_blake3_compress_in_place_avx512:
-        _CET_ENDBR
+        ENDBR
         vmovdqu xmm0, xmmword ptr [rdi]
         vmovdqu xmm1, xmmword ptr [rdi+0x10]
         movzx   eax, r8b
@@ -2491,7 +2481,7 @@ zfs_blake3_compress_in_place_avx512:
 
 .p2align 6
 zfs_blake3_compress_xof_avx512:
-        _CET_ENDBR
+        ENDBR
         vmovdqu xmm0, xmmword ptr [rdi]
         vmovdqu xmm1, xmmword ptr [rdi+0x10]
         movzx   eax, r8b
diff --git a/module/icp/asm-x86_64/blake3/blake3_sse2.S b/module/icp/asm-x86_64/blake3/blake3_sse2.S
index 20689a7dc..c4290aaa8 100644
--- a/module/icp/asm-x86_64/blake3/blake3_sse2.S
+++ b/module/icp/asm-x86_64/blake3/blake3_sse2.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_sse2
 .global zfs_blake3_compress_in_place_sse2
@@ -52,7 +42,7 @@
 
         .p2align  6
 zfs_blake3_hash_many_sse2:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -2050,7 +2040,7 @@ zfs_blake3_hash_many_sse2:
 
 .p2align 6
 zfs_blake3_compress_in_place_sse2:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
@@ -2161,7 +2151,7 @@ zfs_blake3_compress_in_place_sse2:
 
 .p2align 6
 zfs_blake3_compress_xof_sse2:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
diff --git a/module/icp/asm-x86_64/blake3/blake3_sse41.S b/module/icp/asm-x86_64/blake3/blake3_sse41.S
index c5975a4f0..45b90cc9e 100644
--- a/module/icp/asm-x86_64/blake3/blake3_sse41.S
+++ b/module/icp/asm-x86_64/blake3/blake3_sse41.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_compress_in_place_sse41
 .global zfs_blake3_compress_xof_sse41
@@ -52,7 +42,7 @@
 
 .p2align  6
 zfs_blake3_hash_many_sse41:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -1812,7 +1802,7 @@ zfs_blake3_hash_many_sse41:
         jmp     4b
 .p2align 6
 zfs_blake3_compress_in_place_sse41:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
@@ -1911,7 +1901,7 @@ zfs_blake3_compress_in_place_sse41:
         RET
 .p2align 6
 zfs_blake3_compress_xof_sse41:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
diff --git a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
index 6da43ee00..cf17b3768 100644
--- a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
+++ b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
@@ -59,6 +59,7 @@
 .align	32
 _aesni_ctr32_ghash_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
@@ -375,6 +376,7 @@ _aesni_ctr32_ghash_6x:
 .align	32
 _aesni_ctr32_ghash_no_movbe_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
@@ -703,6 +705,7 @@ _aesni_ctr32_ghash_no_movbe_6x:
 .align	32
 aesni_gcm_decrypt:
 .cfi_startproc
+	ENDBR
 	xorq	%r10,%r10
 	cmpq	$0x60,%rdx
 	jb	.Lgcm_dec_abort
@@ -820,6 +823,7 @@ aesni_gcm_decrypt:
 .align	32
 _aesni_ctr32_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	0-128(%rcx),%xmm4
 	vmovdqu	32(%r11),%xmm2
 	leaq	-2(%rbp),%r13	// ICP uses 10,12,14 not 9,11,13 for rounds.
@@ -914,6 +918,7 @@ _aesni_ctr32_6x:
 .align	32
 aesni_gcm_encrypt:
 .cfi_startproc
+	ENDBR
 	xorq	%r10,%r10
 	cmpq	$288,%rdx
 	jb	.Lgcm_enc_abort
diff --git a/module/icp/asm-x86_64/modes/ghash-x86_64.S b/module/icp/asm-x86_64/modes/ghash-x86_64.S
index d7cdaeb36..bf3724a23 100644
--- a/module/icp/asm-x86_64/modes/ghash-x86_64.S
+++ b/module/icp/asm-x86_64/modes/ghash-x86_64.S
@@ -107,6 +107,7 @@
 .align	16
 gcm_gmult_clmul:
 .cfi_startproc
+	ENDBR
 .L_gmult_clmul:
 	movdqu	(%rdi),%xmm0
 	movdqa	.Lbswap_mask(%rip),%xmm5
@@ -161,6 +162,7 @@ gcm_gmult_clmul:
 .align	32
 gcm_init_htab_avx:
 .cfi_startproc
+	ENDBR
 	vzeroupper
 
 	vmovdqu	(%rsi),%xmm2
@@ -274,6 +276,7 @@ gcm_init_htab_avx:
 .align	32
 gcm_gmult_avx:
 .cfi_startproc
+	ENDBR
 	jmp	.L_gmult_clmul
 .cfi_endproc
 .size	gcm_gmult_avx,.-gcm_gmult_avx
@@ -282,6 +285,7 @@ gcm_gmult_avx:
 .align	32
 gcm_ghash_avx:
 .cfi_startproc
+	ENDBR
 	vzeroupper
 
 	vmovdqu	(%rdi),%xmm10
diff --git a/module/icp/asm-x86_64/sha2/sha256_impl.S b/module/icp/asm-x86_64/sha2/sha256_impl.S
index 1391bd59a..60d34b4a3 100644
--- a/module/icp/asm-x86_64/sha2/sha256_impl.S
+++ b/module/icp/asm-x86_64/sha2/sha256_impl.S
@@ -84,6 +84,7 @@ SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
 
 ENTRY_NP(SHA256TransformBlocks)
 .cfi_startproc
+	ENDBR
 	movq	%rsp, %rax
 .cfi_def_cfa_register %rax
 	push	%rbx
diff --git a/module/icp/asm-x86_64/sha2/sha512_impl.S b/module/icp/asm-x86_64/sha2/sha512_impl.S
index e61e96957..ed7fb362a 100644
--- a/module/icp/asm-x86_64/sha2/sha512_impl.S
+++ b/module/icp/asm-x86_64/sha2/sha512_impl.S
@@ -85,6 +85,7 @@ SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
 
 ENTRY_NP(SHA512TransformBlocks)
 .cfi_startproc
+	ENDBR
 	movq	%rsp, %rax
 .cfi_def_cfa_register %rax
 	push	%rbx