Add `--enable-asan` and `--enable-ubsan` switches

`configure` now accepts `--enable-asan` and `--enable-ubsan` switches
which results in passing `-fsanitize=address`
and `-fsanitize=undefined`, respectively, to the compiler. Those
flags are enabled in GitHub workflows for ZTS and zloop. Errors
reported by both instrumentations are corrected, except for:

- Memory leak reporting is (temporarily) suppressed. The cost of
  fixing them is relatively high compared to the gains.

- Checksum computing functions in `module/zcommon/zfs_fletcher*`
  have UBSan errors suppressed. It is completely impractical
  to enforce 64-byte payload alignment there due to performance
  impact.

- There's no ASan heap poisoning in `module/zstd/lib/zstd.c`. A custom
  memory allocator is used there rendering that measure
  unfeasible.

- Memory leaks detection has to be suppressed for `cmd/zvol_id`.
  `zvol_id` is run by udev with the help of `ptrace(2)`. Tracing is
  incompatible with memory leaks detection.

Reviewed-by: Ahelenia Ziemiańska <[email protected]>
Reviewed-by: George Melikov <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: szubersk <[email protected]>
Closes #12928 

12 files changed, 38 insertions, 9 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 13fee6752..51052e801 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -51,6 +51,7 @@ export SYSTEM_FILES_COMMON='arp
     iostat
     kill
     ksh
+    ldd
     ln
     logname
     ls
diff --git a/tests/zfs-tests/include/default.cfg.in b/tests/zfs-tests/include/default.cfg.in
index 1a9cc5a2b..cf382cfe9 100644
--- a/tests/zfs-tests/include/default.cfg.in
+++ b/tests/zfs-tests/include/default.cfg.in
@@ -155,6 +155,21 @@ done
 
 export MAX_PARTITIONS=8
 
+if [ "@ASAN_ENABLED@" = "yes" ]; then
+	export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true
+
+	# TODO
+	# disable memory leaks detection
+	# there are quite many of them and they are not as
+	# destructive to CLI programs as they are to daemons
+	export ASAN_OPTIONS="$ASAN_OPTIONS:detect_leaks=false"
+fi
+
+if [ "@UBSAN_ENABLED@" = "yes" ]; then
+	export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true
+fi
+
+
 case $(uname -o) in
 GNU/Linux)
 	unpack_opts="--sparse -xf"
diff --git a/tests/zfs-tests/tests/functional/alloc_class/alloc_class_013_pos.ksh b/tests/zfs-tests/tests/functional/alloc_class/alloc_class_013_pos.ksh
index 7a0eb5343..624cab88a 100755
--- a/tests/zfs-tests/tests/functional/alloc_class/alloc_class_013_pos.ksh
+++ b/tests/zfs-tests/tests/functional/alloc_class/alloc_class_013_pos.ksh
@@ -41,8 +41,8 @@ log_must display_status "$TESTPOOL"
 #
 
 log_must zfs create -o dedup=on -V 2G $TESTPOOL/$TESTVOL
-
-log_must eval "new_fs $ZVOL_DEVDIR/$TESTPOOL/$TESTVOL >/dev/null 2>&1"
+block_device_wait "$ZVOL_DEVDIR/$TESTPOOL/$TESTVOL"
+log_must eval "new_fs $ZVOL_DEVDIR/$TESTPOOL/$TESTVOL >/dev/null"
 
 sync_pool
 log_must zpool list -v $TESTPOOL
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs/zfs_002_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs/zfs_002_pos.ksh
index 1290d888a..51a7ce1d9 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs/zfs_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs/zfs_002_pos.ksh
@@ -97,7 +97,6 @@ if is_linux; then
 	ulimit -c unlimited
 	echo "$corefile" >/proc/sys/kernel/core_pattern
 	echo 0 >/proc/sys/kernel/core_uses_pid
-	export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 elif is_freebsd; then
 	ulimit -c unlimited
 	savedcorefile=$(sysctl -n kern.corefile)
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_002_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_002_pos.ksh
index caf8a9a2d..e4ffa2a6c 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_002_pos.ksh
@@ -95,7 +95,6 @@ elif is_freebsd; then
 fi
 ulimit -c unlimited
 
-export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 export ZFS_ABORT=yes
 
 for subcmd in "${cmds[@]}" "${badparams[@]}"; do
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_003_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_003_pos.ksh
index 71d73c0f8..6f15c09a5 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_003_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool/zpool_003_pos.ksh
@@ -89,7 +89,6 @@ elif is_freebsd; then
 fi
 ulimit -c unlimited
 
-export ASAN_OPTIONS="abort_on_error=1:disable_coredump=0"
 export ZFS_ABORT=yes
 
 zpool >/dev/null 2>&1
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh
index fbb0c2910..a60e12e14 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh
@@ -35,7 +35,7 @@
 DISK1=${DISKS%% *}
 
 log_must zpool create -f $TESTPOOL $DISK1
-log_must zpool trim $TESTPOOL
+log_must zpool trim -r 1 "$TESTPOOL"
 
 [[ -z "$(trim_progress $TESTPOOL $DISK1)" ]] && \
     log_fail "TRIM did not start"
diff --git a/tests/zfs-tests/tests/functional/pam/pam_basic.ksh b/tests/zfs-tests/tests/functional/pam/pam_basic.ksh
index f146a6e5f..eab819ab8 100755
--- a/tests/zfs-tests/tests/functional/pam/pam_basic.ksh
+++ b/tests/zfs-tests/tests/functional/pam/pam_basic.ksh
@@ -22,6 +22,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 log_mustnot ismounted "$TESTPOOL/pam/${username}"
 keystatus unavailable
 
diff --git a/tests/zfs-tests/tests/functional/pam/pam_nounmount.ksh b/tests/zfs-tests/tests/functional/pam/pam_nounmount.ksh
index eb9976f2f..29ce437b4 100755
--- a/tests/zfs-tests/tests/functional/pam/pam_nounmount.ksh
+++ b/tests/zfs-tests/tests/functional/pam/pam_nounmount.ksh
@@ -22,6 +22,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 log_mustnot ismounted "$TESTPOOL/pam/${username}"
 keystatus unavailable
 
diff --git a/tests/zfs-tests/tests/functional/pam/pam_short_password.ksh b/tests/zfs-tests/tests/functional/pam/pam_short_password.ksh
index 1f72c9468..100f279ff 100755
--- a/tests/zfs-tests/tests/functional/pam/pam_short_password.ksh
+++ b/tests/zfs-tests/tests/functional/pam/pam_short_password.ksh
@@ -27,6 +27,10 @@
 
 . $STF_SUITE/tests/functional/pam/utilities.kshlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 if [[ -z pamservice ]]; then
 	pamservice=pam_zfs_key_test
 fi
diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in
index 1f58d8116..84e20e2e7 100755
--- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in
+++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in
@@ -16,6 +16,10 @@
 
 . $STF_SUITE/include/libtest.shlib
 
+if [ -n "$ASAN_OPTIONS" ]; then
+	export LD_PRELOAD=$(ldd "$(command -v zfs)" | awk '/libasan\.so/ {print $3}')
+fi
+
 #
 # DESCRIPTION:
 #	Verify the libzfs_core Python test suite can be run successfully
diff --git a/tests/zfs-tests/tests/functional/tmpfile/tmpfile_002_pos.c b/tests/zfs-tests/tests/functional/tmpfile/tmpfile_002_pos.c
index 55d939abd..39a72c22a 100644
--- a/tests/zfs-tests/tests/functional/tmpfile/tmpfile_002_pos.c
+++ b/tests/zfs-tests/tests/functional/tmpfile/tmpfile_002_pos.c
@@ -58,7 +58,7 @@ main(void)
 		exit(3);
 	}
 
-	if ((ret = system("sudo zpool freeze $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool freeze $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool freeze\"");
 		else
@@ -69,7 +69,7 @@ main(void)
 
 	close(fd);
 
-	if ((ret = system("sudo zpool export $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool export $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool export\"");
 		else
@@ -78,7 +78,7 @@ main(void)
 		exit(4);
 	}
 
-	if ((ret = system("sudo zpool import $TESTPOOL"))) {
+	if ((ret = system("sudo -E zpool import $TESTPOOL"))) {
 		if (ret == -1)
 			perror("system \"zpool import\"");
 		else