Retry zfs destroy when busy in rsend tests

rsend tests in the test suite frequently create and
destroy datasets. It is possible for zfs destroy to
return an error code indicating the dataset is busy.
Simply use a log_must_busy in these cases to retry
destroying those datasets. Other fixes to rsend test
cases to avoid unmounting and remounting filesystems
and some cleanup.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Giuseppe Di Natale <[email protected]>
Closes #6418 

16 files changed, 46 insertions, 47 deletions

diff --git a/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh b/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
index 2d0054b2c..a396aae4b 100755
--- a/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
+++ b/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
@@ -56,7 +56,7 @@ verify_runnable "both"
 function cleanup {
 
 	for obj in $OBJ_LIST; do
-		datasetexists $obj && log_must zfs destroy -f $obj
+		datasetexists $obj && log_must_busy zfs destroy -f $obj
 	done
 }
 
@@ -112,7 +112,7 @@ for obj in $OBJ_LIST ; do
 		"($resv_size_set != $resv_size_get)"
 	fi
 
-	log_must zfs destroy -f $obj
+	log_must_busy zfs destroy -f $obj
 
 	new_space_avail=`get_prop available $TESTPOOL`
 	new_space_used=`get_prop used $TESTPOOL`
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
index fde4123be..6e2f2ce6f 100644
--- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
+++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
@@ -119,13 +119,13 @@ function cleanup_pool
 	log_must rm -rf $BACKDIR/*
 
 	if is_global_zone ; then
-		log_must zfs destroy -Rf $pool
+		log_must_busy zfs destroy -Rf $pool
 	else
 		typeset list=$(zfs list -H -r -t all -o name $pool)
 		for ds in $list ; do
 			if [[ $ds != $pool ]] ; then
 				if datasetexists $ds ; then
-					log_must zfs destroy -Rf $ds
+					log_must_busy zfs destroy -Rf $ds
 				fi
 			fi
 		done
@@ -296,24 +296,16 @@ function destroy_tree
 	typeset -i ret=0
 	typeset snap
 	for snap in "$@" ; do
-		zfs destroy $snap
-		ret=$?
+		log_must_busy zfs destroy $snap
 
 		typeset ds=${snap%%@*}
 		typeset type=$(get_prop "type" $ds)
 		if [[ $type == "filesystem" ]]; then
 			typeset mntpnt=$(get_prop mountpoint $ds)
-			((ret |= $?))
-
-			if ((ret != 0)); then
-				rm -r $mntpnt/$snap
-				((ret |= $?))
+			if [[ -n $mntpnt ]]; then
+				rm -rf $mntpnt/$snap
 			fi
 		fi
-
-		if ((ret != 0)); then
-			return $ret
-		fi
 	done
 
 	return 0
@@ -516,8 +508,8 @@ function test_fs_setup
 	typeset sendpool=${sendfs%%/*}
 	typeset recvpool=${recvfs%%/*}
 
-	datasetexists $sendfs && log_must zfs destroy -r $sendpool
-	datasetexists $recvfs && log_must zfs destroy -r $recvpool
+	datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
+	datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
 
 	if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
 		mk_files 1000 256 0 $sendfs &
@@ -547,7 +539,7 @@ function test_fs_setup
 	fi
 
 	if datasetexists $streamfs; then
-		log_must zfs destroy -r $streamfs
+		log_must_busy zfs destroy -r $streamfs
 	fi
 	log_must zfs create -o compress=lz4 $sendpool/stream
 }
@@ -664,8 +656,8 @@ function resume_cleanup
 	typeset sendfs=$1
 	typeset streamfs=$2
 
-	datasetexists $sendfs && log_must zfs destroy -r $sendfs
-	datasetexists $streamfs && log_must zfs destroy -r $streamfs
+	datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
+	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 	cleanup_pool $POOL2
 	rm -f /$POOL/initial.zsend /$POOL/incremental.zsend
 }
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
index 11d48ef0d..2ace6737b 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
@@ -59,7 +59,7 @@ if is_global_zone ; then
 	#
 	log_must eval "zfs send -I $POOL2@psnap $POOL2/pclone@final > " \
 		"$BACKDIR/pool-clone-I"
-	log_must zfs destroy -rf $POOL2/pclone
+	log_must_busy zfs destroy -rf $POOL2/pclone
 	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-clone-I"
 	log_must cmp_ds_subs $POOL $POOL2
 	log_must cmp_ds_cont $POOL $POOL2
@@ -73,7 +73,7 @@ dstds=$(get_dst_ds $POOL $POOL2)
 ds=$dstds/$FS/fs1
 log_must eval "zfs send -I $ds/fs2@fsnap $ds/fclone@final > " \
 	"$BACKDIR/fs-clone-I"
-log_must zfs destroy -rf $ds/fclone
+log_must_busy zfs destroy -rf $ds/fclone
 log_must eval "zfs receive -F $ds/fclone < $BACKDIR/fs-clone-I"
 
 log_must cmp_ds_subs $POOL $dstds
@@ -86,7 +86,7 @@ if is_global_zone ; then
 	ds=$POOL2/$FS
 	log_must eval "zfs send -I $ds/vol@vsnap $ds/vclone@final > " \
 		"$BACKDIR/vol-clone-I"
-	log_must zfs destroy -rf $ds/vclone
+	log_must_busy zfs destroy -rf $ds/vclone
 	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-clone-I"
 	log_must cmp_ds_subs $POOL $POOL2
 	log_must cmp_ds_cont $POOL $POOL2
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
index 1578283fc..7d858d92d 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
@@ -48,10 +48,10 @@ verify_runnable "global"
 function cleanup
 {
 	if datasetexists bpool ; then
-		log_must zpool destroy -f bpool
+		log_must_busy zpool destroy -f bpool
 	fi
 	if datasetexists spool ; then
-		log_must zpool destroy -f spool
+		log_must_busy zpool destroy -f spool
 	fi
 }
 
@@ -80,7 +80,7 @@ log_must ismounted spool
 # Test out of space on top filesystem
 #
 log_must mv /bpool/fs/file /bpool
-log_must zfs destroy -rf bpool/fs
+log_must_busy zfs destroy -rf bpool/fs
 
 log_must zfs snapshot bpool@snap
 log_must eval "zfs send -R bpool@snap > $BACKDIR/bpool-R"
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
index 3b6cc2937..9ecd18d87 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
@@ -45,9 +45,6 @@ verify_runnable "both"
 
 function cleanup
 {
-	export __ZFS_POOL_RESTRICT="$POOL $POOL2"
-	log_must zfs unmount -a
-	unset __ZFS_POOL_RESTRICT
 	log_must cleanup_pool $POOL
 	log_must cleanup_pool $POOL2
 
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
index 80644ee20..110e56144 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
@@ -66,11 +66,11 @@ cleanup
 log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R"
 log_must eval "zfs receive -dF $POOL2 < $BACKDIR/pool-final-R"
 
-log_must zfs destroy -Rf $POOL/$FS
-log_must zfs destroy -Rf $POOL/pclone
+log_must_busy zfs destroy -Rf $POOL/$FS
+log_must_busy zfs destroy -Rf $POOL/pclone
 
 if is_global_zone ; then
-	log_must zfs destroy -Rf $POOL/vol
+	log_must_busy zfs destroy -Rf $POOL/vol
 fi
 log_must zfs snapshot -r $POOL@destroy
 
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
index 29bdc4f84..9f1530b45 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
@@ -33,6 +33,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+	log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of full ZFS send/receive with the -D " \
     "(dedup) flag"
 
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
index d74844890..54d56c9a6 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
@@ -35,6 +35,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+	log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of a full and incremental ZFS send/receive " \
     "with the -e (embedded) flag"
 
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
index 487a01426..ff51fecaf 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
@@ -54,10 +54,10 @@ log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
 log_must zfs bookmark $sendfs@a $sendfs#bm_a
-log_must zfs destroy $sendfs@a
+log_must_busy zfs destroy $sendfs@a
 log_must zfs receive -v $recvfs </$POOL/initial.zsend
 resume_test "zfs send -i \#bm_a $sendfs@b" $streamfs $recvfs
-log_must zfs destroy -r -f $sendfs
+log_must_busy zfs destroy -r -f $sendfs
 log_must zfs receive -v $sendfs </$POOL/initial.zsend
 log_must zfs receive -v $sendfs </$POOL/incremental.zsend
 file_check $sendfs $recvfs
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
index e7453a0e1..bcea6b728 100755
--- a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
@@ -50,7 +50,7 @@ streamfs=$POOL/stream
 log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
-log_must zfs unmount $sendfs
+log_must zfs unmount -f $sendfs
 resume_test "zfs send $sendfs" $streamfs $recvfs
 file_check $sendfs $recvfs
 
diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
index 70e200bad..666e11f70 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
@@ -52,8 +52,8 @@ for compress in off gzip; do
 		poolexists $POOL3 && destroy_pool $POOL3
 		log_must zpool create $pool_opt $POOL3 $DISK3
 
-		datasetexists $send_ds && log_must zfs destroy -r $send_ds
-		datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+		datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+		datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
 		log_must zfs create -o compress=$compress $send_ds
 		typeset dir=$(get_prop mountpoint $send_ds)
@@ -63,7 +63,7 @@ for compress in off gzip; do
 		log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c"
 		log_must eval "zfs recv $recv_ds <$BACKDIR/full-c"
 
-		log_must zfs destroy -r $recv_ds
+		log_must_busy zfs destroy -r $recv_ds
 
 		log_must eval "zfs send $send_ds@full >$BACKDIR/full"
 		log_must eval "zfs recv $recv_ds <$BACKDIR/full"
diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
index 53c7ce8bd..614394e52 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
@@ -50,8 +50,8 @@ datasetexists $POOL3 && log_must zpool destroy $POOL3
 log_must zpool create -d $POOL3 $DISK3
 
 for compress in $compress_types; do
-	datasetexists $send_ds && log_must zfs destroy -r $send_ds
-	datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+	datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+	datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
 	log_must zfs create -o compress=$compress $send_ds
 	typeset dir=$(get_prop mountpoint $send_ds)
diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
index e810f712a..3677db83f 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
@@ -56,8 +56,8 @@ log_onexit cleanup_pool $POOL2
 write_compressible $BACKDIR ${megs}m
 
 for compress in $compress_types; do
-	datasetexists $send_ds && log_must zfs destroy -r $send_ds
-	datasetexists $send_vol && log_must zfs destroy -r $send_vol
+	datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+	datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol
 	log_must zfs create -o compress=$compress $send_ds
 	log_must zfs create -V 1g -o compress=$compress $send_vol
 
diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
index 17bf1f7c0..9b886f815 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
@@ -59,7 +59,7 @@ for prop in $(get_rand_compress_any 6); do
 		verify_stream_size $BACKDIR/compressed $sendfs
 
 		log_must rm $BACKDIR/uncompressed $BACKDIR/compressed
-		log_must zfs destroy -r $sendfs
+		log_must_busy zfs destroy -r $sendfs
 	done
 done
 
diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
index 903cefd57..d2f8c51d1 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
@@ -29,7 +29,7 @@
 
 function cleanup
 {
-	log_must zfs destroy -r $vol
+	log_must_busy zfs destroy -r $vol
 	cleanup_pool $POOL2
 }
 
diff --git a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
index ee7421d08..84c0a5e3c 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
@@ -55,8 +55,8 @@ verify_runnable "both"
 
 function cleanup
 {
-	datasetexists $TESTPOOL/128k && log_must zfs destroy $TESTPOOL/128k
-	datasetexists $TESTPOOL/1m && log_must zfs destroy $TESTPOOL/1m
+	datasetexists $TESTPOOL/128k && log_must_busy zfs destroy $TESTPOOL/128k
+	datasetexists $TESTPOOL/1m && log_must_busy zfs destroy $TESTPOOL/1m
 	cleanup_pool $POOL2
 	destroy_pool $POOL3
 }
@@ -153,7 +153,7 @@ function check
 
 	check_recsize $recv_ds $expected_file_bs $expected_recsize
 	$do_size_test && verify_stream_size $stream $recv_ds
-	log_must zfs destroy -r $recv_ds
+	log_must_busy zfs destroy -r $recv_ds
 }
 
 log_assert "Verify compressed send works with datasets of varying recsize."