ZTS: Fix EBUSY volume destroy failures

It's possible for an unrelated process, like blkid, to have the
volume open when 'zfs destroy' is run.  Switch the cleanup functions
to the destroy_dataset() helper which handles this case by retrying
the destroy when the dataset is busy.  This was done not only for
volumes but also for file systems for consistency.

Reviewed-by: Richard Elling <[email protected]>
Reviewed-by: George Melikov <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #7854 

26 files changed, 41 insertions, 52 deletions

diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py
index e1a9011d3..5747f1422 100755
--- a/tests/test-runner/bin/zts-report.py
+++ b/tests/test-runner/bin/zts-report.py
@@ -216,8 +216,6 @@ maybe = {
     'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason],
     'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason],
     'cli_root/zfs_get/zfs_get_009_pos': ['SKIP', '5479'],
-    'cli_root/zfs_receive/receive-o-x_props_override':
-        ['FAIL', known_reason],
     'cli_root/zfs_rename/zfs_rename_006_pos': ['FAIL', '5647'],
     'cli_root/zfs_rename/zfs_rename_009_neg': ['FAIL', '5648'],
     'cli_root/zfs_rollback/zfs_rollback_001_pos': ['FAIL', '6415'],
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_fs.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_fs.ksh
index b2e0e600e..18f246889 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_fs.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_fs.ksh
@@ -22,7 +22,7 @@ fs=$TESTPOOL/$TESTFS/testchild
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy $fs
+	destroy_dataset $fs
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_snap.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_snap.ksh
index 0205e6c11..8650322ad 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_snap.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.destroy_snap.ksh
@@ -22,7 +22,7 @@ snap=$TESTPOOL/$TESTFS@$TESTSNAP
 
 function cleanup
 {
-	datasetexists $snap && log_must zfs destroy $snap
+	destroy_dataset $snap
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_count_and_limit.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_count_and_limit.ksh
index eb559103b..70330c91a 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_count_and_limit.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_count_and_limit.ksh
@@ -29,7 +29,7 @@ snap=$fs@$TESTSNAP
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $fs "-R"
 	log_must rm -rf $fs/foo
 	log_must rm -rf $fs/bar
 }
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_index_props.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_index_props.ksh
index 5fbe18e48..6478fa654 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_index_props.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_index_props.ksh
@@ -27,7 +27,7 @@ fs=$TESTPOOL/$TESTFS/testchild
 snap=$fs@$TESTSNAP
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $fs "-R"
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_mountpoint.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_mountpoint.ksh
index 77555a7e0..f3cf396aa 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_mountpoint.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_mountpoint.ksh
@@ -31,8 +31,8 @@ mnt2=/$fs/mnt2
 
 function cleanup
 {
-	datasetexists $clone && log_must zfs destroy $clone
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $clone
+	destroy_dataset $fs "-R"
 	log_must rm -rf $mnt1
 	log_must rm -rf $mnt2
 }
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_neg.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_neg.ksh
index b1a7b8ac8..17f7a8fd9 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_neg.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_neg.ksh
@@ -25,7 +25,7 @@ verify_runnable "global"
 fs=$TESTPOOL/$TESTFS/testchild
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy $fs
+	destroy_dataset $fs
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_number_props.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_number_props.ksh
index a78873114..942930fa6 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_number_props.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_number_props.ksh
@@ -30,8 +30,8 @@ vol=$TESTPOOL/$TESTVOL
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
-	datasetexists $vol && log_must zfs destroy $vol
+	destroy_dataset $fs "-R"
+	destroy_dataset $vol
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_type.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_type.ksh
index f52eda75a..d040e8962 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_type.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_type.ksh
@@ -28,9 +28,9 @@ snap=$fs@$TESTSNAP
 vol=$TESTPOOL/$TESTVOL
 function cleanup
 {
-	datasetexists $snap && log_must zfs destroy $snap
-	datasetexists $fs && log_must zfs destroy $fs
-	datasetexists $vol && log_must zfs destroy $vol
+	destroy_dataset $snap
+	destroy_dataset $fs
+	destroy_dataset $vol
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_userquota.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_userquota.ksh
index a10154420..80cc4ad41 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_userquota.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_userquota.ksh
@@ -31,8 +31,8 @@ groupid='456'
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy $fs
-	datasetexists $fs1 && log_must zfs destroy $fs1
+	destroy_dataset $fs
+	destroy_dataset $fs1
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_written.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_written.ksh
index 7f94aea24..9755e6f82 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_written.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.get_written.ksh
@@ -30,7 +30,7 @@ dir=/$fs/dir
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $fs "-R"
 	log_must rm -rf $dir
 }
 
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_children.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_children.ksh
index 2afd2e47d..06b82cab4 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_children.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_children.ksh
@@ -32,10 +32,10 @@ TESTCHILD3=$TESTCHILD-3
 
 function cleanup
 {
-	datasetexists $TESTCHILD && log_must zfs destroy $TESTCHILD
-	datasetexists $TESTCHILD1 && log_must zfs destroy $TESTCHILD1
-	datasetexists $TESTCHILD2 && log_must zfs destroy $TESTCHILD2
-	datasetexists $TESTCHILD3 && log_must zfs destroy $TESTCHILD3
+	destroy_dataset $TESTCHILD
+	destroy_dataset $TESTCHILD1
+	destroy_dataset $TESTCHILD2
+	destroy_dataset $TESTCHILD3
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_clones.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_clones.ksh
index ef2bd5ac0..68d053283 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_clones.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_clones.ksh
@@ -27,8 +27,7 @@ log_assert "Listing zfs clones should work correctly."
 
 function cleanup
 {
-	datasetexists $TESTPOOL/$TESTFS@$TESTSNAP && \
-	    log_must zfs destroy -R $TESTPOOL/$TESTFS@$TESTSNAP
+	destroy_dataset $TESTPOOL/$TESTFS@$TESTSNAP "-R"
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_snapshots.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_snapshots.ksh
index 8cd78c513..7bc36606c 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_snapshots.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_snapshots.ksh
@@ -27,14 +27,10 @@ log_assert "Listing zfs snapshots should work correctly."
 
 function cleanup
 {
-	datasetexists $TESTPOOL/$TESTFS@$TESTSNAP && \
-	    log_must zfs destroy $TESTPOOL/$TESTFS@$TESTSNAP
-	datasetexists $TESTPOOL/$TESTFS@$TESTSNAP1 && \
-	    log_must zfs destroy $TESTPOOL/$TESTFS@$TESTSNAP1
-	datasetexists $TESTPOOL/$TESTFS@$TESTSNAP2 && \
-	    log_must zfs destroy $TESTPOOL/$TESTFS@$TESTSNAP2
-	datasetexists $TESTPOOL/$TESTFS@$TESTSNAP3 && \
-	    log_must zfs destroy $TESTPOOL/$TESTFS@$TESTSNAP3
+	destroy_dataset $TESTPOOL/$TESTFS@$TESTSNAP
+	destroy_dataset $TESTPOOL/$TESTFS@$TESTSNAP1
+	destroy_dataset $TESTPOOL/$TESTFS@$TESTSNAP2
+	destroy_dataset $TESTPOOL/$TESTFS@$TESTSNAP3
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_system_props.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_system_props.ksh
index cc0be6d90..24ab65a19 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_system_props.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.list_system_props.ksh
@@ -28,9 +28,9 @@ snap=$fs@$TESTSNAP
 vol=$TESTPOOL/$TESTVOL
 function cleanup
 {
-	datasetexists $snap && log_must zfs destroy $snap
-	datasetexists $fs && log_must zfs destroy $fs
-	datasetexists $vol && log_must zfs destroy $vol
+	destroy_dataset $snap
+	destroy_dataset $fs
+	destroy_dataset $vol
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_conflict.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_conflict.ksh
index 0739940da..b2840377b 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_conflict.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_conflict.ksh
@@ -32,7 +32,7 @@ snap=promote_conflict_snap
 function cleanup
 {
     for to_destroy in $fs $clone; do
-        datasetexists $to_destroy && log_must zfs destroy -R $to_destroy
+        destroy_dataset $to_destroy "-R"
     done
 }
 
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_multiple.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_multiple.ksh
index a3c05fabd..7618a34bc 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_multiple.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_multiple.ksh
@@ -32,7 +32,7 @@ snap2=$clone1@testchild_snap2
 function cleanup
 {
     for to_destroy in $fs $clone1 $clone2; do
-        datasetexists $to_destroy && log_must zfs destroy -R $to_destroy
+        destroy_dataset $to_destroy "-R"
     done
 }
 
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_simple.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_simple.ksh
index 83685c4dc..541c6840e 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_simple.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.promote_simple.ksh
@@ -30,7 +30,7 @@ snap=$fs@$TESTSNAP
 function cleanup
 {
     for to_destroy in $fs $clone; do
-        datasetexists $to_destroy && log_must zfs destroy -R $to_destroy
+        destroy_dataset $to_destroy "-R"
     done
 }
 
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_mult.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_mult.ksh
index e0b917a32..2fbbc73d5 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_mult.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_mult.ksh
@@ -24,8 +24,7 @@ file=$TESTDIR/$TESTFILE0
 
 function cleanup
 {
-	datasetexists $snap1 && log_must zfs destroy $snap1 && \
-	    log_must rm $file
+	destroy_dataset $snap1 && log_must rm $file
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_one.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_one.ksh
index d6396f488..128344301 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_one.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.rollback_one.ksh
@@ -23,9 +23,7 @@ file=$TESTDIR/$TESTFILE0
 
 function cleanup
 {
-	datasetexists $snap && log_must zfs destroy $snap && \
-	    log_must rm $file
-
+	destroy_dataset $snap && log_must rm $file
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_destroy.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_destroy.ksh
index 3691492bf..b0cdfb84f 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_destroy.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_destroy.ksh
@@ -26,7 +26,7 @@ fs=$TESTPOOL/$TESTFS/testchild
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $fs "-R"
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_neg.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_neg.ksh
index c20ed8a61..8d6cd3831 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_neg.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_neg.ksh
@@ -28,7 +28,7 @@ fs2=$TESTPOOL/$TESTFS/testchild2
 function cleanup
 {
 	for fs in $fs1 $fs2; do
-		datasetexists $fs && log_must zfs destroy -R $fs
+		destroy_dataset $fs "-R"
 	done
 }
 
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_recursive.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_recursive.ksh
index f112bf0d2..e3bf10b69 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_recursive.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_recursive.ksh
@@ -28,7 +28,7 @@ snapname=snap
 
 function cleanup
 {
-	datasetexists $rootfs && log_must zfs destroy -R $rootfs
+	destroy_dataset $rootfs "-R"
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_simple.ksh b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_simple.ksh
index a26234278..4a7acadda 100755
--- a/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_simple.ksh
+++ b/tests/zfs-tests/tests/functional/channel_program/synctask_core/tst.snapshot_simple.ksh
@@ -27,7 +27,7 @@ snapname=testsnap
 
 function cleanup
 {
-	datasetexists $fs && log_must zfs destroy -R $fs
+	destroy_dataset $fs "-R"
 }
 
 log_onexit cleanup
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_002_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_002_pos.ksh
index 2498e4a4c..6f36b40bf 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_002_pos.ksh
@@ -47,9 +47,7 @@ function cleanup
 {
 	typeset -i j=0
 	while [[ $j -lt ${#size[*]} ]]; do
-		if datasetexists $TESTPOOL/${TESTVOL}${size[j]}; then
-			log_must zfs destroy $TESTPOOL/${TESTVOL}${size[j]}
-		fi
+		destroy_dataset $TESTPOOL/${TESTVOL}${size[j]}
 		((j = j + 1))
 	done
 }
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/receive-o-x_props_override.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/receive-o-x_props_override.ksh
index 6a0152e81..6f897a96f 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/receive-o-x_props_override.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/receive-o-x_props_override.ksh
@@ -268,7 +268,7 @@ log_must zfs snapshot $orig@snap1
 log_must eval "zfs send $orig@snap1 > $streamfile_full"
 log_mustnot eval "zfs receive -x atime $dest < $streamfile_full"
 log_mustnot eval "zfs receive -o atime=off $dest < $streamfile_full"
-log_must zfs destroy -r -f $orig
+log_must_busy zfs destroy -r -f $orig
 log_must zfs create $orig
 log_must zfs create -V 128K -s $origsub
 log_must zfs snapshot -r $orig@snap1
@@ -279,8 +279,9 @@ log_must eval "check_prop_source $dest atime off local"
 log_must eval "check_prop_source $destsub type volume -"
 log_must eval "check_prop_source $destsub atime - -"
 # Cleanup
-log_must zfs destroy -r -f $orig
-log_must zfs destroy -r -f $dest
+block_device_wait
+log_must_busy zfs destroy -r -f $orig
+log_must_busy zfs destroy -r -f $dest
 
 #
 # 3.8 Verify 'zfs recv -x|-o' works correctly when used in conjunction with -d