Fix ENOSPC when unlinking multiple files from full pool

When unlinking multiple files from a pool at 100% capacity, it was
possible for ENOSPC to be returned after the first unlink.  e.g.

    rm -f /mnt/fs/test1.0.0 /mnt/fs/test1.1.0 /mnt/fs/test1.2.0
    rm: cannot remove '/mnt/fs/test1.1.0': No space left on device
    rm: cannot remove '/mnt/fs/test1.2.0': No space left on device

After waiting for the pending deferred frees from the first unlink to
be processed the remaining files can then be unlinked.  This is caused
by the quota limit in dsl_dir_tempreserve_impl() being temporarily
decreased to the allocatable pool capacity less any deferred free
space.

This is resolved using the existing mechanism of returning ERESTART
when over quota as long as we know enough space will shortly be
available after processing the pending deferred frees.

Reviewed-by: Alexander Motin <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #13172 

3 files changed, 63 insertions, 2 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index eb0d45135..fdc2172b6 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -694,7 +694,7 @@ tags = ['functional', 'nestedfs']
 
 [tests/functional/no_space]
 tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos',
-    'enospc_df']
+    'enospc_df', 'enospc_rm']
 tags = ['functional', 'no_space']
 
 [tests/functional/nopwrite]
diff --git a/tests/zfs-tests/tests/functional/no_space/Makefile.am b/tests/zfs-tests/tests/functional/no_space/Makefile.am
index c2e42bc2a..31584fb17 100644
--- a/tests/zfs-tests/tests/functional/no_space/Makefile.am
+++ b/tests/zfs-tests/tests/functional/no_space/Makefile.am
@@ -5,7 +5,8 @@ dist_pkgdata_SCRIPTS = \
 	enospc_001_pos.ksh \
 	enospc_002_pos.ksh \
 	enospc_003_pos.ksh \
-	enospc_df.ksh
+	enospc_df.ksh \
+	enospc_rm.ksh
 
 dist_pkgdata_DATA = \
 	enospc.cfg
diff --git a/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh
new file mode 100755
index 000000000..065abc759
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh
@@ -0,0 +1,60 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2022 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/no_space/enospc.cfg
+
+#
+# DESCRIPTION:
+# After filling a filesystem, verify the contents can be removed
+# without encountering an ENOSPC error.
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	destroy_pool $TESTPOOL
+	log_must rm -f $all_vdevs
+}
+
+log_onexit cleanup
+
+log_assert "Files can be removed from full file system."
+
+all_vdevs=$(echo $TEST_BASE_DIR/file.{01..12})
+
+log_must truncate -s $MINVDEVSIZE $all_vdevs
+
+log_must zpool create -f $TESTPOOL draid2:8d:2s $all_vdevs
+log_must zfs create $TESTPOOL/$TESTFS
+log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+
+log_note "Writing files until ENOSPC."
+log_mustnot_expect "No space left on device" fio --name=test \
+    --fallocate=none --rw=write --bs=1M --size=1G --numjobs=4 \
+    --sync=1 --directory=$TESTDIR/ --group_reporting
+
+log_must rm $TESTDIR/test.*
+log_must test -z "$(ls -A $TESTDIR)"
+
+log_pass "All files removed without error"