OpenZFS 9330 - stack overflow when creating a deeply nested dataset

Datasets that are deeply nested (~100 levels) are impractical. We just
put a limit of 50 levels to newly created datasets. Existing datasets
should work without a problem.

The problem can be seen by attempting to create a dataset using the -p
option with many levels:

    panic[cpu0]/thread=ffffff01cd282c20: BAD TRAP: type=8 (#df Double fault) rp=ffffffff

    fffffffffbc3aa60 unix:die+100 ()
    fffffffffbc3ab70 unix:trap+157d ()
    ffffff00083d7020 unix:_patch_xrstorq_rbx+196 ()
    ffffff00083d7050 zfs:dbuf_rele+2e ()
    ...
    ffffff00083d7080 zfs:dsl_dir_close+32 ()
    ffffff00083d70b0 zfs:dsl_dir_evict+30 ()
    ffffff00083d70d0 zfs:dbuf_evict_user+4a ()
    ffffff00083d7100 zfs:dbuf_rele_and_unlock+87 ()
    ffffff00083d7130 zfs:dbuf_rele+2e ()
    ... The block above repeats once per directory in the ...
    ... create -p command, working towards the root ...
    ffffff00083db9f0 zfs:dsl_dataset_drop_ref+19 ()
    ffffff00083dba20 zfs:dsl_dataset_rele+42 ()
    ffffff00083dba70 zfs:dmu_objset_prefetch+e4 ()
    ffffff00083dbaa0 zfs:findfunc+23 ()
    ffffff00083dbb80 zfs:dmu_objset_find_spa+38c ()
    ffffff00083dbbc0 zfs:dmu_objset_find+40 ()
    ffffff00083dbc20 zfs:zfs_ioc_snapshot_list_next+4b ()
    ffffff00083dbcc0 zfs:zfsdev_ioctl+347 ()
    ffffff00083dbd00 genunix:cdev_ioctl+45 ()
    ffffff00083dbd40 specfs:spec_ioctl+5a ()
    ffffff00083dbdc0 genunix:fop_ioctl+7b ()
    ffffff00083dbec0 genunix:ioctl+18e ()
    ffffff00083dbf10 unix:brand_sys_sysenter+1c9 ()

Porting notes:
* Added zfs_max_dataset_nesting module option with documentation.
* Updated zfs_rename_014_neg.ksh for Linux.
* Increase the zfs.sh stack warning to 15K.  Enough time has passed
  that 16K can be reasonably assumed to be the default value.  It
  was increased in the 3.15 kernel released in June of 2014.

Authored by: Serapheim Dimitropoulos <[email protected]>
Reviewed by: John Kennedy <[email protected]>
Reviewed by: Matt Ahrens <[email protected]>
Ported-by: Brian Behlendorf <[email protected]>
Approved by: Garrett D'Amore <[email protected]>

OpenZFS-issue: https://www.illumos.org/issues/9330
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/757a75a
Closes #7681

5 files changed, 121 insertions, 3 deletions

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index bd301e328..056b1dddb 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -209,7 +209,7 @@ tests = ['zfs_rename_001_pos', 'zfs_rename_002_pos', 'zfs_rename_003_pos',
     'zfs_rename_004_neg', 'zfs_rename_005_neg', 'zfs_rename_006_pos',
     'zfs_rename_007_pos', 'zfs_rename_008_pos', 'zfs_rename_009_neg',
     'zfs_rename_010_neg', 'zfs_rename_011_pos', 'zfs_rename_012_neg',
-    'zfs_rename_013_pos', 'zfs_rename_encrypted_child',
+    'zfs_rename_013_pos', 'zfs_rename_014_neg', 'zfs_rename_encrypted_child',
     'zfs_rename_to_encrypted']
 tags = ['functional', 'cli_root', 'zfs_rename']
 
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create.cfg b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create.cfg
index c97e44d87..b96908ce1 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create.cfg
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create.cfg
@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2012 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 #
 
 export BYND_MAX_NAME="byondmaxnamelength\
@@ -38,6 +38,12 @@ export BYND_MAX_NAME="byondmaxnamelength\
 012345678901234567890123456789\
 012345678901234567890123456789"
 
+export BYND_NEST_LIMIT="a/a/a/\
+a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/\
+a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/\
+a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/\
+a/a"
+
 # There're 3 different prompt messages while create
 # a volume that great than 1TB on 32-bit
 #       - volume size exceeds limit for this system. (happy gate)
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_009_neg.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_009_neg.ksh
index 28c6f0cc2..b8190626c 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_009_neg.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_create/zfs_create_009_neg.ksh
@@ -43,6 +43,7 @@
 #	*Beyond maximal name length.
 #	*Same property set multiple times via '-o property=value'
 #	*Volume's property set on filesystem
+#	*Exceeding maximum name nesting
 #
 # STRATEGY:
 # 1. Create an array of <filesystem> arguments
@@ -89,7 +90,7 @@ set -A args  "$TESTPOOL/" "$TESTPOOL//blah" "$TESTPOOL/@blah" \
 	"$TESTPOOL/blah*blah" "$TESTPOOL/blah blah" \
 	"-s $TESTPOOL/$TESTFS1" "-b 1092 $TESTPOOL/$TESTFS1" \
 	"-b 64k $TESTPOOL/$TESTFS1" "-s -b 32k $TESTPOOL/$TESTFS1" \
-	"$TESTPOOL/$BYND_MAX_NAME"
+	"$TESTPOOL/$BYND_MAX_NAME" "$TESTPOOL/$BYND_NEST_LIMIT"
 
 log_assert "Verify 'zfs create <filesystem>' fails with bad <filesystem> argument."
 
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_rename/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_rename/Makefile.am
index 352d6d5fe..974538dd9 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_rename/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_rename/Makefile.am
@@ -15,6 +15,7 @@ dist_pkgdata_SCRIPTS = \
 	zfs_rename_011_pos.ksh \
 	zfs_rename_012_neg.ksh \
 	zfs_rename_013_pos.ksh \
+	zfs_rename_014_neg.ksh \
 	zfs_rename_encrypted_child.ksh \
 	zfs_rename_to_encrypted.ksh
 
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_rename/zfs_rename_014_neg.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_rename/zfs_rename_014_neg.ksh
new file mode 100755
index 000000000..7d99e9f69
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_rename/zfs_rename_014_neg.ksh
@@ -0,0 +1,110 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#	zfs rename should work on existing datasets that exceed
+#	zfs_max_dataset_nesting (our nesting limit) except in the
+#	scenario that we try to rename it to something deeper
+#	than it already is.
+#
+# STRATEGY:
+#	1. Create a set of ZFS datasets within our nesting limit.
+#	2. Try renaming one of them on top of the other so its
+#	   children pass the limit - it should fail.
+#	3. Increase the nesting limit.
+#	4. Check that renaming a dataset on top of the other
+#	   cannot exceed the new nesting limit but can exceed
+#	   the old one.
+#	5. Bring back the old nesting limit so you can simulate
+#	   the scenario of existing datasets that exceed our
+#	   nesting limit.
+#	6. Make sure that 'zfs rename' can work only if we are
+#	   trying to keep existing datasets that exceed the limit
+#	   at the same nesting level or less. Making it even
+#	   deeper should not work.
+#
+
+verify_runnable "both"
+
+dsA01="a"
+dsA02="a/a"
+dsA49=$(printf 'a/%.0s' {1..48})"a"
+
+dsB01="b"
+dsB49=$(printf 'b/%.0s' {1..48})"b"
+
+dsC01="c"
+dsC16=$(printf 'c/%.0s' {1..15})"c"
+
+dsB16A=$(printf 'b/%.0s' {1..16})"a"
+dsB15A=$(printf 'b/%.0s' {1..15})"a"
+
+dsB15A47A=$(printf 'b/%.0s' {1..15})$(printf 'a/%.0s' {1..47})"a"
+dsB15A47C=$(printf 'b/%.0s' {1..15})$(printf 'a/%.0s' {1..47})"c"
+
+dsB15A40B=$(printf 'b/%.0s' {1..15})$(printf 'a/%.0s' {1..40})"b"
+dsB15A47B=$(printf 'b/%.0s' {1..15})$(printf 'a/%.0s' {1..47})"b"
+
+function nesting_cleanup
+{
+	log_must zfs destroy -fR $TESTPOOL/$dsA01
+	log_must zfs destroy -fR $TESTPOOL/$dsB01
+	log_must zfs destroy -fR $TESTPOOL/$dsC01
+
+	# If the test fails after increasing the limit and
+	# before resetting it, it will be left at the modified
+	# value for the remaining tests. That's the reason
+	# we reset it again here just in case.
+	log_must set_tunable_impl zfs_max_dataset_nesting 50 Z zcommon
+}
+
+log_onexit nesting_cleanup
+
+log_must zfs create -p $TESTPOOL/$dsA49
+log_must zfs create -p $TESTPOOL/$dsB49
+log_must zfs create -p $TESTPOOL/$dsC16
+
+log_mustnot zfs rename $TESTPOOL/$dsA02 $TESTPOOL/$dsB15A
+
+# extend limit
+log_must set_tunable_impl zfs_max_dataset_nesting 64 Z zcommon
+
+log_mustnot zfs rename $TESTPOOL/$dsA02 $TESTPOOL/$dsB16A
+log_must zfs rename $TESTPOOL/$dsA02 $TESTPOOL/$dsB15A
+
+# bring back old limit
+log_must set_tunable_impl zfs_max_dataset_nesting 50 Z zcommon
+
+log_mustnot zfs rename $TESTPOOL/$dsC01 $TESTPOOL/$dsB15A47C
+log_must zfs rename $TESTPOOL/$dsB15A47A $TESTPOOL/$dsB15A47B
+log_must zfs rename $TESTPOOL/$dsB15A47B $TESTPOOL/$dsB15A40B
+
+log_pass "Verify 'zfs rename' limits datasets so they don't pass " \
+	"the nesting limit. For existing ones that do, it should " \
+	"not allow them to grow anymore."