zdb: add support for object ranges for zdb -d

Allow a range of object identifiers to dump with -d. This may
be useful when dumping a large dataset and you want to break
it up into multiple phases, or to resume where a previous scan
left off. Object type selection flags are supported to reduce
the performance overhead of verbosely dumping unwanted objects,
and to reduce the amount of post-processing work needed to
filter out unwanted objects from zdb output.

This change extends existing syntax in a backward-compatible
way. That is, the base case of a range is to specify a single
object identifier to dump. Ranges and object identifiers can
be intermixed as command line parameters.

Usage synopsis:

    Object ranges take the form <start>:<end>[:<flags>]
        start    Starting object number
        end      Ending object number, or -1 for no upper bound
        flags    Optional flags to select object types:
         A    All objects (this is the default)
         d    ZFS directories
         f    ZFS files
         m    SPA space maps
         z    ZAPs
         -    Negate effect of next flag

Examples:

 # Dump all file objects
 zdb -dd tank/fish 0:-1:f

 # Dump all file and directory objects
 zdb -dd tank/fish 0:-1:fd

 # Dump all types except file and directory objects
 zdb -dd tank/fish 0:-1:A-f-d

 # Dump object IDs in a specific range
 zdb -dd tank/fish 1000:2000

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Reviewed-by: Paul Zuchowski <[email protected]>
Signed-off-by: Ned Bass <[email protected]>
Closes #9832 

4 files changed, 247 insertions, 2 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index a04f5f3cc..e1315902b 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -101,7 +101,7 @@ tags = ['functional', 'clean_mirror']
 [tests/functional/cli_root/zdb]
 tests = ['zdb_001_neg', 'zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos',
     'zdb_005_pos', 'zdb_006_pos', 'zdb_checksum', 'zdb_decompress',
-    'zdb_objset_id']
+    'zdb_object_range_neg', 'zdb_object_range_pos', 'zdb_objset_id']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zdb']
diff --git a/tests/zfs-tests/tests/functional/cli_root/zdb/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zdb/Makefile.am
index edbebb020..e4679ae9f 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zdb/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zdb/Makefile.am
@@ -8,4 +8,6 @@ dist_pkgdata_SCRIPTS = \
 	zdb_006_pos.ksh \
 	zdb_checksum.ksh \
 	zdb_decompress.ksh \
-	zdb_objset_id.ksh
+	zdb_objset_id.ksh \
+	zdb_object_range_neg.ksh \
+	zdb_object_range_pos.ksh
diff --git a/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_neg.ksh b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_neg.ksh
new file mode 100755
index 000000000..430180788
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_neg.ksh
@@ -0,0 +1,72 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+# Copyright (c) 2020 Lawrence Livermore National Security, LLC.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# A badly formed object range parameter passed to zdb -dd should
+# return an error.
+#
+# Strategy:
+# 1. Create a pool
+# 2. Run zdb -dd with assorted invalid object range arguments and
+#    confirm it fails as expected
+# 3. Run zdb -dd with an invalid object identifier and
+#    confirm it fails as expected
+
+function cleanup
+{
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+}
+
+log_assert "Execute zdb using invalid object range parameters."
+log_onexit cleanup
+verify_runnable "both"
+verify_disk_count "$DISKS" 2
+default_mirror_setup_noexit $DISKS
+
+log_must zpool sync
+
+set -A bad_flags a b c   e   g h i j k l   n o p q r s t u v w x y   \
+                   B C D E F G H I J K L M N O P Q R S T U V W X Y Z \
+                 0 1 2 3 4 5 6 7 8 9 _ - + % . , :
+
+typeset -i i=0
+while [[ $i -lt ${#bad_flags[*]} ]]; do
+	log_mustnot zdb -dd $TESTPOOL 0:1:${bad_flags[i]}
+	log_mustnot zdb -dd $TESTPOOL 0:1:A-${bad_flags[i]}
+	((i = i + 1))
+done
+
+set -A bad_ranges ":" "::" ":::" ":0" "0:" "0:1:" "0:1::" "0::f" "0a:1" \
+    "a0:1" "a:1" "0:a" "0:1a" "0:a1" "a:b0" "a:0b" "0:1:A-" "1:0" \
+    "0:1:f:f" "0:1:f:"
+
+i=0
+while [[ $i -lt ${#bad_ranges[*]} ]]; do
+	log_mustnot zdb -dd $TESTPOOL ${bad_ranges[i]}
+	((i = i + 1))
+done
+
+# Specifying a non-existent object identifier returns an error
+obj_id_highest=$(zdb -P -dd $TESTPOOL/$TESTFS 2>/dev/null |
+    egrep "^ +-?([0-9]+ +){7}" | sort -n | tail -n 1 | awk '{print $1}')
+obj_id_invalid=$(( $obj_id_highest + 1 ))
+log_mustnot zdb -dd $TESTPOOL/$TESTFS $obj_id_invalid
+
+log_pass "Badly formed zdb object range parameters fail as expected."
diff --git a/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_pos.ksh
new file mode 100755
index 000000000..b7f47d11a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_pos.ksh
@@ -0,0 +1,171 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+# Copyright (c) 2020 Lawrence Livermore National Security, LLC.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# Object range parameters passed to zdb -dd work correctly.
+#
+# Strategy:
+# 1. Create a pool
+# 2. Create some files
+# 3. Run zdb -dd with assorted object range arguments and verify output
+
+function cleanup
+{
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+}
+
+#
+# Print objects in @dataset with identifiers greater than or equal to
+# @begin and less than or equal to @end, without using object range
+# parameters.
+#
+function get_object_list_range
+{
+	dataset=$1
+	begin=$2
+	end=$3
+	get_object_list $dataset |
+	while read line; do
+		obj=$(echo $line | awk '{print $1}')
+		if [[ $obj -ge $begin && $obj -le $end ]] ; then
+			echo "$line"
+		elif [[ $obj -gt $end ]] ; then
+			break
+		fi
+	done
+}
+
+#
+# Print just the list of objects from 'zdb -dd' with leading whitespace
+# trimmed, discarding other zdb output, sorted by object identifier.
+# Caller must pass in the dataset argument at minimum.
+#
+function get_object_list
+{
+	zdb -P -dd $@ 2>/dev/null |
+	egrep "^ +-?([0-9]+ +){7}" |
+	sed 's/^[[:space:]]*//' |
+	sort -n
+}
+
+log_assert "Verify zdb -dd object range arguments work correctly."
+log_onexit cleanup
+verify_runnable "both"
+verify_disk_count "$DISKS" 2
+default_mirror_setup_noexit $DISKS
+
+for x in $(seq 0 7); do
+	touch $TESTDIR/file$x
+	mkdir $TESTDIR/dir$x
+done
+
+log_must zpool sync
+
+# Get list of all objects, but filter out user/group objects which don't
+# appear when using object or object range arguments
+all_objects=$(get_object_list $TESTPOOL/$TESTFS | grep -v 'used$')
+
+# Range 0:-1 gets all objects
+expected=$all_objects
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:A gets all objects
+expected=$all_objects
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1:A)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:f must output all file objects
+expected=$(grep "ZFS plain file" <<< $all_objects)
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1:f)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:d must output all directory objects
+expected=$(grep "ZFS directory" <<< $all_objects)
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1:d)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:df must output all directory and file objects
+expected=$(grep -e "ZFS directory" -e "ZFS plain file" <<< $all_objects)
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1:df)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:A-f-d must output all non-files and non-directories
+expected=$(grep -v -e "ZFS plain file" -e "ZFS directory" <<< $all_objects)
+actual=$(get_object_list $TESTPOOL/$TESTFS 0:-1:A-f-d)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Specifying multiple ranges works
+set -A obj_ids $(ls -i $TESTDIR | awk '{print $1}' | sort -n)
+start1=${obj_ids[0]}
+end1=${obj_ids[5]}
+start2=${obj_ids[8]}
+end2=${obj_ids[13]}
+expected=$(get_object_list_range $TESTPOOL/$TESTFS $start1 $end1;
+    get_object_list_range $TESTPOOL/$TESTFS $start2 $end2)
+actual=$(get_object_list $TESTPOOL/$TESTFS $start1:$end1 $start2:$end2)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Combining ranges with individual object IDs works
+expected=$(get_object_list_range $TESTPOOL/$TESTFS $start1 $end1;
+    get_object_list $TESTPOOL/$TESTFS $start2 $end2)
+actual=$(get_object_list $TESTPOOL/$TESTFS $start1:$end1 $start2 $end2)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Hex conversion must work for ranges and individual object identifiers
+# (this test uses expected result from previous test).
+start1_hex=$(printf "0x%x" $start1)
+end1_hex=$(printf "0x%x" $end1)
+start2_hex=$(printf "0x%x" $start2)
+end2_hex=$(printf "0x%x" $end2)
+actual=$(get_object_list $TESTPOOL/$TESTFS $start1_hex:$end1_hex \
+    $start2_hex $end2_hex)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Specifying individual object IDs works
+objects="$start1 $end1 $start2 $end2"
+expected="$objects"
+actual=$(get_object_list $TESTPOOL/$TESTFS $objects | awk '{print $1}' | xargs)
+log_must test "$actual" == "$expected"
+
+# Get all objects in the meta-objset to test m (spacemap) and z (zap) flags
+all_mos_objects=$(get_object_list $TESTPOOL 0:-1)
+
+# Range 0:-1:m must output all space map objects
+expected=$(grep "SPA space map" <<< $all_mos_objects)
+actual=$(get_object_list $TESTPOOL 0:-1:m)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:z must output all zap objects
+expected=$(grep "zap" <<< $all_mos_objects)
+actual=$(get_object_list $TESTPOOL 0:-1:z)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:A-m-z must output all non-space maps and non-zaps
+expected=$(grep -v -e "zap" -e "SPA space map" <<< $all_mos_objects)
+actual=$(get_object_list $TESTPOOL 0:-1:A-m-z)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+# Range 0:-1:mz must output all space maps and zaps
+expected=$(grep -e "SPA space map" -e "zap" <<< $all_mos_objects)
+actual=$(get_object_list $TESTPOOL 0:-1:mz)
+log_must test "\n$actual\n" == "\n$expected\n"
+
+log_pass "zdb -dd object range arguments work correctly"