Profiling for perf tests

Stack profiling is quite useful and Linux ZFS test suite does not
current collect that data.

Linux perf is a common tool for this purpose though the perf record
data file can be quite large. With this change, Linux ZFS perf tests
capture perf record data if perf is installed on the system and
PERF_DO_PROFILING environment variable is set.

Reviewed by: John Wren Kennedy <[email protected]>
Reviewed by: Matthew Ahrens <[email protected]>
Signed-off-by: Tony Nguyen <[email protected]>
External-issue: LX-971
Closes #7549 

9 files changed, 39 insertions, 10 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 2a6c3b13d..50eb6bd6b 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -84,6 +84,7 @@ export SYSTEM_FILES='arp
     openssl
     parted
     pax
+    perf
     pgrep
     ping
     pkill
diff --git a/tests/zfs-tests/tests/perf/regression/random_reads.ksh b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
index 5e91fe0a7..5bf269a85 100755
--- a/tests/zfs-tests/tests/perf/regression/random_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
@@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
-        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-            "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-            "iostat")
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
+	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
+	    "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
+	    "iostat" "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")
diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
index 446a440ec..e6d6e3a11 100755
--- a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
@@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-	    "iostat")
+	    "iostat" "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")
diff --git a/tests/zfs-tests/tests/perf/regression/random_writes.ksh b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
index e724aefe2..d85a3d98a 100755
--- a/tests/zfs-tests/tests/perf/regression/random_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
@@ -77,9 +77,12 @@ fi
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-	    "iostat")
+	    "iostat" "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
index e60dbb181..a9c62fe5a 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
@@ -85,9 +85,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+    typeset perf_record_cmd="perf record -F 99 -a -g -q \
+        -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-	    "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+	    "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+	    "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
index 5445f814e..6622ac973 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
@@ -75,9 +75,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-	    "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+	    "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+	    "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
index 5d7175b1a..9ed0e4792 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
@@ -91,9 +91,13 @@ export TESTFS=$PERFPOOL/$TESTCLONE
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-	    "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+	    "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+	    "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
index 5d029280e..edb7a96c9 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
@@ -77,9 +77,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-	    "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+	    "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+	    "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("kstat zfs:0 1" "kstat" "vmstat -T d 1" "vmstat"
 	    "mpstat -T d 1" "mpstat" "iostat -T d -xcnz 1" "iostat"
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
index d5fb89020..01ab80d4a 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
@@ -77,9 +77,12 @@ fi
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
 	export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-	    "iostat")
+	    "iostat" "$perf_record_cmd" "perf")
 else
 	export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")