OpenZFS 9184 - Add ZFS performance test for fixed blocksize random read/write IO

This change introduces a new performance test which does random reads
and writes, but instead of using `bssplit` to determine the block size,
it uses a fixed blocksize. Additionally, some new IO sizes are added to
other tests and timestamp data is recorded with the performance data.

Authored by: Ahmed Gahnem <[email protected]>
Reviewed by: Dan Kimmel <[email protected]>
Reviewed by: John Kennedy <[email protected]>
Reviewed-by: George Melikov <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Tony Nguyen <[email protected]>
Ported-by: John Kennedy <[email protected]>
Signed-off-by: John Wren Kennedy <[email protected]>
Requires-builders: perf

OpenZFS-issue: https://www.illumos.org/issues/9184
OpenZFS-commit: https://github.com/openzfs/openzfs/pull/659
External-issue: DLPX-46724
Closes #7660

14 files changed, 187 insertions, 50 deletions

diff --git a/tests/runfiles/perf-regression.run b/tests/runfiles/perf-regression.run
index d10ff8836..ec081040d 100644
--- a/tests/runfiles/perf-regression.run
+++ b/tests/runfiles/perf-regression.run
@@ -27,6 +27,7 @@ tags = ['perf']
 [tests/perf/regression]
 tests = ['sequential_writes', 'sequential_reads', 'sequential_reads_arc_cached',
     'sequential_reads_arc_cached_clone', 'sequential_reads_dbuf_cached',
-    'random_reads', 'random_writes', 'random_readwrite', 'random_writes_zil']
+    'random_reads', 'random_writes', 'random_readwrite', 'random_writes_zil',
+    'random_readwrite_fixed']
 post =
 tags = ['perf', 'regression']
diff --git a/tests/zfs-tests/tests/perf/fio/Makefile.am b/tests/zfs-tests/tests/perf/fio/Makefile.am
index c4a2a4c3f..619251682 100644
--- a/tests/zfs-tests/tests/perf/fio/Makefile.am
+++ b/tests/zfs-tests/tests/perf/fio/Makefile.am
@@ -3,6 +3,7 @@ dist_pkgdata_DATA = \
 	mkfiles.fio \
 	random_reads.fio \
 	random_readwrite.fio \
+	random_readwrite_fixed.fio \
 	random_writes.fio \
 	sequential_reads.fio \
 	sequential_writes.fio
diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
new file mode 100644
index 000000000..ed4495553
--- /dev/null
+++ b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
@@ -0,0 +1,35 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Delphix. All rights reserved.
+#
+
+[global]
+filename_format=file$jobnum
+nrfiles=16
+group_reporting=1
+fallocate=0
+overwrite=0
+thread=1
+rw=randrw
+rwmixread=70
+time_based=1
+directory=${DIRECTORY}
+runtime=${RUNTIME}
+bs=${BLOCKSIZE}
+ioengine=psync
+sync=${SYNC_TYPE}
+numjobs=${NUMJOBS}
+buffer_compress_percentage=66
+buffer_compress_chunk=4096
+
+[job]
diff --git a/tests/zfs-tests/tests/perf/regression/Makefile.am b/tests/zfs-tests/tests/perf/regression/Makefile.am
index 4f045880f..124e0936a 100644
--- a/tests/zfs-tests/tests/perf/regression/Makefile.am
+++ b/tests/zfs-tests/tests/perf/regression/Makefile.am
@@ -2,6 +2,7 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf/regression
 dist_pkgdata_SCRIPTS = \
 	random_reads.ksh \
 	random_readwrite.ksh \
+	random_readwrite_fixed.ksh \
 	random_writes.ksh \
 	random_writes_zil.ksh \
 	sequential_reads_arc_cached_clone.ksh \
diff --git a/tests/zfs-tests/tests/perf/regression/random_reads.ksh b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
index f4e333696..079a53680 100755
--- a/tests/zfs-tests/tests/perf/regression/random_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
@@ -62,7 +62,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -89,17 +89,17 @@ if is_linux; then
 
         export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
index 00dd07025..5090d1998 100755
--- a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
@@ -89,17 +89,17 @@ if is_linux; then
 
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh
new file mode 100755
index 000000000..38c0669f6
--- /dev/null
+++ b/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh
@@ -0,0 +1,99 @@
+#!/usr/bin/ksh
+# file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Delphix. All rights reserved.
+#
+
+#
+# Description:
+# Trigger fio runs using the random_readwrite_fixed job file. The number of runs and
+# data collected is determined by the PERF_* variables. See do_fio_run for
+# details about these variables.
+#
+# The files to read and write from are created prior to the first fio run,
+# and used for all fio runs. The ARC is cleared with `zinject -a` prior to
+# each run so reads will go to disk.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/perf/perf.shlib
+
+function cleanup
+{
+	# kill fio and iostat
+	pkill fio
+	pkill iostat
+	recreate_perf_pool
+}
+
+trap "log_fail \"Measure IO stats during random read write load\"" SIGTERM
+log_onexit cleanup
+
+recreate_perf_pool
+populate_perf_filesystems
+
+# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
+export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
+
+# Variables for use by fio.
+if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
+	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
+	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
+	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
+	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
+	export PERF_IOSIZES='8k 64k'
+elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
+	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
+	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
+	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
+	export PERF_IOSIZES='8k'
+fi
+
+# Layout the files to be used by the readwrite tests. Create as many files
+# as the largest number of threads. An fio run with fewer threads will use
+# a subset of the available files.
+export NUMJOBS=$(get_max $PERF_NTHREADS)
+export FILE_SIZE=$((TOTAL_SIZE / NUMJOBS))
+export DIRECTORY=$(get_directory)
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+
+# Set up the scripts and output files that will log performance data.
+lun_list=$(pool_to_lun_list $PERFPOOL)
+log_note "Collecting backend IO stats with lun list $lun_list"
+if is_linux; then
+	typeset perf_record_cmd="perf record -F 99 -a -g -q \
+	    -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
+	export collect_scripts=(
+	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
+	    "vmstat -t 1" "vmstat"
+	    "mpstat -P ALL 1" "mpstat"
+	    "iostat -tdxyz 1" "iostat"
+	    "$perf_record_cmd" "perf"
+	)
+else
+	export collect_scripts=(
+	    "kstat zfs:0 1"  "kstat"
+	    "vmstat -T d 1"       "vmstat"
+	    "mpstat -T d 1"       "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
+	    "dtrace -Cs $PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
+	    "dtrace  -s $PERF_SCRIPTS/profile.d"                  "profile"
+	)
+fi
+
+log_note "Random reads and writes with $PERF_RUNTYPE settings"
+do_fio_run random_readwrite_fixed.fio false true
+log_pass "Measure IO stats during random read and write load"
diff --git a/tests/zfs-tests/tests/perf/regression/random_writes.ksh b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
index c84f96506..3101ac993 100755
--- a/tests/zfs-tests/tests/perf/regression/random_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
@@ -61,7 +61,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -80,17 +80,17 @@ if is_linux; then
 
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
index 022021daa..0f3256351 100755
--- a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
@@ -68,9 +68,9 @@ if is_linux; then
 
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
index 93c109e73..37e219627 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
@@ -62,7 +62,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -90,18 +90,18 @@ if is_linux; then
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
-	    "mpstat  -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "vmstat -t 1" "vmstat"
+	    "mpstat -P ALL 1" "mpstat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
index 5ef95c0e0..9528d97d6 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
@@ -49,10 +49,10 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -80,18 +80,18 @@ if is_linux; then
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
index 60f5d750d..f2747640e 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
@@ -55,10 +55,10 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -106,18 +106,18 @@ if is_linux; then
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
 	    "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi
 
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
index d49da6057..884856c2f 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
@@ -84,9 +84,9 @@ if is_linux; then
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
 	    "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch"
-	    "vmstat 1" "vmstat"
-	    "mpstat  -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "vmstat -t 1" "vmstat"
+	    "mpstat -P ALL 1" "mpstat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
diff --git a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
index e6d9ec2d5..8ea4f8419 100755
--- a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
@@ -61,7 +61,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
-	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'}
+	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
@@ -80,17 +80,17 @@ if is_linux; then
 
 	export collect_scripts=(
 	    "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-	    "vmstat 1" "vmstat"
+	    "vmstat -t 1" "vmstat"
 	    "mpstat -P ALL 1" "mpstat"
-	    "iostat -dxyz 1" "iostat"
+	    "iostat -tdxyz 1" "iostat"
 	    "$perf_record_cmd" "perf"
 	)
 else
 	export collect_scripts=(
 	    "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
-	    "vmstat 1" "vmstat"
-	    "mpstat 1" "mpstat"
-	    "iostat -xcnz 1" "iostat"
+	    "vmstat -T d 1" "vmstat"
+	    "mpstat -T d 1" "mpstat"
+	    "iostat -T d -xcnz 1" "iostat"
 	)
 fi