OpenZFS 9185 - Enable testing over NFS in ZFS performance tests

This change makes additions to the ZFS test suite that allows the
performance tests to run over NFS. The test is run and performance data
collected from the server side, while IO is generated on the NFS client.

This has been tested with Linux and illumos NFS clients.

Authored by: Ahmed Ghanem <[email protected]>
Reviewed by: Dan Kimmel <[email protected]>
Reviewed by: John Kennedy <[email protected]>
Reviewed by: Kevin Greene <[email protected]>
Reviewed-by: Richard Elling <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Ported-by: John Kennedy <[email protected]>
Signed-off-by: John Kennedy <[email protected]>

OpenZFS-issue: https://www.illumos.org/issues/9185
Closes #8367

11 files changed, 107 insertions, 2 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 78ba2488d..127a1477d 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -100,6 +100,7 @@ export SYSTEM_FILES='arp
     readlink
     rm
     rmdir
+    scp
     sed
     seq
     setenforce
@@ -110,6 +111,7 @@ export SYSTEM_FILES='arp
     shuf
     sleep
     sort
+    ssh
     stat
     strings
     su
diff --git a/tests/zfs-tests/tests/perf/Makefile.am b/tests/zfs-tests/tests/perf/Makefile.am
index d31bba0e2..68dd31ec1 100644
--- a/tests/zfs-tests/tests/perf/Makefile.am
+++ b/tests/zfs-tests/tests/perf/Makefile.am
@@ -1,5 +1,7 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf
-dist_pkgdata_DATA = perf.shlib
+dist_pkgdata_SCRIPTS = \
+	nfs-sample.cfg \
+	perf.shlib
 
 SUBDIRS = \
 	fio \
diff --git a/tests/zfs-tests/tests/perf/fio/random_reads.fio b/tests/zfs-tests/tests/perf/fio/random_reads.fio
index 79610f9b2..e6e7034e0 100644
--- a/tests/zfs-tests/tests/perf/fio/random_reads.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_reads.fio
@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]
diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
index 7d01c38ad..852d4bed6 100644
--- a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bssplit=4k/50:8k/30:128k/10:1m/10
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096
diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
index ed4495553..67b88c09d 100644
--- a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096
diff --git a/tests/zfs-tests/tests/perf/fio/random_writes.fio b/tests/zfs-tests/tests/perf/fio/random_writes.fio
index 5e2cb3002..90db5ce3b 100644
--- a/tests/zfs-tests/tests/perf/fio/random_writes.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_writes.fio
@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66
diff --git a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
index 33a9a1d89..b4b45e084 100644
--- a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
+++ b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]
diff --git a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
index 65a65910f..714993e92 100644
--- a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
+++ b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66
diff --git a/tests/zfs-tests/tests/perf/nfs-sample.cfg b/tests/zfs-tests/tests/perf/nfs-sample.cfg
new file mode 100644
index 000000000..f7ac2dae3
--- /dev/null
+++ b/tests/zfs-tests/tests/perf/nfs-sample.cfg
@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+#
+# This file is a sample NFS configuration for the performance tests. To use the
+# performance tests over NFS you must have:
+#   - a client machine with fio and sudo installed
+#   - passwordless SSH set up from this host
+#     for delphix and root users to the client
+#   - passwordless sudo for the user on the client
+#
+
+
+# The IP address for the server
+export NFS_SERVER=127.0.0.1
+
+# The IP address for the client
+export NFS_CLIENT=127.0.0.1
+
+# The mountpoint to use inside the client
+export NFS_MOUNT=/var/tmp/nfs
+
+# The user to run the tests as on the client
+export NFS_USER=delphix
+
+# Common NFS client mount options
+export NFS_OPTIONS="-o rw,nosuid,bg,hard,rsize=1048576,wsize=1048576,"
+NFS_OPTIONS+="nointr,timeo=600,proto=tcp,actimeo=0,port=2049"
+
+# illumos NFS client mount options
+# export NFS_OPTIONS="$NFS_OPTIONS,vers=3"
+
+# Linux NFS client mount options
+export NFS_OPTIONS="-t nfs $NFS_OPTIONS,noacl,nfsvers=3"
diff --git a/tests/zfs-tests/tests/perf/perf.shlib b/tests/zfs-tests/tests/perf/perf.shlib
index 7165df759..69e61e9fd 100644
--- a/tests/zfs-tests/tests/perf/perf.shlib
+++ b/tests/zfs-tests/tests/perf/perf.shlib
@@ -112,6 +112,16 @@ function do_fio_run_impl
 	export BLOCKSIZE=$iosize
 	sync
 
+	# When running locally, we want to keep the default behavior of
+	# DIRECT == 0, so only set it when we're running over NFS to
+	# disable client cache for reads.
+	if [[ $NFS -eq 1 ]]; then
+		export DIRECT=1
+		do_setup_nfs $script
+	else
+		export DIRECT=0
+	fi
+
 	# This will be part of the output filename.
 	typeset suffix=$(get_suffix $threads $sync $iosize)
 
@@ -124,7 +134,14 @@ function do_fio_run_impl
 	typeset outfile="$logbase.fio.$suffix"
 
 	# Start the load
-	log_must fio --output $outfile $FIO_SCRIPTS/$script
+	if [[ $NFS -eq 1 ]]; then
+		log_must ssh -t $NFS_USER@$NFS_CLIENT "
+			fio --output /tmp/fio.out /tmp/test.fio
+		"
+		log_must scp $NFS_USER@$NFS_CLIENT:/tmp/fio.out $outfile
+	else
+		log_must fio --output $outfile $FIO_SCRIPTS/$script
+	fi
 }
 
 #
@@ -166,6 +183,34 @@ function do_fio_run
 	done
 }
 
+# This function sets NFS mount on the client and make sure all correct
+# permissions are in place
+#
+function do_setup_nfs
+{
+	typeset script=$1
+	zfs  set sharenfs=on $TESTFS
+	log_must chmod  -R 777 /$TESTFS
+
+	ssh -t $NFS_USER@$NFS_CLIENT "mkdir -m 777 -p $NFS_MOUNT"
+	ssh -t $NFS_USER@$NFS_CLIENT "sudo -S umount $NFS_MOUNT"
+	log_must ssh -t $NFS_USER@$NFS_CLIENT "
+		sudo -S mount $NFS_OPTIONS $NFS_SERVER:/$TESTFS $NFS_MOUNT
+	"
+	#
+	# The variables in the fio script are only available in our current
+	# shell session, so we have to evaluate them here before copying
+	# the resulting script over to the target machine.
+	#
+	export jobnum='$jobnum'
+	while read line; do
+		eval echo "$line"
+	done < $FIO_SCRIPTS/$script > /tmp/test.fio
+	log_must sed -i -e "s%directory.*%directory=$NFS_MOUNT%" /tmp/test.fio
+	log_must scp /tmp/test.fio $NFS_USER@$NFS_CLIENT:/tmp
+	log_must rm /tmp/test.fio
+}
+
 #
 # This function iterates through the value pairs in $PERF_COLLECT_SCRIPTS.
 # The script at index N is launched in the background, with its output
diff --git a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
index 0f3256351..e0b253200 100755
--- a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
@@ -60,6 +60,10 @@ elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 fi
 
+# Until the performance tests over NFS can deal with multiple file systems,
+# force the use of only one file system when testing over NFS.
+[[ $NFS -eq 1 ]] && PERF_NTHREADS_PER_FS='0'
+
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then