Add linux zpios support

Linux kernel implementation of PIOS test app.

Signed-off-by: Brian Behlendorf <[email protected]>

20 files changed, 1794 insertions, 0 deletions

diff --git a/scripts/Makefile.am b/scripts/Makefile.am
index a1dfc3871..ed6e5028f 100644
--- a/scripts/Makefile.am
+++ b/scripts/Makefile.am
@@ -4,11 +4,17 @@ nobase_pkglibexec_SCRIPTS += zconfig.sh
 nobase_pkglibexec_SCRIPTS += zfs.sh
 nobase_pkglibexec_SCRIPTS += zpool-create.sh
 nobase_pkglibexec_SCRIPTS += zpool-config/*
+nobase_pkglibexec_SCRIPTS += zpios.sh
+nobase_pkglibexec_SCRIPTS += zpios-sanity.sh
+nobase_pkglibexec_SCRIPTS += zpios-survey.sh
+nobase_pkglibexec_SCRIPTS += zpios-test/*
+nobase_pkglibexec_SCRIPTS += zpios-profile/*
 EXTRA_DIST = zfs-update.sh $(nobase_pkglibexec_SCRIPTS)
 
 ZFS=${top_srcdir}/scripts/zfs.sh
 ZCONFIG=${top_srcdir}/scripts/zconfig.sh
 ZTEST=${top_builddir}/cmd/ztest/ztest
+ZPIOS_SANITY=${top_srcdir}/scripts/zpios-sanity.sh
 
 check:
 	@echo
@@ -27,3 +33,11 @@ check:
 	@echo
 	@$(ZCONFIG)
 	@echo
+	@echo -n "===================================="
+	@echo -n " ZPIOS "
+	@echo    "===================================="
+	@echo
+	@$(ZFS)
+	@$(ZPIOS_SANITY)
+	@$(ZFS) -u
+	@echo
diff --git a/scripts/common.sh.in b/scripts/common.sh.in
index 0a8399f18..102952fdb 100644
--- a/scripts/common.sh.in
+++ b/scripts/common.sh.in
@@ -39,6 +39,8 @@ sbindir=@sbindir@
 ETCDIR=${ETCDIR:-/etc}
 DEVDIR=${DEVDIR:-/dev/disk/zpool}
 ZPOOLDIR=${ZPOOLDIR:-${pkglibexecdir}/zpool-config}
+ZPIOSDIR=${ZPIOSDIR:-${pkglibexecdir}/zpios-test}
+ZPIOSPROFILEDIR=${ZPIOSPROFILEDIR:-${pkglibexecdir}/zpios-profile}
 
 ZDB=${ZDB:-${sbindir}/zdb}
 ZFS=${ZFS:-${sbindir}/zfs}
@@ -46,10 +48,13 @@ ZINJECT=${ZINJECT:-${sbindir}/zinject}
 ZPOOL=${ZPOOL:-${sbindir}/zpool}
 ZPOOL_ID=${ZPOOL_ID:-${bindir}/zpool_id}
 ZTEST=${ZTEST:-${sbindir}/ztest}
+ZPIOS=${ZPIOS:-${sbindir}/zpios}
 
 COMMON_SH=${COMMON_SH:-${pkglibexecdir}/common.sh}
 ZFS_SH=${ZFS_SH:-${pkglibexecdir}/zfs.sh}
 ZPOOL_CREATE_SH=${ZPOOL_CREATE_SH:-${pkglibexecdir}/zpool-create.sh}
+ZPIOS_SH=${ZPIOS_SH:-${pkglibexecdir}/zpios.sh}
+ZPIOS_SURVEY_SH=${ZPIOS_SURVEY_SH:-${pkglibexecdir}/zpios-survey.sh}
 
 LDMOD=${LDMOD:-/sbin/modprobe}
 LSMOD=${LSMOD:-/sbin/lsmod}
diff --git a/scripts/zpios-profile/zpios-profile-disk.sh b/scripts/zpios-profile/zpios-profile-disk.sh
new file mode 100755
index 000000000..b56ee1ee4
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-disk.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# /proc/diskinfo <after skipping major/minor>
+# Field 1 -- device name
+# Field 2 -- # of reads issued
+# Field 3 -- # of reads merged
+# Field 4 -- # of sectors read
+# Field 5 -- # of milliseconds spent reading
+# Field 6 -- # of writes completed
+# Field 7 -- # of writes merged
+# Field 8 -- # of sectors written
+# Field 9 -- # of milliseconds spent writing
+# Field 10 -- # of I/Os currently in progress
+# Field 11 -- # of milliseconds spent doing I/Os
+# Field 12 -- weighted # of milliseconds spent doing I/Os
+
+PROG=zpios-profile-disk.sh
+
+RUN_PIDS=${0}
+RUN_LOG_DIR=${1}
+RUN_ID=${2}
+
+create_table() {
+	local FIELD=$1
+	local ROW_M=()
+	local ROW_N=()
+	local HEADER=1
+	local STEP=1
+
+	for DISK_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/disk-[0-9]*`; do
+		ROW_M=( ${ROW_N[@]} )
+		ROW_N=( `cat ${DISK_FILE} | grep sd | cut -c11- | cut -f${FIELD} -d' ' | tr "\n" "\t"` )
+
+		if [ $HEADER -eq 1 ]; then
+			echo -n "step, "
+			cat ${DISK_FILE} | grep sd | cut -c11- | cut -f1 -d' ' | tr "\n" ", "
+	                echo "total"
+			HEADER=0
+		fi
+
+		if [ ${#ROW_M[@]} -eq 0 ]; then
+			continue
+		fi
+
+		if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+			echo "Badly formatted profile data in ${DISK_FILE}"
+			break
+		fi
+
+		TOTAL=0
+		echo -n "${STEP}, "
+		for (( i=0; i<${#ROW_N[@]}; i++ )); do
+			DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+			let TOTAL=${TOTAL}+${DELTA}
+			echo -n "${DELTA}, "
+		done
+		echo "${TOTAL}, "
+
+		let STEP=${STEP}+1
+	done
+}
+
+create_table_mbs() {
+	local FIELD=$1
+	local TIME=$2
+	local ROW_M=()
+	local ROW_N=()
+	local HEADER=1
+	local STEP=1
+
+	for DISK_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/disk-[0-9]*`; do
+		ROW_M=( ${ROW_N[@]} )
+		ROW_N=( `cat ${DISK_FILE} | grep sd | cut -c11- | cut -f${FIELD} -d' ' | tr "\n" "\t"` )
+
+		if [ $HEADER -eq 1 ]; then
+			echo -n "step, "
+			cat ${DISK_FILE} | grep sd | cut -c11- | cut -f1 -d' ' | tr "\n" ", "
+	                echo "total"
+			HEADER=0
+		fi
+
+		if [ ${#ROW_M[@]} -eq 0 ]; then
+			continue
+		fi
+
+		if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+			echo "Badly formatted profile data in ${DISK_FILE}"
+			break
+		fi
+
+		TOTAL=0
+		echo -n "${STEP}, "
+		for (( i=0; i<${#ROW_N[@]}; i++ )); do
+			DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+			MBS=`echo "scale=2; ((${DELTA}*512)/${TIME})/(1024*1024)" | bc`
+			TOTAL=`echo "scale=2; ${TOTAL}+${MBS}" | bc`
+			echo -n "${MBS}, "
+		done
+		echo "${TOTAL}, "
+
+		let STEP=${STEP}+1
+	done
+}
+
+echo
+echo "Reads issued per device"
+create_table 2
+echo
+echo "Reads merged per device"
+create_table 3
+echo
+echo "Sectors read per device"
+create_table 4
+echo "MB/s per device"
+create_table_mbs 4 3
+
+echo
+echo "Writes issued per device"
+create_table 6
+echo
+echo "Writes merged per device"
+create_table 7
+echo
+echo "Sectors written per device"
+create_table 8
+echo "MB/s per device"
+create_table_mbs 8 3
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-pids.sh b/scripts/zpios-profile/zpios-profile-pids.sh
new file mode 100755
index 000000000..3514b38e2
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-pids.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+
+PROG=zpios-profile-pids.sh
+
+RUN_PIDS=${0}
+RUN_LOG_DIR=${1}
+RUN_ID=${2}
+
+ROW_M=()
+ROW_N=()
+ROW_N_SCHED=()
+ROW_N_WAIT=()
+
+HEADER=1
+STEP=1
+
+for PID_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/pids-[0-9]*`; do
+	ROW_M=( ${ROW_N[@]} )
+	ROW_N=( 0 0 0 0  0 0 0 0  0 0 0 0  0 0 0 0  0 0 0 )
+	ROW_N_SCHED=( `cat ${PID_FILE} | cut -f15 -d' ' | tr "\n" "\t"` )
+	ROW_N_WAIT=(  `cat ${PID_FILE} | cut -f17 -d' ' | tr "\n" "\t"` )
+	ROW_N_NAMES=( `cat ${PID_FILE} | cut -f2  -d' ' | cut -f2 -d'(' | 
+                       cut -f1 -d')'   | cut -f1  -d'/' | tr "\n" "\t"` )
+
+	for (( i=0; i<${#ROW_N_SCHED[@]}; i++ )); do
+		SUM=`echo "${ROW_N_WAIT[${i}]}+${ROW_N_SCHED[${i}]}" | bc`
+
+		case ${ROW_N_NAMES[${i}]} in
+			zio_taskq)	IDX=0;;
+			zio_req_nul)	IDX=1;;
+			zio_irq_nul)	IDX=2;;
+			zio_req_rd)	IDX=3;;
+			zio_irq_rd)	IDX=4;;
+			zio_req_wr)	IDX=5;;
+			zio_irq_wr)	IDX=6;;
+			zio_req_fr)	IDX=7;;
+			zio_irq_fr)	IDX=8;;
+			zio_req_cm)	IDX=9;;
+			zio_irq_cm)	IDX=10;;
+			zio_req_ctl)	IDX=11;;
+			zio_irq_ctl)	IDX=12;;
+			txg_quiesce)	IDX=13;;
+			txg_sync)	IDX=14;;
+			txg_timelimit)	IDX=15;;
+			arc_reclaim)	IDX=16;;
+			l2arc_feed)	IDX=17;;
+			zpios_io)	IDX=18;;
+			*)		continue;;
+		esac
+
+		let ROW_N[${IDX}]=${ROW_N[${IDX}]}+${SUM}
+	done
+
+	if [ $HEADER -eq 1 ]; then
+		echo "step, zio_taskq, zio_req_nul, zio_irq_nul, "        \
+                     "zio_req_rd, zio_irq_rd, zio_req_wr, zio_irq_wr, "   \
+                     "zio_req_fr, zio_irq_fr, zio_req_cm, zio_irq_cm, "   \
+                     "zio_req_ctl, zio_irq_ctl, txg_quiesce, txg_sync, "  \
+                     "txg_timelimit, arc_reclaim, l2arc_feed, zpios_io, " \
+		     "idle"
+		HEADER=0
+	fi
+
+	if [ ${#ROW_M[@]} -eq 0 ]; then
+		continue
+	fi
+
+	if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+		echo "Badly formatted profile data in ${PID_FILE}"
+		break
+	fi
+
+	# Original values are in jiffies and we expect HZ to be 1000
+	# on most 2.6 systems thus we divide by 10 to get a percentage.
+	IDLE=1000
+        echo -n "${STEP}, "
+	for (( i=0; i<${#ROW_N[@]}; i++ )); do
+		DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+		DELTA_PERCENT=`echo "scale=1; ${DELTA}/10" | bc`
+		let IDLE=${IDLE}-${DELTA}
+		echo -n "${DELTA_PERCENT}, "
+	done
+	ILDE_PERCENT=`echo "scale=1; ${IDLE}/10" | bc`
+	echo "${ILDE_PERCENT}"
+
+	let STEP=${STEP}+1
+done
+
+exit
+
+echo
+echo "Percent of total system time per pid"
+for PID_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/pids-[0-9]*`; do
+	ROW_M=( ${ROW_N[@]} )
+	ROW_N_SCHED=( `cat ${PID_FILE} | cut -f15 -d' ' | tr "\n" "\t"` )
+	ROW_N_WAIT=( `cat ${PID_FILE} | cut -f17 -d' ' | tr "\n" "\t"` )
+
+	for (( i=0; i<${#ROW_N_SCHED[@]}; i++ )); do
+		ROW_N[${i}]=`echo "${ROW_N_WAIT[${i}]}+${ROW_N_SCHED[${i}]}" | bc`
+	done
+
+	if [ $HEADER -eq 1 ]; then
+		echo -n "step, "
+		cat ${PID_FILE} | cut -f2 -d' ' | tr "\n" ", "
+		echo
+		HEADER=0
+	fi
+
+	if [ ${#ROW_M[@]} -eq 0 ]; then
+		continue
+	fi
+
+	if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+		echo "Badly formatted profile data in ${PID_FILE}"
+		break
+	fi
+
+	# Original values are in jiffies and we expect HZ to be 1000
+	# on most 2.6 systems thus we divide by 10 to get a percentage.
+        echo -n "${STEP}, "
+	for (( i=0; i<${#ROW_N[@]}; i++ )); do
+		DELTA=`echo "scale=1; (${ROW_N[${i}]}-${ROW_M[${i}]})/10" | bc`
+		echo -n "${DELTA}, "
+	done
+
+	echo
+	let STEP=${STEP}+1
+done
+
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-post.sh b/scripts/zpios-profile/zpios-profile-post.sh
new file mode 100755
index 000000000..3a454ba04
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-post.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+PROG=zpios-profile-post.sh
+
+RUN_POST=${0}
+RUN_PHASE=${1}
+RUN_DIR=${2}
+RUN_ID=${3}
+RUN_POOL=${4}
+RUN_CHUNK_SIZE=${5}
+RUN_REGION_SIZE=${6}
+RUN_THRD_COUNT=${7}
+RUN_REGION_COUNT=${8}
+RUN_OFFSET=${9}
+RUN_REGION_NOISE=${10}
+RUN_CHUNK_NOISE=${11}
+RUN_THRD_DELAY=${12}
+RUN_FLAGS=${13}
+RUN_RESULT=${14}
+
+# Summarize system time per process
+zpios_profile_post_pids() {
+	${PROFILE_PIDS} ${PROFILE_RUN_CR_PIDS_LOG} >${PROFILE_RUN_CR_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_WR_PIDS_LOG} >${PROFILE_RUN_WR_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_RD_PIDS_LOG} >${PROFILE_RUN_RD_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_RM_PIDS_LOG} >${PROFILE_RUN_RM_PIDS_CSV}
+}
+
+zpios_profile_post_disk() {
+	${PROFILE_DISK} ${PROFILE_RUN_CR_DISK_LOG} >${PROFILE_RUN_CR_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_WR_DISK_LOG} >${PROFILE_RUN_WR_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_RD_DISK_LOG} >${PROFILE_RUN_RD_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_RM_DISK_LOG} >${PROFILE_RUN_RM_DISK_CSV}
+}
+
+# Summarize per device performance
+
+# Stop a user defined profiling script which is gathering additional data
+zpios_profile_post_stop() {
+	local PROFILE_PID=$1
+
+	kill -s SIGHUP `cat ${PROFILE_PID}`
+
+
+	# Sleep waiting for profile script to exit
+	while [ -f ${PROFILE_PID} ]; do
+		sleep 0.01
+	done
+}
+
+zpios_profile_post_proc_stop() {
+	local PROC_DIR=$1
+
+	if [ -f ${PROFILE_ARC_PROC} ]; then
+		cat ${PROFILE_ARC_PROC} >${PROC_DIR}/arcstats.txt
+	fi
+
+	if [ -f ${PROFILE_VDEV_CACHE_PROC} ]; then
+		cat ${PROFILE_VDEV_CACHE_PROC} >${PROC_DIR}/vdev_cache_stats.txt
+	fi
+}
+
+zpios_profile_post_oprofile_stop() {
+	local OPROFILE_LOG=$1
+	local OPROFILE_ARGS="-a -g -l -p ${OPROFILE_KERNEL_DIR},${OPROFILE_SPL_DIR},${OPROFILE_ZFS_DIR}"
+
+	/usr/bin/opcontrol --stop >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opcontrol --dump >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opreport ${OPROFILE_ARGS} >${OPROFILE_LOG} 2>&1
+	/usr/bin/oparchive
+}
+
+zpios_profile_post_create() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_CR_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_CR_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_CR_PID}
+}
+
+zpios_profile_post_write() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_WR_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_WR_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_WR_PID}
+}
+
+zpios_profile_post_read() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_CR_RD_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_RD_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_RD_PID}
+}
+
+zpios_profile_post_remove() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_RM_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_RM_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_RM_PID}
+}
+
+# Source global zpios test configuration
+if [ -f ${RUN_DIR}/zpios-config.sh ]; then
+	. ${RUN_DIR}/zpios-config.sh
+fi
+
+# Source global per-run test configuration
+if [ -f ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh ]; then
+	. ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh
+fi
+
+case "${RUN_PHASE}" in
+	post-run)
+		zpios_profile_post_pids
+		zpios_profile_post_disk
+		;;
+	post-create)
+		zpios_profile_post_create
+		;;
+	post-write)
+		zpios_profile_post_write
+		;;
+	post-read)
+		zpios_profile_post_read
+		;;
+	post-remove)
+		zpios_profile_post_remove
+		;;
+	*)
+		echo "Usage: ${PROG} {post-run|post-create|post-write|post-read|post-remove}"
+		exit 1
+esac
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-pre.sh b/scripts/zpios-profile/zpios-profile-pre.sh
new file mode 100755
index 000000000..a2a885798
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-pre.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+
+PROG=zpios-profile-pre.sh
+
+PROFILE_RDY=0
+trap "PROFILE_RDY=1" SIGHUP
+
+RUN_PRE=${0}
+RUN_PHASE=${1}
+RUN_DIR=${2}
+RUN_ID=${3}
+RUN_POOL=${4}
+RUN_CHUNK_SIZE=${5}
+RUN_REGION_SIZE=${6}
+RUN_THRD_COUNT=${7}
+RUN_REGION_COUNT=${8}
+RUN_OFFSET=${9}
+RUN_REGION_NOISE=${10}
+RUN_CHUNK_NOISE=${11}
+RUN_THRD_DELAY=${12}
+RUN_FLAGS=${13}
+RUN_RESULT=${14}
+
+zpios_profile_pre_run_cfg() {
+cat > ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh << EOF
+#
+# Zpios Profiling Configuration for Run ${RUN_ID}
+#
+
+PROFILE_RUN_DIR=${RUN_DIR}/${RUN_ID}
+
+PROFILE_RUN_CR_DIR=${RUN_DIR}/${RUN_ID}/create
+PROFILE_RUN_CR_PID=${RUN_DIR}/${RUN_ID}/create/profile.pid
+PROFILE_RUN_CR_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/create/oprofile.txt
+PROFILE_RUN_CR_PIDS_LOG=${RUN_DIR}/${RUN_ID}/create/pids.txt
+PROFILE_RUN_CR_PIDS_CSV=${RUN_DIR}/${RUN_ID}/create/pids.csv
+PROFILE_RUN_CR_DISK_LOG=${RUN_DIR}/${RUN_ID}/create/disk.txt
+PROFILE_RUN_CR_DISK_CSV=${RUN_DIR}/${RUN_ID}/create/disk.csv
+
+PROFILE_RUN_WR_DIR=${RUN_DIR}/${RUN_ID}/write
+PROFILE_RUN_WR_PID=${RUN_DIR}/${RUN_ID}/write/profile.pid
+PROFILE_RUN_WR_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/write/oprofile.txt
+PROFILE_RUN_WR_PIDS_LOG=${RUN_DIR}/${RUN_ID}/write/pids.txt
+PROFILE_RUN_WR_PIDS_CSV=${RUN_DIR}/${RUN_ID}/write/pids.csv
+PROFILE_RUN_WR_DISK_LOG=${RUN_DIR}/${RUN_ID}/write/disk.txt
+PROFILE_RUN_WR_DISK_CSV=${RUN_DIR}/${RUN_ID}/write/disk.csv
+
+PROFILE_RUN_RD_DIR=${RUN_DIR}/${RUN_ID}/read
+PROFILE_RUN_RD_PID=${RUN_DIR}/${RUN_ID}/read/profile.pid
+PROFILE_RUN_RD_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/read/oprofile.txt
+PROFILE_RUN_RD_PIDS_LOG=${RUN_DIR}/${RUN_ID}/read/pids.txt
+PROFILE_RUN_RD_PIDS_CSV=${RUN_DIR}/${RUN_ID}/read/pids.csv
+PROFILE_RUN_RD_DISK_LOG=${RUN_DIR}/${RUN_ID}/read/disk.txt
+PROFILE_RUN_RD_DISK_CSV=${RUN_DIR}/${RUN_ID}/read/disk.csv
+
+PROFILE_RUN_RM_DIR=${RUN_DIR}/${RUN_ID}/remove
+PROFILE_RUN_RM_PID=${RUN_DIR}/${RUN_ID}/remove/profile.pid
+PROFILE_RUN_RM_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/remove/oprofile.txt
+PROFILE_RUN_RM_PIDS_LOG=${RUN_DIR}/${RUN_ID}/remove/pids.txt
+PROFILE_RUN_RM_PIDS_CSV=${RUN_DIR}/${RUN_ID}/remove/pids.csv
+PROFILE_RUN_RM_DISK_LOG=${RUN_DIR}/${RUN_ID}/remove/disk.txt
+PROFILE_RUN_RM_DISK_CSV=${RUN_DIR}/${RUN_ID}/remove/disk.csv
+
+# PROFILE_PIDS_LOG=${RUN_DIR}/${RUN_ID}/pids-summary.csv
+# PROFILE_DISK_LOG=${RUN_DIR}/${RUN_ID}/disk-summary.csv
+EOF
+}
+
+zpios_profile_pre_run_args() {
+cat > ${RUN_DIR}/${RUN_ID}/zpios-args.txt << EOF
+#
+# Zpios Arguments for Run ${RUN_ID}
+#
+
+DIR=${RUN_DIR}
+ID=${RUN_ID}
+POOL=${RUN_POOL}
+CHUNK_SIZE=${RUN_CHUNK_SIZE}
+REGION_SIZE=${RUN_REGION_SIZE}
+THRD_COUNT=${RUN_THRD_COUNT}
+REGION_COUNT=${RUN_REGION_COUNT}
+OFFSET=${RUN_OFFSET}
+REGION_NOISE=${RUN_REGION_NOISE}
+CHUNK_NOISE=${RUN_CHUNK_NOISE}
+THRD_DELAY=${RUN_THRD_DELAY}
+FLAGS=${RUN_FLAGS}
+RESULT=${RUN_RESULT}
+EOF
+}
+
+# Spawn a user defined profiling script to gather additional data
+zpios_profile_pre_start() {
+	local PROFILE_PID=$1
+
+	${PROFILE_USER} ${RUN_PHASE} ${RUN_DIR} ${RUN_ID} &
+	echo "$!" >${PROFILE_PID}
+
+	# Sleep waiting for profile script to be ready, it will
+	# signal us via SIGHUP when it is ready to start profiling.
+	while [ ${PROFILE_RDY} -eq 0 ]; do
+		sleep 0.01
+	done
+}
+
+zpios_profile_post_proc_start() { 
+
+        if [ -f ${PROFILE_ARC_PROC} ]; then
+                echo 0 >${PROFILE_ARC_PROC}
+        fi
+
+        if [ -f ${PROFILE_VDEV_CACHE_PROC} ]; then
+                echo 0 >${PROFILE_VDEV_CACHE_PROC}
+        fi
+}
+
+zpios_profile_pre_oprofile_start() {
+	local OPROFILE_LOG=$1
+
+	/usr/bin/opcontrol --reset >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opcontrol --start >>${OPROFILE_LOG} 2>&1
+}
+
+zpios_profile_pre_create() {
+	mkdir ${PROFILE_RUN_CR_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_CR_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_CR_OPROFILE_LOG}
+}
+
+zpios_profile_pre_write() {
+	mkdir ${PROFILE_RUN_WR_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_WR_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_WR_OPROFILE_LOG}
+}
+
+zpios_profile_pre_read() {
+	mkdir ${PROFILE_RUN_RD_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_RD_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_CR_RD_LOG}
+}
+
+zpios_profile_pre_remove() {
+	mkdir ${PROFILE_RUN_RM_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_RM_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_RM_OPROFILE_LOG}
+}
+
+# Source global zpios test configuration
+if [ -f ${RUN_DIR}/zpios-config.sh ]; then
+	. ${RUN_DIR}/zpios-config.sh
+fi
+
+# Source global per-run test configuration
+if [ -f ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh ]; then
+	. ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh
+fi
+
+case "${RUN_PHASE}" in
+	pre-run)
+		mkdir -p ${RUN_DIR}/${RUN_ID}/
+		zpios_profile_pre_run_cfg
+		zpios_profile_pre_run_args
+		;;
+	pre-create)
+		zpios_profile_pre_create
+		;;
+	pre-write)
+		zpios_profile_pre_write
+		;;
+	pre-read)
+		zpios_profile_pre_read
+		;;
+	pre-remove)
+		zpios_profile_pre_remove
+		;;
+	*)
+		echo "Usage: ${PROG} {pre-run|pre-create|pre-write|pre-read|pre-remove}"
+		exit 1
+esac
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile.sh b/scripts/zpios-profile/zpios-profile.sh
new file mode 100755
index 000000000..f4f0ee97f
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile.sh
@@ -0,0 +1,226 @@
+#!/bin/bash
+
+
+PROG=zpios-profile.sh
+
+trap "RUN_DONE=1" SIGHUP
+
+RUN_PHASE=${1}
+RUN_LOG_DIR=${2}
+RUN_ID=${3}
+RUN_DONE=0
+
+POLL_INTERVAL=2.99
+
+# Log these pids, the exact pid numbers will vary from system to system
+# so I harvest pid for all the following type of processes from /proc/<pid>/
+#
+# zio_taskq/#
+# spa_zio_issue/#
+# spa_zio_intr/#
+# txg_quiesce_thr
+# txg_sync_thread
+# txg_timelimit_t
+# arc_reclaim_thr
+# l2arc_feed_thre
+# zpios_io/#
+
+ZIO_TASKQ_PIDS=()
+ZIO_REQ_NUL_PIDS=() 
+ZIO_IRQ_NUL_PIDS=() 
+ZIO_REQ_RD_PIDS=() 
+ZIO_IRQ_RD_PIDS=() 
+ZIO_REQ_WR_PIDS=() 
+ZIO_IRQ_WR_PIDS=()
+ZIO_REQ_FR_PIDS=() 
+ZIO_IRQ_FR_PIDS=()
+ZIO_REQ_CM_PIDS=() 
+ZIO_IRQ_CM_PIDS=()
+ZIO_REQ_CTL_PIDS=() 
+ZIO_IRQ_CTL_PIDS=()
+
+TXG_QUIESCE_PIDS=()
+TXG_SYNC_PIDS=() 
+TXG_TIMELIMIT_PIDS=()
+
+ARC_RECLAIM_PIDS=()
+L2ARC_FEED_PIDS=()
+
+ZPIOS_IO_PIDS=()
+
+show_pids() {
+	echo "* zio_taskq:     { ${ZIO_TASKQ_PIDS[@]} } = ${#ZIO_TASKQ_PIDS[@]}"
+	echo "* zio_req_nul:   { ${ZIO_REQ_NUL_PIDS[@]} } = ${#ZIO_REQ_NUL_PIDS[@]}"
+	echo "* zio_irq_nul:   { ${ZIO_IRQ_NUL_PIDS[@]} } = ${#ZIO_IRQ_NUL_PIDS[@]}"
+	echo "* zio_req_rd:    { ${ZIO_REQ_RD_PIDS[@]} } = ${#ZIO_REQ_RD_PIDS[@]}"
+	echo "* zio_irq_rd:    { ${ZIO_IRQ_RD_PIDS[@]} } = ${#ZIO_IRQ_RD_PIDS[@]}"
+	echo "* zio_req_wr:    { ${ZIO_REQ_WR_PIDS[@]} } = ${#ZIO_REQ_WR_PIDS[@]}"
+	echo "* zio_irq_wr:    { ${ZIO_IRQ_WR_PIDS[@]} } = ${#ZIO_IRQ_WR_PIDS[@]}"
+	echo "* zio_req_fr:    { ${ZIO_REQ_FR_PIDS[@]} } = ${#ZIO_REQ_FR_PIDS[@]}"
+	echo "* zio_irq_fr:    { ${ZIO_IRQ_FR_PIDS[@]} } = ${#ZIO_IRQ_FR_PIDS[@]}"
+	echo "* zio_req_cm:    { ${ZIO_REQ_CM_PIDS[@]} } = ${#ZIO_REQ_CM_PIDS[@]}"
+	echo "* zio_irq_cm:    { ${ZIO_IRQ_CM_PIDS[@]} } = ${#ZIO_IRQ_CM_PIDS[@]}"
+	echo "* zio_req_ctl:   { ${ZIO_REQ_CTL_PIDS[@]} } = ${#ZIO_REQ_CTL_PIDS[@]}"
+	echo "* zio_irq_ctl:   { ${ZIO_IRQ_CTL_PIDS[@]} } = ${#ZIO_IRQ_CTL_PIDS[@]}"
+	echo "* txg_quiesce:   { ${TXG_QUIESCE_PIDS[@]} } = ${#TXG_QUIESCE_PIDS[@]}"
+	echo "* txg_sync:      { ${TXG_SYNC_PIDS[@]} } = ${#TXG_SYNC_PIDS[@]}"
+	echo "* txg_timelimit: { ${TXG_TIMELIMIT_PIDS[@]} } = ${#TXG_TIMELIMIT_PIDS[@]}"
+	echo "* arc_reclaim:   { ${ARC_RECLAIM_PIDS[@]} } = ${#ARC_RECLAIM_PIDS[@]}"
+	echo "* l2arc_feed:    { ${L2ARC_FEED_PIDS[@]} } = ${#L2ARC_FEED_PIDS[@]}"
+	echo "* zpios_io:      { ${ZPIOS_IO_PIDS[@]} } = ${#ZPIOS_IO_PIDS[@]}"
+}
+
+check_pid() {
+	local PID=$1
+	local NAME=$2
+	local TYPE=$3
+	local PIDS=( "$4" )
+        local NAME_STRING=`echo ${NAME} | cut -f1 -d'/'`
+        local NAME_NUMBER=`echo ${NAME} | cut -f2 -d'/'`
+
+	if [ "${NAME_STRING}" == "${TYPE}" ]; then
+		if [ -n "${NAME_NUMBER}" ]; then
+			PIDS[${NAME_NUMBER}]=${PID}
+		else
+			PIDS[${#PIDS[@]}]=${PID}
+
+		fi
+	fi
+
+	echo "${PIDS[@]}"
+}
+
+# NOTE: This whole process is crazy slow but it will do for now
+aquire_pids() {
+	echo "--- Aquiring ZFS pids ---"
+
+	for PID in `ls /proc/ | grep [0-9] | sort -n -u`; do
+		if [ ! -e /proc/${PID}/status ]; then
+			continue
+		fi
+
+		NAME=`cat /proc/${PID}/status  | head -n1 | cut -f2`
+
+		ZIO_TASKQ_PIDS=( `check_pid ${PID} ${NAME} "zio_taskq" \
+		                 "$(echo "${ZIO_TASKQ_PIDS[@]}")"` )
+
+		ZIO_REQ_NUL_PIDS=( `check_pid ${PID} ${NAME} "zio_req_nul" \
+		                   "$(echo "${ZIO_REQ_NUL_PIDS[@]}")"` )
+
+		ZIO_IRQ_NUL_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_nul" \
+		                   "$(echo "${ZIO_IRQ_NUL_PIDS[@]}")"` )
+
+		ZIO_REQ_RD_PIDS=( `check_pid ${PID} ${NAME} "zio_req_rd" \
+		                   "$(echo "${ZIO_REQ_RD_PIDS[@]}")"` )
+
+		ZIO_IRQ_RD_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_rd" \
+		                   "$(echo "${ZIO_IRQ_RD_PIDS[@]}")"` )
+
+		ZIO_REQ_WR_PIDS=( `check_pid ${PID} ${NAME} "zio_req_wr" \
+		                   "$(echo "${ZIO_REQ_WR_PIDS[@]}")"` )
+
+		ZIO_IRQ_WR_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_wr" \
+		                   "$(echo "${ZIO_IRQ_WR_PIDS[@]}")"` )
+
+		ZIO_REQ_FR_PIDS=( `check_pid ${PID} ${NAME} "zio_req_fr" \
+		                   "$(echo "${ZIO_REQ_FR_PIDS[@]}")"` )
+
+		ZIO_IRQ_FR_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_fr" \
+		                   "$(echo "${ZIO_IRQ_FR_PIDS[@]}")"` )
+
+		ZIO_REQ_CM_PIDS=( `check_pid ${PID} ${NAME} "zio_req_cm" \
+		                   "$(echo "${ZIO_REQ_CM_PIDS[@]}")"` )
+
+		ZIO_IRQ_CM_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_cm" \
+		                   "$(echo "${ZIO_IRQ_CM_PIDS[@]}")"` )
+
+		ZIO_REQ_CTL_PIDS=( `check_pid ${PID} ${NAME} "zio_req_ctl" \
+		                   "$(echo "${ZIO_REQ_CTL_PIDS[@]}")"` )
+
+		ZIO_IRQ_CTL_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_ctl" \
+		                   "$(echo "${ZIO_IRQ_CTL_PIDS[@]}")"` )
+
+		TXG_QUIESCE_PIDS=( `check_pid ${PID} ${NAME} "txg_quiesce" \
+		                   "$(echo "${TXG_QUIESCE_PIDS[@]}")"` )
+
+		TXG_SYNC_PIDS=( `check_pid ${PID} ${NAME} "txg_sync" \
+		                "$(echo "${TXG_SYNC_PIDS[@]}")"` )
+
+		TXG_TIMELIMIT_PIDS=( `check_pid ${PID} ${NAME} "txg_timelimit" \
+		                     "$(echo "${TXG_TIMELIMIT_PIDS[@]}")"` )
+
+		ARC_RECLAIM_PIDS=( `check_pid ${PID} ${NAME} "arc_reclaim" \
+                                     "$(echo "${ARC_RECLAIM_PIDS[@]}")"` )
+
+		L2ARC_FEED_PIDS=( `check_pid ${PID} ${NAME} "l2arc_feed" \
+                                  "$(echo "${L2ARC_FEED_PIDS[@]}")"` )
+	done
+
+	# Wait for zpios_io threads to start
+	kill -s SIGHUP ${PPID}
+	echo "* Waiting for zpios_io threads to start"
+	while [ ${RUN_DONE} -eq 0 ]; do
+		ZPIOS_IO_PIDS=( `ps ax | grep zpios_io | grep -v grep | \
+                                 sed 's/^ *//g' | cut -f1 -d' '` )
+		if [ ${#ZPIOS_IO_PIDS[@]} -gt 0 ]; then
+			break;
+		fi
+		sleep 0.1
+	done
+
+	echo "`show_pids`" >${RUN_LOG_DIR}/${RUN_ID}/pids.txt
+}
+
+log_pids() {
+	echo "--- Logging ZFS profile to ${RUN_LOG_DIR}/${RUN_ID}/ ---"
+	ALL_PIDS=( ${ZIO_TASKQ_PIDS[@]}     \
+                   ${ZIO_REQ_NUL_PIDS[@]}   \
+                   ${ZIO_IRQ_NUL_PIDS[@]}   \
+                   ${ZIO_REQ_RD_PID[@]}     \
+                   ${ZIO_IRQ_RD_PIDS[@]}    \
+                   ${ZIO_REQ_WR_PIDS[@]}    \
+                   ${ZIO_IRQ_WR_PIDS[@]}    \
+                   ${ZIO_REQ_FR_PIDS[@]}    \ 
+                   ${ZIO_IRQ_FR_PIDS[@]}    \
+                   ${ZIO_REQ_CM_PIDS[@]}    \ 
+                   ${ZIO_IRQ_CM_PIDS[@]}    \
+                   ${ZIO_REQ_CTL_PIDS[@]}   \
+                   ${ZIO_IRQ_CTL_PIDS[@]}   \
+                   ${TXG_QUIESCE_PIDS[@]}   \
+                   ${TXG_SYNC_PIDS[@]}      \
+                   ${TXG_TIMELIMIT_PIDS[@]} \
+                   ${ARC_RECLAIM_PIDS[@]}   \
+                   ${L2ARC_FEED_PIDS[@]}    \
+                   ${ZPIOS_IO_PIDS[@]} )
+
+	while [ ${RUN_DONE} -eq 0 ]; do
+		NOW=`date +%s.%N`
+		LOG_PIDS="${RUN_LOG_DIR}/${RUN_ID}/pids-${NOW}"
+		LOG_DISK="${RUN_LOG_DIR}/${RUN_ID}/disk-${NOW}"
+
+		for PID in "${ALL_PIDS[@]}"; do
+			if [ -z ${PID} ]; then
+				continue;
+			fi
+
+	        	if [ -e /proc/${PID}/stat ]; then
+	        		cat /proc/${PID}/stat | head -n1 >>${LOG_PIDS}
+			else
+	                	echo "<${PID} exited>" >>${LOG_PIDS}
+	        	fi
+		done
+
+		cat /proc/diskstats >${LOG_DISK}
+
+		NOW2=`date +%s.%N`
+		DELTA=`echo "${POLL_INTERVAL}-(${NOW2}-${NOW})" | bc`
+		sleep ${DELTA}
+	done
+}
+
+aquire_pids
+log_pids
+
+# rm ${PROFILE_PID}
+
+exit 0
diff --git a/scripts/zpios-sanity.sh b/scripts/zpios-sanity.sh
new file mode 100755
index 000000000..194ae82e2
--- /dev/null
+++ b/scripts/zpios-sanity.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+#
+# ZFS/ZPOOL configuration test script.
+
+basedir="$(dirname $0)"
+
+SCRIPT_COMMON=common.sh
+if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
+. "${basedir}/${SCRIPT_COMMON}"
+else
+echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
+fi
+
+PROG=zpios-sanity.sh
+HEADER=
+
+usage() {
+cat << EOF
+USAGE:
+$0 [hvxfc]
+
+DESCRIPTION:
+        ZPIOS sanity tests
+
+OPTIONS:
+        -h      Show this message
+        -v      Verbose
+        -x      Destructive hd/sd/md/dm/ram tests
+	-f      Don't prompt due to -x
+	-c      Cleanup lo+file devices at start
+
+EOF
+}
+
+while getopts 'hvxfc?' OPTION; do
+	case $OPTION in
+	h)
+		usage
+		exit 1
+		;;
+	v)
+		VERBOSE=1
+		;;
+	x)
+		DANGEROUS=1
+		;;
+	f)
+		FORCE=1
+		;;
+	c)
+		CLEANUP=1
+		;;
+	?)
+		usage
+		exit
+		;;
+	esac
+done
+
+if [ $(id -u) != 0 ]; then
+	die "Must run as root"
+fi
+
+# Perform pre-cleanup is requested
+if [ ${CLEANUP} ]; then
+	cleanup_loop_devices
+	rm -f /tmp/zpool.cache.*
+fi
+
+zpios_test() {
+	CONFIG=$1
+	TEST=$2
+	LOG=`mktemp`
+
+	${ZPIOS_SH} -f -c ${CONFIG} -t ${TEST} &>${LOG}
+	if [ $? -ne 0 ]; then
+		if [ ${VERBOSE} ]; then
+			printf "FAIL:     %-13s\n" ${CONFIG}
+			cat ${LOG}
+		else
+			if [ ! ${HEADER} ]; then
+				head -2 ${LOG}
+				HEADER=1
+			fi
+
+			printf "FAIL:     %-13s" ${CONFIG}
+			tail -1 ${LOG}
+		fi
+	else
+		if [ ${VERBOSE} ]; then
+			cat ${LOG}
+		else
+			if [ ! ${HEADER} ]; then
+				head -2 ${LOG}
+				HEADER=1
+			fi
+
+			tail -1 ${LOG}
+		fi
+	fi
+
+	rm -f ${LOG}
+}
+
+if [ ${DANGEROUS} ] && [ ! ${FORCE} ]; then
+	cat << EOF
+The -x option was passed which will result in UNRECOVERABLE DATA LOSS
+on on the following block devices:
+
+  /dev/sd[abcd]
+  /dev/hda
+  /dev/ram0
+  /dev/md0
+  /dev/dm-0
+
+To continue please confirm by entering YES:
+EOF
+	read CONFIRM
+	if [ ${CONFIRM} != "YES" ] && [ ${CONFIRM} != "yes" ]; then
+		exit 0;
+	fi
+fi
+
+#
+# These configurations are all safe and pose no risk to any data on
+# the system which runs them.  They will confine all their IO to a
+# file in /tmp or a loopback device configured to use a file in /tmp.
+#
+SAFE_CONFIGS=(						\
+	file-raid0 file-raid10 file-raidz file-raidz2	\
+	lo-raid0 lo-raid10 lo-raidz lo-raidz2		\
+)
+
+#
+# These configurations are down right dangerous.  They will attempt
+# to use various real block devices on your system which may contain
+# data you car about.  You are STRONGLY advised not to run this unless
+# you are certain there is no data on the system you care about.
+#
+DANGEROUS_CONFIGS=(					\
+	hda-raid0					\
+	sda-raid0					\
+	ram0-raid0					\
+	md0-raid10 md0-raid5				\
+	dm0-raid0					\
+)
+
+for CONFIG in ${SAFE_CONFIGS[*]}; do
+	zpios_test $CONFIG tiny
+done
+
+if [ ${DANGEROUS} ]; then
+	for CONFIG in ${DANGEROUS_CONFIGS[*]}; do
+		zpios_test $CONFIG tiny
+	done
+fi
+
+exit 0
diff --git a/scripts/zpios-survey.sh b/scripts/zpios-survey.sh
new file mode 100755
index 000000000..cb751b467
--- /dev/null
+++ b/scripts/zpios-survey.sh
@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Wrapper script for easily running a survey of zpios based tests
+#
+
+basedir="$(dirname $0)"
+
+SCRIPT_COMMON=common.sh
+if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
+. "${basedir}/${SCRIPT_COMMON}"
+else
+echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
+fi
+
+PROG=zpios-survey.sh
+
+usage() {
+cat << EOF
+USAGE:
+$0 [hvp] <-c config> <-t test>
+
+DESCRIPTION:
+        Helper script for easy zpios survey benchmarking.
+
+OPTIONS:
+        -h      Show this message
+        -v      Verbose
+        -p      Enable profiling
+        -c      Zpool configuration
+        -t      Zpios test
+        -l      Zpios survey log
+
+EOF
+}
+
+print_header() {
+tee -a ${ZPIOS_SURVEY_LOG} << EOF
+
+================================================================
+Test: $1
+EOF
+}
+
+# Baseline performance for an out of the box config with no manual tuning.
+# Ideally, we want everything to be automatically tuned for your system and
+# for this to perform reasonably well.
+zpios_survey_base() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+baseline"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# Disable ZFS's prefetching.  For some reason still not clear to me
+# current prefetching policy is quite bad for a random workload.
+# Allowing the algorithm to detect a random workload and not do 
+# anything may be the way to address this issue.
+zpios_survey_prefetch() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+prefetch"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG}               \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
+		-o "--noprefetch" |                                    \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# Simulating a zerocopy IO path should improve performance by freeing up
+# lots of CPU which is wasted move data between buffers.
+zpios_survey_zerocopy() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+zerocopy"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
+		-o "--zerocopy" |                                      \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# Disabling checksumming should show some (if small) improvement
+# simply due to freeing up a modest amount of CPU.
+zpios_survey_checksum() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+checksum"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
+		-s "set checksum=off" |                                \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# Increasing the pending IO depth also seems to improve things likely
+# at the expense of latency.  This should be explored more because I'm
+# seeing a much bigger impact there that I would have expected.  There
+# may be some low hanging fruit to be found here.
+zpios_survey_pending() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+pending"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG}                  \
+		zfs="zfs_vdev_max_pending=1024" | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# To avoid memory fragmentation issues our slab implementation can be
+# based on a virtual address space.  Interestingly, we take a pretty
+# substantial performance penalty for this somewhere in the low level
+# IO drivers.  If we back the slab with kmem pages we see far better
+# read performance numbers at the cost of memory fragmention and general
+# system instability due to large allocations.  This may be because of
+# an optimization in the low level drivers due to the contigeous kmem
+# based memory.  This needs to be explained.  The good news here is that
+# with zerocopy interfaces added at the DMU layer we could gaurentee
+# kmem based memory for a pool of pages.
+#
+# 0x100 = KMC_KMEM - Force kmem_* based slab
+# 0x200 = KMC_VMEM - Force vmem_* based slab
+zpios_survey_kmem() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+kmem"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG}             \  
+		zfs="zio_bulk_flags=0x100" | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+# Apply all possible turning concurrently to get a best case number
+zpios_survey_all() {
+	TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+all"
+	print_header ${TEST_NAME}
+
+	${ZFS_SH} ${VERBOSE_FLAG}                \  
+		zfs="zfs_vdev_max_pending=1024" \
+		zfs="zio_bulk_flags=0x100" |    \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
+		-o "--noprefetch --zerocopy"                           \
+		-s "set checksum=off" |                                \
+		tee -a ${ZPIOS_SURVEY_LOG}
+	${ZFS_SH} -u ${VERBOSE_FLAG} | \
+		tee -a ${ZPIOS_SURVEY_LOG}
+}
+
+
+PROFILE=
+ZPOOL_NAME=zpios-survey
+ZPOOL_CONFIG=zpool-config.sh
+ZPIOS_TEST=zpios-test.sh
+ZPIOS_SURVEY_LOG=/dev/null
+
+while getopts 'hvpc:t:l:' OPTION; do
+	case $OPTION in
+	h)
+		usage
+		exit 1
+		;;
+	v)
+		VERBOSE=1
+		VERBOSE_FLAG="-v"
+		;;
+	p)
+		PROFILE=1
+		PROFILE_FLAG="-p"
+		;;
+	c)
+		ZPOOL_CONFIG=${OPTARG}
+		;;
+	t)
+		ZPIOS_TEST=${OPTARG}
+		;;
+	l)
+		ZPIOS_SURVEY_LOG=${OPTARG}
+		;;
+	?)
+		usage
+		exit
+		;;
+	esac
+done
+
+if [ $(id -u) != 0 ]; then
+	die "Must run as root"
+fi
+
+zpios_survey_base
+zpios_survey_prefetch
+zpios_survey_zerocopy
+zpios_survey_checksum
+zpios_survey_pending
+zpios_survey_kmem
+zpios_survey_all
+
+exit 0
diff --git a/scripts/zpios-test/16th-8192rc-4rs-1cs-4off.sh b/scripts/zpios-test/16th-8192rc-4rs-1cs-4off.sh
new file mode 100755
index 000000000..cbd9c697a
--- /dev/null
+++ b/scripts/zpios-test/16th-8192rc-4rs-1cs-4off.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Usage: zpios
+#        --threadcount       -t    =values
+#        --threadcount_low   -l    =value
+#        --threadcount_high  -h    =value
+#        --threadcount_incr  -e    =value
+#        --regioncount       -n    =values
+#        --regioncount_low   -i    =value
+#        --regioncount_high  -j    =value
+#        --regioncount_incr  -k    =value
+#        --offset            -o    =values
+#        --offset_low        -m    =value
+#        --offset_high       -q    =value
+#        --offset_incr       -r    =value
+#        --chunksize         -c    =values
+#        --chunksize_low     -a    =value
+#        --chunksize_high    -b    =value
+#        --chunksize_incr    -g    =value
+#        --regionsize        -s    =values
+#        --regionsize_low    -A    =value
+#        --regionsize_high   -B    =value
+#        --regionsize_incr   -C    =value
+#        --load              -L    =dmuio|ssf|fpp
+#        --pool              -p    =pool name
+#        --name              -M    =test name
+#        --cleanup           -x
+#        --prerun            -P    =pre-command
+#        --postrun           -R    =post-command
+#        --log               -G    =log directory
+#        --regionnoise       -I    =shift
+#        --chunknoise        -N    =bytes
+#        --threaddelay       -T    =jiffies
+#        --verify            -V
+#        --zerocopy          -z
+#        --nowait            -O
+#        --human-readable    -H
+#        --verbose           -v    =increase verbosity
+#        --help              -?    =this help
+
+ZPIOS_CMD="${ZPIOS}                                              \
+	--load=dmuio                                             \
+	--pool=${ZPOOL_NAME}                                     \
+	--name=${ZPOOL_CONFIG}                                   \
+	--threadcount=16                                         \
+	--regioncount=8192                                       \
+	--regionsize=4M                                          \
+	--chunksize=1M                                           \
+	--offset=4M                                              \
+	--cleanup                                                \
+	--human-readable                                         \
+	${ZPIOS_OPTIONS}"
+
+zpios_start() {
+	if [ ${VERBOSE} ]; then
+		ZPIOS_CMD="${ZPIOS_CMD} --verbose"
+		echo ${ZPIOS_CMD}
+	fi
+
+	${ZPIOS_CMD} || exit 1
+}
+
+zpios_stop() {
+	[ ${VERBOSE} ] && echo
+}
diff --git a/scripts/zpios-test/1th-16rc-4rs-1cs-4off.sh b/scripts/zpios-test/1th-16rc-4rs-1cs-4off.sh
new file mode 100755
index 000000000..cd3c50b77
--- /dev/null
+++ b/scripts/zpios-test/1th-16rc-4rs-1cs-4off.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Usage: zpios
+#        --threadcount       -t    =values
+#        --threadcount_low   -l    =value
+#        --threadcount_high  -h    =value
+#        --threadcount_incr  -e    =value
+#        --regioncount       -n    =values
+#        --regioncount_low   -i    =value
+#        --regioncount_high  -j    =value
+#        --regioncount_incr  -k    =value
+#        --offset            -o    =values
+#        --offset_low        -m    =value
+#        --offset_high       -q    =value
+#        --offset_incr       -r    =value
+#        --chunksize         -c    =values
+#        --chunksize_low     -a    =value
+#        --chunksize_high    -b    =value
+#        --chunksize_incr    -g    =value
+#        --regionsize        -s    =values
+#        --regionsize_low    -A    =value
+#        --regionsize_high   -B    =value
+#        --regionsize_incr   -C    =value
+#        --load              -L    =dmuio|ssf|fpp
+#        --pool              -p    =pool name
+#        --name              -M    =test name
+#        --cleanup           -x
+#        --prerun            -P    =pre-command
+#        --postrun           -R    =post-command
+#        --log               -G    =log directory
+#        --regionnoise       -I    =shift
+#        --chunknoise        -N    =bytes
+#        --threaddelay       -T    =jiffies
+#        --verify            -V
+#        --zerocopy          -z
+#        --nowait            -O
+#        --human-readable    -H
+#        --verbose           -v    =increase verbosity
+#        --help              -?    =this help
+
+
+ZPIOS_CMD="${ZPIOS}                                              \
+	--load=dmuio                                             \
+	--pool=${ZPOOL_NAME}                                     \
+	--name=${ZPOOL_CONFIG}                                   \
+	--threadcount=1                                          \
+	--regioncount=16                                         \
+	--regionsize=4M                                          \
+	--chunksize=1M                                           \
+	--offset=4M                                              \
+	--cleanup                                                \
+	--human-readable                                         \
+	${ZPIOS_OPTIONS}"
+
+zpios_start() {
+	if [ ${VERBOSE} ]; then
+		ZPIOS_CMD="${ZPIOS_CMD} --verbose"
+		echo ${ZPIOS_CMD}
+	fi
+
+	${ZPIOS_CMD} || exit 1
+}
+
+zpios_stop() {
+	[ ${VERBOSE} ] && echo
+}
diff --git a/scripts/zpios-test/1x256th-65536rc-4rs-1cs-4off.sh b/scripts/zpios-test/1x256th-65536rc-4rs-1cs-4off.sh
new file mode 100755
index 000000000..743e97b64
--- /dev/null
+++ b/scripts/zpios-test/1x256th-65536rc-4rs-1cs-4off.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Usage: zpios
+#        --threadcount       -t    =values
+#        --threadcount_low   -l    =value
+#        --threadcount_high  -h    =value
+#        --threadcount_incr  -e    =value
+#        --regioncount       -n    =values
+#        --regioncount_low   -i    =value
+#        --regioncount_high  -j    =value
+#        --regioncount_incr  -k    =value
+#        --offset            -o    =values
+#        --offset_low        -m    =value
+#        --offset_high       -q    =value
+#        --offset_incr       -r    =value
+#        --chunksize         -c    =values
+#        --chunksize_low     -a    =value
+#        --chunksize_high    -b    =value
+#        --chunksize_incr    -g    =value
+#        --regionsize        -s    =values
+#        --regionsize_low    -A    =value
+#        --regionsize_high   -B    =value
+#        --regionsize_incr   -C    =value
+#        --load              -L    =dmuio|ssf|fpp
+#        --pool              -p    =pool name
+#        --name              -M    =test name
+#        --cleanup           -x
+#        --prerun            -P    =pre-command
+#        --postrun           -R    =post-command
+#        --log               -G    =log directory
+#        --regionnoise       -I    =shift
+#        --chunknoise        -N    =bytes
+#        --threaddelay       -T    =jiffies
+#        --verify            -V
+#        --zerocopy          -z
+#        --nowait            -O
+#        --human-readable    -H
+#        --verbose           -v    =increase verbosity
+#        --help              -?    =this help
+
+ZPIOS_CMD="${ZPIOS}                                              \
+	--load=dmuio                                             \
+	--pool=${ZPOOL_NAME}                                     \
+	--name=${ZPOOL_CONFIG}                                   \
+	--threadcount=1,2,4,8,16,32,64,128,256                   \
+	--regioncount=65536                                      \
+	--regionsize=4M                                          \
+	--chunksize=1M                                           \
+	--offset=4M                                              \
+        --cleanup                                                \
+	--human-readable                                         \
+	${ZPIOS_OPTIONS}"
+
+zpios_start() {
+	if [ ${VERBOSE} ]; then
+		ZPIOS_CMD="${ZPIOS_CMD} --verbose"
+		echo ${ZPIOS_CMD}
+	fi
+
+	${ZPIOS_CMD} || exit 1
+}
+
+zpios_stop() {
+	[ ${VERBOSE} ] && echo
+}
diff --git a/scripts/zpios-test/256th-65536rc-4rs-1cs-4off.sh b/scripts/zpios-test/256th-65536rc-4rs-1cs-4off.sh
new file mode 100755
index 000000000..92a3b77b4
--- /dev/null
+++ b/scripts/zpios-test/256th-65536rc-4rs-1cs-4off.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Usage: zpios
+#        --threadcount       -t    =values
+#        --threadcount_low   -l    =value
+#        --threadcount_high  -h    =value
+#        --threadcount_incr  -e    =value
+#        --regioncount       -n    =values
+#        --regioncount_low   -i    =value
+#        --regioncount_high  -j    =value
+#        --regioncount_incr  -k    =value
+#        --offset            -o    =values
+#        --offset_low        -m    =value
+#        --offset_high       -q    =value
+#        --offset_incr       -r    =value
+#        --chunksize         -c    =values
+#        --chunksize_low     -a    =value
+#        --chunksize_high    -b    =value
+#        --chunksize_incr    -g    =value
+#        --regionsize        -s    =values
+#        --regionsize_low    -A    =value
+#        --regionsize_high   -B    =value
+#        --regionsize_incr   -C    =value
+#        --load              -L    =dmuio|ssf|fpp
+#        --pool              -p    =pool name
+#        --name              -M    =test name
+#        --cleanup           -x
+#        --prerun            -P    =pre-command
+#        --postrun           -R    =post-command
+#        --log               -G    =log directory
+#        --regionnoise       -I    =shift
+#        --chunknoise        -N    =bytes
+#        --threaddelay       -T    =jiffies
+#        --verify            -V
+#        --zerocopy          -z
+#        --nowait            -O
+#        --human-readable    -H
+#        --verbose           -v    =increase verbosity
+#        --help              -?    =this help
+
+ZPIOS_CMD="${ZPIOS}                                              \
+	--load=dmuio                                             \
+	--pool=${ZPOOL_NAME}                                     \
+	--name=${ZPOOL_CONFIG}                                   \
+	--threadcount=256                                        \
+	--regioncount=65536                                      \
+	--regionsize=4M                                          \
+	--chunksize=1M                                           \
+	--offset=4M                                              \
+        --cleanup                                                \
+	--human-readable                                         \
+	${ZPIOS_OPTIONS}"
+
+zpios_start() {
+	if [ ${VERBOSE} ]; then
+		ZPIOS_CMD="${ZPIOS_CMD} --verbose"
+		echo ${ZPIOS_CMD}
+	fi
+
+	${ZPIOS_CMD} || exit 1
+}
+
+zpios_stop() {
+	[ ${VERBOSE} ] && echo
+}
diff --git a/scripts/zpios-test/4th-1024rc-4rs-1cs-4off.sh b/scripts/zpios-test/4th-1024rc-4rs-1cs-4off.sh
new file mode 100755
index 000000000..0db952cd6
--- /dev/null
+++ b/scripts/zpios-test/4th-1024rc-4rs-1cs-4off.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Usage: zpios
+#        --threadcount       -t    =values
+#        --threadcount_low   -l    =value
+#        --threadcount_high  -h    =value
+#        --threadcount_incr  -e    =value
+#        --regioncount       -n    =values
+#        --regioncount_low   -i    =value
+#        --regioncount_high  -j    =value
+#        --regioncount_incr  -k    =value
+#        --offset            -o    =values
+#        --offset_low        -m    =value
+#        --offset_high       -q    =value
+#        --offset_incr       -r    =value
+#        --chunksize         -c    =values
+#        --chunksize_low     -a    =value
+#        --chunksize_high    -b    =value
+#        --chunksize_incr    -g    =value
+#        --regionsize        -s    =values
+#        --regionsize_low    -A    =value
+#        --regionsize_high   -B    =value
+#        --regionsize_incr   -C    =value
+#        --load              -L    =dmuio|ssf|fpp
+#        --pool              -p    =pool name
+#        --name              -M    =test name
+#        --cleanup           -x
+#        --prerun            -P    =pre-command
+#        --postrun           -R    =post-command
+#        --log               -G    =log directory
+#        --regionnoise       -I    =shift
+#        --chunknoise        -N    =bytes
+#        --threaddelay       -T    =jiffies
+#        --verify            -V
+#        --zerocopy          -z
+#        --nowait            -O
+#        --human-readable    -H
+#        --verbose           -v    =increase verbosity
+#        --help              -?    =this help
+
+ZPIOS_CMD="${ZPIOS}                                              \
+	--load=dmuio                                             \
+	--pool=${ZPOOL_NAME}                                     \
+	--name=${ZPOOL_CONFIG}                                   \
+	--threadcount=4                                          \
+	--regioncount=1024                                       \
+	--regionsize=4M                                          \
+	--chunksize=1M                                           \
+	--offset=4M                                              \
+	--cleanup                                                \
+	--human-readable                                         \
+	${ZPIOS_OPTIONS}"
+
+zpios_start() {
+	if [ ${VERBOSE} ]; then
+		ZPIOS_CMD="${ZPIOS_CMD} --verbose"
+		echo ${ZPIOS_CMD}
+	fi
+
+	${ZPIOS_CMD} || exit 1
+}
+
+zpios_stop() {
+	[ ${VERBOSE} ] && echo
+}
diff --git a/scripts/zpios-test/large-thread-survey.sh b/scripts/zpios-test/large-thread-survey.sh
new file mode 120000
index 000000000..90b6e3c47
--- /dev/null
+++ b/scripts/zpios-test/large-thread-survey.sh
@@ -0,0 +1 @@
+1x256th-65536rc-4rs-1cs-4off.sh
\ No newline at end of file
diff --git a/scripts/zpios-test/large.sh b/scripts/zpios-test/large.sh
new file mode 120000
index 000000000..b8e22bf54
--- /dev/null
+++ b/scripts/zpios-test/large.sh
@@ -0,0 +1 @@
+256th-65536rc-4rs-1cs-4off.sh
\ No newline at end of file
diff --git a/scripts/zpios-test/medium.sh b/scripts/zpios-test/medium.sh
new file mode 120000
index 000000000..d81027b73
--- /dev/null
+++ b/scripts/zpios-test/medium.sh
@@ -0,0 +1 @@
+16th-8192rc-4rs-1cs-4off.sh
\ No newline at end of file
diff --git a/scripts/zpios-test/small.sh b/scripts/zpios-test/small.sh
new file mode 120000
index 000000000..cbf03b5ce
--- /dev/null
+++ b/scripts/zpios-test/small.sh
@@ -0,0 +1 @@
+4th-1024rc-4rs-1cs-4off.sh
\ No newline at end of file
diff --git a/scripts/zpios-test/tiny.sh b/scripts/zpios-test/tiny.sh
new file mode 120000
index 000000000..ba8b7cd0c
--- /dev/null
+++ b/scripts/zpios-test/tiny.sh
@@ -0,0 +1 @@
+1th-16rc-4rs-1cs-4off.sh
\ No newline at end of file
diff --git a/scripts/zpios.sh b/scripts/zpios.sh
new file mode 100755
index 000000000..e16a58a3b
--- /dev/null
+++ b/scripts/zpios.sh
@@ -0,0 +1,272 @@
+#!/bin/bash
+#
+# Wrapper script for easily running zpios based tests
+#
+
+basedir="$(dirname $0)"
+
+SCRIPT_COMMON=common.sh
+if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
+. "${basedir}/${SCRIPT_COMMON}"
+else
+echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
+fi
+
+PROG=zpios.sh
+DATE=`date +%Y%m%d-%H%M%S`
+if [ "${ZPIOS_MODULES}" ]; then
+	MODULES=(${ZPIOS_MODULES[*]})
+else
+	MODULES=(zpios)
+fi
+
+usage() {
+cat << EOF
+USAGE:
+$0 [hvp] <-c config> <-t test>
+
+DESCRIPTION:
+        Helper script for easy zpios benchmarking.
+
+OPTIONS:
+        -h      Show this message
+        -v      Verbose
+        -f      Force everything
+        -p      Enable profiling
+        -c      Zpool configuration
+        -t      Zpios test
+        -o      Additional zpios options
+        -l      Additional zpool options
+        -s      Additional zfs options
+
+EOF
+}
+
+print_header() {
+	echo --------------------- ZPIOS RESULTS ----------------------------
+	echo -n "Date: "; date
+	echo -n "Kernel: "; uname -r
+	dmesg | grep "Loaded Solaris Porting Layer" | tail -n1
+	dmesg | grep "Loaded ZFS Filesystem" | tail -n1
+	echo
+}
+
+print_spl_info() {
+	echo --------------------- SPL Tunings ------------------------------
+	${SYSCTL} -A | grep spl
+
+	if [ -d /sys/module/spl/parameters ]; then
+		grep [0-9] /sys/module/spl/parameters/*
+	else
+		grep [0-9] /sys/module/spl/*
+	fi
+
+	echo
+}
+
+print_zfs_info() {
+	echo --------------------- ZFS Tunings ------------------------------
+	${SYSCTL} -A | grep zfs
+
+	if [ -d /sys/module/zfs/parameters ]; then
+		grep [0-9] /sys/module/zfs/parameters/*
+	else
+		grep [0-9] /sys/module/zfs/*
+	fi
+
+	echo
+}
+
+print_stats() {
+	echo ---------------------- Statistics -------------------------------
+	${SYSCTL} -A | grep spl | grep stack_max
+
+	if [ -d /proc/spl/kstat/ ]; then
+		if [ -f /proc/spl/kstat/zfs/arcstats ]; then
+			echo "* ARC"
+			cat /proc/spl/kstat/zfs/arcstats
+			echo
+		fi
+
+		if [ -f /proc/spl/kstat/zfs/vdev_cache_stats ]; then
+			echo "* VDEV Cache"
+			cat /proc/spl/kstat/zfs/vdev_cache_stats
+			echo
+		fi
+	fi
+
+	if [ -f /proc/spl/kmem/slab ]; then
+		echo "* SPL SLAB"
+		cat /proc/spl/kmem/slab
+		echo
+	fi
+
+	echo
+}
+
+check_test() {
+
+	if [ ! -f ${ZPIOS_TEST} ]; then
+		local NAME=`basename ${ZPIOS_TEST} .sh`
+		ERROR="Unknown test '${NAME}', available tests are:\n"
+
+		for TST in `ls ${ZPIOSDIR}/ | grep ".sh"`; do
+			local NAME=`basename ${TST} .sh`
+			ERROR="${ERROR}${NAME}\n"
+		done
+
+		return 1
+	fi
+
+	return 0
+}
+
+zpios_profile_config() {
+cat > ${PROFILE_DIR}/zpios-config.sh << EOF
+#
+# Zpios Profiling Configuration
+#
+
+PROFILE_DIR=/tmp/zpios/${ZPOOL_CONFIG}+${ZPIOS_TEST_ARG}+${DATE}
+PROFILE_PRE=${ZPIOSPROFILEDIR}/zpios-profile-pre.sh
+PROFILE_POST=${ZPIOSPROFILEDIR}/zpios-profile-post.sh
+PROFILE_USER=${ZPIOSPROFILEDIR}/zpios-profile.sh
+PROFILE_PIDS=${ZPIOSPROFILEDIR}/zpios-profile-pids.sh
+PROFILE_DISK=${ZPIOSPROFILEDIR}/zpios-profile-disk.sh
+PROFILE_ARC_PROC=/proc/spl/kstat/zfs/arcstats
+PROFILE_VDEV_CACHE_PROC=/proc/spl/kstat/zfs/vdev_cache_stats
+
+OPROFILE_KERNEL="/boot/vmlinux-`uname -r`"
+OPROFILE_KERNEL_DIR="/lib/modules/`uname -r`/kernel/"
+OPROFILE_SPL_DIR=${SPLBUILD}/module/
+OPROFILE_ZFS_DIR=${MODDIR}
+
+EOF
+}
+
+zpios_profile_start() {
+	PROFILE_DIR=/tmp/zpios/${ZPOOL_CONFIG}+${ZPIOS_TEST_ARG}+${DATE}
+
+	mkdir -p ${PROFILE_DIR}
+	zpios_profile_config
+	. ${PROFILE_DIR}/zpios-config.sh
+
+	ZPIOS_OPTIONS="${ZPIOS_OPTIONS} --log=${PROFILE_DIR}"
+	ZPIOS_OPTIONS="${ZPIOS_OPTIONS} --prerun=${PROFILE_PRE}"
+	ZPIOS_OPTIONS="${ZPIOS_OPTIONS} --postrun=${PROFILE_POST}"
+
+	/usr/bin/opcontrol --init
+	/usr/bin/opcontrol --setup --vmlinux=${OPROFILE_KERNEL}
+}
+
+zpios_profile_stop() {
+	/usr/bin/opcontrol --shutdown
+	/usr/bin/opcontrol --deinit
+}
+
+PROFILE=
+ZPOOL_CONFIG=zpool-config.sh
+ZPIOS_TEST=zpios-test.sh
+ZPOOL_NAME=zpios
+ZPIOS_OPTIONS=
+ZPOOL_OPTIONS=""
+ZFS_OPTIONS=""
+
+while getopts 'hvfpc:t:o:l:s:' OPTION; do
+	case $OPTION in
+	h)
+		usage
+		exit 1
+		;;
+	v)
+		VERBOSE=1
+		VERBOSE_FLAG="-v"
+		;;
+	f)
+		FORCE=1
+		FORCE_FLAG="-f"
+		;;
+	p)
+		PROFILE=1
+		;;
+	c)
+		ZPOOL_CONFIG=${OPTARG}
+		;;
+	t)
+		ZPIOS_TEST_ARG=${OPTARG}
+		ZPIOS_TEST=${ZPIOSDIR}/${OPTARG}.sh
+		;;
+	o)
+		ZPIOS_OPTIONS=${OPTARG}
+		;;
+	l)	# Passed through to zpool-create.sh 
+		ZPOOL_OPTIONS=${OPTARG}
+		;;
+	s)	# Passed through to zpool-create.sh
+		ZFS_OPTIONS=${OPTARG}
+		;;
+	?)
+		usage
+		exit
+		;;
+	esac
+done
+
+if [ $(id -u) != 0 ]; then
+        die "Must run as root"
+fi
+
+# Validate and source your test config
+check_test || die "${ERROR}"
+. ${ZPIOS_TEST}
+
+# Pull in the zpios test module is not loaded.  If this fails it is
+# likely because the full module stack was not yet loaded with zfs.sh
+if check_modules; then
+	if ! load_modules; then
+		die "Run 'zfs.sh' to ensure the full module stack is loaded"
+	fi
+fi
+
+# Wait for device creation
+while [ ! -c /dev/zpios ]; do
+	sleep 1
+done
+
+if [ ${VERBOSE} ]; then
+	print_header
+	print_spl_info
+	print_zfs_info
+fi
+
+# Create the zpool configuration
+${ZPOOL_CREATE_SH} ${VERBOSE_FLAG} ${FORCE_FLAG} \
+	-p ${ZPOOL_NAME} -c ${ZPOOL_CONFIG} \
+	-l "${ZPOOL_OPTIONS}" -s "${ZFS_OPTIONS}" || exit 1
+
+if [ ${PROFILE} ]; then
+	zpios_profile_start
+fi
+
+zpios_start
+zpios_stop
+
+if [ ${PROFILE} ]; then
+	zpios_profile_stop
+fi
+
+if [ ${VERBOSE} ]; then
+	print_stats
+fi
+
+# Destroy the zpool configuration
+${ZPOOL_CREATE_SH} ${VERBOSE_FLAG} ${FORCE_FLAG} \
+	-p ${ZPOOL_NAME} -c ${ZPOOL_CONFIG} -d || exit 1
+
+# Unload the test module stack and wait for device removal
+unload_modules
+while [ -c /dev/zpios ]; do
+	sleep 1
+done
+
+exit 0