Add linux zpios support

Linux kernel implementation of PIOS test app.

Signed-off-by: Brian Behlendorf <[email protected]>

5 files changed, 799 insertions, 0 deletions

diff --git a/scripts/zpios-profile/zpios-profile-disk.sh b/scripts/zpios-profile/zpios-profile-disk.sh
new file mode 100755
index 000000000..b56ee1ee4
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-disk.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# /proc/diskinfo <after skipping major/minor>
+# Field 1 -- device name
+# Field 2 -- # of reads issued
+# Field 3 -- # of reads merged
+# Field 4 -- # of sectors read
+# Field 5 -- # of milliseconds spent reading
+# Field 6 -- # of writes completed
+# Field 7 -- # of writes merged
+# Field 8 -- # of sectors written
+# Field 9 -- # of milliseconds spent writing
+# Field 10 -- # of I/Os currently in progress
+# Field 11 -- # of milliseconds spent doing I/Os
+# Field 12 -- weighted # of milliseconds spent doing I/Os
+
+PROG=zpios-profile-disk.sh
+
+RUN_PIDS=${0}
+RUN_LOG_DIR=${1}
+RUN_ID=${2}
+
+create_table() {
+	local FIELD=$1
+	local ROW_M=()
+	local ROW_N=()
+	local HEADER=1
+	local STEP=1
+
+	for DISK_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/disk-[0-9]*`; do
+		ROW_M=( ${ROW_N[@]} )
+		ROW_N=( `cat ${DISK_FILE} | grep sd | cut -c11- | cut -f${FIELD} -d' ' | tr "\n" "\t"` )
+
+		if [ $HEADER -eq 1 ]; then
+			echo -n "step, "
+			cat ${DISK_FILE} | grep sd | cut -c11- | cut -f1 -d' ' | tr "\n" ", "
+	                echo "total"
+			HEADER=0
+		fi
+
+		if [ ${#ROW_M[@]} -eq 0 ]; then
+			continue
+		fi
+
+		if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+			echo "Badly formatted profile data in ${DISK_FILE}"
+			break
+		fi
+
+		TOTAL=0
+		echo -n "${STEP}, "
+		for (( i=0; i<${#ROW_N[@]}; i++ )); do
+			DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+			let TOTAL=${TOTAL}+${DELTA}
+			echo -n "${DELTA}, "
+		done
+		echo "${TOTAL}, "
+
+		let STEP=${STEP}+1
+	done
+}
+
+create_table_mbs() {
+	local FIELD=$1
+	local TIME=$2
+	local ROW_M=()
+	local ROW_N=()
+	local HEADER=1
+	local STEP=1
+
+	for DISK_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/disk-[0-9]*`; do
+		ROW_M=( ${ROW_N[@]} )
+		ROW_N=( `cat ${DISK_FILE} | grep sd | cut -c11- | cut -f${FIELD} -d' ' | tr "\n" "\t"` )
+
+		if [ $HEADER -eq 1 ]; then
+			echo -n "step, "
+			cat ${DISK_FILE} | grep sd | cut -c11- | cut -f1 -d' ' | tr "\n" ", "
+	                echo "total"
+			HEADER=0
+		fi
+
+		if [ ${#ROW_M[@]} -eq 0 ]; then
+			continue
+		fi
+
+		if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+			echo "Badly formatted profile data in ${DISK_FILE}"
+			break
+		fi
+
+		TOTAL=0
+		echo -n "${STEP}, "
+		for (( i=0; i<${#ROW_N[@]}; i++ )); do
+			DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+			MBS=`echo "scale=2; ((${DELTA}*512)/${TIME})/(1024*1024)" | bc`
+			TOTAL=`echo "scale=2; ${TOTAL}+${MBS}" | bc`
+			echo -n "${MBS}, "
+		done
+		echo "${TOTAL}, "
+
+		let STEP=${STEP}+1
+	done
+}
+
+echo
+echo "Reads issued per device"
+create_table 2
+echo
+echo "Reads merged per device"
+create_table 3
+echo
+echo "Sectors read per device"
+create_table 4
+echo "MB/s per device"
+create_table_mbs 4 3
+
+echo
+echo "Writes issued per device"
+create_table 6
+echo
+echo "Writes merged per device"
+create_table 7
+echo
+echo "Sectors written per device"
+create_table 8
+echo "MB/s per device"
+create_table_mbs 8 3
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-pids.sh b/scripts/zpios-profile/zpios-profile-pids.sh
new file mode 100755
index 000000000..3514b38e2
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-pids.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+
+PROG=zpios-profile-pids.sh
+
+RUN_PIDS=${0}
+RUN_LOG_DIR=${1}
+RUN_ID=${2}
+
+ROW_M=()
+ROW_N=()
+ROW_N_SCHED=()
+ROW_N_WAIT=()
+
+HEADER=1
+STEP=1
+
+for PID_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/pids-[0-9]*`; do
+	ROW_M=( ${ROW_N[@]} )
+	ROW_N=( 0 0 0 0  0 0 0 0  0 0 0 0  0 0 0 0  0 0 0 )
+	ROW_N_SCHED=( `cat ${PID_FILE} | cut -f15 -d' ' | tr "\n" "\t"` )
+	ROW_N_WAIT=(  `cat ${PID_FILE} | cut -f17 -d' ' | tr "\n" "\t"` )
+	ROW_N_NAMES=( `cat ${PID_FILE} | cut -f2  -d' ' | cut -f2 -d'(' | 
+                       cut -f1 -d')'   | cut -f1  -d'/' | tr "\n" "\t"` )
+
+	for (( i=0; i<${#ROW_N_SCHED[@]}; i++ )); do
+		SUM=`echo "${ROW_N_WAIT[${i}]}+${ROW_N_SCHED[${i}]}" | bc`
+
+		case ${ROW_N_NAMES[${i}]} in
+			zio_taskq)	IDX=0;;
+			zio_req_nul)	IDX=1;;
+			zio_irq_nul)	IDX=2;;
+			zio_req_rd)	IDX=3;;
+			zio_irq_rd)	IDX=4;;
+			zio_req_wr)	IDX=5;;
+			zio_irq_wr)	IDX=6;;
+			zio_req_fr)	IDX=7;;
+			zio_irq_fr)	IDX=8;;
+			zio_req_cm)	IDX=9;;
+			zio_irq_cm)	IDX=10;;
+			zio_req_ctl)	IDX=11;;
+			zio_irq_ctl)	IDX=12;;
+			txg_quiesce)	IDX=13;;
+			txg_sync)	IDX=14;;
+			txg_timelimit)	IDX=15;;
+			arc_reclaim)	IDX=16;;
+			l2arc_feed)	IDX=17;;
+			zpios_io)	IDX=18;;
+			*)		continue;;
+		esac
+
+		let ROW_N[${IDX}]=${ROW_N[${IDX}]}+${SUM}
+	done
+
+	if [ $HEADER -eq 1 ]; then
+		echo "step, zio_taskq, zio_req_nul, zio_irq_nul, "        \
+                     "zio_req_rd, zio_irq_rd, zio_req_wr, zio_irq_wr, "   \
+                     "zio_req_fr, zio_irq_fr, zio_req_cm, zio_irq_cm, "   \
+                     "zio_req_ctl, zio_irq_ctl, txg_quiesce, txg_sync, "  \
+                     "txg_timelimit, arc_reclaim, l2arc_feed, zpios_io, " \
+		     "idle"
+		HEADER=0
+	fi
+
+	if [ ${#ROW_M[@]} -eq 0 ]; then
+		continue
+	fi
+
+	if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+		echo "Badly formatted profile data in ${PID_FILE}"
+		break
+	fi
+
+	# Original values are in jiffies and we expect HZ to be 1000
+	# on most 2.6 systems thus we divide by 10 to get a percentage.
+	IDLE=1000
+        echo -n "${STEP}, "
+	for (( i=0; i<${#ROW_N[@]}; i++ )); do
+		DELTA=`echo "${ROW_N[${i}]}-${ROW_M[${i}]}" | bc`
+		DELTA_PERCENT=`echo "scale=1; ${DELTA}/10" | bc`
+		let IDLE=${IDLE}-${DELTA}
+		echo -n "${DELTA_PERCENT}, "
+	done
+	ILDE_PERCENT=`echo "scale=1; ${IDLE}/10" | bc`
+	echo "${ILDE_PERCENT}"
+
+	let STEP=${STEP}+1
+done
+
+exit
+
+echo
+echo "Percent of total system time per pid"
+for PID_FILE in `ls -r --sort=time --time=ctime ${RUN_LOG_DIR}/${RUN_ID}/pids-[0-9]*`; do
+	ROW_M=( ${ROW_N[@]} )
+	ROW_N_SCHED=( `cat ${PID_FILE} | cut -f15 -d' ' | tr "\n" "\t"` )
+	ROW_N_WAIT=( `cat ${PID_FILE} | cut -f17 -d' ' | tr "\n" "\t"` )
+
+	for (( i=0; i<${#ROW_N_SCHED[@]}; i++ )); do
+		ROW_N[${i}]=`echo "${ROW_N_WAIT[${i}]}+${ROW_N_SCHED[${i}]}" | bc`
+	done
+
+	if [ $HEADER -eq 1 ]; then
+		echo -n "step, "
+		cat ${PID_FILE} | cut -f2 -d' ' | tr "\n" ", "
+		echo
+		HEADER=0
+	fi
+
+	if [ ${#ROW_M[@]} -eq 0 ]; then
+		continue
+	fi
+
+	if [ ${#ROW_M[@]} -ne ${#ROW_N[@]} ]; then
+		echo "Badly formatted profile data in ${PID_FILE}"
+		break
+	fi
+
+	# Original values are in jiffies and we expect HZ to be 1000
+	# on most 2.6 systems thus we divide by 10 to get a percentage.
+        echo -n "${STEP}, "
+	for (( i=0; i<${#ROW_N[@]}; i++ )); do
+		DELTA=`echo "scale=1; (${ROW_N[${i}]}-${ROW_M[${i}]})/10" | bc`
+		echo -n "${DELTA}, "
+	done
+
+	echo
+	let STEP=${STEP}+1
+done
+
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-post.sh b/scripts/zpios-profile/zpios-profile-post.sh
new file mode 100755
index 000000000..3a454ba04
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-post.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+PROG=zpios-profile-post.sh
+
+RUN_POST=${0}
+RUN_PHASE=${1}
+RUN_DIR=${2}
+RUN_ID=${3}
+RUN_POOL=${4}
+RUN_CHUNK_SIZE=${5}
+RUN_REGION_SIZE=${6}
+RUN_THRD_COUNT=${7}
+RUN_REGION_COUNT=${8}
+RUN_OFFSET=${9}
+RUN_REGION_NOISE=${10}
+RUN_CHUNK_NOISE=${11}
+RUN_THRD_DELAY=${12}
+RUN_FLAGS=${13}
+RUN_RESULT=${14}
+
+# Summarize system time per process
+zpios_profile_post_pids() {
+	${PROFILE_PIDS} ${PROFILE_RUN_CR_PIDS_LOG} >${PROFILE_RUN_CR_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_WR_PIDS_LOG} >${PROFILE_RUN_WR_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_RD_PIDS_LOG} >${PROFILE_RUN_RD_PIDS_CSV}
+	${PROFILE_PIDS} ${PROFILE_RUN_RM_PIDS_LOG} >${PROFILE_RUN_RM_PIDS_CSV}
+}
+
+zpios_profile_post_disk() {
+	${PROFILE_DISK} ${PROFILE_RUN_CR_DISK_LOG} >${PROFILE_RUN_CR_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_WR_DISK_LOG} >${PROFILE_RUN_WR_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_RD_DISK_LOG} >${PROFILE_RUN_RD_DISK_CSV}
+	${PROFILE_DISK} ${PROFILE_RUN_RM_DISK_LOG} >${PROFILE_RUN_RM_DISK_CSV}
+}
+
+# Summarize per device performance
+
+# Stop a user defined profiling script which is gathering additional data
+zpios_profile_post_stop() {
+	local PROFILE_PID=$1
+
+	kill -s SIGHUP `cat ${PROFILE_PID}`
+
+
+	# Sleep waiting for profile script to exit
+	while [ -f ${PROFILE_PID} ]; do
+		sleep 0.01
+	done
+}
+
+zpios_profile_post_proc_stop() {
+	local PROC_DIR=$1
+
+	if [ -f ${PROFILE_ARC_PROC} ]; then
+		cat ${PROFILE_ARC_PROC} >${PROC_DIR}/arcstats.txt
+	fi
+
+	if [ -f ${PROFILE_VDEV_CACHE_PROC} ]; then
+		cat ${PROFILE_VDEV_CACHE_PROC} >${PROC_DIR}/vdev_cache_stats.txt
+	fi
+}
+
+zpios_profile_post_oprofile_stop() {
+	local OPROFILE_LOG=$1
+	local OPROFILE_ARGS="-a -g -l -p ${OPROFILE_KERNEL_DIR},${OPROFILE_SPL_DIR},${OPROFILE_ZFS_DIR}"
+
+	/usr/bin/opcontrol --stop >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opcontrol --dump >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opreport ${OPROFILE_ARGS} >${OPROFILE_LOG} 2>&1
+	/usr/bin/oparchive
+}
+
+zpios_profile_post_create() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_CR_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_CR_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_CR_PID}
+}
+
+zpios_profile_post_write() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_WR_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_WR_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_WR_PID}
+}
+
+zpios_profile_post_read() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_CR_RD_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_RD_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_RD_PID}
+}
+
+zpios_profile_post_remove() {
+	zpios_profile_post_oprofile_stop ${PROFILE_RUN_RM_OPROFILE_LOG}
+	zpios_profile_post_proc_stop ${PROFILE_RUN_RM_DIR}
+	zpios_profile_post_stop ${PROFILE_RUN_RM_PID}
+}
+
+# Source global zpios test configuration
+if [ -f ${RUN_DIR}/zpios-config.sh ]; then
+	. ${RUN_DIR}/zpios-config.sh
+fi
+
+# Source global per-run test configuration
+if [ -f ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh ]; then
+	. ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh
+fi
+
+case "${RUN_PHASE}" in
+	post-run)
+		zpios_profile_post_pids
+		zpios_profile_post_disk
+		;;
+	post-create)
+		zpios_profile_post_create
+		;;
+	post-write)
+		zpios_profile_post_write
+		;;
+	post-read)
+		zpios_profile_post_read
+		;;
+	post-remove)
+		zpios_profile_post_remove
+		;;
+	*)
+		echo "Usage: ${PROG} {post-run|post-create|post-write|post-read|post-remove}"
+		exit 1
+esac
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile-pre.sh b/scripts/zpios-profile/zpios-profile-pre.sh
new file mode 100755
index 000000000..a2a885798
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile-pre.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+
+PROG=zpios-profile-pre.sh
+
+PROFILE_RDY=0
+trap "PROFILE_RDY=1" SIGHUP
+
+RUN_PRE=${0}
+RUN_PHASE=${1}
+RUN_DIR=${2}
+RUN_ID=${3}
+RUN_POOL=${4}
+RUN_CHUNK_SIZE=${5}
+RUN_REGION_SIZE=${6}
+RUN_THRD_COUNT=${7}
+RUN_REGION_COUNT=${8}
+RUN_OFFSET=${9}
+RUN_REGION_NOISE=${10}
+RUN_CHUNK_NOISE=${11}
+RUN_THRD_DELAY=${12}
+RUN_FLAGS=${13}
+RUN_RESULT=${14}
+
+zpios_profile_pre_run_cfg() {
+cat > ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh << EOF
+#
+# Zpios Profiling Configuration for Run ${RUN_ID}
+#
+
+PROFILE_RUN_DIR=${RUN_DIR}/${RUN_ID}
+
+PROFILE_RUN_CR_DIR=${RUN_DIR}/${RUN_ID}/create
+PROFILE_RUN_CR_PID=${RUN_DIR}/${RUN_ID}/create/profile.pid
+PROFILE_RUN_CR_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/create/oprofile.txt
+PROFILE_RUN_CR_PIDS_LOG=${RUN_DIR}/${RUN_ID}/create/pids.txt
+PROFILE_RUN_CR_PIDS_CSV=${RUN_DIR}/${RUN_ID}/create/pids.csv
+PROFILE_RUN_CR_DISK_LOG=${RUN_DIR}/${RUN_ID}/create/disk.txt
+PROFILE_RUN_CR_DISK_CSV=${RUN_DIR}/${RUN_ID}/create/disk.csv
+
+PROFILE_RUN_WR_DIR=${RUN_DIR}/${RUN_ID}/write
+PROFILE_RUN_WR_PID=${RUN_DIR}/${RUN_ID}/write/profile.pid
+PROFILE_RUN_WR_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/write/oprofile.txt
+PROFILE_RUN_WR_PIDS_LOG=${RUN_DIR}/${RUN_ID}/write/pids.txt
+PROFILE_RUN_WR_PIDS_CSV=${RUN_DIR}/${RUN_ID}/write/pids.csv
+PROFILE_RUN_WR_DISK_LOG=${RUN_DIR}/${RUN_ID}/write/disk.txt
+PROFILE_RUN_WR_DISK_CSV=${RUN_DIR}/${RUN_ID}/write/disk.csv
+
+PROFILE_RUN_RD_DIR=${RUN_DIR}/${RUN_ID}/read
+PROFILE_RUN_RD_PID=${RUN_DIR}/${RUN_ID}/read/profile.pid
+PROFILE_RUN_RD_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/read/oprofile.txt
+PROFILE_RUN_RD_PIDS_LOG=${RUN_DIR}/${RUN_ID}/read/pids.txt
+PROFILE_RUN_RD_PIDS_CSV=${RUN_DIR}/${RUN_ID}/read/pids.csv
+PROFILE_RUN_RD_DISK_LOG=${RUN_DIR}/${RUN_ID}/read/disk.txt
+PROFILE_RUN_RD_DISK_CSV=${RUN_DIR}/${RUN_ID}/read/disk.csv
+
+PROFILE_RUN_RM_DIR=${RUN_DIR}/${RUN_ID}/remove
+PROFILE_RUN_RM_PID=${RUN_DIR}/${RUN_ID}/remove/profile.pid
+PROFILE_RUN_RM_OPROFILE_LOG=${RUN_DIR}/${RUN_ID}/remove/oprofile.txt
+PROFILE_RUN_RM_PIDS_LOG=${RUN_DIR}/${RUN_ID}/remove/pids.txt
+PROFILE_RUN_RM_PIDS_CSV=${RUN_DIR}/${RUN_ID}/remove/pids.csv
+PROFILE_RUN_RM_DISK_LOG=${RUN_DIR}/${RUN_ID}/remove/disk.txt
+PROFILE_RUN_RM_DISK_CSV=${RUN_DIR}/${RUN_ID}/remove/disk.csv
+
+# PROFILE_PIDS_LOG=${RUN_DIR}/${RUN_ID}/pids-summary.csv
+# PROFILE_DISK_LOG=${RUN_DIR}/${RUN_ID}/disk-summary.csv
+EOF
+}
+
+zpios_profile_pre_run_args() {
+cat > ${RUN_DIR}/${RUN_ID}/zpios-args.txt << EOF
+#
+# Zpios Arguments for Run ${RUN_ID}
+#
+
+DIR=${RUN_DIR}
+ID=${RUN_ID}
+POOL=${RUN_POOL}
+CHUNK_SIZE=${RUN_CHUNK_SIZE}
+REGION_SIZE=${RUN_REGION_SIZE}
+THRD_COUNT=${RUN_THRD_COUNT}
+REGION_COUNT=${RUN_REGION_COUNT}
+OFFSET=${RUN_OFFSET}
+REGION_NOISE=${RUN_REGION_NOISE}
+CHUNK_NOISE=${RUN_CHUNK_NOISE}
+THRD_DELAY=${RUN_THRD_DELAY}
+FLAGS=${RUN_FLAGS}
+RESULT=${RUN_RESULT}
+EOF
+}
+
+# Spawn a user defined profiling script to gather additional data
+zpios_profile_pre_start() {
+	local PROFILE_PID=$1
+
+	${PROFILE_USER} ${RUN_PHASE} ${RUN_DIR} ${RUN_ID} &
+	echo "$!" >${PROFILE_PID}
+
+	# Sleep waiting for profile script to be ready, it will
+	# signal us via SIGHUP when it is ready to start profiling.
+	while [ ${PROFILE_RDY} -eq 0 ]; do
+		sleep 0.01
+	done
+}
+
+zpios_profile_post_proc_start() { 
+
+        if [ -f ${PROFILE_ARC_PROC} ]; then
+                echo 0 >${PROFILE_ARC_PROC}
+        fi
+
+        if [ -f ${PROFILE_VDEV_CACHE_PROC} ]; then
+                echo 0 >${PROFILE_VDEV_CACHE_PROC}
+        fi
+}
+
+zpios_profile_pre_oprofile_start() {
+	local OPROFILE_LOG=$1
+
+	/usr/bin/opcontrol --reset >>${OPROFILE_LOG} 2>&1
+	/usr/bin/opcontrol --start >>${OPROFILE_LOG} 2>&1
+}
+
+zpios_profile_pre_create() {
+	mkdir ${PROFILE_RUN_CR_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_CR_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_CR_OPROFILE_LOG}
+}
+
+zpios_profile_pre_write() {
+	mkdir ${PROFILE_RUN_WR_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_WR_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_WR_OPROFILE_LOG}
+}
+
+zpios_profile_pre_read() {
+	mkdir ${PROFILE_RUN_RD_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_RD_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_CR_RD_LOG}
+}
+
+zpios_profile_pre_remove() {
+	mkdir ${PROFILE_RUN_RM_DIR}
+	zpios_profile_pre_start ${PROFILE_RUN_RM_PID}
+	zpios_profile_post_proc_start
+	zpios_profile_pre_oprofile_start ${PROFILE_RUN_RM_OPROFILE_LOG}
+}
+
+# Source global zpios test configuration
+if [ -f ${RUN_DIR}/zpios-config.sh ]; then
+	. ${RUN_DIR}/zpios-config.sh
+fi
+
+# Source global per-run test configuration
+if [ -f ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh ]; then
+	. ${RUN_DIR}/${RUN_ID}/zpios-config-run.sh
+fi
+
+case "${RUN_PHASE}" in
+	pre-run)
+		mkdir -p ${RUN_DIR}/${RUN_ID}/
+		zpios_profile_pre_run_cfg
+		zpios_profile_pre_run_args
+		;;
+	pre-create)
+		zpios_profile_pre_create
+		;;
+	pre-write)
+		zpios_profile_pre_write
+		;;
+	pre-read)
+		zpios_profile_pre_read
+		;;
+	pre-remove)
+		zpios_profile_pre_remove
+		;;
+	*)
+		echo "Usage: ${PROG} {pre-run|pre-create|pre-write|pre-read|pre-remove}"
+		exit 1
+esac
+
+exit 0
diff --git a/scripts/zpios-profile/zpios-profile.sh b/scripts/zpios-profile/zpios-profile.sh
new file mode 100755
index 000000000..f4f0ee97f
--- /dev/null
+++ b/scripts/zpios-profile/zpios-profile.sh
@@ -0,0 +1,226 @@
+#!/bin/bash
+
+
+PROG=zpios-profile.sh
+
+trap "RUN_DONE=1" SIGHUP
+
+RUN_PHASE=${1}
+RUN_LOG_DIR=${2}
+RUN_ID=${3}
+RUN_DONE=0
+
+POLL_INTERVAL=2.99
+
+# Log these pids, the exact pid numbers will vary from system to system
+# so I harvest pid for all the following type of processes from /proc/<pid>/
+#
+# zio_taskq/#
+# spa_zio_issue/#
+# spa_zio_intr/#
+# txg_quiesce_thr
+# txg_sync_thread
+# txg_timelimit_t
+# arc_reclaim_thr
+# l2arc_feed_thre
+# zpios_io/#
+
+ZIO_TASKQ_PIDS=()
+ZIO_REQ_NUL_PIDS=() 
+ZIO_IRQ_NUL_PIDS=() 
+ZIO_REQ_RD_PIDS=() 
+ZIO_IRQ_RD_PIDS=() 
+ZIO_REQ_WR_PIDS=() 
+ZIO_IRQ_WR_PIDS=()
+ZIO_REQ_FR_PIDS=() 
+ZIO_IRQ_FR_PIDS=()
+ZIO_REQ_CM_PIDS=() 
+ZIO_IRQ_CM_PIDS=()
+ZIO_REQ_CTL_PIDS=() 
+ZIO_IRQ_CTL_PIDS=()
+
+TXG_QUIESCE_PIDS=()
+TXG_SYNC_PIDS=() 
+TXG_TIMELIMIT_PIDS=()
+
+ARC_RECLAIM_PIDS=()
+L2ARC_FEED_PIDS=()
+
+ZPIOS_IO_PIDS=()
+
+show_pids() {
+	echo "* zio_taskq:     { ${ZIO_TASKQ_PIDS[@]} } = ${#ZIO_TASKQ_PIDS[@]}"
+	echo "* zio_req_nul:   { ${ZIO_REQ_NUL_PIDS[@]} } = ${#ZIO_REQ_NUL_PIDS[@]}"
+	echo "* zio_irq_nul:   { ${ZIO_IRQ_NUL_PIDS[@]} } = ${#ZIO_IRQ_NUL_PIDS[@]}"
+	echo "* zio_req_rd:    { ${ZIO_REQ_RD_PIDS[@]} } = ${#ZIO_REQ_RD_PIDS[@]}"
+	echo "* zio_irq_rd:    { ${ZIO_IRQ_RD_PIDS[@]} } = ${#ZIO_IRQ_RD_PIDS[@]}"
+	echo "* zio_req_wr:    { ${ZIO_REQ_WR_PIDS[@]} } = ${#ZIO_REQ_WR_PIDS[@]}"
+	echo "* zio_irq_wr:    { ${ZIO_IRQ_WR_PIDS[@]} } = ${#ZIO_IRQ_WR_PIDS[@]}"
+	echo "* zio_req_fr:    { ${ZIO_REQ_FR_PIDS[@]} } = ${#ZIO_REQ_FR_PIDS[@]}"
+	echo "* zio_irq_fr:    { ${ZIO_IRQ_FR_PIDS[@]} } = ${#ZIO_IRQ_FR_PIDS[@]}"
+	echo "* zio_req_cm:    { ${ZIO_REQ_CM_PIDS[@]} } = ${#ZIO_REQ_CM_PIDS[@]}"
+	echo "* zio_irq_cm:    { ${ZIO_IRQ_CM_PIDS[@]} } = ${#ZIO_IRQ_CM_PIDS[@]}"
+	echo "* zio_req_ctl:   { ${ZIO_REQ_CTL_PIDS[@]} } = ${#ZIO_REQ_CTL_PIDS[@]}"
+	echo "* zio_irq_ctl:   { ${ZIO_IRQ_CTL_PIDS[@]} } = ${#ZIO_IRQ_CTL_PIDS[@]}"
+	echo "* txg_quiesce:   { ${TXG_QUIESCE_PIDS[@]} } = ${#TXG_QUIESCE_PIDS[@]}"
+	echo "* txg_sync:      { ${TXG_SYNC_PIDS[@]} } = ${#TXG_SYNC_PIDS[@]}"
+	echo "* txg_timelimit: { ${TXG_TIMELIMIT_PIDS[@]} } = ${#TXG_TIMELIMIT_PIDS[@]}"
+	echo "* arc_reclaim:   { ${ARC_RECLAIM_PIDS[@]} } = ${#ARC_RECLAIM_PIDS[@]}"
+	echo "* l2arc_feed:    { ${L2ARC_FEED_PIDS[@]} } = ${#L2ARC_FEED_PIDS[@]}"
+	echo "* zpios_io:      { ${ZPIOS_IO_PIDS[@]} } = ${#ZPIOS_IO_PIDS[@]}"
+}
+
+check_pid() {
+	local PID=$1
+	local NAME=$2
+	local TYPE=$3
+	local PIDS=( "$4" )
+        local NAME_STRING=`echo ${NAME} | cut -f1 -d'/'`
+        local NAME_NUMBER=`echo ${NAME} | cut -f2 -d'/'`
+
+	if [ "${NAME_STRING}" == "${TYPE}" ]; then
+		if [ -n "${NAME_NUMBER}" ]; then
+			PIDS[${NAME_NUMBER}]=${PID}
+		else
+			PIDS[${#PIDS[@]}]=${PID}
+
+		fi
+	fi
+
+	echo "${PIDS[@]}"
+}
+
+# NOTE: This whole process is crazy slow but it will do for now
+aquire_pids() {
+	echo "--- Aquiring ZFS pids ---"
+
+	for PID in `ls /proc/ | grep [0-9] | sort -n -u`; do
+		if [ ! -e /proc/${PID}/status ]; then
+			continue
+		fi
+
+		NAME=`cat /proc/${PID}/status  | head -n1 | cut -f2`
+
+		ZIO_TASKQ_PIDS=( `check_pid ${PID} ${NAME} "zio_taskq" \
+		                 "$(echo "${ZIO_TASKQ_PIDS[@]}")"` )
+
+		ZIO_REQ_NUL_PIDS=( `check_pid ${PID} ${NAME} "zio_req_nul" \
+		                   "$(echo "${ZIO_REQ_NUL_PIDS[@]}")"` )
+
+		ZIO_IRQ_NUL_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_nul" \
+		                   "$(echo "${ZIO_IRQ_NUL_PIDS[@]}")"` )
+
+		ZIO_REQ_RD_PIDS=( `check_pid ${PID} ${NAME} "zio_req_rd" \
+		                   "$(echo "${ZIO_REQ_RD_PIDS[@]}")"` )
+
+		ZIO_IRQ_RD_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_rd" \
+		                   "$(echo "${ZIO_IRQ_RD_PIDS[@]}")"` )
+
+		ZIO_REQ_WR_PIDS=( `check_pid ${PID} ${NAME} "zio_req_wr" \
+		                   "$(echo "${ZIO_REQ_WR_PIDS[@]}")"` )
+
+		ZIO_IRQ_WR_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_wr" \
+		                   "$(echo "${ZIO_IRQ_WR_PIDS[@]}")"` )
+
+		ZIO_REQ_FR_PIDS=( `check_pid ${PID} ${NAME} "zio_req_fr" \
+		                   "$(echo "${ZIO_REQ_FR_PIDS[@]}")"` )
+
+		ZIO_IRQ_FR_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_fr" \
+		                   "$(echo "${ZIO_IRQ_FR_PIDS[@]}")"` )
+
+		ZIO_REQ_CM_PIDS=( `check_pid ${PID} ${NAME} "zio_req_cm" \
+		                   "$(echo "${ZIO_REQ_CM_PIDS[@]}")"` )
+
+		ZIO_IRQ_CM_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_cm" \
+		                   "$(echo "${ZIO_IRQ_CM_PIDS[@]}")"` )
+
+		ZIO_REQ_CTL_PIDS=( `check_pid ${PID} ${NAME} "zio_req_ctl" \
+		                   "$(echo "${ZIO_REQ_CTL_PIDS[@]}")"` )
+
+		ZIO_IRQ_CTL_PIDS=( `check_pid ${PID} ${NAME} "zio_irq_ctl" \
+		                   "$(echo "${ZIO_IRQ_CTL_PIDS[@]}")"` )
+
+		TXG_QUIESCE_PIDS=( `check_pid ${PID} ${NAME} "txg_quiesce" \
+		                   "$(echo "${TXG_QUIESCE_PIDS[@]}")"` )
+
+		TXG_SYNC_PIDS=( `check_pid ${PID} ${NAME} "txg_sync" \
+		                "$(echo "${TXG_SYNC_PIDS[@]}")"` )
+
+		TXG_TIMELIMIT_PIDS=( `check_pid ${PID} ${NAME} "txg_timelimit" \
+		                     "$(echo "${TXG_TIMELIMIT_PIDS[@]}")"` )
+
+		ARC_RECLAIM_PIDS=( `check_pid ${PID} ${NAME} "arc_reclaim" \
+                                     "$(echo "${ARC_RECLAIM_PIDS[@]}")"` )
+
+		L2ARC_FEED_PIDS=( `check_pid ${PID} ${NAME} "l2arc_feed" \
+                                  "$(echo "${L2ARC_FEED_PIDS[@]}")"` )
+	done
+
+	# Wait for zpios_io threads to start
+	kill -s SIGHUP ${PPID}
+	echo "* Waiting for zpios_io threads to start"
+	while [ ${RUN_DONE} -eq 0 ]; do
+		ZPIOS_IO_PIDS=( `ps ax | grep zpios_io | grep -v grep | \
+                                 sed 's/^ *//g' | cut -f1 -d' '` )
+		if [ ${#ZPIOS_IO_PIDS[@]} -gt 0 ]; then
+			break;
+		fi
+		sleep 0.1
+	done
+
+	echo "`show_pids`" >${RUN_LOG_DIR}/${RUN_ID}/pids.txt
+}
+
+log_pids() {
+	echo "--- Logging ZFS profile to ${RUN_LOG_DIR}/${RUN_ID}/ ---"
+	ALL_PIDS=( ${ZIO_TASKQ_PIDS[@]}     \
+                   ${ZIO_REQ_NUL_PIDS[@]}   \
+                   ${ZIO_IRQ_NUL_PIDS[@]}   \
+                   ${ZIO_REQ_RD_PID[@]}     \
+                   ${ZIO_IRQ_RD_PIDS[@]}    \
+                   ${ZIO_REQ_WR_PIDS[@]}    \
+                   ${ZIO_IRQ_WR_PIDS[@]}    \
+                   ${ZIO_REQ_FR_PIDS[@]}    \ 
+                   ${ZIO_IRQ_FR_PIDS[@]}    \
+                   ${ZIO_REQ_CM_PIDS[@]}    \ 
+                   ${ZIO_IRQ_CM_PIDS[@]}    \
+                   ${ZIO_REQ_CTL_PIDS[@]}   \
+                   ${ZIO_IRQ_CTL_PIDS[@]}   \
+                   ${TXG_QUIESCE_PIDS[@]}   \
+                   ${TXG_SYNC_PIDS[@]}      \
+                   ${TXG_TIMELIMIT_PIDS[@]} \
+                   ${ARC_RECLAIM_PIDS[@]}   \
+                   ${L2ARC_FEED_PIDS[@]}    \
+                   ${ZPIOS_IO_PIDS[@]} )
+
+	while [ ${RUN_DONE} -eq 0 ]; do
+		NOW=`date +%s.%N`
+		LOG_PIDS="${RUN_LOG_DIR}/${RUN_ID}/pids-${NOW}"
+		LOG_DISK="${RUN_LOG_DIR}/${RUN_ID}/disk-${NOW}"
+
+		for PID in "${ALL_PIDS[@]}"; do
+			if [ -z ${PID} ]; then
+				continue;
+			fi
+
+	        	if [ -e /proc/${PID}/stat ]; then
+	        		cat /proc/${PID}/stat | head -n1 >>${LOG_PIDS}
+			else
+	                	echo "<${PID} exited>" >>${LOG_PIDS}
+	        	fi
+		done
+
+		cat /proc/diskstats >${LOG_DISK}
+
+		NOW2=`date +%s.%N`
+		DELTA=`echo "${POLL_INTERVAL}-(${NOW2}-${NOW})" | bc`
+		sleep ${DELTA}
+	done
+}
+
+aquire_pids
+log_pids
+
+# rm ${PROFILE_PID}
+
+exit 0