Python 2 and 3 compatibility

With Python 2 (slowly) approaching EOL and its removal from distribitions
already being planned (Fedora), the existing Python 2 code needs to be
transitioned to Python 3.  This patch stack updates the Python code to
be compatible with Python 2.7, 3.4, 3.5, 3.6, and 3.7.

Reviewed-by: John Ramsden <[email protected]>
Reviewed-by: Neal Gompa <[email protected]>
Reviewed-by: loli10K <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: John Wren Kennedy <[email protected]>
Reviewed-by: Antonio Russo <[email protected]>
Closes #8096

13 files changed, 78 insertions, 102 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 5efcb6102..8ced03e93 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -146,10 +146,10 @@ export ZFS_FILES='zdb
     zpool
     ztest
     raidz_test
-    arc_summary.py
-    arc_summary3.py
-    arcstat.py
-    dbufstat.py
+    arc_summary
+    arc_summary3
+    arcstat
+    dbufstat
     zed
     zgenhostid
     zstreamdump'
diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh
index 5ceff962d..7ec9eaf4c 100755
--- a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh
@@ -37,7 +37,7 @@
 # 2. Store output from dbufs kstat
 # 3. Store output from dbufstats kstat
 # 4. Compare stats presented in dbufstats with stat generated using
-#    dbufstat.py and the dbufs kstat output
+#    dbufstat and the dbufs kstat output
 #
 
 DBUFSTATS_FILE=$(mktemp $TEST_BASE_DIR/dbufstats.out.XXXXXX)
@@ -56,7 +56,7 @@ function testdbufstat # stat_name dbufstat_filter
         [[ -n "$2" ]] && filter="-F $2"
 
 	from_dbufstat=$(grep -w "$name" "$DBUFSTATS_FILE" | awk '{ print $3 }')
-	from_dbufs=$(dbufstat.py -bxn -i "$DBUFS_FILE" "$filter" | wc -l)
+	from_dbufs=$(dbufstat -bxn -i "$DBUFS_FILE" "$filter" | wc -l)
 
 	within_tolerance $from_dbufstat $from_dbufs 9 \
 	    || log_fail "Stat $name exceeded tolerance"
diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh
index e256bfabe..dc30b6606 100755
--- a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh
@@ -62,18 +62,18 @@ objid=$(stat --format="%i" "$TESTDIR/file")
 log_note "Object ID for $TESTDIR/file is $objid"
 
 log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE"
-dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
-mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
-mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
+dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
+mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
+mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
 log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu"
 verify_ne "0" "$mru" "mru count"
 verify_eq "0" "$mfu" "mfu count"
 
 log_must eval "cat $TESTDIR/file > /dev/null"
 log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE"
-dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
-mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
-mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
+dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l)
+mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l)
+mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l)
 log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu"
 verify_ne "0" "$mfu" "mfu count"
 
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am b/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am
index ff7b4906f..29c034290 100644
--- a/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/Makefile.am
@@ -46,7 +46,6 @@ dist_pkgdata_SCRIPTS = \
 	arcstat_001_pos.ksh \
 	arc_summary_001_pos.ksh \
 	arc_summary_002_neg.ksh \
-	arc_summary3_001_pos.ksh \
 	dbufstat_001_pos.ksh
 
 dist_pkgdata_DATA = \
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh
deleted file mode 100755
index 22dceaaf4..000000000
--- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh
+++ /dev/null
@@ -1,56 +0,0 @@
-#! /bin/ksh -p
-#
-# CDDL HEADER START
-#
-# The contents of this file are subject to the terms of the
-# Common Development and Distribution License (the "License").
-# You may not use this file except in compliance with the License.
-#
-# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-# or http://www.opensolaris.org/os/licensing.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-#
-# When distributing Covered Code, include this CDDL HEADER in each
-# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-# If applicable, add the following below this CDDL HEADER, with the
-# fields enclosed by brackets "[]" replaced with your own identifying
-# information: Portions Copyright [yyyy] [name of copyright owner]
-#
-# CDDL HEADER END
-#
-
-#
-# Copyright (c) 2015 by Lawrence Livermore National Security, LLC.
-# All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-
-# Keep the following test until Python 3 is installed on all test systems,
-# then remove
-python3 -V 2>&1 > /dev/null
-if (( $? )); then
-        log_unsupported "Python3 is not installed"
-fi
-
-
-# Some systems have Python 3 installed, but only older versions that don't
-# have the subprocess.run() functionality. We catch these with a separate
-# test. Remove this when all systems have reached 3.5 or greater
-VERSIONPYTEST=$(python3 -V)
-if [[ ${VERSIONPYTEST:9:1} -lt 5 ]]; then
-        log_unsupported "Python3 must be version 3.5 or greater"
-fi
-
-
-set -A args  "" "-a" "-d" "-p 1" "-g" "-s arc" "-r"
-log_assert "arc_summary3.py generates output and doesn't return an error code"
-
-typeset -i i=0
-while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arc_summary3.py ${args[i]} > /dev/null"
-        ((i = i + 1))
-done
-
-log_pass "arc_summary3.py generates output and doesn't return an error code"
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh
index 6653b9c1a..a445fbb48 100755
--- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh
@@ -27,17 +27,34 @@
 
 . $STF_SUITE/include/libtest.shlib
 
-set -A args  "" "-a" "-d" "-p 1"
+log_assert "arc_summary generates output and doesn't return an error code"
 
-log_assert "arc_summary.py generates output and doesn't return an error code"
+# Depending on which version of arc_summary is installed some command
+# line options may not be available.  The python3 version includes
+# several additional flags.
+python3 -V 2>&1 > /dev/null
+if (( $? )); then
+	# Some systems have Python 3 installed, but only older versions
+	# that don't have the subprocess.run() functionality. We catch
+	# these with a separate test. Remove this when all systems have
+	# reached 3.5 or greater
+	VERSIONPYTEST=$(python3 -V)
+	if [[ ${VERSIONPYTEST:9:1} -lt 5 ]]; then
+		set -A args  "" "-a" "-d" "-p 1"
+	else
+		set -A args  "" "-a" "-d" "-p 1" "-g" "-s arc" "-r"
+	fi
+else
+	set -A args  "" "-a" "-d" "-p 1"
+fi
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arc_summary.py ${args[i]} > /dev/null"
-        ((i = i + 1))
+	log_must eval "arc_summary ${args[i]} > /dev/null"
+	((i = i + 1))
 done
 
-log_must eval "arc_summary.py | head > /dev/null"
-log_must eval "arc_summary.py | head -1 > /dev/null"
+log_must eval "arc_summary | head > /dev/null"
+log_must eval "arc_summary | head -1 > /dev/null"
 
-log_pass "arc_summary.py generates output and doesn't return an error code"
+log_pass "arc_summary generates output and doesn't return an error code"
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh
index e63552feb..de747fba8 100755
--- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh
@@ -27,12 +27,12 @@
 
 . $STF_SUITE/include/libtest.shlib
 
-typeset args=("-x" "-r" "-5" "-p 7" "--err" "-@")
+typeset args=("-x" "-5" "-p 7" "--err" "-@")
 
-log_assert "arc_summary.py generates an error code with invalid options"
+log_assert "arc_summary generates an error code with invalid options"
 
 for arg in "${args[@]}"; do
-        log_mustnot eval "arc_summary.py $arg > /dev/null"
+        log_mustnot eval "arc_summary $arg > /dev/null"
 done
 
-log_pass "arc_summary.py generates an error code with invalid options"
+log_pass "arc_summary generates an error code with invalid options"
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh
index c8a89f8c4..ab574731f 100755
--- a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh
@@ -30,12 +30,12 @@
 set -A args  "" "-s \",\"" "-x" "-v" \
     "-f time,hit%,dh%,ph%,mh%"
 
-log_assert "arcstat.py generates output and doesn't return an error code"
+log_assert "arcstat generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "arcstat.py ${args[i]} > /dev/null"
+        log_must eval "arcstat ${args[i]} > /dev/null"
         ((i = i + 1))
 done
-log_pass "arcstat.py generates output and doesn't return an error code"
+log_pass "arcstat generates output and doesn't return an error code"
 
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/cleanup.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/cleanup.ksh
index 874a9fd2d..e3dc8c179 100755
--- a/tests/zfs-tests/tests/functional/cli_user/misc/cleanup.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/cleanup.ksh
@@ -32,19 +32,20 @@
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_user/misc/misc.cfg
 
-if poolexists $TESTPOOL.virt
-then
+if poolexists $TESTPOOL.virt; then
 	log_must zpool destroy $TESTPOOL.virt
 fi
 
-if poolexists v1-pool
-then
+if poolexists v1-pool; then
 	log_must zpool destroy v1-pool
 fi
 
-if [[ -f $TEST_BASE_DIR/zfstest_datastream.dat ]]
-then
-	log_must rm -f $TEST_BASE_DIR/zfstest_datastream.dat
-fi
+log_must rm -f $TEST_BASE_DIR/zfstest_datastream.dat
+log_must rm -f $TEST_BASE_DIR/disk1.dat $TEST_BASE_DIR/disk2.dat \
+    $TEST_BASE_DIR/disk3.dat $TEST_BASE_DIR/disk-additional.dat \
+    $TEST_BASE_DIR/disk-export.dat $TEST_BASE_DIR/disk-offline.dat \
+    $TEST_BASE_DIR/disk-spare1.dat $TEST_BASE_DIR/disk-spare2.dat
+log_must rm -f $TEST_BASE_DIR/zfs-pool-v1.dat \
+    $TEST_BASE_DIR/zfs-pool-v1.dat.bz2
 
 default_cleanup
diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh
index 1c267d6af..95f0598c6 100755
--- a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh
@@ -29,15 +29,15 @@
 
 set -A args  "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" "-n"
 
-log_assert "dbufstat.py generates output and doesn't return an error code"
+log_assert "dbufstat generates output and doesn't return an error code"
 
 typeset -i i=0
 while [[ $i -lt ${#args[*]} ]]; do
-        log_must eval "dbufstat.py ${args[i]} > /dev/null"
+        log_must eval "dbufstat ${args[i]} > /dev/null"
         ((i = i + 1))
 done
 
-# A simple test of dbufstat.py filter functionality
-log_must eval "dbufstat.py -F object=10,dbc=1,pool=$TESTPOOL > /dev/null"
+# A simple test of dbufstat filter functionality
+log_must eval "dbufstat -F object=10,dbc=1,pool=$TESTPOOL > /dev/null"
 
-log_pass "dbufstat.py generates output and doesn't return an error code"
+log_pass "dbufstat generates output and doesn't return an error code"
diff --git a/tests/zfs-tests/tests/functional/pyzfs/.gitignore b/tests/zfs-tests/tests/functional/pyzfs/.gitignore
new file mode 100644
index 000000000..bcbe0573e
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/pyzfs/.gitignore
@@ -0,0 +1 @@
+pyzfs_unittest.ksh
diff --git a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am
index 61cb3d074..0a27adecc 100644
--- a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am
+++ b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am
@@ -1,4 +1,18 @@
-pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs
-
-dist_pkgdata_SCRIPTS = \
+pkgpyzfsdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs
+pkgpyzfs_SCRIPTS = \
 	pyzfs_unittest.ksh
+
+EXTRA_DIST = \
+	pyzfs_unittest.ksh.in
+
+#
+# The pyzfs module is built either for Python 2 or Python 3.  In order
+# to properly test it the unit tests must be updated to the matching vesion.
+#
+$(pkgpyzfs_SCRIPTS):%:%.in
+	-$(SED) -e 's,@PYTHON\@,$(PYTHON),g' \
+		$< >'$@'
+	-chmod 775 $@
+
+distclean-local::
+	-$(RM) $(pkgpyzfs_SCRIPTS)
diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in
index fb4b60361..4ca610e5f 100755
--- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh
+++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in
@@ -28,7 +28,7 @@
 verify_runnable "global"
 
 # Verify that the required dependencies for testing are installed.
-python -c "import cffi" 2>/dev/null
+@PYTHON@ -c "import cffi" 2>/dev/null
 if [ $? -eq 1 ]; then
 	log_unsupported "python-cffi not found by Python"
 fi
@@ -37,7 +37,7 @@ fi
 # only if pyzfs was not installed due to missing, build-time, dependencies; if
 # we cannot load "libzfs_core" due to other reasons, for instance an API/ABI
 # mismatch, we want to report it.
-python -c '
+@PYTHON@ -c '
 import pkgutil, sys
 sys.exit(pkgutil.find_loader("libzfs_core") is None)'
 if [ $? -eq 1 ]; then
@@ -47,7 +47,7 @@ fi
 log_assert "Verify the nvlist and libzfs_core Python unittest run successfully"
 
 # NOTE: don't use log_must() here because it makes output unreadable
-python -m unittest --verbose \
+@PYTHON@ -m unittest --verbose \
 	libzfs_core.test.test_nvlist.TestNVList \
 	libzfs_core.test.test_libzfs_core.ZFSTest
 if [ $? -ne 0 ]; then