From 6e72a5b9b61066146deafda39ab8158c559f5f15 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 31 Oct 2018 09:22:59 -0700 Subject: pyzfs: python3 support (build system) Almost all of the Python code in the respository has been updated to be compatibile with Python 2.6, Python 3.4, or newer. The only exceptions are arc_summery3.py which requires Python 3, and pyzfs which requires at least Python 2.7. This allows us to maintain a single version of the code and support most default versions of python. This change does the following: * Sets the default shebang for all Python scripts to python3. If only Python 2 is available, then at install time scripts which are compatible with Python 2 will have their shebangs replaced with /usr/bin/python. This is done for compatibility until Python 2 goes end of life. Since only the installed versions are changed this means Python 3 must be installed on the system for test-runner when testing in-tree. * Added --with-python=<2|3|3.4,etc> configure option which sets the PYTHON environment variable to target a specific python version. By default the newest installed version of Python will be used or the preferred distribution version when creating pacakges. * Fixed --enable-pyzfs configure checks so they are run when --enable-pyzfs=check and --enable-pyzfs=yes. * Enabled pyzfs for Python 3.4 and newer, which is now supported. * Renamed pyzfs package to python-pyzfs and updated to install in the appropriate site location. For example, when building with --with-python=3.4 a python34-pyzfs will be created which installs in /usr/lib/python3.4/site-packages/. * Renamed the following python scripts according to the Fedora guidance for packaging utilities in /bin - dbufstat.py -> dbufstat - arcstat.py -> arcstat - arc_summary.py -> arc_summary - arc_summary3.py -> arc_summary3 * Updated python-cffi package name. On CentOS 6, CentOS 7, and Amazon Linux it's called python-cffi, not python2-cffi. For Python3 it's called python3-cffi or python3x-cffi. * Install one version of arc_summary. Depending on the version of Python available install either arc_summary2 or arc_summary3 as arc_summary. The user output is only slightly different. Reviewed-by: John Ramsden Reviewed-by: Neal Gompa Reviewed-by: loli10K Signed-off-by: Brian Behlendorf Closes #8096 --- tests/test-runner/bin/Makefile.am | 11 +++++ tests/test-runner/bin/test-runner.py | 2 + tests/test-runner/bin/zts-report.py | 4 +- tests/zfs-tests/include/commands.cfg | 8 +-- .../tests/functional/arc/dbufstats_001_pos.ksh | 4 +- .../tests/functional/arc/dbufstats_002_pos.ksh | 12 ++--- .../cli_user/misc/arc_summary3_001_pos.ksh | 6 +-- .../cli_user/misc/arc_summary_001_pos.ksh | 10 ++-- .../cli_user/misc/arc_summary_002_neg.ksh | 6 +-- .../functional/cli_user/misc/arcstat_001_pos.ksh | 6 +-- .../functional/cli_user/misc/dbufstat_001_pos.ksh | 10 ++-- tests/zfs-tests/tests/functional/pyzfs/.gitignore | 1 + tests/zfs-tests/tests/functional/pyzfs/Makefile.am | 20 ++++++-- .../tests/functional/pyzfs/pyzfs_unittest.ksh | 57 ---------------------- .../tests/functional/pyzfs/pyzfs_unittest.ksh.in | 57 ++++++++++++++++++++++ 15 files changed, 122 insertions(+), 92 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/pyzfs/.gitignore delete mode 100755 tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh create mode 100755 tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in (limited to 'tests') diff --git a/tests/test-runner/bin/Makefile.am b/tests/test-runner/bin/Makefile.am index e843e4e09..30c564e55 100644 --- a/tests/test-runner/bin/Makefile.am +++ b/tests/test-runner/bin/Makefile.am @@ -2,3 +2,14 @@ pkgdatadir = $(datadir)/@PACKAGE@/test-runner/bin dist_pkgdata_SCRIPTS = \ test-runner.py \ zts-report.py +# +# These scripts are compatibile with both Python 2.6 and 3.4. As such the +# python 3 shebang can be replaced at install time when targeting a python +# 2 system. This allows us to maintain a single version of the source. +# +if USING_PYTHON_2 +install-data-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(pkgdatadir)/test-runner.py \ + $(DESTDIR)$(pkgdatadir)/zts-report.py +endif diff --git a/tests/test-runner/bin/test-runner.py b/tests/test-runner/bin/test-runner.py index 7ef8a87ed..2e26fa266 100755 --- a/tests/test-runner/bin/test-runner.py +++ b/tests/test-runner/bin/test-runner.py @@ -15,6 +15,8 @@ # Copyright (c) 2012, 2015 by Delphix. All rights reserved. # Copyright (c) 2017 Datto Inc. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# # some python 2.7 system don't have a configparser shim try: diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py index 950295601..4e51bc94e 100755 --- a/tests/test-runner/bin/zts-report.py +++ b/tests/test-runner/bin/zts-report.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # This file and its contents are supplied under the terms of the @@ -15,6 +15,8 @@ # Copyright (c) 2017 by Delphix. All rights reserved. # Copyright (c) 2018 by Lawrence Livermore National Security, LLC. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import os import re diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 5efcb6102..8ced03e93 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -146,10 +146,10 @@ export ZFS_FILES='zdb zpool ztest raidz_test - arc_summary.py - arc_summary3.py - arcstat.py - dbufstat.py + arc_summary + arc_summary3 + arcstat + dbufstat zed zgenhostid zstreamdump' diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh index 5ceff962d..7ec9eaf4c 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh @@ -37,7 +37,7 @@ # 2. Store output from dbufs kstat # 3. Store output from dbufstats kstat # 4. Compare stats presented in dbufstats with stat generated using -# dbufstat.py and the dbufs kstat output +# dbufstat and the dbufs kstat output # DBUFSTATS_FILE=$(mktemp $TEST_BASE_DIR/dbufstats.out.XXXXXX) @@ -56,7 +56,7 @@ function testdbufstat # stat_name dbufstat_filter [[ -n "$2" ]] && filter="-F $2" from_dbufstat=$(grep -w "$name" "$DBUFSTATS_FILE" | awk '{ print $3 }') - from_dbufs=$(dbufstat.py -bxn -i "$DBUFS_FILE" "$filter" | wc -l) + from_dbufs=$(dbufstat -bxn -i "$DBUFS_FILE" "$filter" | wc -l) within_tolerance $from_dbufstat $from_dbufs 9 \ || log_fail "Stat $name exceeded tolerance" diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh index e256bfabe..dc30b6606 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh @@ -62,18 +62,18 @@ objid=$(stat --format="%i" "$TESTDIR/file") log_note "Object ID for $TESTDIR/file is $objid" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mru" "mru count" verify_eq "0" "$mfu" "mfu count" log_must eval "cat $TESTDIR/file > /dev/null" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mfu" "mfu count" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh index 22dceaaf4..ff090baee 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh @@ -45,12 +45,12 @@ fi set -A args "" "-a" "-d" "-p 1" "-g" "-s arc" "-r" -log_assert "arc_summary3.py generates output and doesn't return an error code" +log_assert "arc_summary3 generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary3.py ${args[i]} > /dev/null" + log_must eval "arc_summary3 ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arc_summary3.py generates output and doesn't return an error code" +log_pass "arc_summary3 generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh index 6653b9c1a..8736b18d8 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-a" "-d" "-p 1" -log_assert "arc_summary.py generates output and doesn't return an error code" +log_assert "arc_summary generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary.py ${args[i]} > /dev/null" + log_must eval "arc_summary ${args[i]} > /dev/null" ((i = i + 1)) done -log_must eval "arc_summary.py | head > /dev/null" -log_must eval "arc_summary.py | head -1 > /dev/null" +log_must eval "arc_summary | head > /dev/null" +log_must eval "arc_summary | head -1 > /dev/null" -log_pass "arc_summary.py generates output and doesn't return an error code" +log_pass "arc_summary generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh index e63552feb..477b7ca08 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh @@ -29,10 +29,10 @@ typeset args=("-x" "-r" "-5" "-p 7" "--err" "-@") -log_assert "arc_summary.py generates an error code with invalid options" +log_assert "arc_summary generates an error code with invalid options" for arg in "${args[@]}"; do - log_mustnot eval "arc_summary.py $arg > /dev/null" + log_mustnot eval "arc_summary $arg > /dev/null" done -log_pass "arc_summary.py generates an error code with invalid options" +log_pass "arc_summary generates an error code with invalid options" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh index c8a89f8c4..ab574731f 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh @@ -30,12 +30,12 @@ set -A args "" "-s \",\"" "-x" "-v" \ "-f time,hit%,dh%,ph%,mh%" -log_assert "arcstat.py generates output and doesn't return an error code" +log_assert "arcstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arcstat.py ${args[i]} > /dev/null" + log_must eval "arcstat ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arcstat.py generates output and doesn't return an error code" +log_pass "arcstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh index 1c267d6af..95f0598c6 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" "-n" -log_assert "dbufstat.py generates output and doesn't return an error code" +log_assert "dbufstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "dbufstat.py ${args[i]} > /dev/null" + log_must eval "dbufstat ${args[i]} > /dev/null" ((i = i + 1)) done -# A simple test of dbufstat.py filter functionality -log_must eval "dbufstat.py -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" +# A simple test of dbufstat filter functionality +log_must eval "dbufstat -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" -log_pass "dbufstat.py generates output and doesn't return an error code" +log_pass "dbufstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/pyzfs/.gitignore b/tests/zfs-tests/tests/functional/pyzfs/.gitignore new file mode 100644 index 000000000..bcbe0573e --- /dev/null +++ b/tests/zfs-tests/tests/functional/pyzfs/.gitignore @@ -0,0 +1 @@ +pyzfs_unittest.ksh diff --git a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am index 61cb3d074..0a27adecc 100644 --- a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am +++ b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am @@ -1,4 +1,18 @@ -pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs - -dist_pkgdata_SCRIPTS = \ +pkgpyzfsdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs +pkgpyzfs_SCRIPTS = \ pyzfs_unittest.ksh + +EXTRA_DIST = \ + pyzfs_unittest.ksh.in + +# +# The pyzfs module is built either for Python 2 or Python 3. In order +# to properly test it the unit tests must be updated to the matching vesion. +# +$(pkgpyzfs_SCRIPTS):%:%.in + -$(SED) -e 's,@PYTHON\@,$(PYTHON),g' \ + $< >'$@' + -chmod 775 $@ + +distclean-local:: + -$(RM) $(pkgpyzfs_SCRIPTS) diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh deleted file mode 100755 index fb4b60361..000000000 --- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/ksh -p -# -# This file and its contents are supplied under the terms of the -# Common Development and Distribution License ("CDDL"), version 1.0. -# You may only use this file in accordance with the terms of version -# 1.0 of the CDDL. -# -# A full copy of the text of the CDDL should have accompanied this -# source. A copy of the CDDL is also available via the Internet at -# http://www.illumos.org/license/CDDL. -# - -# -# Copyright 2018, loli10K . All rights reserved. -# - -. $STF_SUITE/include/libtest.shlib - -# -# DESCRIPTION: -# Verify the libzfs_core Python test suite can be run successfully -# -# STRATEGY: -# 1. Run the nvlist and libzfs_core Python unittest -# 2. Verify the exit code is 0 (no errors) -# - -verify_runnable "global" - -# Verify that the required dependencies for testing are installed. -python -c "import cffi" 2>/dev/null -if [ $? -eq 1 ]; then - log_unsupported "python-cffi not found by Python" -fi - -# We don't just try to "import libzfs_core" because we want to skip these tests -# only if pyzfs was not installed due to missing, build-time, dependencies; if -# we cannot load "libzfs_core" due to other reasons, for instance an API/ABI -# mismatch, we want to report it. -python -c ' -import pkgutil, sys -sys.exit(pkgutil.find_loader("libzfs_core") is None)' -if [ $? -eq 1 ]; then - log_unsupported "libzfs_core not found by Python" -fi - -log_assert "Verify the nvlist and libzfs_core Python unittest run successfully" - -# NOTE: don't use log_must() here because it makes output unreadable -python -m unittest --verbose \ - libzfs_core.test.test_nvlist.TestNVList \ - libzfs_core.test.test_libzfs_core.ZFSTest -if [ $? -ne 0 ]; then - log_fail "Python unittest completed with errors" -fi - -log_pass "Python unittest completed without errors" diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in new file mode 100755 index 000000000..4ca610e5f --- /dev/null +++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in @@ -0,0 +1,57 @@ +#!/bin/ksh -p +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2018, loli10K . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Verify the libzfs_core Python test suite can be run successfully +# +# STRATEGY: +# 1. Run the nvlist and libzfs_core Python unittest +# 2. Verify the exit code is 0 (no errors) +# + +verify_runnable "global" + +# Verify that the required dependencies for testing are installed. +@PYTHON@ -c "import cffi" 2>/dev/null +if [ $? -eq 1 ]; then + log_unsupported "python-cffi not found by Python" +fi + +# We don't just try to "import libzfs_core" because we want to skip these tests +# only if pyzfs was not installed due to missing, build-time, dependencies; if +# we cannot load "libzfs_core" due to other reasons, for instance an API/ABI +# mismatch, we want to report it. +@PYTHON@ -c ' +import pkgutil, sys +sys.exit(pkgutil.find_loader("libzfs_core") is None)' +if [ $? -eq 1 ]; then + log_unsupported "libzfs_core not found by Python" +fi + +log_assert "Verify the nvlist and libzfs_core Python unittest run successfully" + +# NOTE: don't use log_must() here because it makes output unreadable +@PYTHON@ -m unittest --verbose \ + libzfs_core.test.test_nvlist.TestNVList \ + libzfs_core.test.test_libzfs_core.ZFSTest +if [ $? -ne 0 ]; then + log_fail "Python unittest completed with errors" +fi + +log_pass "Python unittest completed without errors" -- cgit v1.2.3