diff options
author | Brian Behlendorf <[email protected]> | 2018-10-31 09:22:59 -0700 |
---|---|---|
committer | Brian Behlendorf <[email protected]> | 2019-01-06 10:39:41 -0800 |
commit | 6e72a5b9b61066146deafda39ab8158c559f5f15 (patch) | |
tree | 9439fbf7ba661e5d12fec2699a3d9c554791cc3b | |
parent | 4b1c4062d050e2cfa609e1040384d1f3b5f04f52 (diff) |
pyzfs: python3 support (build system)
Almost all of the Python code in the respository has been updated
to be compatibile with Python 2.6, Python 3.4, or newer. The only
exceptions are arc_summery3.py which requires Python 3, and pyzfs
which requires at least Python 2.7. This allows us to maintain a
single version of the code and support most default versions of
python. This change does the following:
* Sets the default shebang for all Python scripts to python3. If
only Python 2 is available, then at install time scripts which
are compatible with Python 2 will have their shebangs replaced
with /usr/bin/python. This is done for compatibility until
Python 2 goes end of life. Since only the installed versions
are changed this means Python 3 must be installed on the system
for test-runner when testing in-tree.
* Added --with-python=<2|3|3.4,etc> configure option which sets
the PYTHON environment variable to target a specific python
version. By default the newest installed version of Python
will be used or the preferred distribution version when
creating pacakges.
* Fixed --enable-pyzfs configure checks so they are run when
--enable-pyzfs=check and --enable-pyzfs=yes.
* Enabled pyzfs for Python 3.4 and newer, which is now supported.
* Renamed pyzfs package to python<VERSION>-pyzfs and updated to
install in the appropriate site location. For example, when
building with --with-python=3.4 a python34-pyzfs will be
created which installs in /usr/lib/python3.4/site-packages/.
* Renamed the following python scripts according to the Fedora
guidance for packaging utilities in /bin
- dbufstat.py -> dbufstat
- arcstat.py -> arcstat
- arc_summary.py -> arc_summary
- arc_summary3.py -> arc_summary3
* Updated python-cffi package name. On CentOS 6, CentOS 7, and
Amazon Linux it's called python-cffi, not python2-cffi. For
Python3 it's called python3-cffi or python3x-cffi.
* Install one version of arc_summary. Depending on the version
of Python available install either arc_summary2 or arc_summary3
as arc_summary. The user output is only slightly different.
Reviewed-by: John Ramsden <[email protected]>
Reviewed-by: Neal Gompa <[email protected]>
Reviewed-by: loli10K <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #8096
29 files changed, 339 insertions, 162 deletions
diff --git a/.gitignore b/.gitignore index 3a8cb2e86..6367ebf77 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ *.swp *.gcno *.gcda +*.pyc +*.pyo .deps .libs .dirstamp diff --git a/cmd/arc_summary/Makefile.am b/cmd/arc_summary/Makefile.am index ac7b0d48d..a83edffad 100644 --- a/cmd/arc_summary/Makefile.am +++ b/cmd/arc_summary/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = arc_summary.py arc_summary3.py +EXTRA_DIST = arc_summary2 arc_summary3 + +if USING_PYTHON_2 +dist_bin_SCRIPTS = arc_summary2 +install-exec-hook: + mv $(DESTDIR)$(bindir)/arc_summary2 $(DESTDIR)$(bindir)/arc_summary +endif + +if USING_PYTHON_3 +dist_bin_SCRIPTS = arc_summary3 +install-exec-hook: + mv $(DESTDIR)$(bindir)/arc_summary3 $(DESTDIR)$(bindir)/arc_summary +endif diff --git a/cmd/arc_summary/arc_summary.py b/cmd/arc_summary/arc_summary2 index 642c94b69..ab4a3c574 100755 --- a/cmd/arc_summary/arc_summary.py +++ b/cmd/arc_summary/arc_summary2 @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # $Id: arc_summary.pl,v 388:e27800740aa2 2011-07-08 02:53:29Z jhell $ # @@ -35,6 +35,8 @@ # Note some of this code uses older code (eg getopt instead of argparse, # subprocess.Popen() instead of subprocess.run()) because we need to support # some very old versions of Python. +# + """Print statistics on the ZFS Adjustable Replacement Cache (ARC) Provides basic information on the ARC, its efficiency, the L2ARC (if present), @@ -1005,7 +1007,7 @@ def zfs_header(): def usage(): """Print usage information""" - sys.stdout.write("Usage: arc_summary.py [-h] [-a] [-d] [-p PAGE]\n\n") + sys.stdout.write("Usage: arc_summary [-h] [-a] [-d] [-p PAGE]\n\n") sys.stdout.write("\t -h, --help : " "Print this help message and exit\n") sys.stdout.write("\t -a, --alternate : " @@ -1018,10 +1020,10 @@ def usage(): "should be an integer between 1 and " + str(len(unSub)) + "\n\n") sys.stdout.write("Examples:\n") - sys.stdout.write("\tarc_summary.py -a\n") - sys.stdout.write("\tarc_summary.py -p 4\n") - sys.stdout.write("\tarc_summary.py -ad\n") - sys.stdout.write("\tarc_summary.py --page=2\n") + sys.stdout.write("\tarc_summary -a\n") + sys.stdout.write("\tarc_summary -p 4\n") + sys.stdout.write("\tarc_summary -ad\n") + sys.stdout.write("\tarc_summary --page=2\n") def main(): diff --git a/cmd/arc_summary/arc_summary3.py b/cmd/arc_summary/arc_summary3 index e70f2a35e..e67cd90f7 100755 --- a/cmd/arc_summary/arc_summary3.py +++ b/cmd/arc_summary/arc_summary3 @@ -346,7 +346,7 @@ def get_version(request): error_msg = '(ERROR: "{0}" requested)'.format(request) return error_msg - # The original arc_summary.py called /sbin/modinfo/{spl,zfs} to get + # The original arc_summary called /sbin/modinfo/{spl,zfs} to get # the version information. We switch to /sys/module/{spl,zfs}/version # to make sure we get what is really loaded in the kernel command = ["cat", "/sys/module/{0}/version".format(request)] @@ -374,7 +374,7 @@ def print_header(): """ # datetime is now recommended over time but we keep the exact formatting - # from the older version of arc_summary.py in case there are scripts + # from the older version of arc_summary in case there are scripts # that expect it in this way daydate = time.strftime(DATE_FORMAT) spc_date = LINE_LENGTH-len(daydate) @@ -586,7 +586,7 @@ def section_archits(kstats_dict): # For some reason, anon_hits can turn negative, which is weird. Until we # have figured out why this happens, we just hide the problem, following - # the behavior of the original arc_summary.py + # the behavior of the original arc_summary. if anon_hits >= 0: prt_i2('Anonymously used:', f_perc(anon_hits, arc_stats['hits']), f_hits(anon_hits)) diff --git a/cmd/arcstat/Makefile.am b/cmd/arcstat/Makefile.am index 8987b2414..462e9a619 100644 --- a/cmd/arcstat/Makefile.am +++ b/cmd/arcstat/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = arcstat.py +dist_bin_SCRIPTS = arcstat + +# +# The arcstat script is compatibile with both Python 2.6 and 3.4. +# As such the python 3 shebang can be replaced at install time when +# targeting a python 2 system. This allows us to maintain a single +# version of the source. +# +if USING_PYTHON_2 +install-exec-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(bindir)/arcstat +endif diff --git a/cmd/arcstat/arcstat.py b/cmd/arcstat/arcstat index a2c52ddb3..57a2d621f 100755 --- a/cmd/arcstat/arcstat.py +++ b/cmd/arcstat/arcstat @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Print out ZFS ARC Statistics exported via kstat(1) # For a definition of fields, or usage, use arctstat.pl -v @@ -42,7 +42,8 @@ # @hdr is the array of fields that needs to be printed, so we # just iterate over this array and print the values using our pretty printer. # - +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import sys import time @@ -109,7 +110,7 @@ opfile = None sep = " " # Default separator is 2 spaces version = "0.4" l2exist = False -cmd = ("Usage: arcstat.py [-hvx] [-f fields] [-o file] [-s string] [interval " +cmd = ("Usage: arcstat [-hvx] [-f fields] [-o file] [-s string] [interval " "[count]]\n") cur = {} d = {} @@ -138,10 +139,10 @@ def usage(): sys.stderr.write("\t -s : Override default field separator with custom " "character or string\n") sys.stderr.write("\nExamples:\n") - sys.stderr.write("\tarcstat.py -o /tmp/a.log 2 10\n") - sys.stderr.write("\tarcstat.py -s \",\" -o /tmp/a.log 2 10\n") - sys.stderr.write("\tarcstat.py -v\n") - sys.stderr.write("\tarcstat.py -f time,hit%,dh%,ph%,mh% 1\n") + sys.stderr.write("\tarcstat -o /tmp/a.log 2 10\n") + sys.stderr.write("\tarcstat -s \",\" -o /tmp/a.log 2 10\n") + sys.stderr.write("\tarcstat -v\n") + sys.stderr.write("\tarcstat -f time,hit%,dh%,ph%,mh% 1\n") sys.stderr.write("\n") sys.exit(1) diff --git a/cmd/dbufstat/Makefile.am b/cmd/dbufstat/Makefile.am index 19bffb020..968a76077 100644 --- a/cmd/dbufstat/Makefile.am +++ b/cmd/dbufstat/Makefile.am @@ -1 +1,13 @@ -dist_bin_SCRIPTS = dbufstat.py +dist_bin_SCRIPTS = dbufstat + +# +# The dbufstat script is compatibile with both Python 2.6 and 3.4. +# As such the python 3 shebang can be replaced at install time when +# targeting a python 2 system. This allows us to maintain a single +# version of the source. +# +if USING_PYTHON_2 +install-exec-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(bindir)/dbufstat +endif diff --git a/cmd/dbufstat/dbufstat.py b/cmd/dbufstat/dbufstat index 5e2217a54..e6c947fbc 100755 --- a/cmd/dbufstat/dbufstat.py +++ b/cmd/dbufstat/dbufstat @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Print out statistics for all cached dmu buffers. This information # is available through the dbufs kstat and may be post-processed as @@ -27,6 +27,8 @@ # Copyright (C) 2013 Lawrence Livermore National Security, LLC. # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import sys import getopt @@ -106,7 +108,7 @@ cols = { hdr = None xhdr = None sep = " " # Default separator is 2 spaces -cmd = ("Usage: dbufstat.py [-bdhnrtvx] [-i file] [-f fields] [-o file] " +cmd = ("Usage: dbufstat [-bdhnrtvx] [-i file] [-f fields] [-o file] " "[-s string] [-F filter]\n") raw = 0 @@ -167,11 +169,11 @@ def usage(): "character or string\n") sys.stderr.write("\t -F : Filter output by value or regex\n") sys.stderr.write("\nExamples:\n") - sys.stderr.write("\tdbufstat.py -d -o /tmp/d.log\n") - sys.stderr.write("\tdbufstat.py -t -s \",\" -o /tmp/t.log\n") - sys.stderr.write("\tdbufstat.py -v\n") - sys.stderr.write("\tdbufstat.py -d -f pool,object,objset,dsize,cached\n") - sys.stderr.write("\tdbufstat.py -bx -F dbc=1,objset=54,pool=testpool\n") + sys.stderr.write("\tdbufstat -d -o /tmp/d.log\n") + sys.stderr.write("\tdbufstat -t -s \",\" -o /tmp/t.log\n") + sys.stderr.write("\tdbufstat -v\n") + sys.stderr.write("\tdbufstat -d -f pool,object,objset,dsize,cached\n") + sys.stderr.write("\tdbufstat -bx -F dbc=1,objset=54,pool=testpool\n") sys.stderr.write("\n") sys.exit(1) diff --git a/config/always-python.m4 b/config/always-python.m4 new file mode 100644 index 000000000..858ab7b01 --- /dev/null +++ b/config/always-python.m4 @@ -0,0 +1,102 @@ +dnl # +dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false]) +dnl # +dnl # Verify Python version +dnl # +AC_DEFUN([ZFS_AC_PYTHON_VERSION], [ + ver_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"` + AS_IF([test "$ver_check" = "True"], [ + m4_ifvaln([$2], [$2]) + ], [ + m4_ifvaln([$3], [$3]) + ]) +]) + +dnl # +dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false]) +dnl # +dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE +dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html +dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS. +dnl # +AC_DEFUN([ZFS_AC_PYTHON_MODULE], [ + PYTHON_NAME=`basename $PYTHON` + AC_MSG_CHECKING([for $PYTHON_NAME module: $1]) + AS_IF([$PYTHON -c "import $1" 2>/dev/null], [ + AC_MSG_RESULT(yes) + m4_ifvaln([$2], [$2]) + ], [ + AC_MSG_RESULT(no) + m4_ifvaln([$3], [$3]) + ]) +]) + +dnl # +dnl # The majority of the python scripts are written to be compatible +dnl # with Python 2.6 and Python 3.4. Therefore, they may be installed +dnl # and used with either interpreter. This option is intended to +dnl # to provide a method to specify the default system version, and +dnl # set the PYTHON environment variable accordingly. +dnl # +AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [ + AC_ARG_WITH([python], + AC_HELP_STRING([--with-python[=VERSION]], + [default system python version @<:@default=check@:>@]), + [with_python=$withval], + [with_python=check]) + + AS_CASE([$with_python], + [check], + [AS_IF([test -x /usr/bin/python3], + [PYTHON="python3"], + [AS_IF([test -x /usr/bin/python2], + [PYTHON="python2"], + [PYTHON=""] + )] + )], + [2*], [PYTHON="python${with_python}"], + [*python2*], [PYTHON="${with_python}"], + [3*], [PYTHON="python${with_python}"], + [*python3*], [PYTHON="${with_python}"], + [no], [PYTHON=""], + [AC_MSG_ERROR([Unknown --with-python value '$with_python'])] + ) + + AS_IF([$PYTHON --version >/dev/null 2>&1], [ /bin/true ], [ + AC_MSG_ERROR([Cannot find $PYTHON in your system path]) + ]) + + AM_PATH_PYTHON([2.6], [], [:]) + AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :]) + AM_CONDITIONAL([USING_PYTHON_2], [test "${PYTHON_VERSION:0:2}" = "2."]) + AM_CONDITIONAL([USING_PYTHON_3], [test "${PYTHON_VERSION:0:2}" = "3."]) + + dnl # + dnl # Minimum supported Python versions for utilities: + dnl # Python 2.6.x, or Python 3.4.x + dnl # + AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [ + ZFS_AC_PYTHON_VERSION([>= '2.6'], [ /bin/true ], + [AC_MSG_ERROR("Python >= 2.6.x is not available")]) + ]) + + AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [ + ZFS_AC_PYTHON_VERSION([>= '3.4'], [ /bin/true ], + [AC_MSG_ERROR("Python >= 3.4.x is not available")]) + ]) + + dnl # + dnl # Request that packages be built for a specific Python version. + dnl # + AS_IF([test $with_python != check], [ + PYTHON_PKG_VERSION=`echo ${PYTHON} | tr -d 'a-zA-Z.'` + DEFINE_PYTHON_PKG_VERSION='--define "__use_python_pkg_version '${PYTHON_PKG_VERSION}'"' + DEFINE_PYTHON_VERSION='--define "__use_python '${PYTHON}'"' + ], [ + DEFINE_PYTHON_VERSION='' + DEFINE_PYTHON_PKG_VERSION='' + ]) + + AC_SUBST(DEFINE_PYTHON_VERSION) + AC_SUBST(DEFINE_PYTHON_PKG_VERSION) +]) diff --git a/config/always-pyzfs.m4 b/config/always-pyzfs.m4 index c50acb099..d74d6f1a7 100644 --- a/config/always-pyzfs.m4 +++ b/config/always-pyzfs.m4 @@ -1,80 +1,44 @@ dnl # -dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false]) +dnl # Determines if pyzfs can be built, requires Python 2.7 or latter. dnl # -dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE -dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html -dnl # -AC_DEFUN([ZFS_AC_PYTHON_MODULE],[ - PYTHON_NAME=`basename $PYTHON` - AC_MSG_CHECKING([for $PYTHON_NAME module: $1]) - $PYTHON -c "import $1" 2>/dev/null - if test $? -eq 0; - then - AC_MSG_RESULT(yes) - m4_ifvaln([$2], [$2]) - else - AC_MSG_RESULT(no) - m4_ifvaln([$3], [$3]) - fi -]) - -dnl # -dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false]) -dnl # -dnl # Verify Python version -dnl # -AC_DEFUN([ZFS_AC_PYTHON_VERSION], [ - AC_MSG_CHECKING([for a version of Python $1]) - version_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"` - if test "$version_check" = "True"; - then - AC_MSG_RESULT(yes) - m4_ifvaln([$2], [$2]) - else - AC_MSG_RESULT(no) - m4_ifvaln([$3], [$3]) - fi - -]) - AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ - PYTHON_REQUIRED_VERSION="<= '2.7.x'" - AC_ARG_ENABLE([pyzfs], AC_HELP_STRING([--enable-pyzfs], [install libzfs_core python bindings @<:@default=check@:>@]), [enable_pyzfs=$enableval], [enable_pyzfs=check]) - AM_PATH_PYTHON([2.7], [], [ + dnl # + dnl # Packages for pyzfs specifically enabled/disabled. + dnl # + AS_IF([test "x$enable_pyzfs" != xcheck], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python >= 2.7 is not installed") - ], [test ! "x$enable_pyzfs" = xno], [ - enable_pyzfs=no + DEFINE_PYZFS='--with pyzfs' + ], [ + DEFINE_PYZFS='--without pyzfs' ]) + ], [ + DEFINE_PYZFS='' ]) - AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != :]) + AC_SUBST(DEFINE_PYZFS) dnl # - dnl # Python 2.7.x is supported, other versions (3.5) are not yet + dnl # Require python-devel libraries dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ - ZFS_AC_PYTHON_VERSION([$PYTHON_REQUIRED_VERSION], [], [ - AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION is not available") - ], [test ! "x$enable_pyzfs" = xno], [ - enable_pyzfs=no + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ + AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [ + PYTHON_REQUIRED_VERSION=">= '2.7.0'" + ], [ + AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [ + PYTHON_REQUIRED_VERSION=">= '3.4.0'" + ], [ + AC_MSG_ERROR("Python $PYTHON_VERSION unknown") ]) ]) - ]) - dnl # - dnl # Require python-devel libraries - dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("Python development library is not available") + AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION development library is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -84,10 +48,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ dnl # dnl # Python "setuptools" module is required to build and install pyzfs dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ ZFS_AC_PYTHON_MODULE([setuptools], [], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python-setuptools is not installed") + AC_MSG_ERROR("Python $PYTHON_VERSION setuptools is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -97,10 +61,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ dnl # dnl # Python "cffi" module is required to run pyzfs dnl # - AS_IF([test "x$enable_pyzfs" = xcheck], [ + AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [ ZFS_AC_PYTHON_MODULE([cffi], [], [ AS_IF([test "x$enable_pyzfs" = xyes], [ - AC_MSG_ERROR("python-cffi is not installed") + AC_MSG_ERROR("Python $PYTHON_VERSION cffi is not installed") ], [test ! "x$enable_pyzfs" = xno], [ enable_pyzfs=no ]) @@ -114,12 +78,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [ AM_CONDITIONAL([PYZFS_ENABLED], [test x$enable_pyzfs = xyes]) AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs]) - - AS_IF([test "x$enable_pyzfs" = xyes], [ - DEFINE_PYZFS='--define "_pyzfs 1"' - ],[ - DEFINE_PYZFS='' - ]) - AC_SUBST(DEFINE_PYZFS) AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG]) + + AC_MSG_CHECKING([whether to enable pyzfs: ]) + AC_MSG_RESULT($enable_pyzfs) ]) diff --git a/config/deb.am b/config/deb.am index eb4e5bbda..e405547aa 100644 --- a/config/deb.am +++ b/config/deb.am @@ -47,7 +47,7 @@ deb-utils: deb-local rpm-utils pkg7=$${name}-test-$${version}.$${arch}.rpm; \ pkg8=$${name}-dracut-$${version}.$${arch}.rpm; \ pkg9=$${name}-initramfs-$${version}.$${arch}.rpm; \ - pkg10=pyzfs-$${version}.noarch.rpm; \ + pkg10=`ls python*-pyzfs-$${version}* | tail -1`; \ ## Arguments need to be passed to dh_shlibdeps. Alien provides no mechanism ## to do this, so we install a shim onto the path which calls the real ## dh_shlibdeps with the required arguments. diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 1d47b0384..6e305996e 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -160,6 +160,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [ ZFS_AC_CONFIG_ALWAYS_CC_ASAN ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD ZFS_AC_CONFIG_ALWAYS_ARCH + ZFS_AC_CONFIG_ALWAYS_PYTHON ZFS_AC_CONFIG_ALWAYS_PYZFS ]) @@ -264,10 +265,13 @@ AC_DEFUN([ZFS_AC_RPM], [ RPM_DEFINE_UTIL+=' $(DEFINE_INITRAMFS)' RPM_DEFINE_UTIL+=' $(DEFINE_SYSTEMD)' RPM_DEFINE_UTIL+=' $(DEFINE_PYZFS)' + RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_VERSION)' + RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_PKG_VERSION)' - dnl # Override default lib directory on Debian/Ubuntu systems. The provided - dnl # /usr/lib/rpm/platform/<arch>/macros files do not specify the correct - dnl # path for multiarch systems as described by the packaging guidelines. + dnl # Override default lib directory on Debian/Ubuntu systems. The + dnl # provided /usr/lib/rpm/platform/<arch>/macros files do not + dnl # specify the correct path for multiarch systems as described + dnl # by the packaging guidelines. dnl # dnl # https://wiki.ubuntu.com/MultiarchSpec dnl # https://wiki.debian.org/Multiarch/Implementation diff --git a/contrib/pyzfs/Makefile.am b/contrib/pyzfs/Makefile.am index f27216a77..36290661f 100644 --- a/contrib/pyzfs/Makefile.am +++ b/contrib/pyzfs/Makefile.am @@ -27,7 +27,7 @@ install-exec-local: $(PYTHON) $(srcdir)/setup.py install \ --prefix $(prefix) \ --root $(DESTDIR)/ \ - --install-lib $(pythondir) \ + --install-lib $(pythonsitedir) \ --single-version-externally-managed \ --verbose diff --git a/contrib/pyzfs/setup.py b/contrib/pyzfs/setup.py index e76ffbf82..3ff6c04c6 100644 --- a/contrib/pyzfs/setup.py +++ b/contrib/pyzfs/setup.py @@ -29,8 +29,13 @@ setup( "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 2 :: Only", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: System :: Filesystems", "Topic :: Software Development :: Libraries", ], @@ -48,7 +53,7 @@ setup( setup_requires=[ "cffi", ], - python_requires='>=2.7,<3', + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<4', zip_safe=False, test_suite="libzfs_core.test", ) diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index 55edbc83f..533792989 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -53,16 +53,6 @@ %bcond_with asan %bcond_with systemd -# Python permits the !/usr/bin/python shebang for scripts that are cross -# compatible between python2 and python3, but Fedora 28 does not. Fedora -# wants us to choose python3 for cross-compatible scripts. Since we want -# to support python2 and python3 users, exclude our scripts from Fedora 28's -# RPM build check, so that we don't get a bunch of build warnings. -# -# Details: https://github.com/zfsonlinux/zfs/issues/7360 -# -%global __brp_mangle_shebangs_exclude_from arc_summary.py|arcstat.py|dbufstat.py|test-runner.py|zts-report.py - # Generic enable switch for systemd %if %{with systemd} %define _systemd 1 @@ -85,6 +75,32 @@ %define _systemd 1 %endif +# When not specified default to distribution provided version. This +# is normally Python 3, but for RHEL <= 7 only Python 2 is provided. +%if %{undefined __use_python} +%if 0%{?rhel} && 0%{?rhel} <= 7 +%define __python /usr/bin/python2 +%define __python_pkg_version 2 +%define __python_cffi_pkg python-cffi +%else +%define __python /usr/bin/python3 +%define __python_pkg_version 3 +%define __python_cffi_pkg python3-cffi +%endif +%else +%define __python %{__use_python} +%define __python_pkg_version %{__use_python_pkg_version} +%define __python_cffi_pkg python%{__python_pkg_version}-cffi +%endif + +# By default python-pyzfs is enabled, with the exception of +# RHEL 6 which by default uses Python 2.6 which is too old. +%if 0%{?rhel} == 6 +%bcond_with pyzfs +%else +%bcond_without pyzfs +%endif + Name: @PACKAGE@ Version: @VERSION@ Release: @RELEASE@%{?dist} @@ -135,7 +151,7 @@ Requires: util-linux Requires: sysstat %description -This package contains the ZFS command line utilities. +This package contains the core ZFS command line utilities. %package -n libzpool2 Summary: Native ZFS pool library for Linux @@ -219,6 +235,7 @@ Requires: acl Requires: sudo Requires: sysstat Requires: libaio +Requires: python%{__python_pkg_version} %if 0%{?rhel}%{?fedora}%{?suse_version} BuildRequires: libaio-devel %endif @@ -240,23 +257,23 @@ Requires: grep This package contains a dracut module used to construct an initramfs image which is ZFS aware. -%if 0%{?_pyzfs} -%package -n pyzfs -Summary: Python wrapper for libzfs_core +%if %{with pyzfs} +%package -n python%{__python_pkg_version}-pyzfs +Summary: Python %{python_version} wrapper for libzfs_core Group: Development/Languages/Python License: Apache-2.0 BuildArch: noarch Requires: libzfs2 = %{version} Requires: libnvpair1 = %{version} Requires: libffi -Requires: python >= 2.7 -Requires: python-cffi +Requires: python%{__python_pkg_version} +Requires: %{__python_cffi_pkg} %if 0%{?rhel}%{?fedora}%{?suse_version} -BuildRequires: python-devel +BuildRequires: python%{__python_pkg_version}-devel BuildRequires: libffi-devel %endif -%description -n pyzfs +%description -n python%{__python_pkg_version}-pyzfs This package provides a python wrapper for the libzfs_core C library. %endif @@ -299,6 +316,12 @@ image which is ZFS aware. %define systemd --enable-sysvinit --disable-systemd %endif +%if %{with pyzfs} + %define pyzfs --enable-pyzfs +%else + %define pyzfs --disable-pyzfs +%endif + %setup -q %build @@ -307,11 +330,13 @@ image which is ZFS aware. --with-udevdir=%{_udevdir} \ --with-udevruledir=%{_udevruledir} \ --with-dracutdir=%{_dracutdir} \ + --with-python=%{__python} \ --disable-static \ %{debug} \ %{debuginfo} \ %{asan} \ - %{systemd} + %{systemd}\ + %{pyzfs} make %{?_smp_mflags} %install @@ -379,12 +404,20 @@ systemctl --system daemon-reload >/dev/null || true %endif %files +# Core utilities %{_sbindir}/* -%{_bindir}/* -%{_libexecdir}/%{name} +%{_bindir}/raidz_test +%{_bindir}/zgenhostid +# Optional Python 2/3 scripts +%{_bindir}/arc_summary +%{_bindir}/arcstat +%{_bindir}/dbufstat +# Man pages %{_mandir}/man1/* %{_mandir}/man5/* %{_mandir}/man8/* +# Configuration files and scripts +%{_libexecdir}/%{name} %{_udevdir}/vdev_id %{_udevdir}/zvol_id %{_udevdir}/rules.d/* @@ -426,8 +459,8 @@ systemctl --system daemon-reload >/dev/null || true %doc contrib/dracut/README.dracut.markdown %{_dracutdir}/modules.d/* -%if 0%{?_pyzfs} -%files -n pyzfs +%if %{with pyzfs} +%files -n python%{__python_pkg_version}-pyzfs %doc contrib/pyzfs/README %doc contrib/pyzfs/LICENSE %defattr(-,root,root,-) diff --git a/tests/test-runner/bin/Makefile.am b/tests/test-runner/bin/Makefile.am index e843e4e09..30c564e55 100644 --- a/tests/test-runner/bin/Makefile.am +++ b/tests/test-runner/bin/Makefile.am @@ -2,3 +2,14 @@ pkgdatadir = $(datadir)/@PACKAGE@/test-runner/bin dist_pkgdata_SCRIPTS = \ test-runner.py \ zts-report.py +# +# These scripts are compatibile with both Python 2.6 and 3.4. As such the +# python 3 shebang can be replaced at install time when targeting a python +# 2 system. This allows us to maintain a single version of the source. +# +if USING_PYTHON_2 +install-data-hook: + sed --in-place 's|^#!/usr/bin/python3|#!/usr/bin/python2|' \ + $(DESTDIR)$(pkgdatadir)/test-runner.py \ + $(DESTDIR)$(pkgdatadir)/zts-report.py +endif diff --git a/tests/test-runner/bin/test-runner.py b/tests/test-runner/bin/test-runner.py index 7ef8a87ed..2e26fa266 100755 --- a/tests/test-runner/bin/test-runner.py +++ b/tests/test-runner/bin/test-runner.py @@ -15,6 +15,8 @@ # Copyright (c) 2012, 2015 by Delphix. All rights reserved. # Copyright (c) 2017 Datto Inc. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# # some python 2.7 system don't have a configparser shim try: diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py index 950295601..4e51bc94e 100755 --- a/tests/test-runner/bin/zts-report.py +++ b/tests/test-runner/bin/zts-report.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # This file and its contents are supplied under the terms of the @@ -15,6 +15,8 @@ # Copyright (c) 2017 by Delphix. All rights reserved. # Copyright (c) 2018 by Lawrence Livermore National Security, LLC. # +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# import os import re diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 5efcb6102..8ced03e93 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -146,10 +146,10 @@ export ZFS_FILES='zdb zpool ztest raidz_test - arc_summary.py - arc_summary3.py - arcstat.py - dbufstat.py + arc_summary + arc_summary3 + arcstat + dbufstat zed zgenhostid zstreamdump' diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh index 5ceff962d..7ec9eaf4c 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_001_pos.ksh @@ -37,7 +37,7 @@ # 2. Store output from dbufs kstat # 3. Store output from dbufstats kstat # 4. Compare stats presented in dbufstats with stat generated using -# dbufstat.py and the dbufs kstat output +# dbufstat and the dbufs kstat output # DBUFSTATS_FILE=$(mktemp $TEST_BASE_DIR/dbufstats.out.XXXXXX) @@ -56,7 +56,7 @@ function testdbufstat # stat_name dbufstat_filter [[ -n "$2" ]] && filter="-F $2" from_dbufstat=$(grep -w "$name" "$DBUFSTATS_FILE" | awk '{ print $3 }') - from_dbufs=$(dbufstat.py -bxn -i "$DBUFS_FILE" "$filter" | wc -l) + from_dbufs=$(dbufstat -bxn -i "$DBUFS_FILE" "$filter" | wc -l) within_tolerance $from_dbufstat $from_dbufs 9 \ || log_fail "Stat $name exceeded tolerance" diff --git a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh index e256bfabe..dc30b6606 100755 --- a/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/arc/dbufstats_002_pos.ksh @@ -62,18 +62,18 @@ objid=$(stat --format="%i" "$TESTDIR/file") log_note "Object ID for $TESTDIR/file is $objid" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mru" "mru count" verify_eq "0" "$mfu" "mfu count" log_must eval "cat $TESTDIR/file > /dev/null" log_must eval "cat /proc/spl/kstat/zfs/dbufs > $DBUFS_FILE" -dbuf=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) -mru=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) -mfu=$(dbufstat.py -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) +dbuf=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid" | wc -l) +mru=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=1" | wc -l) +mfu=$(dbufstat -bxn -i "$DBUFS_FILE" -F "object=$objid,list=3" | wc -l) log_note "dbuf count is $dbuf, mru count is $mru, mfu count is $mfu" verify_ne "0" "$mfu" "mfu count" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh index 22dceaaf4..ff090baee 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary3_001_pos.ksh @@ -45,12 +45,12 @@ fi set -A args "" "-a" "-d" "-p 1" "-g" "-s arc" "-r" -log_assert "arc_summary3.py generates output and doesn't return an error code" +log_assert "arc_summary3 generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary3.py ${args[i]} > /dev/null" + log_must eval "arc_summary3 ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arc_summary3.py generates output and doesn't return an error code" +log_pass "arc_summary3 generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh index 6653b9c1a..8736b18d8 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-a" "-d" "-p 1" -log_assert "arc_summary.py generates output and doesn't return an error code" +log_assert "arc_summary generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arc_summary.py ${args[i]} > /dev/null" + log_must eval "arc_summary ${args[i]} > /dev/null" ((i = i + 1)) done -log_must eval "arc_summary.py | head > /dev/null" -log_must eval "arc_summary.py | head -1 > /dev/null" +log_must eval "arc_summary | head > /dev/null" +log_must eval "arc_summary | head -1 > /dev/null" -log_pass "arc_summary.py generates output and doesn't return an error code" +log_pass "arc_summary generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh index e63552feb..477b7ca08 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arc_summary_002_neg.ksh @@ -29,10 +29,10 @@ typeset args=("-x" "-r" "-5" "-p 7" "--err" "-@") -log_assert "arc_summary.py generates an error code with invalid options" +log_assert "arc_summary generates an error code with invalid options" for arg in "${args[@]}"; do - log_mustnot eval "arc_summary.py $arg > /dev/null" + log_mustnot eval "arc_summary $arg > /dev/null" done -log_pass "arc_summary.py generates an error code with invalid options" +log_pass "arc_summary generates an error code with invalid options" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh index c8a89f8c4..ab574731f 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/arcstat_001_pos.ksh @@ -30,12 +30,12 @@ set -A args "" "-s \",\"" "-x" "-v" \ "-f time,hit%,dh%,ph%,mh%" -log_assert "arcstat.py generates output and doesn't return an error code" +log_assert "arcstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "arcstat.py ${args[i]} > /dev/null" + log_must eval "arcstat ${args[i]} > /dev/null" ((i = i + 1)) done -log_pass "arcstat.py generates output and doesn't return an error code" +log_pass "arcstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh index 1c267d6af..95f0598c6 100755 --- a/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_user/misc/dbufstat_001_pos.ksh @@ -29,15 +29,15 @@ set -A args "" "-b" "-d" "-r" "-v" "-s \",\"" "-x" "-n" -log_assert "dbufstat.py generates output and doesn't return an error code" +log_assert "dbufstat generates output and doesn't return an error code" typeset -i i=0 while [[ $i -lt ${#args[*]} ]]; do - log_must eval "dbufstat.py ${args[i]} > /dev/null" + log_must eval "dbufstat ${args[i]} > /dev/null" ((i = i + 1)) done -# A simple test of dbufstat.py filter functionality -log_must eval "dbufstat.py -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" +# A simple test of dbufstat filter functionality +log_must eval "dbufstat -F object=10,dbc=1,pool=$TESTPOOL > /dev/null" -log_pass "dbufstat.py generates output and doesn't return an error code" +log_pass "dbufstat generates output and doesn't return an error code" diff --git a/tests/zfs-tests/tests/functional/pyzfs/.gitignore b/tests/zfs-tests/tests/functional/pyzfs/.gitignore new file mode 100644 index 000000000..bcbe0573e --- /dev/null +++ b/tests/zfs-tests/tests/functional/pyzfs/.gitignore @@ -0,0 +1 @@ +pyzfs_unittest.ksh diff --git a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am index 61cb3d074..0a27adecc 100644 --- a/tests/zfs-tests/tests/functional/pyzfs/Makefile.am +++ b/tests/zfs-tests/tests/functional/pyzfs/Makefile.am @@ -1,4 +1,18 @@ -pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs - -dist_pkgdata_SCRIPTS = \ +pkgpyzfsdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/pyzfs +pkgpyzfs_SCRIPTS = \ pyzfs_unittest.ksh + +EXTRA_DIST = \ + pyzfs_unittest.ksh.in + +# +# The pyzfs module is built either for Python 2 or Python 3. In order +# to properly test it the unit tests must be updated to the matching vesion. +# +$(pkgpyzfs_SCRIPTS):%:%.in + -$(SED) -e 's,@PYTHON\@,$(PYTHON),g' \ + $< >'$@' + -chmod 775 $@ + +distclean-local:: + -$(RM) $(pkgpyzfs_SCRIPTS) diff --git a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in index fb4b60361..4ca610e5f 100755 --- a/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh +++ b/tests/zfs-tests/tests/functional/pyzfs/pyzfs_unittest.ksh.in @@ -28,7 +28,7 @@ verify_runnable "global" # Verify that the required dependencies for testing are installed. -python -c "import cffi" 2>/dev/null +@PYTHON@ -c "import cffi" 2>/dev/null if [ $? -eq 1 ]; then log_unsupported "python-cffi not found by Python" fi @@ -37,7 +37,7 @@ fi # only if pyzfs was not installed due to missing, build-time, dependencies; if # we cannot load "libzfs_core" due to other reasons, for instance an API/ABI # mismatch, we want to report it. -python -c ' +@PYTHON@ -c ' import pkgutil, sys sys.exit(pkgutil.find_loader("libzfs_core") is None)' if [ $? -eq 1 ]; then @@ -47,7 +47,7 @@ fi log_assert "Verify the nvlist and libzfs_core Python unittest run successfully" # NOTE: don't use log_must() here because it makes output unreadable -python -m unittest --verbose \ +@PYTHON@ -m unittest --verbose \ libzfs_core.test.test_nvlist.TestNVList \ libzfs_core.test.test_libzfs_core.ZFSTest if [ $? -ne 0 ]; then |