pyzfs: python3 support (build system)

Almost all of the Python code in the respository has been updated
to be compatibile with Python 2.6, Python 3.4, or newer.  The only
exceptions are arc_summery3.py which requires Python 3, and pyzfs
which requires at least Python 2.7.  This allows us to maintain a
single version of the code and support most default versions of
python.  This change does the following:

* Sets the default shebang for all Python scripts to python3.  If
  only Python 2 is available, then at install time scripts which
  are compatible with Python 2 will have their shebangs replaced
  with /usr/bin/python.  This is done for compatibility until
  Python 2 goes end of life.  Since only the installed versions
  are changed this means Python 3 must be installed on the system
  for test-runner when testing in-tree.

* Added --with-python=<2|3|3.4,etc> configure option which sets
  the PYTHON environment variable to target a specific python
  version.  By default the newest installed version of Python
  will be used or the preferred distribution version when
  creating pacakges.

* Fixed --enable-pyzfs configure checks so they are run when
  --enable-pyzfs=check and --enable-pyzfs=yes.

* Enabled pyzfs for Python 3.4 and newer, which is now supported.

* Renamed pyzfs package to python<VERSION>-pyzfs and updated to
  install in the appropriate site location.  For example, when
  building with --with-python=3.4 a python34-pyzfs will be
  created which installs in /usr/lib/python3.4/site-packages/.

* Renamed the following python scripts according to the Fedora
  guidance for packaging utilities in /bin

  - dbufstat.py     -> dbufstat
  - arcstat.py      -> arcstat
  - arc_summary.py  -> arc_summary
  - arc_summary3.py -> arc_summary3

* Updated python-cffi package name.  On CentOS 6, CentOS 7, and
  Amazon Linux it's called python-cffi, not python2-cffi.  For
  Python3 it's called python3-cffi or python3x-cffi.

* Install one version of arc_summary.  Depending on the version
  of Python available install either arc_summary2 or arc_summary3
  as arc_summary.  The user output is only slightly different.

Reviewed-by: John Ramsden <[email protected]>
Reviewed-by: Neal Gompa <[email protected]>
Reviewed-by: loli10K <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #8096

4 files changed, 138 insertions, 72 deletions

diff --git a/config/always-python.m4 b/config/always-python.m4
new file mode 100644
index 000000000..858ab7b01
--- /dev/null
+++ b/config/always-python.m4
@@ -0,0 +1,102 @@
+dnl #
+dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false])
+dnl #
+dnl # Verify Python version
+dnl #
+AC_DEFUN([ZFS_AC_PYTHON_VERSION], [
+	ver_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"`
+	AS_IF([test "$ver_check" = "True"], [
+		m4_ifvaln([$2], [$2])
+	], [
+		m4_ifvaln([$3], [$3])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
+dnl #
+dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
+dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
+dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS.
+dnl #
+AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
+	PYTHON_NAME=`basename $PYTHON`
+	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
+	AS_IF([$PYTHON -c "import $1" 2>/dev/null], [
+		AC_MSG_RESULT(yes)
+		m4_ifvaln([$2], [$2])
+	], [
+		AC_MSG_RESULT(no)
+		m4_ifvaln([$3], [$3])
+	])
+])
+
+dnl #
+dnl # The majority of the python scripts are written to be compatible
+dnl # with Python 2.6 and Python 3.4.  Therefore, they may be installed
+dnl # and used with either interpreter.  This option is intended to
+dnl # to provide a method to specify the default system version, and
+dnl # set the PYTHON environment variable accordingly.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
+	AC_ARG_WITH([python],
+		AC_HELP_STRING([--with-python[=VERSION]],
+		[default system python version @<:@default=check@:>@]),
+		[with_python=$withval],
+		[with_python=check])
+
+	AS_CASE([$with_python],
+		[check],
+		[AS_IF([test -x /usr/bin/python3],
+			[PYTHON="python3"],
+			[AS_IF([test -x /usr/bin/python2],
+				[PYTHON="python2"],
+				[PYTHON=""]
+			)]
+		)],
+		[2*], [PYTHON="python${with_python}"],
+		[*python2*], [PYTHON="${with_python}"],
+		[3*], [PYTHON="python${with_python}"],
+		[*python3*], [PYTHON="${with_python}"],
+		[no], [PYTHON=""],
+		[AC_MSG_ERROR([Unknown --with-python value '$with_python'])]
+	)
+
+	AS_IF([$PYTHON --version >/dev/null 2>&1], [ /bin/true ], [
+		AC_MSG_ERROR([Cannot find $PYTHON in your system path])
+	])
+
+	AM_PATH_PYTHON([2.6], [], [:])
+	AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :])
+	AM_CONDITIONAL([USING_PYTHON_2], [test "${PYTHON_VERSION:0:2}" = "2."])
+	AM_CONDITIONAL([USING_PYTHON_3], [test "${PYTHON_VERSION:0:2}" = "3."])
+
+	dnl #
+	dnl # Minimum supported Python versions for utilities:
+	dnl # Python 2.6.x, or Python 3.4.x
+	dnl #
+	AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [
+		ZFS_AC_PYTHON_VERSION([>= '2.6'], [ /bin/true ],
+			[AC_MSG_ERROR("Python >= 2.6.x is not available")])
+	])
+
+	AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [
+		ZFS_AC_PYTHON_VERSION([>= '3.4'], [ /bin/true ],
+			[AC_MSG_ERROR("Python >= 3.4.x is not available")])
+	])
+
+	dnl #
+	dnl # Request that packages be built for a specific Python version.
+	dnl #
+	AS_IF([test $with_python != check], [
+		PYTHON_PKG_VERSION=`echo ${PYTHON} | tr -d 'a-zA-Z.'`
+		DEFINE_PYTHON_PKG_VERSION='--define "__use_python_pkg_version '${PYTHON_PKG_VERSION}'"'
+		DEFINE_PYTHON_VERSION='--define "__use_python '${PYTHON}'"'
+	], [
+		DEFINE_PYTHON_VERSION=''
+		DEFINE_PYTHON_PKG_VERSION=''
+	])
+
+	AC_SUBST(DEFINE_PYTHON_VERSION)
+	AC_SUBST(DEFINE_PYTHON_PKG_VERSION)
+])
diff --git a/config/always-pyzfs.m4 b/config/always-pyzfs.m4
index c50acb099..d74d6f1a7 100644
--- a/config/always-pyzfs.m4
+++ b/config/always-pyzfs.m4
@@ -1,80 +1,44 @@
 dnl #
-dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
+dnl # Determines if pyzfs can be built, requires Python 2.7 or latter.
 dnl #
-dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
-dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
-dnl #
-AC_DEFUN([ZFS_AC_PYTHON_MODULE],[
-	PYTHON_NAME=`basename $PYTHON`
-	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
-	$PYTHON -c "import $1" 2>/dev/null
-	if test $? -eq 0;
-	then
-		AC_MSG_RESULT(yes)
-		m4_ifvaln([$2], [$2])
-	else
-		AC_MSG_RESULT(no)
-		m4_ifvaln([$3], [$3])
-	fi
-])
-
-dnl #
-dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false])
-dnl #
-dnl # Verify Python version
-dnl #
-AC_DEFUN([ZFS_AC_PYTHON_VERSION], [
-	AC_MSG_CHECKING([for a version of Python $1])
-	version_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"`
-	if test "$version_check" = "True";
-	then
-		AC_MSG_RESULT(yes)
-		m4_ifvaln([$2], [$2])
-	else
-		AC_MSG_RESULT(no)
-		m4_ifvaln([$3], [$3])
-	fi
-
-])
-
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
-	PYTHON_REQUIRED_VERSION="<= '2.7.x'"
-
 	AC_ARG_ENABLE([pyzfs],
 		AC_HELP_STRING([--enable-pyzfs],
 		[install libzfs_core python bindings @<:@default=check@:>@]),
 		[enable_pyzfs=$enableval],
 		[enable_pyzfs=check])
 
-	AM_PATH_PYTHON([2.7], [], [
+	dnl #
+	dnl # Packages for pyzfs specifically enabled/disabled.
+	dnl #
+	AS_IF([test "x$enable_pyzfs" != xcheck], [
 		AS_IF([test "x$enable_pyzfs" = xyes], [
-			AC_MSG_ERROR("python >= 2.7 is not installed")
-		], [test ! "x$enable_pyzfs" = xno], [
-			enable_pyzfs=no
+			DEFINE_PYZFS='--with pyzfs'
+		], [
+			DEFINE_PYZFS='--without pyzfs'
 		])
+	], [
+		DEFINE_PYZFS=''
 	])
-	AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != :])
+	AC_SUBST(DEFINE_PYZFS)
 
 	dnl #
-	dnl # Python 2.7.x is supported, other versions (3.5) are not yet
+	dnl # Require python-devel libraries
 	dnl #
-	AS_IF([test "x$enable_pyzfs" = xcheck], [
-		ZFS_AC_PYTHON_VERSION([$PYTHON_REQUIRED_VERSION], [], [
-			AS_IF([test "x$enable_pyzfs" = xyes], [
-				AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION is not available")
-			], [test ! "x$enable_pyzfs" = xno], [
-				enable_pyzfs=no
+	AS_IF([test "x$enable_pyzfs" = xcheck  -o "x$enable_pyzfs" = xyes], [
+		AS_IF([test "${PYTHON_VERSION:0:2}" = "2."], [
+			PYTHON_REQUIRED_VERSION=">= '2.7.0'"
+		], [
+			AS_IF([test "${PYTHON_VERSION:0:2}" = "3."], [
+				PYTHON_REQUIRED_VERSION=">= '3.4.0'"
+			], [
+				AC_MSG_ERROR("Python $PYTHON_VERSION unknown")
 			])
 		])
-	])
 
-	dnl #
-	dnl # Require python-devel libraries
-	dnl #
-	AS_IF([test "x$enable_pyzfs" = xcheck], [
 		AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
-				AC_MSG_ERROR("Python development library is not available")
+				AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION development library is not installed")
 			], [test ! "x$enable_pyzfs" = xno], [
 				enable_pyzfs=no
 			])
@@ -84,10 +48,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	dnl #
 	dnl # Python "setuptools" module is required to build and install pyzfs
 	dnl #
-	AS_IF([test "x$enable_pyzfs" = xcheck], [
+	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
 		ZFS_AC_PYTHON_MODULE([setuptools], [], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
-				AC_MSG_ERROR("python-setuptools is not installed")
+				AC_MSG_ERROR("Python $PYTHON_VERSION setuptools is not installed")
 			], [test ! "x$enable_pyzfs" = xno], [
 				enable_pyzfs=no
 			])
@@ -97,10 +61,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	dnl #
 	dnl # Python "cffi" module is required to run pyzfs
 	dnl #
-	AS_IF([test "x$enable_pyzfs" = xcheck], [
+	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
 		ZFS_AC_PYTHON_MODULE([cffi], [], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
-				AC_MSG_ERROR("python-cffi is not installed")
+				AC_MSG_ERROR("Python $PYTHON_VERSION cffi is not installed")
 			], [test ! "x$enable_pyzfs" = xno], [
 				enable_pyzfs=no
 			])
@@ -114,12 +78,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 
 	AM_CONDITIONAL([PYZFS_ENABLED], [test x$enable_pyzfs = xyes])
 	AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs])
-
-	AS_IF([test "x$enable_pyzfs" = xyes], [
-		DEFINE_PYZFS='--define "_pyzfs 1"'
-	],[
-		DEFINE_PYZFS=''
-	])
-	AC_SUBST(DEFINE_PYZFS)
 	AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG])
+
+	AC_MSG_CHECKING([whether to enable pyzfs: ])
+	AC_MSG_RESULT($enable_pyzfs)
 ])
diff --git a/config/deb.am b/config/deb.am
index eb4e5bbda..e405547aa 100644
--- a/config/deb.am
+++ b/config/deb.am
@@ -47,7 +47,7 @@ deb-utils: deb-local rpm-utils
 	pkg7=$${name}-test-$${version}.$${arch}.rpm; \
 	pkg8=$${name}-dracut-$${version}.$${arch}.rpm; \
 	pkg9=$${name}-initramfs-$${version}.$${arch}.rpm; \
-	pkg10=pyzfs-$${version}.noarch.rpm; \
+	pkg10=`ls python*-pyzfs-$${version}* | tail -1`; \
 ## Arguments need to be passed to dh_shlibdeps. Alien provides no mechanism
 ## to do this, so we install a shim onto the path which calls the real
 ## dh_shlibdeps with the required arguments.
diff --git a/config/zfs-build.m4 b/config/zfs-build.m4
index 1d47b0384..6e305996e 100644
--- a/config/zfs-build.m4
+++ b/config/zfs-build.m4
@@ -160,6 +160,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_ARCH
+	ZFS_AC_CONFIG_ALWAYS_PYTHON
 	ZFS_AC_CONFIG_ALWAYS_PYZFS
 ])
 
@@ -264,10 +265,13 @@ AC_DEFUN([ZFS_AC_RPM], [
 	RPM_DEFINE_UTIL+=' $(DEFINE_INITRAMFS)'
 	RPM_DEFINE_UTIL+=' $(DEFINE_SYSTEMD)'
 	RPM_DEFINE_UTIL+=' $(DEFINE_PYZFS)'
+	RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_VERSION)'
+	RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_PKG_VERSION)'
 
-	dnl # Override default lib directory on Debian/Ubuntu systems.  The provided
-	dnl # /usr/lib/rpm/platform/<arch>/macros files do not specify the correct
-	dnl # path for multiarch systems as described by the packaging guidelines.
+	dnl # Override default lib directory on Debian/Ubuntu systems.  The
+	dnl # provided /usr/lib/rpm/platform/<arch>/macros files do not
+	dnl # specify the correct path for multiarch systems as described
+	dnl # by the packaging guidelines.
 	dnl #
 	dnl # https://wiki.ubuntu.com/MultiarchSpec
 	dnl # https://wiki.debian.org/Multiarch/Implementation