Adopt pyzfs from ClusterHQ

This commit introduces several changes:

 * Update LICENSE and project information

 * Give a good PEP8 talk to existing Python source code

 * Add RPM/DEB packaging for pyzfs

 * Fix some outstanding issues with the existing pyzfs code caused by
   changes in the ABI since the last time the code was updated

 * Integrate pyzfs Python unittest with the ZFS Test Suite

 * Add missing libzfs_core functions: lzc_change_key,
   lzc_channel_program, lzc_channel_program_nosync, lzc_load_key,
   lzc_receive_one, lzc_receive_resumable, lzc_receive_with_cmdprops,
   lzc_receive_with_header, lzc_reopen, lzc_send_resume, lzc_sync,
   lzc_unload_key, lzc_remap

Note: this commit slightly changes zfs_ioc_unload_key() ABI. This allow
to differentiate the case where we tried to unload a key on a
non-existing dataset (ENOENT) from the situation where a dataset has
no key loaded: this is consistent with the "change" case where trying
to zfs_ioc_change_key() from a dataset with no key results in EACCES.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: loli10K <[email protected]>
Closes #7230

4 files changed, 475 insertions, 2 deletions

diff --git a/config/always-pyzfs.m4 b/config/always-pyzfs.m4
new file mode 100644
index 000000000..1df11a5d7
--- /dev/null
+++ b/config/always-pyzfs.m4
@@ -0,0 +1,125 @@
+dnl #
+dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
+dnl #
+dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
+dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
+dnl #
+AC_DEFUN([ZFS_AC_PYTHON_MODULE],[
+	PYTHON_NAME=`basename $PYTHON`
+	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
+	$PYTHON -c "import $1" 2>/dev/null
+	if test $? -eq 0;
+	then
+		AC_MSG_RESULT(yes)
+		m4_ifvaln([$2], [$2])
+	else
+		AC_MSG_RESULT(no)
+		m4_ifvaln([$3], [$3])
+	fi
+])
+
+dnl #
+dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false])
+dnl #
+dnl # Verify Python version
+dnl #
+AC_DEFUN([ZFS_AC_PYTHON_VERSION], [
+	AC_MSG_CHECKING([for a version of Python $1])
+	version_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"`
+	if test "$version_check" = "True";
+	then
+		AC_MSG_RESULT(yes)
+		m4_ifvaln([$2], [$2])
+	else
+		AC_MSG_RESULT(no)
+		m4_ifvaln([$3], [$3])
+	fi
+
+])
+
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
+	PYTHON_REQUIRED_VERSION="<= '2.7.x'"
+
+	AC_ARG_ENABLE([pyzfs],
+		AC_HELP_STRING([--enable-pyzfs],
+		[install libzfs_core python bindings @<:@default=check@:>@]),
+		[enable_pyzfs=$enableval],
+		[enable_pyzfs=check])
+
+	AM_PATH_PYTHON([2.7], [], [
+		AS_IF([test ! "x$enable_pyzfs" = xyes], [
+			AC_MSG_ERROR("python >= 2.7 is not installed")
+		], [test ! "x$enable_pyzfs" = xno], [
+			enable_pyzfs=no
+		])
+	])
+	AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != :])
+
+	dnl #
+	dnl # Python 2.7.x is supported, other versions (3.5) are not yet
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck], [
+		ZFS_AC_PYTHON_VERSION([$PYTHON_REQUIRED_VERSION], [], [
+			AS_IF([test "x$enable_pyzfs" = xyes], [
+				AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION is not available")
+			], [test ! "x$enable_pyzfs" = xno], [
+				enable_pyzfs=no
+			])
+		])
+	])
+
+	dnl #
+	dnl # Require python-devel libraries
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck], [
+		AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [
+			AS_IF([test "x$enable_pyzfs" = xyes], [
+				AC_MSG_ERROR("Python development library is not available")
+			], [test ! "x$enable_pyzfs" = xno], [
+				enable_pyzfs=no
+			])
+		])
+	])
+
+	dnl #
+	dnl # Python "setuptools" module is required to build and install pyzfs
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck], [
+		ZFS_AC_PYTHON_MODULE([setuptools], [], [
+			AS_IF([test "x$enable_pyzfs" = xyes], [
+				AC_MSG_ERROR("python-setuptools is not installed")
+			], [test ! "x$enable_pyzfs" = xno], [
+				enable_pyzfs=no
+			])
+		])
+	])
+
+	dnl #
+	dnl # Python "cffi" module is required to run pyzfs
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck], [
+		ZFS_AC_PYTHON_MODULE([cffi], [], [
+			AS_IF([test "x$enable_pyzfs" = xyes], [
+				AC_MSG_ERROR("python-cffi is not installed")
+			], [test ! "x$enable_pyzfs" = xno], [
+				enable_pyzfs=no
+			])
+		])
+	])
+
+	dnl #
+	dnl # Set enable_pyzfs to 'yes' if every check passed
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck], [enable_pyzfs=yes])
+
+	AM_CONDITIONAL([PYZFS_ENABLED], [test x$enable_pyzfs = xyes])
+	AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs])
+
+	AS_IF([test "x$enable_pyzfs" = xyes], [
+		DEFINE_PYZFS='--define "_pyzfs 1"'
+	],[
+		DEFINE_PYZFS=''
+	])
+	AC_SUBST(DEFINE_PYZFS)
+	AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG])
+])
diff --git a/config/ax_python_devel.m4 b/config/ax_python_devel.m4
new file mode 100644
index 000000000..c51b45b7d
--- /dev/null
+++ b/config/ax_python_devel.m4
@@ -0,0 +1,345 @@
+# ===========================================================================
+#     https://www.gnu.org/software/autoconf-archive/ax_python_devel.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_PYTHON_DEVEL([version], [action-if-not-found])
+#
+# DESCRIPTION
+#
+#   Note: Defines as a precious variable "PYTHON_VERSION". Don't override it
+#   in your configure.ac.
+#
+#   Note: this is a slightly modified version of the original AX_PYTHON_DEVEL
+#   macro which accepts an additional [action-if-not-found] argument. This
+#   allow to detect if Python development is available without aborting the
+#   configure phase with an hard error in case it is not.
+#
+#   This macro checks for Python and tries to get the include path to
+#   'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LIBS) output
+#   variables. It also exports $(PYTHON_EXTRA_LIBS) and
+#   $(PYTHON_EXTRA_LDFLAGS) for embedding Python in your code.
+#
+#   You can search for some particular version of Python by passing a
+#   parameter to this macro, for example ">= '2.3.1'", or "== '2.4'". Please
+#   note that you *have* to pass also an operator along with the version to
+#   match, and pay special attention to the single quotes surrounding the
+#   version number. Don't use "PYTHON_VERSION" for this: that environment
+#   variable is declared as precious and thus reserved for the end-user.
+#
+#   This macro should work for all versions of Python >= 2.1.0. As an end
+#   user, you can disable the check for the python version by setting the
+#   PYTHON_NOVERSIONCHECK environment variable to something else than the
+#   empty string.
+#
+#   If you need to use this macro for an older Python version, please
+#   contact the authors. We're always open for feedback.
+#
+# LICENSE
+#
+#   Copyright (c) 2009 Sebastian Huber <[email protected]>
+#   Copyright (c) 2009 Alan W. Irwin
+#   Copyright (c) 2009 Rafael Laboissiere <[email protected]>
+#   Copyright (c) 2009 Andrew Collier
+#   Copyright (c) 2009 Matteo Settenvini <[email protected]>
+#   Copyright (c) 2009 Horst Knorr <[email protected]>
+#   Copyright (c) 2013 Daniel Mullner <[email protected]>
+#   Copyright (c) 2018 loli10K <[email protected]>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 21
+
+AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL])
+AC_DEFUN([AX_PYTHON_DEVEL],[
+	#
+	# Allow the use of a (user set) custom python version
+	#
+	AC_ARG_VAR([PYTHON_VERSION],[The installed Python
+		version to use, for example '2.3'. This string
+		will be appended to the Python interpreter
+		canonical name.])
+
+	AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]])
+	if test -z "$PYTHON"; then
+		m4_ifvaln([$2],[$2],[
+			AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path])
+			PYTHON_VERSION=""
+		])
+	fi
+
+	#
+	# Check for a version of Python >= 2.1.0
+	#
+	AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
+	ac_supports_python_ver=`$PYTHON -c "import sys; \
+		ver = sys.version.split ()[[0]]; \
+		print (ver >= '2.1.0')"`
+	if test "$ac_supports_python_ver" != "True"; then
+		if test -z "$PYTHON_NOVERSIONCHECK"; then
+			AC_MSG_RESULT([no])
+			m4_ifvaln([$2],[$2],[
+				AC_MSG_FAILURE([
+This version of the AC@&t@_PYTHON_DEVEL macro
+doesn't work properly with versions of Python before
+2.1.0. You may need to re-run configure, setting the
+variables PYTHON_CPPFLAGS, PYTHON_LIBS, PYTHON_SITE_PKG,
+PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand.
+Moreover, to disable this check, set PYTHON_NOVERSIONCHECK
+to something else than an empty string.
+])
+			])
+		else
+			AC_MSG_RESULT([skip at user request])
+		fi
+	else
+		AC_MSG_RESULT([yes])
+	fi
+
+	#
+	# if the macro parameter ``version'' is set, honour it
+	#
+	if test -n "$1"; then
+		AC_MSG_CHECKING([for a version of Python $1])
+		ac_supports_python_ver=`$PYTHON -c "import sys; \
+			ver = sys.version.split ()[[0]]; \
+			print (ver $1)"`
+		if test "$ac_supports_python_ver" = "True"; then
+		   AC_MSG_RESULT([yes])
+		else
+			AC_MSG_RESULT([no])
+			m4_ifvaln([$2],[$2],[
+				AC_MSG_ERROR([this package requires Python $1.
+If you have it installed, but it isn't the default Python
+interpreter in your system path, please pass the PYTHON_VERSION
+variable to configure. See ``configure --help'' for reference.
+])
+				PYTHON_VERSION=""
+			])
+		fi
+	fi
+
+	#
+	# Check if you have distutils, else fail
+	#
+	AC_MSG_CHECKING([for the distutils Python package])
+	ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`
+	if test $? -eq 0; then
+		AC_MSG_RESULT([yes])
+	else
+		AC_MSG_RESULT([no])
+		m4_ifvaln([$2],[$2],[
+			AC_MSG_ERROR([cannot import Python module "distutils".
+Please check your Python installation. The error was:
+$ac_distutils_result])
+			PYTHON_VERSION=""
+		])
+	fi
+
+	#
+	# Check for Python include path
+	#
+	AC_MSG_CHECKING([for Python include path])
+	if test -z "$PYTHON_CPPFLAGS"; then
+		python_path=`$PYTHON -c "import distutils.sysconfig; \
+			print (distutils.sysconfig.get_python_inc ());"`
+		plat_python_path=`$PYTHON -c "import distutils.sysconfig; \
+			print (distutils.sysconfig.get_python_inc (plat_specific=1));"`
+		if test -n "${python_path}"; then
+			if test "${plat_python_path}" != "${python_path}"; then
+				python_path="-I$python_path -I$plat_python_path"
+			else
+				python_path="-I$python_path"
+			fi
+		fi
+		PYTHON_CPPFLAGS=$python_path
+	fi
+	AC_MSG_RESULT([$PYTHON_CPPFLAGS])
+	AC_SUBST([PYTHON_CPPFLAGS])
+
+	#
+	# Check for Python library path
+	#
+	AC_MSG_CHECKING([for Python library path])
+	if test -z "$PYTHON_LIBS"; then
+		# (makes two attempts to ensure we've got a version number
+		# from the interpreter)
+		ac_python_version=`cat<<EOD | $PYTHON -
+
+# join all versioning strings, on some systems
+# major/minor numbers could be in different list elements
+from distutils.sysconfig import *
+e = get_config_var('VERSION')
+if e is not None:
+	print(e)
+EOD`
+
+		if test -z "$ac_python_version"; then
+			if test -n "$PYTHON_VERSION"; then
+				ac_python_version=$PYTHON_VERSION
+			else
+				ac_python_version=`$PYTHON -c "import sys; \
+					print (sys.version[[:3]])"`
+			fi
+		fi
+
+		# Make the versioning information available to the compiler
+		AC_DEFINE_UNQUOTED([HAVE_PYTHON], ["$ac_python_version"],
+                                   [If available, contains the Python version number currently in use.])
+
+		# First, the library directory:
+		ac_python_libdir=`cat<<EOD | $PYTHON -
+
+# There should be only one
+import distutils.sysconfig
+e = distutils.sysconfig.get_config_var('LIBDIR')
+if e is not None:
+	print (e)
+EOD`
+
+		# Now, for the library:
+		ac_python_library=`cat<<EOD | $PYTHON -
+
+import distutils.sysconfig
+c = distutils.sysconfig.get_config_vars()
+if 'LDVERSION' in c:
+	print ('python'+c[['LDVERSION']])
+else:
+	print ('python'+c[['VERSION']])
+EOD`
+
+		# This small piece shamelessly adapted from PostgreSQL python macro;
+		# credits goes to momjian, I think. I'd like to put the right name
+		# in the credits, if someone can point me in the right direction... ?
+		#
+		if test -n "$ac_python_libdir" -a -n "$ac_python_library"
+		then
+			# use the official shared library
+			ac_python_library=`echo "$ac_python_library" | sed "s/^lib//"`
+			PYTHON_LIBS="-L$ac_python_libdir -l$ac_python_library"
+		else
+			# old way: use libpython from python_configdir
+			ac_python_libdir=`$PYTHON -c \
+			  "from distutils.sysconfig import get_python_lib as f; \
+			  import os; \
+			  print (os.path.join(f(plat_specific=1, standard_lib=1), 'config'));"`
+			PYTHON_LIBS="-L$ac_python_libdir -lpython$ac_python_version"
+		fi
+
+		if test -z "PYTHON_LIBS"; then
+			m4_ifvaln([$2],[$2],[
+				AC_MSG_ERROR([
+  Cannot determine location of your Python DSO. Please check it was installed with
+  dynamic libraries enabled, or try setting PYTHON_LIBS by hand.
+				])
+			])
+		fi
+	fi
+	AC_MSG_RESULT([$PYTHON_LIBS])
+	AC_SUBST([PYTHON_LIBS])
+
+	#
+	# Check for site packages
+	#
+	AC_MSG_CHECKING([for Python site-packages path])
+	if test -z "$PYTHON_SITE_PKG"; then
+		PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
+			print (distutils.sysconfig.get_python_lib(0,0));"`
+	fi
+	AC_MSG_RESULT([$PYTHON_SITE_PKG])
+	AC_SUBST([PYTHON_SITE_PKG])
+
+	#
+	# libraries which must be linked in when embedding
+	#
+	AC_MSG_CHECKING(python extra libraries)
+	if test -z "$PYTHON_EXTRA_LIBS"; then
+	   PYTHON_EXTRA_LIBS=`$PYTHON -c "import distutils.sysconfig; \
+                conf = distutils.sysconfig.get_config_var; \
+                print (conf('LIBS') + ' ' + conf('SYSLIBS'))"`
+	fi
+	AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
+	AC_SUBST(PYTHON_EXTRA_LIBS)
+
+	#
+	# linking flags needed when embedding
+	#
+	AC_MSG_CHECKING(python extra linking flags)
+	if test -z "$PYTHON_EXTRA_LDFLAGS"; then
+		PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import distutils.sysconfig; \
+			conf = distutils.sysconfig.get_config_var; \
+			print (conf('LINKFORSHARED'))"`
+	fi
+	AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])
+	AC_SUBST(PYTHON_EXTRA_LDFLAGS)
+
+	#
+	# final check to see if everything compiles alright
+	#
+	AC_MSG_CHECKING([consistency of all components of python development environment])
+	# save current global flags
+	ac_save_LIBS="$LIBS"
+	ac_save_LDFLAGS="$LDFLAGS"
+	ac_save_CPPFLAGS="$CPPFLAGS"
+	LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS $PYTHON_EXTRA_LIBS"
+	LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS"
+	CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS"
+	AC_LANG_PUSH([C])
+	AC_LINK_IFELSE([
+		AC_LANG_PROGRAM([[#include <Python.h>]],
+				[[Py_Initialize();]])
+		],[pythonexists=yes],[pythonexists=no])
+	AC_LANG_POP([C])
+	# turn back to default flags
+	CPPFLAGS="$ac_save_CPPFLAGS"
+	LIBS="$ac_save_LIBS"
+	LDFLAGS="$ac_save_LDFLAGS"
+
+	AC_MSG_RESULT([$pythonexists])
+
+        if test ! "x$pythonexists" = "xyes"; then
+		m4_ifvaln([$2],[$2],[
+			AC_MSG_FAILURE([
+  Could not link test program to Python. Maybe the main Python library has been
+  installed in some non-standard library path. If so, pass it to configure,
+  via the LIBS environment variable.
+  Example: ./configure LIBS="-L/usr/non-standard-path/python/lib"
+  ============================================================================
+   ERROR!
+   You probably have to install the development version of the Python package
+   for your distribution.  The exact name of this package varies among them.
+  ============================================================================
+	   ])
+			PYTHON_VERSION=""
+		])
+	fi
+
+	#
+	# all done!
+	#
+])
diff --git a/config/deb.am b/config/deb.am
index 58ab96e18..eb4e5bbda 100644
--- a/config/deb.am
+++ b/config/deb.am
@@ -47,6 +47,7 @@ deb-utils: deb-local rpm-utils
 	pkg7=$${name}-test-$${version}.$${arch}.rpm; \
 	pkg8=$${name}-dracut-$${version}.$${arch}.rpm; \
 	pkg9=$${name}-initramfs-$${version}.$${arch}.rpm; \
+	pkg10=pyzfs-$${version}.noarch.rpm; \
 ## Arguments need to be passed to dh_shlibdeps. Alien provides no mechanism
 ## to do this, so we install a shim onto the path which calls the real
 ## dh_shlibdeps with the required arguments.
@@ -62,10 +63,10 @@ deb-utils: deb-local rpm-utils
 	env PATH=$${path_prepend}:$${PATH} \
 	fakeroot $(ALIEN) --bump=0 --scripts --to-deb --target=$$debarch \
 	    $$pkg1 $$pkg2 $$pkg3 $$pkg4 $$pkg5 $$pkg6 $$pkg7 \
-	    $$pkg8 $$pkg9; \
+	    $$pkg8 $$pkg9 $$pkg10; \
 	$(RM) $${path_prepend}/dh_shlibdeps; \
 	rmdir $${path_prepend}; \
 	$(RM) $$pkg1 $$pkg2 $$pkg3 $$pkg4 $$pkg5 $$pkg6 $$pkg7 \
-	    $$pkg8 $$pkg9;
+	    $$pkg8 $$pkg9 $$pkg10;
 
 deb: deb-kmod deb-dkms deb-utils
diff --git a/config/zfs-build.m4 b/config/zfs-build.m4
index d9b052e27..49a4096d6 100644
--- a/config/zfs-build.m4
+++ b/config/zfs-build.m4
@@ -103,6 +103,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_ARCH
+	ZFS_AC_CONFIG_ALWAYS_PYZFS
 ])
 
 AC_DEFUN([ZFS_AC_CONFIG], [
@@ -204,6 +205,7 @@ AC_DEFUN([ZFS_AC_RPM], [
 	])
 	RPM_DEFINE_UTIL+=' $(DEFINE_INITRAMFS)'
 	RPM_DEFINE_UTIL+=' $(DEFINE_SYSTEMD)'
+	RPM_DEFINE_UTIL+=' $(DEFINE_PYZFS)'
 
 	dnl # Override default lib directory on Debian/Ubuntu systems.  The provided
 	dnl # /usr/lib/rpm/platform/<arch>/macros files do not specify the correct