Illumos Crypto Port module added to enable native encryption in zfs

A port of the Illumos Crypto Framework to a Linux kernel module (found
in module/icp). This is needed to do the actual encryption work. We cannot
use the Linux kernel's built in crypto api because it is only exported to
GPL-licensed modules. Having the ICP also means the crypto code can run on
any of the other kernels under OpenZFS. I ended up porting over most of the
internals of the framework, which means that porting over other API calls (if
we need them) should be fairly easy. Specifically, I have ported over the API
functions related to encryption, digests, macs, and crypto templates. The ICP
is able to use assembly-accelerated encryption on amd64 machines and AES-NI
instructions on Intel chips that support it. There are place-holder
directories for similar assembly optimizations for other architectures
(although they have not been written).

Signed-off-by: Tom Caputi <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Issue #4329

3 files changed, 95 insertions, 28 deletions

diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
index b40e9787c..e73268b73 100644
--- a/lib/libzpool/Makefile.am
+++ b/lib/libzpool/Makefile.am
@@ -126,7 +126,8 @@ nodist_libzpool_la_SOURCES = \
 libzpool_la_LIBADD = \
 	$(top_builddir)/lib/libunicode/libunicode.la \
 	$(top_builddir)/lib/libuutil/libuutil.la \
-	$(top_builddir)/lib/libnvpair/libnvpair.la
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libicp/libicp.la
 
 libzpool_la_LIBADD += $(ZLIB)
 libzpool_la_LDFLAGS = -version-info 2:0:0
diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
index 93bfb0403..a68911451 100644
--- a/lib/libzpool/kernel.c
+++ b/lib/libzpool/kernel.c
@@ -41,6 +41,7 @@
 #include <sys/time.h>
 #include <sys/systeminfo.h>
 #include <zfs_fletcher.h>
+#include <sys/crypto/icp.h>
 
 /*
  * Emulation of kernel services in userland.
@@ -1113,9 +1114,96 @@ lowbit64(uint64_t i)
 	return (h);
 }
 
+/*
+ * Find highest one bit set.
+ * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
+ * High order bit is 31 (or 63 in _LP64 kernel).
+ */
+int
+highbit(ulong_t i)
+{
+register int h = 1;
+
+	if (i == 0)
+		return (0);
+#ifdef _LP64
+	if (i & 0xffffffff00000000ul) {
+		h += 32; i >>= 32;
+	}
+#endif
+	if (i & 0xffff0000) {
+		h += 16; i >>= 16;
+	}
+	if (i & 0xff00) {
+		h += 8; i >>= 8;
+	}
+	if (i & 0xf0) {
+		h += 4; i >>= 4;
+	}
+	if (i & 0xc) {
+		h += 2; i >>= 2;
+	}
+	if (i & 0x2) {
+		h += 1;
+	}
+	return (h);
+}
+
+/*
+ * Find lowest one bit set.
+ *     Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
+ * Low order bit is 0.
+ */
+int
+lowbit(ulong_t i)
+{
+	register int h = 1;
+
+	if (i == 0)
+		return (0);
+
+#ifdef _LP64
+	if (!(i & 0xffffffff)) {
+		h += 32; i >>= 32;
+	}
+#endif
+	if (!(i & 0xffff)) {
+		h += 16; i >>= 16;
+	}
+	if (!(i & 0xff)) {
+		h += 8; i >>= 8;
+	}
+	if (!(i & 0xf)) {
+		h += 4; i >>= 4;
+	}
+	if (!(i & 0x3)) {
+		h += 2; i >>= 2;
+	}
+	if (!(i & 0x1)) {
+		h += 1;
+	}
+	return (h);
+}
 
 static int random_fd = -1, urandom_fd = -1;
 
+void
+random_init(void)
+{
+	VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
+	VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+}
+
+void
+random_fini(void)
+{
+	close(random_fd);
+	close(urandom_fd);
+
+	random_fd = -1;
+	urandom_fd = -1;
+}
+
 static int
 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
 {
@@ -1228,12 +1316,13 @@ kernel_init(int mode)
 	(void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
 	    (mode & FWRITE) ? get_system_hostid() : 0);
 
-	VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
-	VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+	random_init();
+
 	VERIFY0(uname(&hw_utsname));
 
 	thread_init();
 	system_taskq_init();
+	icp_init();
 
 	spa_init(mode);
 
@@ -1248,14 +1337,11 @@ kernel_fini(void)
 	fletcher_4_fini();
 	spa_fini();
 
+	icp_fini();
 	system_taskq_fini();
 	thread_fini();
 
-	close(random_fd);
-	close(urandom_fd);
-
-	random_fd = -1;
-	urandom_fd = -1;
+	random_fini();
 }
 
 uid_t
diff --git a/lib/libzpool/taskq.c b/lib/libzpool/taskq.c
index 1d50d0edc..e42e4261f 100644
--- a/lib/libzpool/taskq.c
+++ b/lib/libzpool/taskq.c
@@ -34,26 +34,6 @@ int taskq_now;
 taskq_t *system_taskq;
 
 #define	TASKQ_ACTIVE	0x00010000
-#define	TASKQ_NAMELEN	31
-
-struct taskq {
-	char		tq_name[TASKQ_NAMELEN + 1];
-	kmutex_t	tq_lock;
-	krwlock_t	tq_threadlock;
-	kcondvar_t	tq_dispatch_cv;
-	kcondvar_t	tq_wait_cv;
-	kthread_t	**tq_threadlist;
-	int		tq_flags;
-	int		tq_active;
-	int		tq_nthreads;
-	int		tq_nalloc;
-	int		tq_minalloc;
-	int		tq_maxalloc;
-	kcondvar_t	tq_maxalloc_cv;
-	int		tq_maxalloc_wait;
-	taskq_ent_t	*tq_freelist;
-	taskq_ent_t	tq_task;
-};
 
 static taskq_ent_t *
 task_alloc(taskq_t *tq, int tqflags)