Illumos Crypto Port module added to enable native encryption in zfs

A port of the Illumos Crypto Framework to a Linux kernel module (found
in module/icp). This is needed to do the actual encryption work. We cannot
use the Linux kernel's built in crypto api because it is only exported to
GPL-licensed modules. Having the ICP also means the crypto code can run on
any of the other kernels under OpenZFS. I ended up porting over most of the
internals of the framework, which means that porting over other API calls (if
we need them) should be fairly easy. Specifically, I have ported over the API
functions related to encryption, digests, macs, and crypto templates. The ICP
is able to use assembly-accelerated encryption on amd64 machines and AES-NI
instructions on Intel chips that support it. There are place-holder
directories for similar assembly optimizations for other architectures
(although they have not been written).

Signed-off-by: Tom Caputi <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Issue #4329

8 files changed, 200 insertions, 32 deletions

diff --git a/lib/Makefile.am b/lib/Makefile.am
index 8e7caf2a1..3aec922c9 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -1,7 +1,7 @@
 # NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries
-# These five libraries are intermediary build components.
-SUBDIRS = libspl libavl libefi libshare libunicode
+# These six libraries are intermediary build components.
+SUBDIRS = libspl libavl libefi libshare libunicode libicp
 
 # These four libraries, which are installed as the final build product,
-# incorporate the five convenience libraries given above.
+# incorporate the six convenience libraries given above.
 SUBDIRS += libuutil libnvpair libzpool libzfs_core libzfs
diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am
new file mode 100644
index 000000000..41457fd52
--- /dev/null
+++ b/lib/libicp/Makefile.am
@@ -0,0 +1,78 @@
+include $(top_srcdir)/config/Rules.am
+
+VPATH = \
+	$(top_srcdir)/module/icp \
+	$(top_srcdir)/lib/libicp
+
+AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+
+DEFAULT_INCLUDES += \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/module/icp/include \
+	-I$(top_srcdir)/lib/libspl/include
+
+noinst_LTLIBRARIES = libicp.la
+
+if TARGET_ASM_X86_64
+ASM_SOURCES_C = asm-x86_64/aes/aeskey.c
+ASM_SOURCES_AS = \
+	asm-x86_64/aes/aes_amd64.S \
+	asm-x86_64/aes/aes_intel.S \
+	asm-x86_64/modes/gcm_intel.S \
+	asm-x86_64/sha1/sha1-x86_64.S \
+	asm-x86_64/sha2/sha256_impl.S
+endif
+
+if TARGET_ASM_I386
+ASM_SOURCES_C =
+ASM_SOURCES_AS =
+endif
+	
+if TARGET_ASM_GENERIC
+ASM_SOURCES_C =
+ASM_SOURCES_AS =
+endif
+
+USER_C =
+
+USER_ASM =
+
+KERNEL_C = \
+	spi/kcf_spi.c \
+	api/kcf_ctxops.c \
+	api/kcf_digest.c \
+	api/kcf_cipher.c \
+	api/kcf_miscapi.c \
+	api/kcf_mac.c \
+	algs/aes/aes_impl.c \
+	algs/aes/aes_modes.c \
+	algs/modes/modes.c \
+	algs/modes/cbc.c \
+	algs/modes/gcm.c \
+	algs/modes/ctr.c \
+	algs/modes/ccm.c \
+	algs/modes/ecb.c \
+	algs/sha1/sha1.c \
+	algs/sha2/sha2.c \
+	illumos-crypto.c \
+	io/aes.c \
+	io/sha1_mod.c \
+	io/sha2_mod.c \
+	os/modhash.c \
+	os/modconf.c \
+	core/kcf_sched.c \
+	core/kcf_prov_lib.c \
+	core/kcf_callprov.c \
+	core/kcf_mech_tabs.c \
+	core/kcf_prov_tabs.c \
+	$(ASM_SOURCES_C)
+
+KERNEL_ASM = $(ASM_SOURCES_AS)
+
+nodist_libicp_la_SOURCES = \
+	$(USER_C) \
+	$(USER_ASM) \
+	$(KERNEL_C) \
+	$(KERNEL_ASM)
+	
+libicp_la_LIBADD = -lrt
diff --git a/lib/libspl/include/sys/byteorder.h b/lib/libspl/include/sys/byteorder.h
index 528d2d208..7ef1c4295 100644
--- a/lib/libspl/include/sys/byteorder.h
+++ b/lib/libspl/include/sys/byteorder.h
@@ -128,6 +128,28 @@ extern	in_port_t ntohs(in_port_t);
 #define	BE_64(x)	BSWAP_64(x)
 #endif
 
+#ifdef _BIG_ENDIAN
+static __inline__ uint64_t
+htonll(uint64_t n) {
+	return (n);
+}
+
+static __inline__ uint64_t
+ntohll(uint64_t n) {
+	return (n);
+}
+#else
+static __inline__ uint64_t
+htonll(uint64_t n) {
+	return ((((uint64_t)htonl(n)) << 32) + htonl(n >> 32));
+}
+
+static __inline__ uint64_t
+ntohll(uint64_t n) {
+	return ((((uint64_t)ntohl(n)) << 32) + ntohl(n >> 32));
+}
+#endif
+
 /*
  * Macros to read unaligned values from a specific byte order to
  * native byte order
diff --git a/lib/libspl/include/sys/file.h b/lib/libspl/include/sys/file.h
index 163a4dca6..b5d985bda 100644
--- a/lib/libspl/include/sys/file.h
+++ b/lib/libspl/include/sys/file.h
@@ -33,7 +33,7 @@
 
 #define	FREAD	1
 #define	FWRITE	2
-// #define FAPPEND  8
+// #define	FAPPEND  8
 
 #define	FCREAT	O_CREAT
 #define	FTRUNC	O_TRUNC
diff --git a/lib/libspl/include/sys/types.h b/lib/libspl/include/sys/types.h
index 8996af96a..c58b2d566 100644
--- a/lib/libspl/include/sys/types.h
+++ b/lib/libspl/include/sys/types.h
@@ -64,6 +64,7 @@ typedef int		major_t;
 typedef int		minor_t;
 
 typedef ushort_t o_mode_t; /* old file attribute type */
+typedef short		index_t;
 
 /*
  * Definitions remaining from previous partial support for 64-bit file
diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
index b40e9787c..e73268b73 100644
--- a/lib/libzpool/Makefile.am
+++ b/lib/libzpool/Makefile.am
@@ -126,7 +126,8 @@ nodist_libzpool_la_SOURCES = \
 libzpool_la_LIBADD = \
 	$(top_builddir)/lib/libunicode/libunicode.la \
 	$(top_builddir)/lib/libuutil/libuutil.la \
-	$(top_builddir)/lib/libnvpair/libnvpair.la
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libicp/libicp.la
 
 libzpool_la_LIBADD += $(ZLIB)
 libzpool_la_LDFLAGS = -version-info 2:0:0
diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
index 93bfb0403..a68911451 100644
--- a/lib/libzpool/kernel.c
+++ b/lib/libzpool/kernel.c
@@ -41,6 +41,7 @@
 #include <sys/time.h>
 #include <sys/systeminfo.h>
 #include <zfs_fletcher.h>
+#include <sys/crypto/icp.h>
 
 /*
  * Emulation of kernel services in userland.
@@ -1113,9 +1114,96 @@ lowbit64(uint64_t i)
 	return (h);
 }
 
+/*
+ * Find highest one bit set.
+ * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
+ * High order bit is 31 (or 63 in _LP64 kernel).
+ */
+int
+highbit(ulong_t i)
+{
+register int h = 1;
+
+	if (i == 0)
+		return (0);
+#ifdef _LP64
+	if (i & 0xffffffff00000000ul) {
+		h += 32; i >>= 32;
+	}
+#endif
+	if (i & 0xffff0000) {
+		h += 16; i >>= 16;
+	}
+	if (i & 0xff00) {
+		h += 8; i >>= 8;
+	}
+	if (i & 0xf0) {
+		h += 4; i >>= 4;
+	}
+	if (i & 0xc) {
+		h += 2; i >>= 2;
+	}
+	if (i & 0x2) {
+		h += 1;
+	}
+	return (h);
+}
+
+/*
+ * Find lowest one bit set.
+ *     Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
+ * Low order bit is 0.
+ */
+int
+lowbit(ulong_t i)
+{
+	register int h = 1;
+
+	if (i == 0)
+		return (0);
+
+#ifdef _LP64
+	if (!(i & 0xffffffff)) {
+		h += 32; i >>= 32;
+	}
+#endif
+	if (!(i & 0xffff)) {
+		h += 16; i >>= 16;
+	}
+	if (!(i & 0xff)) {
+		h += 8; i >>= 8;
+	}
+	if (!(i & 0xf)) {
+		h += 4; i >>= 4;
+	}
+	if (!(i & 0x3)) {
+		h += 2; i >>= 2;
+	}
+	if (!(i & 0x1)) {
+		h += 1;
+	}
+	return (h);
+}
 
 static int random_fd = -1, urandom_fd = -1;
 
+void
+random_init(void)
+{
+	VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
+	VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+}
+
+void
+random_fini(void)
+{
+	close(random_fd);
+	close(urandom_fd);
+
+	random_fd = -1;
+	urandom_fd = -1;
+}
+
 static int
 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
 {
@@ -1228,12 +1316,13 @@ kernel_init(int mode)
 	(void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
 	    (mode & FWRITE) ? get_system_hostid() : 0);
 
-	VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
-	VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+	random_init();
+
 	VERIFY0(uname(&hw_utsname));
 
 	thread_init();
 	system_taskq_init();
+	icp_init();
 
 	spa_init(mode);
 
@@ -1248,14 +1337,11 @@ kernel_fini(void)
 	fletcher_4_fini();
 	spa_fini();
 
+	icp_fini();
 	system_taskq_fini();
 	thread_fini();
 
-	close(random_fd);
-	close(urandom_fd);
-
-	random_fd = -1;
-	urandom_fd = -1;
+	random_fini();
 }
 
 uid_t
diff --git a/lib/libzpool/taskq.c b/lib/libzpool/taskq.c
index 1d50d0edc..e42e4261f 100644
--- a/lib/libzpool/taskq.c
+++ b/lib/libzpool/taskq.c
@@ -34,26 +34,6 @@ int taskq_now;
 taskq_t *system_taskq;
 
 #define	TASKQ_ACTIVE	0x00010000
-#define	TASKQ_NAMELEN	31
-
-struct taskq {
-	char		tq_name[TASKQ_NAMELEN + 1];
-	kmutex_t	tq_lock;
-	krwlock_t	tq_threadlock;
-	kcondvar_t	tq_dispatch_cv;
-	kcondvar_t	tq_wait_cv;
-	kthread_t	**tq_threadlist;
-	int		tq_flags;
-	int		tq_active;
-	int		tq_nthreads;
-	int		tq_nalloc;
-	int		tq_minalloc;
-	int		tq_maxalloc;
-	kcondvar_t	tq_maxalloc_cv;
-	int		tq_maxalloc_wait;
-	taskq_ent_t	*tq_freelist;
-	taskq_ent_t	tq_task;
-};
 
 static taskq_ent_t *
 task_alloc(taskq_t *tq, int tqflags)