1 files changed, 216 insertions, 138 deletions

diff --git a/module/icp/algs/blake3/blake3_impl.c b/module/icp/algs/blake3/blake3_impl.c
index 10741c82d..5276fd88f 100644
--- a/module/icp/algs/blake3/blake3_impl.c
+++ b/module/icp/algs/blake3/blake3_impl.c
@@ -28,7 +28,7 @@
 
 #include "blake3_impl.h"
 
-static const blake3_impl_ops_t *const blake3_impls[] = {
+static const blake3_ops_t *const blake3_impls[] = {
 	&blake3_generic_impl,
 #if defined(__aarch64__) || \
 	(defined(__x86_64) && defined(HAVE_SSE2)) || \
@@ -48,160 +48,199 @@ static const blake3_impl_ops_t *const blake3_impls[] = {
 #endif
 };
 
-/* this pointer holds current ops for implementation */
-static const blake3_impl_ops_t *blake3_selected_impl = &blake3_generic_impl;
-
-/* special implementation selections */
+/* Select BLAKE3 implementation */
 #define	IMPL_FASTEST	(UINT32_MAX)
-#define	IMPL_CYCLE	(UINT32_MAX-1)
-#define	IMPL_USER	(UINT32_MAX-2)
-#define	IMPL_PARAM	(UINT32_MAX-3)
+#define	IMPL_CYCLE	(UINT32_MAX - 1)
 
-#define	IMPL_READ(i) (*(volatile uint32_t *) &(i))
-static uint32_t icp_blake3_impl = IMPL_FASTEST;
+#define	IMPL_READ(i)	(*(volatile uint32_t *) &(i))
 
-#define	BLAKE3_IMPL_NAME_MAX	16
+/* Indicate that benchmark has been done */
+static boolean_t blake3_initialized = B_FALSE;
 
-/* id of fastest implementation */
-static uint32_t blake3_fastest_id = 0;
+/* Implementation that contains the fastest methods */
+static blake3_ops_t blake3_fastest_impl = {
+	.name = "fastest"
+};
 
-/* currently used id */
-static uint32_t blake3_current_id = 0;
+/* Hold all supported implementations */
+static const blake3_ops_t *blake3_supp_impls[ARRAY_SIZE(blake3_impls)];
+static uint32_t blake3_supp_impls_cnt = 0;
 
-/* id of module parameter (-1 == unused) */
-static int blake3_param_id = -1;
+/* Currently selected implementation */
+static uint32_t blake3_impl_chosen = IMPL_FASTEST;
 
-/* return number of supported implementations */
-int
-blake3_get_impl_count(void)
+static struct blake3_impl_selector {
+	const char *name;
+	uint32_t sel;
+} blake3_impl_selectors[] = {
+	{ "cycle",	IMPL_CYCLE },
+	{ "fastest",	IMPL_FASTEST }
+};
+
+/* check the supported implementations */
+static void blake3_impl_init(void)
 {
-	static int impls = 0;
-	int i;
+	int i, c;
+
+	/* init only once */
+	if (likely(blake3_initialized))
+		return;
 
-	if (impls)
-		return (impls);
+	/* move supported implementations into blake3_supp_impls */
+	for (i = 0, c = 0; i < ARRAY_SIZE(blake3_impls); i++) {
+		const blake3_ops_t *impl = blake3_impls[i];
 
-	for (i = 0; i < ARRAY_SIZE(blake3_impls); i++) {
-		if (!blake3_impls[i]->is_supported()) continue;
-		impls++;
+		if (impl->is_supported && impl->is_supported())
+			blake3_supp_impls[c++] = impl;
 	}
+	blake3_supp_impls_cnt = c;
 
-	return (impls);
+	/* first init generic impl, may be changed via set_fastest() */
+	memcpy(&blake3_fastest_impl, blake3_impls[0],
+	    sizeof (blake3_fastest_impl));
+	blake3_initialized = B_TRUE;
 }
 
-/* return id of selected implementation */
-int
-blake3_get_impl_id(void)
+/* get number of supported implementations */
+uint32_t
+blake3_impl_getcnt(void)
+{
+	blake3_impl_init();
+	return (blake3_supp_impls_cnt);
+}
+
+/* get id of selected implementation */
+uint32_t
+blake3_impl_getid(void)
 {
-	return (blake3_current_id);
+	return (IMPL_READ(blake3_impl_chosen));
 }
 
-/* return name of selected implementation */
+/* get name of selected implementation */
 const char *
-blake3_get_impl_name(void)
+blake3_impl_getname(void)
 {
-	return (blake3_selected_impl->name);
+	uint32_t impl = IMPL_READ(blake3_impl_chosen);
+
+	blake3_impl_init();
+	switch (impl) {
+	case IMPL_FASTEST:
+		return ("fastest");
+	case IMPL_CYCLE:
+		return ("cycle");
+	default:
+		return (blake3_supp_impls[impl]->name);
+	}
 }
 
 /* setup id as fastest implementation */
 void
-blake3_set_impl_fastest(uint32_t id)
+blake3_impl_set_fastest(uint32_t id)
 {
-	blake3_fastest_id = id;
+	/* setup fastest impl */
+	memcpy(&blake3_fastest_impl, blake3_supp_impls[id],
+	    sizeof (blake3_fastest_impl));
 }
 
 /* set implementation by id */
 void
-blake3_set_impl_id(uint32_t id)
+blake3_impl_setid(uint32_t id)
 {
-	int i, cid;
-
-	/* select fastest */
-	if (id == IMPL_FASTEST)
-		id = blake3_fastest_id;
-
-	/* select next or first */
-	if (id == IMPL_CYCLE)
-		id = (++blake3_current_id) % blake3_get_impl_count();
-
-	/* 0..N for the real impl */
-	for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
-		if (!blake3_impls[i]->is_supported()) continue;
-		if (cid == id) {
-			blake3_current_id = cid;
-			blake3_selected_impl = blake3_impls[i];
-			return;
-		}
-		cid++;
+	blake3_impl_init();
+	switch (id) {
+	case IMPL_FASTEST:
+		atomic_swap_32(&blake3_impl_chosen, IMPL_FASTEST);
+		break;
+	case IMPL_CYCLE:
+		atomic_swap_32(&blake3_impl_chosen, IMPL_CYCLE);
+		break;
+	default:
+		ASSERT3U(id, >=, 0);
+		ASSERT3U(id, <, blake3_supp_impls_cnt);
+		atomic_swap_32(&blake3_impl_chosen, id);
+		break;
 	}
 }
 
 /* set implementation by name */
 int
-blake3_set_impl_name(const char *name)
+blake3_impl_setname(const char *val)
 {
-	int i, cid;
-
-	if (strcmp(name, "fastest") == 0) {
-		atomic_swap_32(&icp_blake3_impl, IMPL_FASTEST);
-		blake3_set_impl_id(IMPL_FASTEST);
-		return (0);
-	} else if (strcmp(name, "cycle") == 0) {
-		atomic_swap_32(&icp_blake3_impl, IMPL_CYCLE);
-		blake3_set_impl_id(IMPL_CYCLE);
-		return (0);
+	uint32_t impl = IMPL_READ(blake3_impl_chosen);
+	size_t val_len;
+	int i, err = -EINVAL;
+
+	blake3_impl_init();
+	val_len = strlen(val);
+	while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
+		val_len--;
+
+	/* check mandatory implementations */
+	for (i = 0; i < ARRAY_SIZE(blake3_impl_selectors); i++) {
+		const char *name = blake3_impl_selectors[i].name;
+
+		if (val_len == strlen(name) &&
+		    strncmp(val, name, val_len) == 0) {
+			impl = blake3_impl_selectors[i].sel;
+			err = 0;
+			break;
+		}
 	}
 
-	for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
-		if (!blake3_impls[i]->is_supported()) continue;
-		if (strcmp(name, blake3_impls[i]->name) == 0) {
-			if (icp_blake3_impl == IMPL_PARAM) {
-				blake3_param_id = cid;
-				return (0);
+	if (err != 0 && blake3_initialized) {
+		/* check all supported implementations */
+		for (i = 0; i < blake3_supp_impls_cnt; i++) {
+			const char *name = blake3_supp_impls[i]->name;
+
+			if (val_len == strlen(name) &&
+			    strncmp(val, name, val_len) == 0) {
+				impl = i;
+				err = 0;
+				break;
 			}
-			blake3_selected_impl = blake3_impls[i];
-			blake3_current_id = cid;
-			return (0);
 		}
-		cid++;
 	}
 
-	return (-EINVAL);
+	if (err == 0) {
+		atomic_swap_32(&blake3_impl_chosen, impl);
+	}
+
+	return (err);
 }
 
-/* setup implementation */
-void
-blake3_setup_impl(void)
+const blake3_ops_t *
+blake3_impl_get_ops(void)
 {
-	switch (IMPL_READ(icp_blake3_impl)) {
-	case IMPL_PARAM:
-		blake3_set_impl_id(blake3_param_id);
-		atomic_swap_32(&icp_blake3_impl, IMPL_USER);
-		break;
+	const blake3_ops_t *ops = NULL;
+	uint32_t impl = IMPL_READ(blake3_impl_chosen);
+
+	blake3_impl_init();
+	switch (impl) {
 	case IMPL_FASTEST:
-		blake3_set_impl_id(IMPL_FASTEST);
+		ASSERT(blake3_initialized);
+		ops = &blake3_fastest_impl;
 		break;
 	case IMPL_CYCLE:
-		blake3_set_impl_id(IMPL_CYCLE);
+		/* Cycle through supported implementations */
+		ASSERT(blake3_initialized);
+		ASSERT3U(blake3_supp_impls_cnt, >, 0);
+		static uint32_t cycle_count = 0;
+		uint32_t idx = (++cycle_count) % blake3_supp_impls_cnt;
+		ops = blake3_supp_impls[idx];
 		break;
 	default:
-		blake3_set_impl_id(blake3_current_id);
+		ASSERT3U(blake3_supp_impls_cnt, >, 0);
+		ASSERT3U(impl, <, blake3_supp_impls_cnt);
+		ops = blake3_supp_impls[impl];
 		break;
 	}
-}
 
-/* return selected implementation */
-const blake3_impl_ops_t *
-blake3_impl_get_ops(void)
-{
-	/* each call to ops will cycle */
-	if (icp_blake3_impl == IMPL_CYCLE)
-		blake3_set_impl_id(IMPL_CYCLE);
-
-	return (blake3_selected_impl);
+	ASSERT3P(ops, !=, NULL);
+	return (ops);
 }
 
 #if defined(_KERNEL)
+
 void **blake3_per_cpu_ctx;
 
 void
@@ -215,6 +254,9 @@ blake3_per_cpu_ctx_init(void)
 		blake3_per_cpu_ctx[i] = kmem_alloc(sizeof (BLAKE3_CTX),
 		    KM_SLEEP);
 	}
+
+	/* init once in kernel mode */
+	blake3_impl_init();
 }
 
 void
@@ -227,58 +269,94 @@ blake3_per_cpu_ctx_fini(void)
 	memset(blake3_per_cpu_ctx, 0, max_ncpus * sizeof (void *));
 	kmem_free(blake3_per_cpu_ctx, max_ncpus * sizeof (void *));
 }
-#endif
 
-#if defined(_KERNEL) && defined(__linux__)
+#define	IMPL_FMT(impl, i)	(((impl) == (i)) ? "[%s] " : "%s ")
+
+#if defined(__linux__)
+
 static int
-icp_blake3_impl_set(const char *name, zfs_kernel_param_t *kp)
+blake3_param_get(char *buffer, zfs_kernel_param_t *unused)
 {
-	char req_name[BLAKE3_IMPL_NAME_MAX];
-	size_t i;
+	const uint32_t impl = IMPL_READ(blake3_impl_chosen);
+	char *fmt;
+	int cnt = 0;
 
-	/* sanitize input */
-	i = strnlen(name, BLAKE3_IMPL_NAME_MAX);
-	if (i == 0 || i >= BLAKE3_IMPL_NAME_MAX)
-		return (-EINVAL);
+	/* cycling */
+	fmt = IMPL_FMT(impl, IMPL_CYCLE);
+	cnt += sprintf(buffer + cnt, fmt, "cycle");
+
+	/* list fastest */
+	fmt = IMPL_FMT(impl, IMPL_FASTEST);
+	cnt += sprintf(buffer + cnt, fmt, "fastest");
+
+	/* list all supported implementations */
+	for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) {
+		fmt = IMPL_FMT(impl, i);
+		cnt += sprintf(buffer + cnt, fmt,
+		    blake3_supp_impls[i]->name);
+	}
 
-	strlcpy(req_name, name, BLAKE3_IMPL_NAME_MAX);
-	while (i > 0 && isspace(req_name[i-1]))
-		i--;
-	req_name[i] = '\0';
+	return (cnt);
+}
 
-	atomic_swap_32(&icp_blake3_impl, IMPL_PARAM);
-	return (blake3_set_impl_name(req_name));
+static int
+blake3_param_set(const char *val, zfs_kernel_param_t *unused)
+{
+	(void) unused;
+	return (blake3_impl_setname(val));
 }
 
+#elif defined(__FreeBSD__)
+
+#include <sys/sbuf.h>
+
 static int
-icp_blake3_impl_get(char *buffer, zfs_kernel_param_t *kp)
+blake3_param(ZFS_MODULE_PARAM_ARGS)
 {
-	int i, cid, cnt = 0;
-	char *fmt;
+	int err;
 
-	/* cycling */
-	fmt = (icp_blake3_impl == IMPL_CYCLE) ? "[cycle] " : "cycle ";
-	cnt += sprintf(buffer + cnt, fmt);
-
-	/* fastest one */
-	fmt = (icp_blake3_impl == IMPL_FASTEST) ? "[fastest] " : "fastest ";
-	cnt += sprintf(buffer + cnt, fmt);
-
-	/* user selected */
-	for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
-		if (!blake3_impls[i]->is_supported()) continue;
-		fmt = (icp_blake3_impl == IMPL_USER &&
-		    cid == blake3_current_id) ? "[%s] " : "%s ";
-		cnt += sprintf(buffer + cnt, fmt, blake3_impls[i]->name);
-		cid++;
+	if (req->newptr == NULL) {
+		const uint32_t impl = IMPL_READ(blake3_impl_chosen);
+		const int init_buflen = 64;
+		const char *fmt;
+		struct sbuf *s;
+
+		s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
+
+		/* cycling */
+		fmt = IMPL_FMT(impl, IMPL_CYCLE);
+		(void) sbuf_printf(s, fmt, "cycle");
+
+		/* list fastest */
+		fmt = IMPL_FMT(impl, IMPL_FASTEST);
+		(void) sbuf_printf(s, fmt, "fastest");
+
+		/* list all supported implementations */
+		for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) {
+			fmt = IMPL_FMT(impl, i);
+			(void) sbuf_printf(s, fmt, blake3_supp_impls[i]->name);
+		}
+
+		err = sbuf_finish(s);
+		sbuf_delete(s);
+
+		return (err);
 	}
 
-	buffer[cnt] = 0;
+	char buf[16];
 
-	return (cnt);
+	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
+	if (err) {
+		return (err);
+	}
+
+	return (-blake3_impl_setname(buf));
 }
+#endif
+
+#undef IMPL_FMT
 
-module_param_call(icp_blake3_impl, icp_blake3_impl_set, icp_blake3_impl_get,
-    NULL, 0644);
-MODULE_PARM_DESC(icp_blake3_impl, "Select BLAKE3 implementation.");
+ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, blake3_impl,
+    blake3_param_set, blake3_param_get, ZMOD_RW, \
+	"Select BLAKE3 implementation.");
 #endif