aboutsummaryrefslogtreecommitdiffstats
path: root/module/os
diff options
context:
space:
mode:
authorBrian Behlendorf <behlendorf1@llnl.gov>2019-10-24 10:17:33 -0700
committerGitHub <noreply@github.com>2019-10-24 10:17:33 -0700
commit10fa254539ec41c6b043785d4e7ab34bce383b9f (patch)
treec26798d9a42546096cb5bf2b0afeb4ef5257d48e /module/os
parentb834b58ae61063f526ce82df348440e31c58c232 (diff)
Linux 4.14, 4.19, 5.0+ compat: SIMD save/restore
Contrary to initial testing we cannot rely on these kernels to invalidate the per-cpu FPU state and restore the FPU registers. Nor can we guarantee that the kernel won't modify the FPU state which we saved in the task struck. Therefore, the kfpu_begin() and kfpu_end() functions have been updated to save and restore the FPU state using our own dedicated per-cpu FPU state variables. This has the additional advantage of allowing us to use the FPU again in user threads. So we remove the code which was added to use task queues to ensure some functions ran in kernel threads. Reviewed-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> Reviewed-by: Tony Hutter <hutter2@llnl.gov> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Issue #9346 Closes #9403
Diffstat (limited to 'module/os')
-rw-r--r--module/os/linux/spl/spl-taskq.c2
-rw-r--r--module/os/linux/spl/spl-thread.c2
-rw-r--r--module/os/linux/zfs/zio_crypt.c145
3 files changed, 32 insertions, 117 deletions
diff --git a/module/os/linux/spl/spl-taskq.c b/module/os/linux/spl/spl-taskq.c
index 2e6280084..8910c109e 100644
--- a/module/os/linux/spl/spl-taskq.c
+++ b/module/os/linux/spl/spl-taskq.c
@@ -28,7 +28,6 @@
#include <sys/taskq.h>
#include <sys/kmem.h>
#include <sys/tsd.h>
-#include <sys/simd.h>
int spl_taskq_thread_bind = 0;
module_param(spl_taskq_thread_bind, int, 0644);
@@ -854,7 +853,6 @@ taskq_thread(void *args)
sigfillset(&blocked);
sigprocmask(SIG_BLOCK, &blocked, NULL);
flush_signals(current);
- kfpu_initialize();
tsd_set(taskq_tsd, tq);
spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
diff --git a/module/os/linux/spl/spl-thread.c b/module/os/linux/spl/spl-thread.c
index 29de9252a..0352a31ea 100644
--- a/module/os/linux/spl/spl-thread.c
+++ b/module/os/linux/spl/spl-thread.c
@@ -27,7 +27,6 @@
#include <sys/thread.h>
#include <sys/kmem.h>
#include <sys/tsd.h>
-#include <sys/simd.h>
/*
* Thread interfaces
@@ -55,7 +54,6 @@ thread_generic_wrapper(void *arg)
args = tp->tp_args;
set_current_state(tp->tp_state);
set_user_nice((kthread_t *)current, PRIO_TO_NICE(tp->tp_pri));
- kfpu_initialize();
kmem_free(tp->tp_name, tp->tp_name_size);
kmem_free(tp, sizeof (thread_priv_t));
diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c
index 5b4aa664c..96dabe55a 100644
--- a/module/os/linux/zfs/zio_crypt.c
+++ b/module/os/linux/zfs/zio_crypt.c
@@ -25,8 +25,6 @@
#include <sys/zio.h>
#include <sys/zil.h>
#include <sys/sha2.h>
-#include <sys/simd.h>
-#include <sys/spa_impl.h>
#include <sys/hkdf.h>
#include <sys/qat.h>
@@ -376,7 +374,7 @@ error:
* plaintext / ciphertext alone.
*/
static int
-zio_do_crypt_uio_impl(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
+zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
{
@@ -476,75 +474,9 @@ error:
return (ret);
}
-typedef struct crypt_uio_arg {
- boolean_t cu_encrypt;
- uint64_t cu_crypt;
- crypto_key_t *cu_key;
- crypto_ctx_template_t cu_tmpl;
- uint8_t *cu_ivbuf;
- uint_t cu_datalen;
- uio_t *cu_puio;
- uio_t *cu_cuio;
- uint8_t *cu_authbuf;
- uint_t cu_auth_len;
- int cu_error;
-} crypt_uio_arg_t;
-
-static void
-zio_do_crypt_uio_func(void *arg)
-{
- crypt_uio_arg_t *cu = (crypt_uio_arg_t *)arg;
-
- cu->cu_error = zio_do_crypt_uio_impl(cu->cu_encrypt, cu->cu_crypt,
- cu->cu_key, cu->cu_tmpl, cu->cu_ivbuf, cu->cu_datalen,
- cu->cu_puio, cu->cu_cuio, cu->cu_authbuf, cu->cu_auth_len);
-}
-
-static int
-zio_do_crypt_uio(spa_t *spa, boolean_t encrypt, uint64_t crypt,
- crypto_key_t *key, crypto_ctx_template_t tmpl, uint8_t *ivbuf,
- uint_t datalen, uio_t *puio, uio_t *cuio, uint8_t *authbuf,
- uint_t auth_len)
-{
- int error;
-
- /*
- * Dispatch to the I/O pipeline as required by the context in order
- * to take advantage of the SIMD optimization when available.
- */
- if (kfpu_allowed()) {
- error = zio_do_crypt_uio_impl(encrypt, crypt, key, tmpl,
- ivbuf, datalen, puio, cuio, authbuf, auth_len);
- } else {
- crypt_uio_arg_t *cu;
-
- cu = kmem_alloc(sizeof (*cu), KM_SLEEP);
- cu->cu_encrypt = encrypt;
- cu->cu_crypt = crypt;
- cu->cu_key = key;
- cu->cu_tmpl = tmpl;
- cu->cu_ivbuf = ivbuf;
- cu->cu_datalen = datalen;
- cu->cu_puio = puio;
- cu->cu_cuio = cuio;
- cu->cu_authbuf = authbuf;
- cu->cu_auth_len = auth_len;
- cu->cu_error = 0;
-
- spa_taskq_dispatch_sync(spa,
- encrypt ? ZIO_TYPE_WRITE : ZIO_TYPE_READ,
- ZIO_TASKQ_ISSUE, zio_do_crypt_uio_func, cu, TQ_SLEEP);
-
- error = cu->cu_error;
- kmem_free(cu, sizeof (*cu));
- }
-
- return (error);
-}
-
int
-zio_crypt_key_wrap(spa_t *spa, crypto_key_t *cwkey, zio_crypt_key_t *key,
- uint8_t *iv, uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
+zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
+ uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
{
int ret;
uio_t puio, cuio;
@@ -601,7 +533,7 @@ zio_crypt_key_wrap(spa_t *spa, crypto_key_t *cwkey, zio_crypt_key_t *key,
cuio.uio_segflg = UIO_SYSSPACE;
/* encrypt the keys and store the resulting ciphertext and mac */
- ret = zio_do_crypt_uio(spa, B_TRUE, crypt, cwkey, NULL, iv, enc_len,
+ ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
&puio, &cuio, (uint8_t *)aad, aad_len);
if (ret != 0)
goto error;
@@ -612,33 +544,12 @@ error:
return (ret);
}
-static void
-zio_crypt_create_ctx_templates(void *arg)
-{
- zio_crypt_key_t *key = (zio_crypt_key_t *)arg;
- crypto_mechanism_t mech;
- int ret;
-
- mech.cm_type = crypto_mech2id(
- zio_crypt_table[key->zk_crypt].ci_mechname);
-
- ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
- &key->zk_current_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_current_tmpl = NULL;
-
- mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
- ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
- &key->zk_hmac_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_hmac_tmpl = NULL;
-}
-
int
-zio_crypt_key_unwrap(spa_t *spa, crypto_key_t *cwkey, uint64_t crypt,
- uint64_t version, uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata,
- uint8_t *iv, uint8_t *mac, zio_crypt_key_t *key)
+zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
+ uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
+ uint8_t *mac, zio_crypt_key_t *key)
{
+ crypto_mechanism_t mech;
uio_t puio, cuio;
uint64_t aad[3];
iovec_t plain_iovecs[2], cipher_iovecs[3];
@@ -685,7 +596,7 @@ zio_crypt_key_unwrap(spa_t *spa, crypto_key_t *cwkey, uint64_t crypt,
cuio.uio_segflg = UIO_SYSSPACE;
/* decrypt the keys and store the result in the output buffers */
- ret = zio_do_crypt_uio(spa, B_FALSE, crypt, cwkey, NULL, iv, enc_len,
+ ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
&puio, &cuio, (uint8_t *)aad, aad_len);
if (ret != 0)
goto error;
@@ -711,18 +622,27 @@ zio_crypt_key_unwrap(spa_t *spa, crypto_key_t *cwkey, uint64_t crypt,
key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
+ /*
+ * Initialize the crypto templates. It's ok if this fails because
+ * this is just an optimization.
+ */
+ mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
+ ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
+ &key->zk_current_tmpl, KM_SLEEP);
+ if (ret != CRYPTO_SUCCESS)
+ key->zk_current_tmpl = NULL;
+
+ mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
+ ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
+ &key->zk_hmac_tmpl, KM_SLEEP);
+ if (ret != CRYPTO_SUCCESS)
+ key->zk_hmac_tmpl = NULL;
+
key->zk_crypt = crypt;
key->zk_version = version;
key->zk_guid = guid;
key->zk_salt_count = 0;
- /*
- * Initialize the crypto templates in the context they will be
- * primarily used. It's ok if this fails, it's just an optimization.
- */
- spa_taskq_dispatch_sync(spa, ZIO_TYPE_READ, ZIO_TASKQ_ISSUE,
- zio_crypt_create_ctx_templates, key, TQ_SLEEP);
-
return (0);
error:
@@ -1941,7 +1861,7 @@ error:
* Primary encryption / decryption entrypoint for zio data.
*/
int
-zio_do_crypt_data(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
+zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
boolean_t *no_crypt)
@@ -2028,8 +1948,8 @@ zio_do_crypt_data(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
goto error;
/* perform the encryption / decryption in software */
- ret = zio_do_crypt_uio(spa, encrypt, key->zk_crypt, ckey, tmpl, iv,
- enc_len, &puio, &cuio, authbuf, auth_len);
+ ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
+ &puio, &cuio, authbuf, auth_len);
if (ret != 0)
goto error;
@@ -2065,10 +1985,9 @@ error:
* linear buffers.
*/
int
-zio_do_crypt_abd(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
- dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
- uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd,
- boolean_t *no_crypt)
+zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
+ boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
+ uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
{
int ret;
void *ptmp, *ctmp;
@@ -2081,7 +2000,7 @@ zio_do_crypt_abd(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
ctmp = abd_borrow_buf_copy(cabd, datalen);
}
- ret = zio_do_crypt_data(spa, encrypt, key, ot, byteswap, salt, iv, mac,
+ ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
datalen, ptmp, ctmp, no_crypt);
if (ret != 0)
goto error;