Add support for decryption faults in zinject

This patch adds the ability for zinject to trigger decryption
and authentication faults in the ZIO and ARC layers. This
functionality is exposed via the new "decrypt" error type, which
may be provided for "data" object types.

This patch also refactors some of the core encryption / decryption
functions so that they have consistent prototypes, handle errors
consistently, and do not have unused arguments.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Tom Caputi <[email protected]>
Closes #7474 

5 files changed, 121 insertions, 98 deletions

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index fa7f62d99..6662e0fae 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -1922,10 +1922,9 @@ error:
  * also decompress the data.
  */
 static int
-arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
+arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
 {
 	int ret;
-	dsl_crypto_key_t *dck = NULL;
 	abd_t *cabd = NULL;
 	void *tmp = NULL;
 	boolean_t no_crypt = B_FALSE;
@@ -1936,25 +1935,9 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
 
 	arc_hdr_alloc_abd(hdr, B_FALSE);
 
-	/*
-	 * We must be careful to use the passed-in dsobj value here and
-	 * not the value in b_dsobj. b_dsobj is meant to be a best guess for
-	 * the L2ARC, which has the luxury of being able to fail without real
-	 * consequences (the data simply won't make it to the L2ARC). In
-	 * reality, the dsobj stored in the header may belong to a dataset
-	 * that has been unmounted or otherwise disowned, meaning the key
-	 * won't be accessible via that dsobj anymore.
-	 */
-	ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck);
-	if (ret != 0) {
-		ret = SET_ERROR(EACCES);
-		goto error;
-	}
-
-	ret = zio_do_crypt_abd(B_FALSE, &dck->dck_key,
-	    hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_ot,
-	    hdr->b_crypt_hdr.b_iv, hdr->b_crypt_hdr.b_mac,
-	    HDR_GET_PSIZE(hdr), bswap, hdr->b_l1hdr.b_pabd,
+	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
+	    B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
+	    hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
 	    hdr->b_crypt_hdr.b_rabd, &no_crypt);
 	if (ret != 0)
 		goto error;
@@ -1994,14 +1977,10 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
 		hdr->b_l1hdr.b_pabd = cabd;
 	}
 
-	spa_keystore_dsl_key_rele(spa, dck, FTAG);
-
 	return (0);
 
 error:
 	arc_hdr_free_abd(hdr, B_FALSE);
-	if (dck != NULL)
-		spa_keystore_dsl_key_rele(spa, dck, FTAG);
 	if (cabd != NULL)
 		arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);
 
@@ -2015,7 +1994,7 @@ error:
  */
 static int
 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
-    uint64_t dsobj, boolean_t noauth)
+    const zbookmark_phys_t *zb, boolean_t noauth)
 {
 	int ret;
 
@@ -2029,7 +2008,7 @@ arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
 		 * The caller requested authenticated data but our data has
 		 * not been authenticated yet. Verify the MAC now if we can.
 		 */
-		ret = arc_hdr_authenticate(hdr, spa, dsobj);
+		ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
 		if (ret != 0)
 			goto error;
 	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
@@ -2038,7 +2017,7 @@ arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
 		 * unencrypted version was requested we take this opportunity
 		 * to store the decrypted version in the header for future use.
 		 */
-		ret = arc_hdr_decrypt(hdr, spa, dsobj);
+		ret = arc_hdr_decrypt(hdr, spa, zb);
 		if (ret != 0)
 			goto error;
 	}
@@ -2094,7 +2073,8 @@ arc_buf_untransform_in_place(arc_buf_t *buf, kmutex_t *hash_lock)
  * the correct-sized data buffer.
  */
 static int
-arc_buf_fill(arc_buf_t *buf, spa_t *spa, uint64_t dsobj, arc_fill_flags_t flags)
+arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
+    arc_fill_flags_t flags)
 {
 	int error = 0;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
@@ -2131,7 +2111,7 @@ arc_buf_fill(arc_buf_t *buf, spa_t *spa, uint64_t dsobj, arc_fill_flags_t flags)
 	 */
 	if (HDR_PROTECTED(hdr)) {
 		error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
-		    dsobj, !!(flags & ARC_FILL_NOAUTH));
+		    zb, !!(flags & ARC_FILL_NOAUTH));
 		if (error != 0) {
 			arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
 			return (error);
@@ -2272,13 +2252,14 @@ arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
 	if (in_place)
 		flags |= ARC_FILL_IN_PLACE;
 
-	ret = arc_buf_fill(buf, spa, zb->zb_objset, flags);
+	ret = arc_buf_fill(buf, spa, zb, flags);
 	if (ret == ECKSUM) {
 		/*
 		 * Convert authentication and decryption errors to EIO
 		 * (and generate an ereport) before leaving the ARC.
 		 */
 		ret = SET_ERROR(EIO);
+		spa_log_error(spa, zb);
 		zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
 		    spa, NULL, zb, NULL, 0, 0);
 	}
@@ -2813,8 +2794,8 @@ arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
  * copy was made successfully, or an error code otherwise.
  */
 static int
-arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj, void *tag,
-    boolean_t encrypted, boolean_t compressed, boolean_t noauth,
+arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
+    void *tag, boolean_t encrypted, boolean_t compressed, boolean_t noauth,
     boolean_t fill, arc_buf_t **ret)
 {
 	arc_buf_t *buf;
@@ -2906,7 +2887,8 @@ arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj, void *tag,
 	 * decompress the data.
 	 */
 	if (fill) {
-		return (arc_buf_fill(buf, spa, dsobj, flags));
+		ASSERT3P(zb, !=, NULL);
+		return (arc_buf_fill(buf, spa, zb, flags));
 	}
 
 	return (0);
@@ -3588,7 +3570,7 @@ arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
 	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
 
 	arc_buf_t *buf = NULL;
-	VERIFY0(arc_buf_alloc_impl(hdr, spa, 0, tag, B_FALSE, B_FALSE,
+	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
 	    B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 
@@ -3613,7 +3595,7 @@ arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
 	ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
 
 	arc_buf_t *buf = NULL;
-	VERIFY0(arc_buf_alloc_impl(hdr, spa, 0, tag, B_FALSE,
+	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
 	    B_TRUE, B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
@@ -3667,7 +3649,7 @@ arc_alloc_raw_buf(spa_t *spa, void *tag, uint64_t dsobj, boolean_t byteorder,
 	 * arc_write_ready().
 	 */
 	buf = NULL;
-	VERIFY0(arc_buf_alloc_impl(hdr, spa, dsobj, tag, B_TRUE, B_TRUE,
+	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
 	    B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
@@ -5814,7 +5796,7 @@ arc_read_done(zio_t *zio)
 			continue;
 
 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,
-		    acb->acb_dsobj, acb->acb_private, acb->acb_encrypted,
+		    &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
 		    acb->acb_compressed, acb->acb_noauth, B_TRUE,
 		    &acb->acb_buf);
 		if (error != 0) {
@@ -5829,7 +5811,7 @@ arc_read_done(zio_t *zio)
 		 * encryption key wasn't loaded
 		 */
 		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
-		    error != ENOENT);
+		    error != EACCES);
 
 		/*
 		 * If we failed to decrypt, report an error now (as the zio
@@ -5838,11 +5820,10 @@ arc_read_done(zio_t *zio)
 		if (error == ECKSUM) {
 			ASSERT(BP_IS_PROTECTED(bp));
 			error = SET_ERROR(EIO);
-			spa_log_error(zio->io_spa, &zio->io_bookmark);
 			if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
+				spa_log_error(zio->io_spa, &acb->acb_zb);
 				zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
-				    zio->io_spa, NULL, &zio->io_bookmark, zio,
-				    0, 0);
+				    zio->io_spa, NULL, &acb->acb_zb, zio, 0, 0);
 			}
 		}
 
@@ -6004,7 +5985,7 @@ top:
 				acb->acb_compressed = compressed_read;
 				acb->acb_encrypted = encrypted_read;
 				acb->acb_noauth = noauth_read;
-				acb->acb_dsobj = zb->zb_objset;
+				acb->acb_zb = *zb;
 				if (pio != NULL)
 					acb->acb_zio_dummy = zio_null(pio,
 					    spa, NULL, NULL, NULL, zio_flags);
@@ -6049,18 +6030,22 @@ top:
 			ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
 
 			/* Get a buf with the desired data in it. */
-			rc = arc_buf_alloc_impl(hdr, spa, zb->zb_objset,
-			    private, encrypted_read, compressed_read,
-			    noauth_read, B_TRUE, &buf);
+			rc = arc_buf_alloc_impl(hdr, spa, zb, private,
+			    encrypted_read, compressed_read, noauth_read,
+			    B_TRUE, &buf);
 			if (rc == ECKSUM) {
 				/*
 				 * Convert authentication and decryption errors
-				 * to EIO (and generate an ereport) before
-				 * leaving the ARC.
+				 * to EIO (and generate an ereport if needed)
+				 * before leaving the ARC.
 				 */
 				rc = SET_ERROR(EIO);
-				zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
-				    spa, NULL, zb, NULL, 0, 0);
+				if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
+					spa_log_error(spa, zb);
+					zfs_ereport_post(
+					    FM_EREPORT_ZFS_AUTHENTICATION,
+					    spa, NULL, zb, NULL, 0, 0);
+				}
 			}
 			if (rc != 0) {
 				(void) remove_reference(hdr, hash_lock,
@@ -6071,7 +6056,7 @@ top:
 
 			/* assert any errors weren't due to unloaded keys */
 			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
-			    rc != ENOENT);
+			    rc != EACCES);
 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
 		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
@@ -6223,7 +6208,7 @@ top:
 		acb->acb_compressed = compressed_read;
 		acb->acb_encrypted = encrypted_read;
 		acb->acb_noauth = noauth_read;
-		acb->acb_dsobj = zb->zb_objset;
+		acb->acb_zb = *zb;
 
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 		hdr->b_l1hdr.b_acb = acb;
@@ -8078,7 +8063,6 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
 	spa_t *spa = zio->io_spa;
 	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
 	blkptr_t *bp = zio->io_bp;
-	dsl_crypto_key_t *dck = NULL;
 	uint8_t salt[ZIO_DATA_SALT_LEN];
 	uint8_t iv[ZIO_DATA_IV_LEN];
 	uint8_t mac[ZIO_DATA_MAC_LEN];
@@ -8099,31 +8083,20 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
 	 * until arc_read_done().
 	 */
 	if (BP_IS_ENCRYPTED(bp)) {
-		abd_t *eabd = arc_get_data_abd(hdr,
-		    arc_hdr_size(hdr), hdr);
+		abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr);
 
 		zio_crypt_decode_params_bp(bp, salt, iv);
 		zio_crypt_decode_mac_bp(bp, mac);
 
-		ret = spa_keystore_lookup_key(spa,
-		    cb->l2rcb_zb.zb_objset, FTAG, &dck);
+		ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
+		    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
+		    salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
+		    hdr->b_l1hdr.b_pabd, &no_crypt);
 		if (ret != 0) {
 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
 			goto error;
 		}
 
-		ret = zio_do_crypt_abd(B_FALSE, &dck->dck_key,
-		    salt, BP_GET_TYPE(bp), iv, mac, HDR_GET_PSIZE(hdr),
-		    BP_SHOULD_BYTESWAP(bp), eabd, hdr->b_l1hdr.b_pabd,
-		    &no_crypt);
-		if (ret != 0) {
-			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
-			spa_keystore_dsl_key_rele(spa, dck, FTAG);
-			goto error;
-		}
-
-		spa_keystore_dsl_key_rele(spa, dck, FTAG);
-
 		/*
 		 * If we actually performed decryption, replace b_pabd
 		 * with the decrypted data. Otherwise we can just throw
@@ -8529,9 +8502,9 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
 			goto error;
 
 		ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
-		    hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_ot,
-		    hdr->b_crypt_hdr.b_iv, mac, psize, bswap, to_write,
-		    eabd, &no_crypt);
+		    hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
+		    hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
+		    &no_crypt);
 		if (ret != 0)
 			goto error;
 
diff --git a/module/zfs/dsl_crypt.c b/module/zfs/dsl_crypt.c
index 579b32c42..e2067448c 100644
--- a/module/zfs/dsl_crypt.c
+++ b/module/zfs/dsl_crypt.c
@@ -2661,23 +2661,23 @@ error:
  * these fields to populate pabd (the plaintext).
  */
 int
-spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, uint64_t dsobj,
-    const blkptr_t *bp, uint64_t txgid, uint_t datalen, abd_t *pabd,
-    abd_t *cabd, uint8_t *iv, uint8_t *mac, uint8_t *salt, boolean_t *no_crypt)
+spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, const zbookmark_phys_t *zb,
+    dmu_object_type_t ot, boolean_t dedup, boolean_t bswap, uint8_t *salt,
+    uint8_t *iv, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd,
+    boolean_t *no_crypt)
 {
 	int ret;
-	dmu_object_type_t ot = BP_GET_TYPE(bp);
 	dsl_crypto_key_t *dck = NULL;
 	uint8_t *plainbuf = NULL, *cipherbuf = NULL;
 
 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION));
-	ASSERT(!BP_IS_EMBEDDED(bp));
-	ASSERT(BP_IS_ENCRYPTED(bp));
 
 	/* look up the key from the spa's keystore */
-	ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck);
-	if (ret != 0)
+	ret = spa_keystore_lookup_key(spa, zb->zb_objset, FTAG, &dck);
+	if (ret != 0) {
+		ret = SET_ERROR(EACCES);
 		return (ret);
+	}
 
 	if (encrypt) {
 		plainbuf = abd_borrow_buf_copy(pabd, datalen);
@@ -2696,7 +2696,7 @@ spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, uint64_t dsobj,
 	 * at allocation time in zio_alloc_zil(). On decryption, we simply use
 	 * the provided values.
 	 */
-	if (encrypt && ot != DMU_OT_INTENT_LOG && !BP_GET_DEDUP(bp)) {
+	if (encrypt && ot != DMU_OT_INTENT_LOG && !dedup) {
 		ret = zio_crypt_key_get_salt(&dck->dck_key, salt);
 		if (ret != 0)
 			goto error;
@@ -2704,7 +2704,7 @@ spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, uint64_t dsobj,
 		ret = zio_crypt_generate_iv(iv);
 		if (ret != 0)
 			goto error;
-	} else if (encrypt && BP_GET_DEDUP(bp)) {
+	} else if (encrypt && dedup) {
 		ret = zio_crypt_generate_iv_salt_dedup(&dck->dck_key,
 		    plainbuf, datalen, iv, salt);
 		if (ret != 0)
@@ -2712,8 +2712,17 @@ spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, uint64_t dsobj,
 	}
 
 	/* call lower level function to perform encryption / decryption */
-	ret = zio_do_crypt_data(encrypt, &dck->dck_key, salt, ot, iv, mac,
-	    datalen, BP_SHOULD_BYTESWAP(bp), plainbuf, cipherbuf, no_crypt);
+	ret = zio_do_crypt_data(encrypt, &dck->dck_key, ot, bswap, salt, iv,
+	    mac, datalen, plainbuf, cipherbuf, no_crypt);
+
+	/*
+	 * Handle injected decryption faults. Unfortunately, we cannot inject
+	 * faults for dnode blocks because we might trigger the panic in
+	 * dbuf_prepare_encrypted_dnode_leaf(), which exists because syncing
+	 * context is not prepared to handle malicious decryption failures.
+	 */
+	if (zio_injection_enabled && !encrypt && ot != DMU_OT_DNODE && ret == 0)
+		ret = zio_handle_decrypt_injection(spa, zb, ot, ECKSUM);
 	if (ret != 0)
 		goto error;
 
diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index b585368be..6822505f1 100644
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -449,6 +449,10 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 		}
 		abd_copy(data, zio->io_abd, size);
 
+		if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) {
+			ret = zio_handle_decrypt_injection(spa,
+			    &zio->io_bookmark, ot, ECKSUM);
+		}
 		if (ret != 0)
 			goto error;
 
@@ -468,6 +472,10 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 			zio_crypt_decode_mac_bp(bp, mac);
 			ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj,
 			    zio->io_abd, size, mac);
+			if (zio_injection_enabled && ret == 0) {
+				ret = zio_handle_decrypt_injection(spa,
+				    &zio->io_bookmark, ot, ECKSUM);
+			}
 		}
 		abd_copy(data, zio->io_abd, size);
 
@@ -487,8 +495,9 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 		zio_crypt_decode_mac_bp(bp, mac);
 	}
 
-	ret = spa_do_crypt_abd(B_FALSE, spa, dsobj, bp, bp->blk_birth,
-	    size, data, zio->io_abd, iv, mac, salt, &no_crypt);
+	ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp),
+	    BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data,
+	    zio->io_abd, &no_crypt);
 	if (no_crypt)
 		abd_copy(data, zio->io_abd, size);
 
@@ -499,7 +508,7 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
 
 error:
 	/* assert that the key was found unless this was speculative */
-	ASSERT(ret != ENOENT || (zio->io_flags & ZIO_FLAG_SPECULATIVE));
+	ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE));
 
 	/*
 	 * If there was a decryption / authentication error return EIO as
@@ -508,6 +517,7 @@ error:
 	if (ret == ECKSUM) {
 		zio->io_error = SET_ERROR(EIO);
 		if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
+			spa_log_error(spa, &zio->io_bookmark);
 			zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
 			    spa, NULL, &zio->io_bookmark, zio, 0, 0);
 		}
@@ -3906,8 +3916,9 @@ zio_encrypt(zio_t *zio)
 	}
 
 	/* Perform the encryption. This should not fail */
-	VERIFY0(spa_do_crypt_abd(B_TRUE, spa, dsobj, bp, zio->io_txg,
-	    psize, zio->io_abd, eabd, iv, mac, salt, &no_crypt));
+	VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark,
+	    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
+	    salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt));
 
 	/* encode encryption metadata into the bp */
 	if (ot == DMU_OT_INTENT_LOG) {
diff --git a/module/zfs/zio_crypt.c b/module/zfs/zio_crypt.c
index d9e88404f..2e6176782 100644
--- a/module/zfs/zio_crypt.c
+++ b/module/zfs/zio_crypt.c
@@ -1860,9 +1860,9 @@ error:
  * Primary encryption / decryption entrypoint for zio data.
  */
 int
-zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,
-    dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen,
-    boolean_t byteswap, uint8_t *plainbuf, uint8_t *cipherbuf,
+zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
+    dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
+    uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
     boolean_t *no_crypt)
 {
 	int ret;
@@ -1984,9 +1984,9 @@ error:
  * linear buffers.
  */
 int
-zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,
-    dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen,
-    boolean_t byteswap, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
+zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
+    boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
+    uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
 {
 	int ret;
 	void *ptmp, *ctmp;
@@ -1999,8 +1999,8 @@ zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,
 		ctmp = abd_borrow_buf_copy(cabd, datalen);
 	}
 
-	ret = zio_do_crypt_data(encrypt, key, salt, ot, iv, mac,
-	    datalen, byteswap, ptmp, ctmp, no_crypt);
+	ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
+	    datalen, ptmp, ctmp, no_crypt);
 	if (ret != 0)
 		goto error;
 
diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c
index 62ca41bf4..26f255c7b 100644
--- a/module/zfs/zio_inject.c
+++ b/module/zfs/zio_inject.c
@@ -123,7 +123,7 @@ freq_triggered(uint32_t frequency)
  * Returns true if the given record matches the I/O in progress.
  */
 static boolean_t
-zio_match_handler(zbookmark_phys_t *zb, uint64_t type,
+zio_match_handler(const zbookmark_phys_t *zb, uint64_t type,
     zinject_record_t *record, int error)
 {
 	/*
@@ -179,6 +179,36 @@ zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type)
 }
 
 /*
+ * Inject a decryption failure. Decryption failures can occur in
+ * both the ARC and the ZIO layers.
+ */
+int
+zio_handle_decrypt_injection(spa_t *spa, const zbookmark_phys_t *zb,
+    uint64_t type, int error)
+{
+	int ret = 0;
+	inject_handler_t *handler;
+
+	rw_enter(&inject_lock, RW_READER);
+
+	for (handler = list_head(&inject_handlers); handler != NULL;
+	    handler = list_next(&inject_handlers, handler)) {
+
+		if (spa != handler->zi_spa ||
+		    handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)
+			continue;
+
+		if (zio_match_handler(zb, type, &handler->zi_record, error)) {
+			ret = error;
+			break;
+		}
+	}
+
+	rw_exit(&inject_lock);
+	return (ret);
+}
+
+/*
  * Determine if the I/O in question should return failure.  Returns the errno
  * to be returned to the caller.
  */