aboutsummaryrefslogtreecommitdiffstats
path: root/module/zfs/arc.c
diff options
context:
space:
mode:
Diffstat (limited to 'module/zfs/arc.c')
-rw-r--r--module/zfs/arc.c77
1 files changed, 51 insertions, 26 deletions
diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 992e57ce6..6256fed2b 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -3155,13 +3155,14 @@ arc_buf_destroy_impl(arc_buf_t *buf)
hdr->b_crypt_hdr.b_ebufcnt -= 1;
/*
- * if we have no more encrypted buffers and we've already
+ * If we have no more encrypted buffers and we've already
* gotten a copy of the decrypted data we can free b_rabd to
* save some space.
*/
if (hdr->b_crypt_hdr.b_ebufcnt == 0 && HDR_HAS_RABD(hdr) &&
- hdr->b_l1hdr.b_pabd != NULL)
+ hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
arc_hdr_free_abd(hdr, B_TRUE);
+ }
}
arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
@@ -3716,9 +3717,8 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
arc_hdr_free_abd(hdr, B_FALSE);
}
- if (HDR_HAS_RABD(hdr)) {
+ if (HDR_HAS_RABD(hdr))
arc_hdr_free_abd(hdr, B_TRUE);
- }
}
ASSERT3P(hdr->b_hash_next, ==, NULL);
@@ -5746,16 +5746,15 @@ arc_read_done(zio_t *zio)
callback_cnt++;
int error = arc_buf_alloc_impl(hdr, zio->io_spa,
- zio->io_bookmark.zb_objset, acb->acb_private,
- acb->acb_encrypted, acb->acb_compressed, acb->acb_noauth,
- no_zio_error, &acb->acb_buf);
+ acb->acb_dsobj, acb->acb_private, acb->acb_encrypted,
+ acb->acb_compressed, acb->acb_noauth, no_zio_error,
+ &acb->acb_buf);
/*
- * assert non-speculative zios didn't fail because an
+ * Assert non-speculative zios didn't fail because an
* encryption key wasn't loaded
*/
- ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
- error == 0 || error != ENOENT);
+ ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) || error == 0);
/*
* If we failed to decrypt, report an error now (as the zio
@@ -5778,10 +5777,8 @@ arc_read_done(zio_t *zio)
}
hdr->b_l1hdr.b_acb = NULL;
arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
- if (callback_cnt == 0) {
- ASSERT(HDR_PREFETCH(hdr) || HDR_HAS_RABD(hdr));
+ if (callback_cnt == 0)
ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
- }
ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
callback_list != NULL);
@@ -5943,6 +5940,9 @@ top:
acb->acb_done = done;
acb->acb_private = private;
acb->acb_compressed = compressed_read;
+ acb->acb_encrypted = encrypted_read;
+ acb->acb_noauth = noauth_read;
+ acb->acb_dsobj = zb->zb_objset;
if (pio != NULL)
acb->acb_zio_dummy = zio_null(pio,
spa, NULL, NULL, NULL, zio_flags);
@@ -5981,9 +5981,7 @@ top:
rc = arc_buf_alloc_impl(hdr, spa, zb->zb_objset,
private, encrypted_read, compressed_read,
noauth_read, B_TRUE, &buf);
-
- ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
- rc == 0 || rc != ENOENT);
+ ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) || rc == 0);
} else if (*arc_flags & ARC_FLAG_PREFETCH &&
refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
@@ -6008,7 +6006,7 @@ top:
uint64_t addr = 0;
boolean_t devw = B_FALSE;
uint64_t size;
- void *hdr_abd;
+ abd_t *hdr_abd;
/*
* Gracefully handle a damaged logical block size as a
@@ -6131,6 +6129,7 @@ top:
acb->acb_compressed = compressed_read;
acb->acb_encrypted = encrypted_read;
acb->acb_noauth = noauth_read;
+ acb->acb_dsobj = zb->zb_objset;
ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
hdr->b_l1hdr.b_acb = acb;
@@ -6698,6 +6697,9 @@ arc_write_ready(zio_t *zio)
HDR_SET_PSIZE(hdr, psize);
arc_hdr_set_compress(hdr, compress);
+ if (zio->io_error != 0 || psize == 0)
+ goto out;
+
/*
* Fill the hdr with data. If the buffer is encrypted we have no choice
* but to copy the data into b_radb. If the hdr is compressed, the data
@@ -6713,6 +6715,7 @@ arc_write_ready(zio_t *zio)
* the data into it; otherwise, we share the data directly if we can.
*/
if (ARC_BUF_ENCRYPTED(buf)) {
+ ASSERT3U(psize, >, 0);
ASSERT(ARC_BUF_COMPRESSED(buf));
arc_hdr_alloc_abd(hdr, B_TRUE);
abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
@@ -6745,6 +6748,7 @@ arc_write_ready(zio_t *zio)
arc_share_buf(hdr, buf);
}
+out:
arc_hdr_verify(hdr, bp);
spl_fstrans_unmark(cookie);
}
@@ -7956,9 +7960,15 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
*/
ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
+ ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
- /* If the data was encrypted, decrypt it now */
- if (HDR_ENCRYPTED(hdr)) {
+ /*
+ * If the data was encrypted, decrypt it now. Note that
+ * we must check the bp here and not the hdr, since the
+ * hdr does not have its encryption parameters updated
+ * until arc_read_done().
+ */
+ if (BP_IS_ENCRYPTED(bp)) {
abd_t *eabd = arc_get_data_abd(hdr,
arc_hdr_size(hdr), hdr);
@@ -8084,7 +8094,16 @@ l2arc_read_done(zio_t *zio)
*/
abd_free(cb->l2rcb_abd);
zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
- zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
+
+ if (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
+ (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT)) {
+ ASSERT(HDR_HAS_RABD(hdr));
+ zio->io_abd = zio->io_orig_abd =
+ hdr->b_crypt_hdr.b_rabd;
+ } else {
+ ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
+ zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
+ }
}
ASSERT3P(zio->io_abd, !=, NULL);
@@ -8321,7 +8340,7 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
dsl_crypto_key_t *dck = NULL;
uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
- boolean_t no_crypt;
+ boolean_t no_crypt = B_FALSE;
ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
!HDR_COMPRESSION_ENABLED(hdr)) ||
@@ -8333,6 +8352,15 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
* and copy the data. This may be done to elimiate a depedency on a
* shared buffer or to reallocate the buffer to match asize.
*/
+ if (HDR_HAS_RABD(hdr) && asize != psize) {
+ ASSERT3U(size, ==, psize);
+ to_write = abd_alloc_for_io(asize, ismd);
+ abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, size);
+ if (size != asize)
+ abd_zero_off(to_write, size, asize - size);
+ goto out;
+ }
+
if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
!HDR_ENCRYPTED(hdr)) {
ASSERT3U(size, ==, psize);
@@ -8377,11 +8405,8 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
if (ret != 0)
goto error;
- if (no_crypt) {
- spa_keystore_dsl_key_rele(spa, dck, FTAG);
- abd_free(eabd);
- goto out;
- }
+ if (no_crypt)
+ abd_copy(eabd, to_write, psize);
if (psize != asize)
abd_zero_off(eabd, psize, asize - psize);
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-03 06:55:37 +0000