Add ASSERT to debug encryption key mapping issues

This patch simply adds an ASSERT that confirms that the last
decrypting reference on a dataset waits until the dataset is
no longer dirty. This should help to debug issues where the
ZIO layer cannot find encryption keys after a dataset has been
disowned.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Tom Caputi <[email protected]>
Closes #7637 

3 files changed, 24 insertions, 0 deletions

diff --git a/include/sys/dmu.h b/include/sys/dmu.h
index e5950be93..44c026286 100644
--- a/include/sys/dmu.h
+++ b/include/sys/dmu.h
@@ -557,6 +557,7 @@ boolean_t dmu_buf_try_add_ref(dmu_buf_t *, objset_t *os, uint64_t object,
 
 void dmu_buf_rele(dmu_buf_t *db, void *tag);
 uint64_t dmu_buf_refcount(dmu_buf_t *db);
+uint64_t dmu_buf_user_refcount(dmu_buf_t *db);
 
 /*
  * dmu_buf_hold_array holds the DMU buffers which contain all bytes in a
diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c
index 2411c8d4f..e3738db52 100644
--- a/module/zfs/dbuf.c
+++ b/module/zfs/dbuf.c
@@ -3306,6 +3306,20 @@ dbuf_refcount(dmu_buf_impl_t *db)
 	return (refcount_count(&db->db_holds));
 }
 
+uint64_t
+dmu_buf_user_refcount(dmu_buf_t *db_fake)
+{
+	uint64_t holds;
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+
+	mutex_enter(&db->db_mtx);
+	ASSERT3U(refcount_count(&db->db_holds), >=, db->db_dirtycnt);
+	holds = refcount_count(&db->db_holds) - db->db_dirtycnt;
+	mutex_exit(&db->db_mtx);
+
+	return (holds);
+}
+
 void *
 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
     dmu_buf_user_t *new_user)
diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c
index 7a4721e17..9db6d1e0b 100644
--- a/module/zfs/dsl_dataset.c
+++ b/module/zfs/dsl_dataset.c
@@ -791,6 +791,15 @@ dsl_dataset_rele_flags(dsl_dataset_t *ds, ds_hold_flags_t flags, void *tag)
 	    (flags & DS_HOLD_FLAG_DECRYPT)) {
 		(void) spa_keystore_remove_mapping(ds->ds_dir->dd_pool->dp_spa,
 		    ds->ds_object, ds);
+
+		/*
+		 * Encrypted datasets require that users only release their
+		 * decrypting reference after the dirty data has actually
+		 * been written out. This ensures that the mapping exists
+		 * when it is needed to write out dirty data.
+		 */
+		ASSERT(dmu_buf_user_refcount(ds->ds_dbuf) != 0 ||
+		    !dsl_dataset_is_dirty(ds));
 	}
 
 	dmu_buf_rele(ds->ds_dbuf, tag);