Use wmsum for arc, abd, dbuf and zfetch statistics. (#12172)

wmsum was designed exactly for cases like these with many updates
and rare reads.  It allows to completely avoid atomic operations on
congested global variables.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Mark Maybee <[email protected]>
Signed-off-by: Alexander Motin <[email protected]>
Sponsored-By: iXsystems, Inc.
Closes #12172

3 files changed, 561 insertions, 187 deletions

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 11285aa46..b0468159d 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -600,6 +600,8 @@ arc_stats_t arc_stats = {
 	{ "abd_chunk_waste_size",	KSTAT_DATA_UINT64 },
 };
 
+arc_sums_t arc_sums;
+
 #define	ARCSTAT_MAX(stat, val) {					\
 	uint64_t m;							\
 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
@@ -607,9 +609,6 @@ arc_stats_t arc_stats = {
 		continue;						\
 }
 
-#define	ARCSTAT_MAXSTAT(stat) \
-	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
-
 /*
  * We define a macro to allow ARC hits/misses to be easily broken down by
  * two separate conditions, giving a total of four different subtypes for
@@ -671,37 +670,8 @@ arc_state_t	*arc_mfu;
 /* max size for dnodes */
 #define	arc_dnode_size_limit	ARCSTAT(arcstat_dnode_limit)
 #define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
-#define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
 #define	arc_need_free	ARCSTAT(arcstat_need_free) /* waiting to be evicted */
 
-/* size of all b_rabd's in entire arc */
-#define	arc_raw_size	ARCSTAT(arcstat_raw_size)
-/* compressed size of entire arc */
-#define	arc_compressed_size	ARCSTAT(arcstat_compressed_size)
-/* uncompressed size of entire arc */
-#define	arc_uncompressed_size	ARCSTAT(arcstat_uncompressed_size)
-/* number of bytes in the arc from arc_buf_t's */
-#define	arc_overhead_size	ARCSTAT(arcstat_overhead_size)
-
-/*
- * There are also some ARC variables that we want to export, but that are
- * updated so often that having the canonical representation be the statistic
- * variable causes a performance bottleneck. We want to use aggsum_t's for these
- * instead, but still be able to export the kstat in the same way as before.
- * The solution is to always use the aggsum version, except in the kstat update
- * callback.
- */
-aggsum_t arc_size;
-aggsum_t arc_meta_used;
-wmsum_t astat_data_size;
-wmsum_t astat_metadata_size;
-wmsum_t astat_dbuf_size;
-aggsum_t astat_dnode_size;
-wmsum_t astat_bonus_size;
-wmsum_t astat_hdr_size;
-aggsum_t astat_l2_hdr_size;
-wmsum_t astat_abd_chunk_waste_size;
-
 hrtime_t arc_growtime;
 list_t arc_prune_list;
 kmutex_t arc_prune_mtx;
@@ -819,9 +789,6 @@ uint64_t zfs_crc64_table[256];
  */
 #define	L2ARC_FEED_TYPES	4
 
-#define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
-#define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
-
 /* L2ARC Performance Tunables */
 unsigned long l2arc_write_max = L2ARC_WRITE_SIZE;	/* def max write size */
 unsigned long l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra warmup write */
@@ -1085,9 +1052,9 @@ buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
 
 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
 	}
-
-	ARCSTAT_BUMP(arcstat_hash_elements);
-	ARCSTAT_MAXSTAT(arcstat_hash_elements);
+	uint64_t he = atomic_inc_64_nv(
+	    &arc_stats.arcstat_hash_elements.value.ui64);
+	ARCSTAT_MAX(arcstat_hash_elements_max, he);
 
 	return (NULL);
 }
@@ -1111,7 +1078,7 @@ buf_hash_remove(arc_buf_hdr_t *hdr)
 	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
 
 	/* collect some hash table performance data */
-	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
+	atomic_dec_64(&arc_stats.arcstat_hash_elements.value.ui64);
 
 	if (buf_hash_table.ht_table[idx] &&
 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
@@ -2646,25 +2613,25 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 	default:
 		break;
 	case ARC_SPACE_DATA:
-		wmsum_add(&astat_data_size, space);
+		ARCSTAT_INCR(arcstat_data_size, space);
 		break;
 	case ARC_SPACE_META:
-		wmsum_add(&astat_metadata_size, space);
+		ARCSTAT_INCR(arcstat_metadata_size, space);
 		break;
 	case ARC_SPACE_BONUS:
-		wmsum_add(&astat_bonus_size, space);
+		ARCSTAT_INCR(arcstat_bonus_size, space);
 		break;
 	case ARC_SPACE_DNODE:
-		aggsum_add(&astat_dnode_size, space);
+		aggsum_add(&arc_sums.arcstat_dnode_size, space);
 		break;
 	case ARC_SPACE_DBUF:
-		wmsum_add(&astat_dbuf_size, space);
+		ARCSTAT_INCR(arcstat_dbuf_size, space);
 		break;
 	case ARC_SPACE_HDRS:
-		wmsum_add(&astat_hdr_size, space);
+		ARCSTAT_INCR(arcstat_hdr_size, space);
 		break;
 	case ARC_SPACE_L2HDRS:
-		aggsum_add(&astat_l2_hdr_size, space);
+		aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
 		break;
 	case ARC_SPACE_ABD_CHUNK_WASTE:
 		/*
@@ -2673,14 +2640,14 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 		 * scatter ABD's come from the ARC, because other users are
 		 * very short-lived.
 		 */
-		wmsum_add(&astat_abd_chunk_waste_size, space);
+		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
 		break;
 	}
 
 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
-		aggsum_add(&arc_meta_used, space);
+		aggsum_add(&arc_sums.arcstat_meta_used, space);
 
-	aggsum_add(&arc_size, space);
+	aggsum_add(&arc_sums.arcstat_size, space);
 }
 
 void
@@ -2692,45 +2659,41 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 	default:
 		break;
 	case ARC_SPACE_DATA:
-		wmsum_add(&astat_data_size, -space);
+		ARCSTAT_INCR(arcstat_data_size, -space);
 		break;
 	case ARC_SPACE_META:
-		wmsum_add(&astat_metadata_size, -space);
+		ARCSTAT_INCR(arcstat_metadata_size, -space);
 		break;
 	case ARC_SPACE_BONUS:
-		wmsum_add(&astat_bonus_size, -space);
+		ARCSTAT_INCR(arcstat_bonus_size, -space);
 		break;
 	case ARC_SPACE_DNODE:
-		aggsum_add(&astat_dnode_size, -space);
+		aggsum_add(&arc_sums.arcstat_dnode_size, -space);
 		break;
 	case ARC_SPACE_DBUF:
-		wmsum_add(&astat_dbuf_size, -space);
+		ARCSTAT_INCR(arcstat_dbuf_size, -space);
 		break;
 	case ARC_SPACE_HDRS:
-		wmsum_add(&astat_hdr_size, -space);
+		ARCSTAT_INCR(arcstat_hdr_size, -space);
 		break;
 	case ARC_SPACE_L2HDRS:
-		aggsum_add(&astat_l2_hdr_size, -space);
+		aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
 		break;
 	case ARC_SPACE_ABD_CHUNK_WASTE:
-		wmsum_add(&astat_abd_chunk_waste_size, -space);
+		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
 		break;
 	}
 
 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE) {
-		ASSERT(aggsum_compare(&arc_meta_used, space) >= 0);
-		/*
-		 * We use the upper bound here rather than the precise value
-		 * because the arc_meta_max value doesn't need to be
-		 * precise. It's only consumed by humans via arcstats.
-		 */
-		if (arc_meta_max < aggsum_upper_bound(&arc_meta_used))
-			arc_meta_max = aggsum_upper_bound(&arc_meta_used);
-		aggsum_add(&arc_meta_used, -space);
+		ASSERT(aggsum_compare(&arc_sums.arcstat_meta_used,
+		    space) >= 0);
+		ARCSTAT_MAX(arcstat_meta_max,
+		    aggsum_upper_bound(&arc_sums.arcstat_meta_used));
+		aggsum_add(&arc_sums.arcstat_meta_used, -space);
 	}
 
-	ASSERT(aggsum_compare(&arc_size, space) >= 0);
-	aggsum_add(&arc_size, -space);
+	ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
+	aggsum_add(&arc_sums.arcstat_size, -space);
 }
 
 /*
@@ -4246,9 +4209,10 @@ arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
 		 * Request that 10% of the LRUs be scanned by the superblock
 		 * shrinker.
 		 */
-		if (type == ARC_BUFC_DATA && aggsum_compare(&astat_dnode_size,
-		    arc_dnode_size_limit) > 0) {
-			arc_prune_async((aggsum_upper_bound(&astat_dnode_size) -
+		if (type == ARC_BUFC_DATA && aggsum_compare(
+		    &arc_sums.arcstat_dnode_size, arc_dnode_size_limit) > 0) {
+			arc_prune_async((aggsum_upper_bound(
+			    &arc_sums.arcstat_dnode_size) -
 			    arc_dnode_size_limit) / sizeof (dnode_t) /
 			    zfs_arc_dnode_reduce_percent);
 		}
@@ -4478,7 +4442,7 @@ restart:
 }
 
 /*
- * Evict metadata buffers from the cache, such that arc_meta_used is
+ * Evict metadata buffers from the cache, such that arcstat_meta_used is
  * capped by the arc_meta_limit tunable.
  */
 static uint64_t
@@ -4599,7 +4563,7 @@ arc_evict_type(arc_state_t *state)
 }
 
 /*
- * Evict buffers from the cache, such that arc_size is capped by arc_c.
+ * Evict buffers from the cache, such that arcstat_size is capped by arc_c.
  */
 static uint64_t
 arc_evict(void)
@@ -4607,8 +4571,8 @@ arc_evict(void)
 	uint64_t total_evicted = 0;
 	uint64_t bytes;
 	int64_t target;
-	uint64_t asize = aggsum_value(&arc_size);
-	uint64_t ameta = aggsum_value(&arc_meta_used);
+	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
+	uint64_t ameta = aggsum_value(&arc_sums.arcstat_meta_used);
 
 	/*
 	 * If we're over arc_meta_limit, we want to correct that before
@@ -4668,8 +4632,8 @@ arc_evict(void)
 	/*
 	 * Re-sum ARC stats after the first round of evictions.
 	 */
-	asize = aggsum_value(&arc_size);
-	ameta = aggsum_value(&arc_meta_used);
+	asize = aggsum_value(&arc_sums.arcstat_size);
+	ameta = aggsum_value(&arc_sums.arcstat_meta_used);
 
 
 	/*
@@ -4783,7 +4747,7 @@ arc_flush(spa_t *spa, boolean_t retry)
 void
 arc_reduce_target_size(int64_t to_free)
 {
-	uint64_t asize = aggsum_value(&arc_size);
+	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
 
 	/*
 	 * All callers want the ARC to actually evict (at least) this much
@@ -4836,8 +4800,8 @@ arc_kmem_reap_soon(void)
 	extern kmem_cache_t	*zio_data_buf_cache[];
 
 #ifdef _KERNEL
-	if ((aggsum_compare(&arc_meta_used, arc_meta_limit) >= 0) &&
-	    zfs_arc_meta_prune) {
+	if ((aggsum_compare(&arc_sums.arcstat_meta_used,
+	    arc_meta_limit) >= 0) && zfs_arc_meta_prune) {
 		/*
 		 * We are exceeding our meta-data cache limit.
 		 * Prune some entries to release holds on meta-data.
@@ -4940,7 +4904,7 @@ arc_evict_cb(void *arg, zthr_t *zthr)
 	 */
 	mutex_enter(&arc_evict_lock);
 	arc_evict_needed = !zthr_iscancelled(arc_evict_zthr) &&
-	    evicted > 0 && aggsum_compare(&arc_size, arc_c) > 0;
+	    evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
 	if (!arc_evict_needed) {
 		/*
 		 * We're either no longer overflowing, or we
@@ -5153,7 +5117,7 @@ arc_adapt(int bytes, arc_state_t *state)
 	 * cache size, increment the target cache size
 	 */
 	ASSERT3U(arc_c, >=, 2ULL << SPA_MAXBLOCKSHIFT);
-	if (aggsum_upper_bound(&arc_size) >=
+	if (aggsum_upper_bound(&arc_sums.arcstat_size) >=
 	    arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
 		atomic_add_64(&arc_c, (int64_t)bytes);
 		if (arc_c > arc_c_max)
@@ -5186,7 +5150,8 @@ arc_is_overflowing(void)
 	 * in the ARC. In practice, that's in the tens of MB, which is low
 	 * enough to be safe.
 	 */
-	return (aggsum_lower_bound(&arc_size) >= (int64_t)arc_c + overflow);
+	return (aggsum_lower_bound(&arc_sums.arcstat_size) >=
+	    (int64_t)arc_c + overflow);
 }
 
 static abd_t *
@@ -5355,7 +5320,7 @@ arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag,
 		 * If we are growing the cache, and we are adding anonymous
 		 * data, and we have outgrown arc_p, update arc_p
 		 */
-		if (aggsum_upper_bound(&arc_size) < arc_c &&
+		if (aggsum_upper_bound(&arc_sums.arcstat_size) < arc_c &&
 		    hdr->b_l1hdr.b_state == arc_anon &&
 		    (zfs_refcount_count(&arc_anon->arcs_size) +
 		    zfs_refcount_count(&arc_mru->arcs_size) > arc_p))
@@ -7240,55 +7205,219 @@ arc_kstat_update(kstat_t *ksp, int rw)
 {
 	arc_stats_t *as = ksp->ks_data;
 
-	if (rw == KSTAT_WRITE) {
+	if (rw == KSTAT_WRITE)
 		return (SET_ERROR(EACCES));
-	} else {
-		arc_kstat_update_state(arc_anon,
-		    &as->arcstat_anon_size,
-		    &as->arcstat_anon_evictable_data,
-		    &as->arcstat_anon_evictable_metadata);
-		arc_kstat_update_state(arc_mru,
-		    &as->arcstat_mru_size,
-		    &as->arcstat_mru_evictable_data,
-		    &as->arcstat_mru_evictable_metadata);
-		arc_kstat_update_state(arc_mru_ghost,
-		    &as->arcstat_mru_ghost_size,
-		    &as->arcstat_mru_ghost_evictable_data,
-		    &as->arcstat_mru_ghost_evictable_metadata);
-		arc_kstat_update_state(arc_mfu,
-		    &as->arcstat_mfu_size,
-		    &as->arcstat_mfu_evictable_data,
-		    &as->arcstat_mfu_evictable_metadata);
-		arc_kstat_update_state(arc_mfu_ghost,
-		    &as->arcstat_mfu_ghost_size,
-		    &as->arcstat_mfu_ghost_evictable_data,
-		    &as->arcstat_mfu_ghost_evictable_metadata);
-
-		ARCSTAT(arcstat_size) = aggsum_value(&arc_size);
-		ARCSTAT(arcstat_meta_used) = aggsum_value(&arc_meta_used);
-		ARCSTAT(arcstat_data_size) = wmsum_value(&astat_data_size);
-		ARCSTAT(arcstat_metadata_size) =
-		    wmsum_value(&astat_metadata_size);
-		ARCSTAT(arcstat_hdr_size) = wmsum_value(&astat_hdr_size);
-		ARCSTAT(arcstat_l2_hdr_size) = aggsum_value(&astat_l2_hdr_size);
-		ARCSTAT(arcstat_dbuf_size) = wmsum_value(&astat_dbuf_size);
+
+	as->arcstat_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_hits);
+	as->arcstat_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_misses);
+	as->arcstat_demand_data_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_data_hits);
+	as->arcstat_demand_data_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_data_misses);
+	as->arcstat_demand_metadata_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
+	as->arcstat_demand_metadata_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
+	as->arcstat_prefetch_data_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
+	as->arcstat_prefetch_data_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
+	as->arcstat_prefetch_metadata_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
+	as->arcstat_prefetch_metadata_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
+	as->arcstat_mru_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_mru_hits);
+	as->arcstat_mru_ghost_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
+	as->arcstat_mfu_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_mfu_hits);
+	as->arcstat_mfu_ghost_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
+	as->arcstat_deleted.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_deleted);
+	as->arcstat_mutex_miss.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_mutex_miss);
+	as->arcstat_access_skip.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_access_skip);
+	as->arcstat_evict_skip.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_skip);
+	as->arcstat_evict_not_enough.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_not_enough);
+	as->arcstat_evict_l2_cached.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_cached);
+	as->arcstat_evict_l2_eligible.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
+	as->arcstat_evict_l2_eligible_mfu.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
+	as->arcstat_evict_l2_eligible_mru.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
+	as->arcstat_evict_l2_ineligible.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
+	as->arcstat_evict_l2_skip.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_evict_l2_skip);
+	as->arcstat_hash_collisions.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_hash_collisions);
+	as->arcstat_hash_chains.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_hash_chains);
+	as->arcstat_size.value.ui64 =
+	    aggsum_value(&arc_sums.arcstat_size);
+	as->arcstat_compressed_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_compressed_size);
+	as->arcstat_uncompressed_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_uncompressed_size);
+	as->arcstat_overhead_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_overhead_size);
+	as->arcstat_hdr_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_hdr_size);
+	as->arcstat_data_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_data_size);
+	as->arcstat_metadata_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_metadata_size);
+	as->arcstat_dbuf_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_dbuf_size);
 #if defined(COMPAT_FREEBSD11)
-		ARCSTAT(arcstat_other_size) = wmsum_value(&astat_bonus_size) +
-		    aggsum_value(&astat_dnode_size) +
-		    wmsum_value(&astat_dbuf_size);
+	as->arcstat_other_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_bonus_size) +
+	    aggsum_value(&arc_sums.arcstat_dnode_size) +
+	    wmsum_value(&arc_sums.arcstat_dbuf_size);
 #endif
-		ARCSTAT(arcstat_dnode_size) = aggsum_value(&astat_dnode_size);
-		ARCSTAT(arcstat_bonus_size) = wmsum_value(&astat_bonus_size);
-		ARCSTAT(arcstat_abd_chunk_waste_size) =
-		    wmsum_value(&astat_abd_chunk_waste_size);
 
-		as->arcstat_memory_all_bytes.value.ui64 =
-		    arc_all_memory();
-		as->arcstat_memory_free_bytes.value.ui64 =
-		    arc_free_memory();
-		as->arcstat_memory_available_bytes.value.i64 =
-		    arc_available_memory();
-	}
+	arc_kstat_update_state(arc_anon,
+	    &as->arcstat_anon_size,
+	    &as->arcstat_anon_evictable_data,
+	    &as->arcstat_anon_evictable_metadata);
+	arc_kstat_update_state(arc_mru,
+	    &as->arcstat_mru_size,
+	    &as->arcstat_mru_evictable_data,
+	    &as->arcstat_mru_evictable_metadata);
+	arc_kstat_update_state(arc_mru_ghost,
+	    &as->arcstat_mru_ghost_size,
+	    &as->arcstat_mru_ghost_evictable_data,
+	    &as->arcstat_mru_ghost_evictable_metadata);
+	arc_kstat_update_state(arc_mfu,
+	    &as->arcstat_mfu_size,
+	    &as->arcstat_mfu_evictable_data,
+	    &as->arcstat_mfu_evictable_metadata);
+	arc_kstat_update_state(arc_mfu_ghost,
+	    &as->arcstat_mfu_ghost_size,
+	    &as->arcstat_mfu_ghost_evictable_data,
+	    &as->arcstat_mfu_ghost_evictable_metadata);
+
+	as->arcstat_dnode_size.value.ui64 =
+	    aggsum_value(&arc_sums.arcstat_dnode_size);
+	as->arcstat_bonus_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_bonus_size);
+	as->arcstat_l2_hits.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_hits);
+	as->arcstat_l2_misses.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_misses);
+	as->arcstat_l2_prefetch_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
+	as->arcstat_l2_mru_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_mru_asize);
+	as->arcstat_l2_mfu_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
+	as->arcstat_l2_bufc_data_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
+	as->arcstat_l2_bufc_metadata_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
+	as->arcstat_l2_feeds.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_feeds);
+	as->arcstat_l2_rw_clash.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rw_clash);
+	as->arcstat_l2_read_bytes.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_read_bytes);
+	as->arcstat_l2_write_bytes.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_write_bytes);
+	as->arcstat_l2_writes_sent.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_writes_sent);
+	as->arcstat_l2_writes_done.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_writes_done);
+	as->arcstat_l2_writes_error.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_writes_error);
+	as->arcstat_l2_writes_lock_retry.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
+	as->arcstat_l2_evict_lock_retry.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
+	as->arcstat_l2_evict_reading.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_evict_reading);
+	as->arcstat_l2_evict_l1cached.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
+	as->arcstat_l2_free_on_write.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_free_on_write);
+	as->arcstat_l2_abort_lowmem.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
+	as->arcstat_l2_cksum_bad.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
+	as->arcstat_l2_io_error.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_io_error);
+	as->arcstat_l2_lsize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_lsize);
+	as->arcstat_l2_psize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_psize);
+	as->arcstat_l2_hdr_size.value.ui64 =
+	    aggsum_value(&arc_sums.arcstat_l2_hdr_size);
+	as->arcstat_l2_log_blk_writes.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
+	as->arcstat_l2_log_blk_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
+	as->arcstat_l2_log_blk_count.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
+	as->arcstat_l2_rebuild_success.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
+	as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
+	as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
+	as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
+	as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
+	as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
+	as->arcstat_l2_rebuild_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
+	as->arcstat_l2_rebuild_asize.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
+	as->arcstat_l2_rebuild_bufs.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
+	as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
+	as->arcstat_l2_rebuild_log_blks.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
+	as->arcstat_memory_throttle_count.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_memory_throttle_count);
+	as->arcstat_memory_direct_count.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_memory_direct_count);
+	as->arcstat_memory_indirect_count.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_memory_indirect_count);
+
+	as->arcstat_memory_all_bytes.value.ui64 =
+	    arc_all_memory();
+	as->arcstat_memory_free_bytes.value.ui64 =
+	    arc_free_memory();
+	as->arcstat_memory_available_bytes.value.i64 =
+	    arc_available_memory();
+
+	as->arcstat_prune.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_prune);
+	as->arcstat_meta_used.value.ui64 =
+	    aggsum_value(&arc_sums.arcstat_meta_used);
+	as->arcstat_async_upgrade_sync.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
+	as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
+	as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
+	as->arcstat_raw_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_raw_size);
+	as->arcstat_cached_only_in_progress.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
+	as->arcstat_abd_chunk_waste_size.value.ui64 =
+	    wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);
 
 	return (0);
 }
@@ -7511,16 +7640,93 @@ arc_state_init(void)
 	zfs_refcount_create(&arc_mfu_ghost->arcs_size);
 	zfs_refcount_create(&arc_l2c_only->arcs_size);
 
-	aggsum_init(&arc_meta_used, 0);
-	aggsum_init(&arc_size, 0);
-	wmsum_init(&astat_data_size, 0);
-	wmsum_init(&astat_metadata_size, 0);
-	wmsum_init(&astat_hdr_size, 0);
-	aggsum_init(&astat_l2_hdr_size, 0);
-	wmsum_init(&astat_bonus_size, 0);
-	aggsum_init(&astat_dnode_size, 0);
-	wmsum_init(&astat_dbuf_size, 0);
-	wmsum_init(&astat_abd_chunk_waste_size, 0);
+	wmsum_init(&arc_sums.arcstat_hits, 0);
+	wmsum_init(&arc_sums.arcstat_misses, 0);
+	wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
+	wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
+	wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
+	wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
+	wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
+	wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
+	wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
+	wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
+	wmsum_init(&arc_sums.arcstat_mru_hits, 0);
+	wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
+	wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
+	wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
+	wmsum_init(&arc_sums.arcstat_deleted, 0);
+	wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
+	wmsum_init(&arc_sums.arcstat_access_skip, 0);
+	wmsum_init(&arc_sums.arcstat_evict_skip, 0);
+	wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
+	wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
+	wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
+	wmsum_init(&arc_sums.arcstat_hash_chains, 0);
+	aggsum_init(&arc_sums.arcstat_size, 0);
+	wmsum_init(&arc_sums.arcstat_compressed_size, 0);
+	wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
+	wmsum_init(&arc_sums.arcstat_overhead_size, 0);
+	wmsum_init(&arc_sums.arcstat_hdr_size, 0);
+	wmsum_init(&arc_sums.arcstat_data_size, 0);
+	wmsum_init(&arc_sums.arcstat_metadata_size, 0);
+	wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
+	aggsum_init(&arc_sums.arcstat_dnode_size, 0);
+	wmsum_init(&arc_sums.arcstat_bonus_size, 0);
+	wmsum_init(&arc_sums.arcstat_l2_hits, 0);
+	wmsum_init(&arc_sums.arcstat_l2_misses, 0);
+	wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
+	wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
+	wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
+	wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
+	wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
+	wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
+	wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
+	wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
+	wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
+	wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
+	wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
+	wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
+	wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
+	wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
+	wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_psize, 0);
+	aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
+	wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
+	wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
+	wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
+	wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
+	wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
+	wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
+	wmsum_init(&arc_sums.arcstat_prune, 0);
+	aggsum_init(&arc_sums.arcstat_meta_used, 0);
+	wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
+	wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
+	wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
+	wmsum_init(&arc_sums.arcstat_raw_size, 0);
+	wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
+	wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);
 
 	arc_anon->arcs_state = ARC_STATE_ANON;
 	arc_mru->arcs_state = ARC_STATE_MRU;
@@ -7564,16 +7770,93 @@ arc_state_fini(void)
 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
 
-	aggsum_fini(&arc_meta_used);
-	aggsum_fini(&arc_size);
-	wmsum_fini(&astat_data_size);
-	wmsum_fini(&astat_metadata_size);
-	wmsum_fini(&astat_hdr_size);
-	aggsum_fini(&astat_l2_hdr_size);
-	wmsum_fini(&astat_bonus_size);
-	aggsum_fini(&astat_dnode_size);
-	wmsum_fini(&astat_dbuf_size);
-	wmsum_fini(&astat_abd_chunk_waste_size);
+	wmsum_fini(&arc_sums.arcstat_hits);
+	wmsum_fini(&arc_sums.arcstat_misses);
+	wmsum_fini(&arc_sums.arcstat_demand_data_hits);
+	wmsum_fini(&arc_sums.arcstat_demand_data_misses);
+	wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
+	wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
+	wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
+	wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
+	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
+	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
+	wmsum_fini(&arc_sums.arcstat_mru_hits);
+	wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
+	wmsum_fini(&arc_sums.arcstat_mfu_hits);
+	wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
+	wmsum_fini(&arc_sums.arcstat_deleted);
+	wmsum_fini(&arc_sums.arcstat_mutex_miss);
+	wmsum_fini(&arc_sums.arcstat_access_skip);
+	wmsum_fini(&arc_sums.arcstat_evict_skip);
+	wmsum_fini(&arc_sums.arcstat_evict_not_enough);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
+	wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
+	wmsum_fini(&arc_sums.arcstat_hash_collisions);
+	wmsum_fini(&arc_sums.arcstat_hash_chains);
+	aggsum_fini(&arc_sums.arcstat_size);
+	wmsum_fini(&arc_sums.arcstat_compressed_size);
+	wmsum_fini(&arc_sums.arcstat_uncompressed_size);
+	wmsum_fini(&arc_sums.arcstat_overhead_size);
+	wmsum_fini(&arc_sums.arcstat_hdr_size);
+	wmsum_fini(&arc_sums.arcstat_data_size);
+	wmsum_fini(&arc_sums.arcstat_metadata_size);
+	wmsum_fini(&arc_sums.arcstat_dbuf_size);
+	aggsum_fini(&arc_sums.arcstat_dnode_size);
+	wmsum_fini(&arc_sums.arcstat_bonus_size);
+	wmsum_fini(&arc_sums.arcstat_l2_hits);
+	wmsum_fini(&arc_sums.arcstat_l2_misses);
+	wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_feeds);
+	wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
+	wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
+	wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
+	wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
+	wmsum_fini(&arc_sums.arcstat_l2_writes_done);
+	wmsum_fini(&arc_sums.arcstat_l2_writes_error);
+	wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
+	wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
+	wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
+	wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
+	wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
+	wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
+	wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
+	wmsum_fini(&arc_sums.arcstat_l2_io_error);
+	wmsum_fini(&arc_sums.arcstat_l2_lsize);
+	wmsum_fini(&arc_sums.arcstat_l2_psize);
+	aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
+	wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
+	wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
+	wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
+	wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
+	wmsum_fini(&arc_sums.arcstat_memory_direct_count);
+	wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
+	wmsum_fini(&arc_sums.arcstat_prune);
+	aggsum_fini(&arc_sums.arcstat_meta_used);
+	wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
+	wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
+	wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
+	wmsum_fini(&arc_sums.arcstat_raw_size);
+	wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
+	wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
 }
 
 uint64_t
@@ -7623,8 +7906,6 @@ arc_init(void)
 
 	/* Set min to 1/2 of arc_c_min */
 	arc_meta_min = 1ULL << SPA_MAXBLOCKSHIFT;
-	/* Initialize maximum observed usage to zero */
-	arc_meta_max = 0;
 	/*
 	 * Set arc_meta_limit to a percent of arc_c_max with a floor of
 	 * arc_meta_min, and a ceiling of arc_c_max.
@@ -8350,7 +8631,7 @@ top:
 		}
 	}
 
-	atomic_inc_64(&l2arc_writes_done);
+	ARCSTAT_BUMP(arcstat_l2_writes_done);
 	list_remove(buflist, head);
 	ASSERT(!HDR_HAS_L1HDR(head));
 	kmem_cache_free(hdr_l2only_cache, head);
@@ -9322,7 +9603,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
 static boolean_t
 l2arc_hdr_limit_reached(void)
 {
-	int64_t s = aggsum_upper_bound(&astat_l2_hdr_size);
+	int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);
 
 	return (arc_reclaim_needed() || (s > arc_meta_limit * 3 / 4) ||
 	    (s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
@@ -9656,8 +9937,6 @@ l2arc_init(void)
 {
 	l2arc_thread_exit = 0;
 	l2arc_ndev = 0;
-	l2arc_writes_sent = 0;
-	l2arc_writes_done = 0;
 
 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c
index 9d741545f..5b072f026 100644
--- a/module/zfs/dbuf.c
+++ b/module/zfs/dbuf.c
@@ -52,6 +52,7 @@
 #include <sys/vdev.h>
 #include <cityhash.h>
 #include <sys/spa_impl.h>
+#include <sys/wmsum.h>
 
 kstat_t *dbuf_ksp;
 
@@ -135,8 +136,22 @@ dbuf_stats_t dbuf_stats = {
 	{ "metadata_cache_overflow",		KSTAT_DATA_UINT64 }
 };
 
+struct {
+	wmsum_t cache_count;
+	wmsum_t cache_total_evicts;
+	wmsum_t cache_levels[DN_MAX_LEVELS];
+	wmsum_t cache_levels_bytes[DN_MAX_LEVELS];
+	wmsum_t hash_hits;
+	wmsum_t hash_misses;
+	wmsum_t hash_collisions;
+	wmsum_t hash_chains;
+	wmsum_t hash_insert_race;
+	wmsum_t metadata_cache_count;
+	wmsum_t metadata_cache_overflow;
+} dbuf_sums;
+
 #define	DBUF_STAT_INCR(stat, val)	\
-	atomic_add_64(&dbuf_stats.stat.value.ui64, (val));
+	wmsum_add(&dbuf_sums.stat, val);
 #define	DBUF_STAT_DECR(stat, val)	\
 	DBUF_STAT_INCR(stat, -(val));
 #define	DBUF_STAT_BUMP(stat)		\
@@ -297,8 +312,6 @@ dbuf_dest(void *vdb, void *unused)
  */
 static dbuf_hash_table_t dbuf_hash_table;
 
-static uint64_t dbuf_hash_count;
-
 /*
  * We use Cityhash for this. It's fast, and has good hash properties without
  * requiring any large static buffers.
@@ -409,8 +422,8 @@ dbuf_hash_insert(dmu_buf_impl_t *db)
 	db->db_hash_next = h->hash_table[idx];
 	h->hash_table[idx] = db;
 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
-	atomic_inc_64(&dbuf_hash_count);
-	DBUF_STAT_MAX(hash_elements_max, dbuf_hash_count);
+	uint64_t he = atomic_inc_64_nv(&dbuf_stats.hash_elements.value.ui64);
+	DBUF_STAT_MAX(hash_elements_max, he);
 
 	return (NULL);
 }
@@ -483,7 +496,7 @@ dbuf_hash_remove(dmu_buf_impl_t *db)
 	    h->hash_table[idx]->db_hash_next == NULL)
 		DBUF_STAT_BUMPDOWN(hash_chains);
 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
-	atomic_dec_64(&dbuf_hash_count);
+	atomic_dec_64(&dbuf_stats.hash_elements.value.ui64);
 }
 
 typedef enum {
@@ -767,19 +780,40 @@ dbuf_kstat_update(kstat_t *ksp, int rw)
 {
 	dbuf_stats_t *ds = ksp->ks_data;
 
-	if (rw == KSTAT_WRITE) {
+	if (rw == KSTAT_WRITE)
 		return (SET_ERROR(EACCES));
-	} else {
-		ds->metadata_cache_size_bytes.value.ui64 = zfs_refcount_count(
-		    &dbuf_caches[DB_DBUF_METADATA_CACHE].size);
-		ds->cache_size_bytes.value.ui64 =
-		    zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size);
-		ds->cache_target_bytes.value.ui64 = dbuf_cache_target_bytes();
-		ds->cache_hiwater_bytes.value.ui64 = dbuf_cache_hiwater_bytes();
-		ds->cache_lowater_bytes.value.ui64 = dbuf_cache_lowater_bytes();
-		ds->hash_elements.value.ui64 = dbuf_hash_count;
-	}
 
+	ds->cache_count.value.ui64 =
+	    wmsum_value(&dbuf_sums.cache_count);
+	ds->cache_size_bytes.value.ui64 =
+	    zfs_refcount_count(&dbuf_caches[DB_DBUF_CACHE].size);
+	ds->cache_target_bytes.value.ui64 = dbuf_cache_target_bytes();
+	ds->cache_hiwater_bytes.value.ui64 = dbuf_cache_hiwater_bytes();
+	ds->cache_lowater_bytes.value.ui64 = dbuf_cache_lowater_bytes();
+	ds->cache_total_evicts.value.ui64 =
+	    wmsum_value(&dbuf_sums.cache_total_evicts);
+	for (int i = 0; i < DN_MAX_LEVELS; i++) {
+		ds->cache_levels[i].value.ui64 =
+		    wmsum_value(&dbuf_sums.cache_levels[i]);
+		ds->cache_levels_bytes[i].value.ui64 =
+		    wmsum_value(&dbuf_sums.cache_levels_bytes[i]);
+	}
+	ds->hash_hits.value.ui64 =
+	    wmsum_value(&dbuf_sums.hash_hits);
+	ds->hash_misses.value.ui64 =
+	    wmsum_value(&dbuf_sums.hash_misses);
+	ds->hash_collisions.value.ui64 =
+	    wmsum_value(&dbuf_sums.hash_collisions);
+	ds->hash_chains.value.ui64 =
+	    wmsum_value(&dbuf_sums.hash_chains);
+	ds->hash_insert_race.value.ui64 =
+	    wmsum_value(&dbuf_sums.hash_insert_race);
+	ds->metadata_cache_count.value.ui64 =
+	    wmsum_value(&dbuf_sums.metadata_cache_count);
+	ds->metadata_cache_size_bytes.value.ui64 = zfs_refcount_count(
+	    &dbuf_caches[DB_DBUF_METADATA_CACHE].size);
+	ds->metadata_cache_overflow.value.ui64 =
+	    wmsum_value(&dbuf_sums.metadata_cache_overflow);
 	return (0);
 }
 
@@ -846,6 +880,20 @@ retry:
 	dbuf_cache_evict_thread = thread_create(NULL, 0, dbuf_evict_thread,
 	    NULL, 0, &p0, TS_RUN, minclsyspri);
 
+	wmsum_init(&dbuf_sums.cache_count, 0);
+	wmsum_init(&dbuf_sums.cache_total_evicts, 0);
+	for (i = 0; i < DN_MAX_LEVELS; i++) {
+		wmsum_init(&dbuf_sums.cache_levels[i], 0);
+		wmsum_init(&dbuf_sums.cache_levels_bytes[i], 0);
+	}
+	wmsum_init(&dbuf_sums.hash_hits, 0);
+	wmsum_init(&dbuf_sums.hash_misses, 0);
+	wmsum_init(&dbuf_sums.hash_collisions, 0);
+	wmsum_init(&dbuf_sums.hash_chains, 0);
+	wmsum_init(&dbuf_sums.hash_insert_race, 0);
+	wmsum_init(&dbuf_sums.metadata_cache_count, 0);
+	wmsum_init(&dbuf_sums.metadata_cache_overflow, 0);
+
 	dbuf_ksp = kstat_create("zfs", 0, "dbufstats", "misc",
 	    KSTAT_TYPE_NAMED, sizeof (dbuf_stats) / sizeof (kstat_named_t),
 	    KSTAT_FLAG_VIRTUAL);
@@ -908,6 +956,20 @@ dbuf_fini(void)
 		kstat_delete(dbuf_ksp);
 		dbuf_ksp = NULL;
 	}
+
+	wmsum_fini(&dbuf_sums.cache_count);
+	wmsum_fini(&dbuf_sums.cache_total_evicts);
+	for (i = 0; i < DN_MAX_LEVELS; i++) {
+		wmsum_fini(&dbuf_sums.cache_levels[i]);
+		wmsum_fini(&dbuf_sums.cache_levels_bytes[i]);
+	}
+	wmsum_fini(&dbuf_sums.hash_hits);
+	wmsum_fini(&dbuf_sums.hash_misses);
+	wmsum_fini(&dbuf_sums.hash_collisions);
+	wmsum_fini(&dbuf_sums.hash_chains);
+	wmsum_fini(&dbuf_sums.hash_insert_race);
+	wmsum_fini(&dbuf_sums.metadata_cache_count);
+	wmsum_fini(&dbuf_sums.metadata_cache_overflow);
 }
 
 /*
@@ -3708,9 +3770,11 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting)
 				db->db_caching_status = dcs;
 
 				multilist_insert(&dbuf_caches[dcs].cache, db);
+				uint64_t db_size = db->db.db_size;
 				size = zfs_refcount_add_many(
-				    &dbuf_caches[dcs].size,
-				    db->db.db_size, db);
+				    &dbuf_caches[dcs].size, db_size, db);
+				uint8_t db_level = db->db_level;
+				mutex_exit(&db->db_mtx);
 
 				if (dcs == DB_DBUF_METADATA_CACHE) {
 					DBUF_STAT_BUMP(metadata_cache_count);
@@ -3718,16 +3782,14 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting)
 					    metadata_cache_size_bytes_max,
 					    size);
 				} else {
-					DBUF_STAT_BUMP(
-					    cache_levels[db->db_level]);
 					DBUF_STAT_BUMP(cache_count);
-					DBUF_STAT_INCR(
-					    cache_levels_bytes[db->db_level],
-					    db->db.db_size);
 					DBUF_STAT_MAX(cache_size_bytes_max,
 					    size);
+					DBUF_STAT_BUMP(cache_levels[db_level]);
+					DBUF_STAT_INCR(
+					    cache_levels_bytes[db_level],
+					    db_size);
 				}
-				mutex_exit(&db->db_mtx);
 
 				if (dcs == DB_DBUF_CACHE && !evicting)
 					dbuf_evict_notify(size);
diff --git a/module/zfs/dmu_zfetch.c b/module/zfs/dmu_zfetch.c
index 3d7407016..4a323fa99 100644
--- a/module/zfs/dmu_zfetch.c
+++ b/module/zfs/dmu_zfetch.c
@@ -34,6 +34,7 @@
 #include <sys/dmu.h>
 #include <sys/dbuf.h>
 #include <sys/kstat.h>
+#include <sys/wmsum.h>
 
 /*
  * This tunable disables predictive prefetch.  Note that it leaves "prescient"
@@ -69,27 +70,54 @@ static zfetch_stats_t zfetch_stats = {
 	{ "io_issued",		KSTAT_DATA_UINT64 },
 };
 
-#define	ZFETCHSTAT_BUMP(stat) \
-	atomic_inc_64(&zfetch_stats.stat.value.ui64)
+struct {
+	wmsum_t zfetchstat_hits;
+	wmsum_t zfetchstat_misses;
+	wmsum_t zfetchstat_max_streams;
+	wmsum_t zfetchstat_io_issued;
+} zfetch_sums;
+
+#define	ZFETCHSTAT_BUMP(stat)					\
+	wmsum_add(&zfetch_sums.stat, 1)
 #define	ZFETCHSTAT_ADD(stat, val)				\
-	atomic_add_64(&zfetch_stats.stat.value.ui64, val)
-#define	ZFETCHSTAT_SET(stat, val)				\
-	zfetch_stats.stat.value.ui64 = val
-#define	ZFETCHSTAT_GET(stat)					\
-	zfetch_stats.stat.value.ui64
+	wmsum_add(&zfetch_sums.stat, val)
 
 
 kstat_t		*zfetch_ksp;
 
+static int
+zfetch_kstats_update(kstat_t *ksp, int rw)
+{
+	zfetch_stats_t *zs = ksp->ks_data;
+
+	if (rw == KSTAT_WRITE)
+		return (EACCES);
+	zs->zfetchstat_hits.value.ui64 =
+	    wmsum_value(&zfetch_sums.zfetchstat_hits);
+	zs->zfetchstat_misses.value.ui64 =
+	    wmsum_value(&zfetch_sums.zfetchstat_misses);
+	zs->zfetchstat_max_streams.value.ui64 =
+	    wmsum_value(&zfetch_sums.zfetchstat_max_streams);
+	zs->zfetchstat_io_issued.value.ui64 =
+	    wmsum_value(&zfetch_sums.zfetchstat_io_issued);
+	return (0);
+}
+
 void
 zfetch_init(void)
 {
+	wmsum_init(&zfetch_sums.zfetchstat_hits, 0);
+	wmsum_init(&zfetch_sums.zfetchstat_misses, 0);
+	wmsum_init(&zfetch_sums.zfetchstat_max_streams, 0);
+	wmsum_init(&zfetch_sums.zfetchstat_io_issued, 0);
+
 	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
 	    KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
 	    KSTAT_FLAG_VIRTUAL);
 
 	if (zfetch_ksp != NULL) {
 		zfetch_ksp->ks_data = &zfetch_stats;
+		zfetch_ksp->ks_update = zfetch_kstats_update;
 		kstat_install(zfetch_ksp);
 	}
 }
@@ -101,6 +129,11 @@ zfetch_fini(void)
 		kstat_delete(zfetch_ksp);
 		zfetch_ksp = NULL;
 	}
+
+	wmsum_fini(&zfetch_sums.zfetchstat_hits);
+	wmsum_fini(&zfetch_sums.zfetchstat_misses);
+	wmsum_fini(&zfetch_sums.zfetchstat_max_streams);
+	wmsum_fini(&zfetch_sums.zfetchstat_io_issued);
 }
 
 /*