aboutsummaryrefslogtreecommitdiffstats
path: root/module
diff options
context:
space:
mode:
authorAlexander Motin <[email protected]>2021-08-17 11:44:34 -0400
committerTony Hutter <[email protected]>2021-09-14 14:31:01 -0700
commit5afc35b69824db01b36418e8f091ffeaeaeb98c9 (patch)
tree87550992a8d5f8d743a25ac3c8c754ce6490d0fb /module
parentc6c0d30016ff344a91aacc8057d5fb392c5ea9c7 (diff)
Use more atomics in refcounts
Use atomic_load_64() for zfs_refcount_count() to prevent torn reads on 32-bit platforms. On 64-bit ones it should not change anything. When built with ZFS_DEBUG but running without tracking enabled use atomics instead of mutexes same as for builds without ZFS_DEBUG. Since rc_tracked can't change live we can check it without lock. Reviewed-by: Brian Behlendorf <[email protected]> Reviewed-by: Matthew Ahrens <[email protected]> Signed-off-by: Alexander Motin <[email protected]> Closes #12420
Diffstat (limited to 'module')
-rw-r--r--module/zfs/refcount.c51
1 files changed, 22 insertions, 29 deletions
diff --git a/module/zfs/refcount.c b/module/zfs/refcount.c
index a3877b8d1..354e021d9 100644
--- a/module/zfs/refcount.c
+++ b/module/zfs/refcount.c
@@ -112,13 +112,13 @@ zfs_refcount_destroy(zfs_refcount_t *rc)
int
zfs_refcount_is_zero(zfs_refcount_t *rc)
{
- return (rc->rc_count == 0);
+ return (zfs_refcount_count(rc) == 0);
}
int64_t
zfs_refcount_count(zfs_refcount_t *rc)
{
- return (rc->rc_count);
+ return (atomic_load_64(&rc->rc_count));
}
int64_t
@@ -127,15 +127,18 @@ zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
reference_t *ref = NULL;
int64_t count;
- if (rc->rc_tracked) {
- ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
- ref->ref_holder = holder;
- ref->ref_number = number;
+ if (!rc->rc_tracked) {
+ count = atomic_add_64_nv(&(rc)->rc_count, number);
+ ASSERT3U(count, >=, number);
+ return (count);
}
+
+ ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
+ ref->ref_holder = holder;
+ ref->ref_number = number;
mutex_enter(&rc->rc_mtx);
ASSERT3U(rc->rc_count, >=, 0);
- if (rc->rc_tracked)
- list_insert_head(&rc->rc_list, ref);
+ list_insert_head(&rc->rc_list, ref);
rc->rc_count += number;
count = rc->rc_count;
mutex_exit(&rc->rc_mtx);
@@ -156,16 +159,14 @@ zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
reference_t *ref;
int64_t count;
- mutex_enter(&rc->rc_mtx);
- ASSERT3U(rc->rc_count, >=, number);
-
if (!rc->rc_tracked) {
- rc->rc_count -= number;
- count = rc->rc_count;
- mutex_exit(&rc->rc_mtx);
+ count = atomic_add_64_nv(&(rc)->rc_count, -number);
+ ASSERT3S(count, >=, 0);
return (count);
}
+ mutex_enter(&rc->rc_mtx);
+ ASSERT3U(rc->rc_count, >=, number);
for (ref = list_head(&rc->rc_list); ref;
ref = list_next(&rc->rc_list, ref)) {
if (ref->ref_holder == holder && ref->ref_number == number) {
@@ -242,12 +243,10 @@ zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
reference_t *ref;
boolean_t found = B_FALSE;
- mutex_enter(&rc->rc_mtx);
- if (!rc->rc_tracked) {
- mutex_exit(&rc->rc_mtx);
+ if (!rc->rc_tracked)
return;
- }
+ mutex_enter(&rc->rc_mtx);
for (ref = list_head(&rc->rc_list); ref;
ref = list_next(&rc->rc_list, ref)) {
if (ref->ref_holder == current_holder &&
@@ -279,13 +278,10 @@ zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
{
reference_t *ref;
- mutex_enter(&rc->rc_mtx);
-
- if (!rc->rc_tracked) {
- mutex_exit(&rc->rc_mtx);
- return (rc->rc_count > 0);
- }
+ if (!rc->rc_tracked)
+ return (zfs_refcount_count(rc) > 0);
+ mutex_enter(&rc->rc_mtx);
for (ref = list_head(&rc->rc_list); ref;
ref = list_next(&rc->rc_list, ref)) {
if (ref->ref_holder == holder) {
@@ -307,13 +303,10 @@ zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
{
reference_t *ref;
- mutex_enter(&rc->rc_mtx);
-
- if (!rc->rc_tracked) {
- mutex_exit(&rc->rc_mtx);
+ if (!rc->rc_tracked)
return (B_TRUE);
- }
+ mutex_enter(&rc->rc_mtx);
for (ref = list_head(&rc->rc_list); ref;
ref = list_next(&rc->rc_list, ref)) {
if (ref->ref_holder == holder) {