Record skipped MMP writes in multihost_history

Once per pass through the MMP thread's loop, the vdev tree is walked to
find a suitable leaf to write the next MMP block to.  If no such leaf is
found, the thread sleeps for a while and resumes at the top of the loop.

Add an entry to multihost_history when no leaf can be found, and record
the reason in the error column.  The error code for such entries is a
bitfield, displayed in hex:

0x1  At least one vdev (interior or leaf) was not writeable.
0x2  At least one writeable leaf vdev was found, but it had a pending
MMP write.

timestamp = the time in seconds since the epoch when no leaf could be
found originally.

duration = the time (in ns) during which no MMP block was written for
this reason.  This does not include the preceeding inter-write period
nor the following inter-write period.

vdev_guid = the number of sequential cycles of the MMP thread looop when
this occurred.

Sample output, truncated to fit:

For records of skipped MMP writes the right-most column, vdev_path, is
reported as "-".

id   txg  timestamp   error  duration    mmp_delay  vdev_guid     ...
936  11   1520036441  0      146264      891422313  1740883117838 ...
937  11   1520036441  0      163956      888356657  7320395061548 ...
938  11   1520036442  0      130690      885314969  7320395061548 ...
939  11   1520036442  0      2001068577  882296582  1740883117838 ...
940  11   1520036443  0      161806      882296582  7320395061548 ...
941  11   1520036443  0x2    0           998020546  1             ...
942  11   1520036444  0      136585      998020546  7320395061548 ...
943  11   1520036444  0x2    0           998020257  1             ...
944  11   1520036445  5      2002662964  994160219  1740883117838 ...
945  11   1520036445  0x2    998073118   994160219  3             ...
946  11   1520036447  0      247136      994160219  7320395061548 ...

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Olaf Faaland <[email protected]>
Closes #7212

1 files changed, 89 insertions, 25 deletions

diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c
index a668e8dbd..cb0aa53d6 100644
--- a/module/zfs/mmp.c
+++ b/module/zfs/mmp.c
@@ -200,28 +200,33 @@ mmp_thread_stop(spa_t *spa)
 	mmp->mmp_thread_exiting = 0;
 }
 
-/*
- * Choose a leaf vdev to write an MMP block to.  It must not have an
- * outstanding mmp write (if so then there is a problem, and a new write will
- * also block).  If there is no usable leaf in this subtree return NULL,
- * otherwise return a pointer to the leaf.
- *
- * When walking the subtree, a random child is chosen as the starting point so
- * that when the tree is healthy, the leaf chosen will be random with even
- * distribution.  If there are unhealthy vdevs in the tree, the distribution
- * will be really poor only if a large proportion of the vdevs are unhealthy,
- * in which case there are other more pressing problems.
- */
+typedef enum mmp_vdev_state_flag {
+	MMP_FAIL_NOT_WRITABLE	= (1 << 0),
+	MMP_FAIL_WRITE_PENDING	= (1 << 1),
+} mmp_vdev_state_flag_t;
+
 static vdev_t *
-mmp_random_leaf(vdev_t *vd)
+mmp_random_leaf_impl(vdev_t *vd, int *fail_mask)
 {
 	int child_idx;
 
-	if (!vdev_writeable(vd))
+	if (!vdev_writeable(vd)) {
+		*fail_mask |= MMP_FAIL_NOT_WRITABLE;
 		return (NULL);
+	}
 
-	if (vd->vdev_ops->vdev_op_leaf)
-		return (vd->vdev_mmp_pending == 0 ? vd : NULL);
+	if (vd->vdev_ops->vdev_op_leaf) {
+		vdev_t *ret;
+
+		if (vd->vdev_mmp_pending != 0) {
+			*fail_mask |= MMP_FAIL_WRITE_PENDING;
+			ret = NULL;
+		} else {
+			ret = vd;
+		}
+
+		return (ret);
+	}
 
 	child_idx = spa_get_random(vd->vdev_children);
 	for (int offset = vd->vdev_children; offset > 0; offset--) {
@@ -229,7 +234,7 @@ mmp_random_leaf(vdev_t *vd)
 		vdev_t *child = vd->vdev_child[(child_idx + offset) %
 		    vd->vdev_children];
 
-		leaf = mmp_random_leaf(child);
+		leaf = mmp_random_leaf_impl(child, fail_mask);
 		if (leaf)
 			return (leaf);
 	}
@@ -237,6 +242,44 @@ mmp_random_leaf(vdev_t *vd)
 	return (NULL);
 }
 
+/*
+ * Find a leaf vdev to write an MMP block to.  It must not have an outstanding
+ * mmp write (if so a new write will also likely block).  If there is no usable
+ * leaf in the tree rooted at in_vd, a nonzero error value is returned, and
+ * *out_vd is unchanged.
+ *
+ * The error value returned is a bit field.
+ *
+ * MMP_FAIL_WRITE_PENDING
+ * If set, one or more leaf vdevs are writeable, but have an MMP write which has
+ * not yet completed.
+ *
+ * MMP_FAIL_NOT_WRITABLE
+ * If set, one or more vdevs are not writeable.  The children of those vdevs
+ * were not examined.
+ *
+ * Assuming in_vd points to a tree, a random subtree will be chosen to start.
+ * That subtree, and successive ones, will be walked until a usable leaf has
+ * been found, or all subtrees have been examined (except that the children of
+ * un-writeable vdevs are not examined).
+ *
+ * If the leaf vdevs in the tree are healthy, the distribution of returned leaf
+ * vdevs will be even.  If there are unhealthy leaves, the following leaves
+ * (child_index % index_children) will be chosen more often.
+ */
+
+static int
+mmp_random_leaf(vdev_t *in_vd, vdev_t **out_vd)
+{
+	int error_mask = 0;
+	vdev_t *vd = mmp_random_leaf_impl(in_vd, &error_mask);
+
+	if (error_mask == 0)
+		*out_vd = vd;
+
+	return (error_mask);
+}
+
 static void
 mmp_write_done(zio_t *zio)
 {
@@ -319,8 +362,8 @@ mmp_write_uberblock(spa_t *spa)
 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
 	mmp_thread_t *mmp = &spa->spa_mmp;
 	uberblock_t *ub;
-	vdev_t *vd;
-	int label;
+	vdev_t *vd = NULL;
+	int label, error;
 	uint64_t offset;
 
 	hrtime_t lock_acquire_time = gethrtime();
@@ -330,13 +373,33 @@ mmp_write_uberblock(spa_t *spa)
 		zfs_dbgmsg("SCL_STATE acquisition took %llu ns\n",
 		    (u_longlong_t)lock_acquire_time);
 
-	vd = mmp_random_leaf(spa->spa_root_vdev);
-	if (vd == NULL) {
+	error = mmp_random_leaf(spa->spa_root_vdev, &vd);
+
+	mutex_enter(&mmp->mmp_io_lock);
+
+	/*
+	 * spa_mmp_history has two types of entries:
+	 * Issued MMP write: records time issued, error status, etc.
+	 * Skipped MMP write: an MMP write could not be issued because no
+	 * suitable leaf vdev was available.  See comment above struct
+	 * spa_mmp_history for details.
+	 */
+
+	if (error) {
+		if (mmp->mmp_skip_error == error) {
+			spa_mmp_history_set_skip(spa, mmp->mmp_kstat_id - 1);
+		} else {
+			mmp->mmp_skip_error = error;
+			spa_mmp_history_add(spa, mmp->mmp_ub.ub_txg,
+			    gethrestime_sec(), mmp->mmp_delay, NULL, 0,
+			    mmp->mmp_kstat_id++, error);
+		}
+		mutex_exit(&mmp->mmp_io_lock);
 		spa_config_exit(spa, SCL_STATE, FTAG);
 		return;
 	}
 
-	mutex_enter(&mmp->mmp_io_lock);
+	mmp->mmp_skip_error = 0;
 
 	if (mmp->mmp_zio_root == NULL)
 		mmp->mmp_zio_root = zio_root(spa, NULL, NULL,
@@ -347,13 +410,14 @@ mmp_write_uberblock(spa_t *spa)
 	ub->ub_mmp_magic = MMP_MAGIC;
 	ub->ub_mmp_delay = mmp->mmp_delay;
 	vd->vdev_mmp_pending = gethrtime();
-	vd->vdev_mmp_kstat_id = mmp->mmp_kstat_id++;
+	vd->vdev_mmp_kstat_id = mmp->mmp_kstat_id;
 
 	zio_t *zio  = zio_null(mmp->mmp_zio_root, spa, NULL, NULL, NULL, flags);
 	abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
 	abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));
 	abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));
 
+	mmp->mmp_kstat_id++;
 	mutex_exit(&mmp->mmp_io_lock);
 
 	offset = VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) -
@@ -364,8 +428,8 @@ mmp_write_uberblock(spa_t *spa)
 	    VDEV_UBERBLOCK_SIZE(vd), mmp_write_done, mmp,
 	    flags | ZIO_FLAG_DONT_PROPAGATE);
 
-	spa_mmp_history_add(ub->ub_txg, ub->ub_timestamp, ub->ub_mmp_delay, vd,
-	    label, vd->vdev_mmp_kstat_id);
+	(void) spa_mmp_history_add(spa, ub->ub_txg, ub->ub_timestamp,
+	    ub->ub_mmp_delay, vd, label, vd->vdev_mmp_kstat_id, 0);
 
 	zio_nowait(zio);
 }