ZTS: Retry in import_rewind_config_changed.ksh

As explained by the disclaimer in the test case,

    "This test can fail since nothing guarantees that old
    MOS blocks aren't overwritten."

This behavior is expected and correct, but results in a
flaky test case which is problematic for the CI.  The best
we can do to resolve this is to retry the sub-test which
failed when the MOS blocks have clearly been overwritten.

When testing failures were rare enough that a single retry
should normally be sufficient.  However, we allow up to
five for good measure.

Reviewed by: George Melikov <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #13119

2 files changed, 49 insertions, 11 deletions

diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in
index e8100d0aa..559e98dd0 100755
--- a/tests/test-runner/bin/zts-report.py.in
+++ b/tests/test-runner/bin/zts-report.py.in
@@ -224,8 +224,6 @@ maybe = {
     'cli_root/zfs_unshare/setup': ['SKIP', share_reason],
     'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason],
     'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'],
-    'cli_root/zpool_import/import_rewind_config_changed':
-        ['FAIL', rewind_reason],
     'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'],
     'cli_root/zpool_initialize/zpool_initialize_import_export':
         ['FAIL', '11948'],
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
index 3ac8c104f..d79c757d2 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
@@ -61,6 +61,7 @@ function test_common
 	typeset detachvdev="${4:-}"
 	typeset removevdev="${5:-}"
 	typeset finalpool="${6:-}"
+	typeset retval=1
 
 	typeset poolcheck="$poolcreate"
 
@@ -120,19 +121,30 @@ function test_common
 	# while having a checkpoint, we take it after the
 	# operation that changes the config.
 	#
+	# However, it is possible the MOS data was overwritten
+	# in which case the pool will either be unimportable, or
+	# may have been rewound prior to the data being written.
+	# In which case an error is returned and test_common()
+	# is retried by the caller to minimize false positives.
+	#
 	log_must zpool checkpoint $TESTPOOL1
 
 	log_must overwrite_data $TESTPOOL1 ""
 
 	log_must zpool export $TESTPOOL1
 
-	log_must zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1
-	log_must check_pool_config $TESTPOOL1 "$poolcheck"
+	zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1
+	if (( $? == 0 )); then
+		verify_data_md5sums $MD5FILE
+		if (( $? == 0 )); then
+			retval=0
+		fi
 
-	log_must verify_data_md5sums $MD5FILE
+		log_must check_pool_config $TESTPOOL1 "$poolcheck"
+		log_must zpool destroy $TESTPOOL1
+	fi
 
 	# Cleanup
-	log_must zpool destroy $TESTPOOL1
 	if [[ -n $pathstochange ]]; then
 		for dev in $pathstochange; do
 			log_must mv "${dev}_new" $dev
@@ -143,6 +155,7 @@ function test_common
 	log_must zpool destroy $TESTPOOL2
 
 	log_note ""
+	return $retval
 }
 
 function test_add_vdevs
@@ -152,7 +165,12 @@ function test_add_vdevs
 
 	log_note "$0: pool '$poolcreate', add $addvdevs."
 
-	test_common "$poolcreate" "$addvdevs"
+	for retry in $(seq 1 5); do
+		test_common "$poolcreate" "$addvdevs" && return
+		log_note "Retry $retry / 5 for test_add_vdevs()"
+	done
+
+	log_fail "Exhausted all 5 retries for test_add_vdevs()"
 }
 
 function test_attach_vdev
@@ -163,7 +181,12 @@ function test_attach_vdev
 
 	log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto."
 
-	test_common "$poolcreate" "" "$attachto $attachvdev"
+	for retry in $(seq 1 5); do
+		test_common "$poolcreate" "" "$attachto $attachvdev" && return
+		log_note "Retry $retry / 5 for test_attach_vdev()"
+	done
+
+	log_fail "Exhausted all 5 retries for test_attach_vdev()"
 }
 
 function test_detach_vdev
@@ -173,7 +196,12 @@ function test_detach_vdev
 
 	log_note "$0: pool '$poolcreate', detach $detachvdev."
 
-	test_common "$poolcreate" "" "" "$detachvdev"
+	for retry in $(seq 1 5); do
+		test_common "$poolcreate" "" "" "$detachvdev" && return
+		log_note "Retry $retry / 5 for test_detach_vdev()"
+	done
+
+	log_fail "Exhausted all 5 retries for test_detach_vdev()"
 }
 
 function test_attach_detach_vdev
@@ -186,7 +214,13 @@ function test_attach_detach_vdev
 	log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto," \
 	    "then detach $detachvdev."
 
-	test_common "$poolcreate" "" "$attachto $attachvdev" "$detachvdev"
+	for retry in $(seq 1 5); do
+		test_common "$poolcreate" "" "$attachto $attachvdev" \
+		    "$detachvdev" && return
+		log_note "Retry $retry / 5 for test_attach_detach_vdev()"
+	done
+
+	log_fail "Exhausted all 5 retries for test_attach_detach_vdev()"
 }
 
 function test_remove_vdev
@@ -197,7 +231,13 @@ function test_remove_vdev
 
 	log_note "$0: pool '$poolcreate', remove $removevdev."
 
-	test_common "$poolcreate" "" "" "" "$removevdev" "$finalpool"
+	for retry in $(seq 1 5); do
+		test_common "$poolcreate" "" "" "" "$removevdev" \
+		    "$finalpool" && return
+		log_note "Retry $retry / 5 for test_remove_vdev()"
+	done
+
+	log_fail "Exhausted all 5 retries for test_remove_vdev()"
 }
 
 # Record txg history