From 64bdf63f5cceb6ea6b7e199c79a8ac3e37cdbf13 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jan 2019 14:10:02 -0800 Subject: ztest: split block reconstruction Increase the default allowed number of reconstruction attempts. There's not an exact right number for this setting. It needs to be set large enough to cover any realistic failure scenarios and small enough to avoid stalling the IO pipeline and invoking the dead man detection. The current value of 256 was empirically determined to be too low based on multi-day runs of ztest. The fault injection code would inject more damage than could be reconstructed given the relatively small number of attempts. However, in all observed cases the block could be reconstructed using a slightly higher limit. Based on local testing increasing the default value to 4096 was determined to strike the best balance. Checking all combinations takes less than 10s in the worst case, and has so far eliminated the vast majority of false positives detected by ztest. This delay is roughly on par with how long retries may be performed to a misbehaving HDD and was deemed to be reasonable. Better to err on the side of a brief delay rather than fail to reconstruct the data. Lastly, the -Y flag has been added to zdb to make it easy to try all possible combinations when performing split block reconstruction. For badly damaged blocks with 18 splits, they can be fully enumerated within a few minutes. This has been done to ensure permanent errors are never incorrectly reported when ztest verifies the pool with zdb. Reviewed by: Tom Caputi Reviewed by: Matt Ahrens Reviewed by: Serapheim Dimitropoulos Signed-off-by: Brian Behlendorf Closes #8271 --- cmd/zdb/zdb.c | 9 ++++++++- cmd/ztest/ztest.c | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'cmd') diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 1a9303d12..c436139a2 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -100,6 +100,7 @@ extern int zfs_recover; extern uint64_t zfs_arc_max, zfs_arc_meta_limit; extern int zfs_vdev_async_read_max_active; extern boolean_t spa_load_verify_dryrun; +extern int zfs_reconstruct_indirect_combinations_max; static const char cmdname[] = "zdb"; uint8_t dump_opt[256]; @@ -215,6 +216,8 @@ usage(void) "dump all read blocks into specified directory\n"); (void) fprintf(stderr, " -X attempt extreme rewind (does not " "work with dataset)\n"); + (void) fprintf(stderr, " -Y attempt all reconstruction " + "combinations for split blocks\n"); (void) fprintf(stderr, "Specify an option more than once (e.g. -bb) " "to make only that option verbose\n"); (void) fprintf(stderr, "Default is to dump everything non-verbosely\n"); @@ -5871,7 +5874,7 @@ main(int argc, char **argv) spa_config_path = spa_config_path_env; while ((c = getopt(argc, argv, - "AbcCdDeEFGhiI:klLmMo:Op:PqRsSt:uU:vVx:X")) != -1) { + "AbcCdDeEFGhiI:klLmMo:Op:PqRsSt:uU:vVx:XY")) != -1) { switch (c) { case 'b': case 'c': @@ -5903,6 +5906,10 @@ main(int argc, char **argv) case 'X': dump_opt[c]++; break; + case 'Y': + zfs_reconstruct_indirect_combinations_max = INT_MAX; + zfs_deadman_enabled = 0; + break; /* NB: Sort single match options below. */ case 'I': max_inflight = strtoull(optarg, NULL, 0); diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c index 678e29ff6..3686fab76 100644 --- a/cmd/ztest/ztest.c +++ b/cmd/ztest/ztest.c @@ -6447,8 +6447,7 @@ ztest_run_zdb(char *pool) ztest_get_zdb_bin(bin, len); (void) sprintf(zdb, - "%s -bcc%s%s -G -d -U %s " - "-o zfs_reconstruct_indirect_combinations_max=65536 %s", + "%s -bcc%s%s -G -d -Y -U %s %s", bin, ztest_opts.zo_verbose >= 3 ? "s" : "", ztest_opts.zo_verbose >= 4 ? "v" : "", -- cgit v1.2.3