diff options
| -rw-r--r-- | cmd/zinject/zinject.c | 34 | ||||
| -rw-r--r-- | include/sys/zio.h | 2 | ||||
| -rw-r--r-- | man/man8/zinject.8 | 1 | ||||
| -rw-r--r-- | module/zfs/zio.c | 4 | ||||
| -rw-r--r-- | module/zfs/zio_inject.c | 52 |
5 files changed, 76 insertions, 17 deletions
diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index ccd3534d0..d66a959bc 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -36,12 +36,15 @@ * * Errors can be injected into a particular vdev using the '-d' option. This * option takes a path or vdev GUID to uniquely identify the device within a - * pool. There are two types of errors that can be injected, EIO and ENXIO, - * that can be controlled through the '-e' option. The default is ENXIO. For - * EIO failures, any attempt to read data from the device will return EIO, but - * subsequent attempt to reopen the device will succeed. For ENXIO failures, - * any attempt to read from the device will return EIO, but any attempt to - * reopen the device will also return ENXIO. + * pool. There are four types of errors that can be injected, IO, ENXIO, + * ECHILD, and EILSEQ. These can be controlled through the '-e' option and the + * default is ENXIO. For EIO failures, any attempt to read data from the device + * will return EIO, but a subsequent attempt to reopen the device will succeed. + * For ENXIO failures, any attempt to read from the device will return EIO, but + * any attempt to reopen the device will also return ENXIO. The EILSEQ failures + * only apply to read operations (-T read) and will flip a bit after the device + * has read the original data. + * * For label faults, the -L option must be specified. This allows faults * to be injected into either the nvlist, uberblock, pad1, or pad2 region * of all the labels for the specified device. @@ -231,11 +234,12 @@ usage(void) "\t\tspa_vdev_exit() will trigger a panic.\n" "\n" "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n" - "\t [-T <read|write|free|claim|all>] [-f frequency] pool\n" + "\t\t[-T <read|write|free|claim|all>] [-f frequency] pool\n\n" "\t\tInject a fault into a particular device or the device's\n" "\t\tlabel. Label injection can either be 'nvlist', 'uber',\n " "\t\t'pad1', or 'pad2'.\n" - "\t\t'errno' can be 'nxio' (the default), 'io', or 'dtl'.\n" + "\t\t'errno' can be 'nxio' (the default), 'io', 'dtl', or\n" + "\t\t'corrupt' (bit flip).\n" "\t\t'frequency' is a value between 0.0001 and 100.0 that limits\n" "\t\tdevice error injection to a percentage of the IOs.\n" "\n" @@ -774,6 +778,8 @@ main(int argc, char **argv) error = ENXIO; } else if (strcasecmp(optarg, "dtl") == 0) { error = ECHILD; + } else if (strcasecmp(optarg, "corrupt") == 0) { + error = EILSEQ; } else { (void) fprintf(stderr, "invalid error type " "'%s': must be 'io', 'checksum' or " @@ -981,7 +987,15 @@ main(int argc, char **argv) if (error == ECKSUM) { (void) fprintf(stderr, "device error type must be " - "'io' or 'nxio'\n"); + "'io', 'nxio' or 'corrupt'\n"); + libzfs_fini(g_zfs); + return (1); + } + + if (error == EILSEQ && + (record.zi_freq == 0 || io_type != ZIO_TYPE_READ)) { + (void) fprintf(stderr, "device corrupt errors require " + "io type read and a frequency value\n"); libzfs_fini(g_zfs); return (1); } @@ -1109,7 +1123,7 @@ main(int argc, char **argv) return (2); } - if (error == ENXIO) { + if (error == ENXIO || error == EILSEQ) { (void) fprintf(stderr, "data error type must be " "'checksum' or 'io'\n"); libzfs_fini(g_zfs); diff --git a/include/sys/zio.h b/include/sys/zio.h index f7baa270b..30c4ee3ce 100644 --- a/include/sys/zio.h +++ b/include/sys/zio.h @@ -619,6 +619,8 @@ extern int zio_clear_fault(int id); extern void zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type); extern int zio_handle_fault_injection(zio_t *zio, int error); extern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error); +extern int zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1, + int err2); extern int zio_handle_label_injection(zio_t *zio, int error); extern void zio_handle_ignored_writes(zio_t *zio); extern hrtime_t zio_handle_io_delay(zio_t *zio); diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index 50fecfb64..6b6c3733b 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -108,6 +108,7 @@ A vdev specified by path or GUID. .BI "\-e" " device_error" Specify .BR "checksum" " for an ECKSUM error," +.BR "corrupt" " to flip a bit in the data after a read," .BR "dtl" " for an ECHILD error," .BR "io" " for an EIO error where reopening the device will succeed, or" .BR "nxio" " for an ENXIO error where reopening the device will fail." diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 959b9a5a8..057a1405f 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -3472,8 +3472,8 @@ zio_vdev_io_done(zio_t *zio) vdev_cache_write(zio); if (zio_injection_enabled && zio->io_error == 0) - zio->io_error = zio_handle_device_injection(vd, - zio, EIO); + zio->io_error = zio_handle_device_injections(vd, zio, + EIO, EILSEQ); if (zio_injection_enabled && zio->io_error == 0) zio->io_error = zio_handle_label_injection(zio, EIO); diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index 4a4d431e3..e1ea825d7 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -271,9 +271,24 @@ zio_handle_label_injection(zio_t *zio, int error) return (ret); } +/*ARGSUSED*/ +static int +zio_inject_bitflip_cb(void *data, size_t len, void *private) +{ + ASSERTV(zio_t *zio = private); + uint8_t *buffer = data; + uint_t byte = spa_get_random(len); -int -zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) + ASSERT(zio->io_type == ZIO_TYPE_READ); + + /* flip a single random bit in an abd data buffer */ + buffer[byte] ^= 1 << spa_get_random(8); + + return (1); /* stop after first flip */ +} + +static int +zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) { inject_handler_t *handler; int ret = 0; @@ -311,7 +326,8 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) handler->zi_record.zi_iotype != zio->io_type) continue; - if (handler->zi_record.zi_error == error) { + if (handler->zi_record.zi_error == err1 || + handler->zi_record.zi_error == err2) { /* * limit error injection if requested */ @@ -322,7 +338,7 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) * For a failed open, pretend like the device * has gone away. */ - if (error == ENXIO) + if (err1 == ENXIO) vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED; @@ -335,7 +351,21 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) zio != NULL) zio->io_flags |= ZIO_FLAG_IO_RETRY; - ret = error; + /* + * EILSEQ means flip a bit after a read + */ + if (handler->zi_record.zi_error == EILSEQ) { + if (zio == NULL) + break; + + /* locate buffer data and flip a bit */ + (void) abd_iterate_func(zio->io_abd, 0, + zio->io_size, zio_inject_bitflip_cb, + zio); + break; + } + + ret = handler->zi_record.zi_error; break; } if (handler->zi_record.zi_error == ENXIO) { @@ -350,6 +380,18 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) return (ret); } +int +zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error) +{ + return (zio_handle_device_injection_impl(vd, zio, error, INT_MAX)); +} + +int +zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1, int err2) +{ + return (zio_handle_device_injection_impl(vd, zio, err1, err2)); +} + /* * Simulate hardware that ignores cache flushes. For requested number * of seconds nix the actual writing to disk. |