diff options
56 files changed, 2646 insertions, 250 deletions
diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 55a6c4fa1..21f8ea87c 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -2165,6 +2165,13 @@ dump_uberblock(uberblock_t *ub, const char *header, const char *footer) (void) printf("\tguid_sum = %llu\n", (u_longlong_t)ub->ub_guid_sum); (void) printf("\ttimestamp = %llu UTC = %s", (u_longlong_t)ub->ub_timestamp, asctime(localtime(×tamp))); + + (void) printf("\tmmp_magic = %016llx\n", + (u_longlong_t)ub->ub_mmp_magic); + if (ub->ub_mmp_magic == MMP_MAGIC) + (void) printf("\tmmp_delay = %0llu\n", + (u_longlong_t)ub->ub_mmp_delay); + if (dump_opt['u'] >= 4) { char blkbuf[BP_SPRINTF_LEN]; snprintf_blkptr(blkbuf, sizeof (blkbuf), &ub->ub_rootbp); @@ -2529,6 +2536,11 @@ dump_label_uberblocks(label_t *label, uint64_t ashift, int label_num) if ((dump_opt['u'] < 3) && (first_label(rec) != label_num)) continue; + if ((dump_opt['u'] < 4) && + (ub->ub_mmp_magic == MMP_MAGIC) && ub->ub_mmp_delay && + (i >= VDEV_UBERBLOCK_COUNT(&vd) - MMP_BLOCKS_PER_LABEL)) + continue; + print_label_header(label, label_num); (void) snprintf(header, ZDB_MAX_UB_HEADER_SIZE, " Uberblock[%d]\n", i); @@ -4125,89 +4137,6 @@ zdb_embedded_block(char *thing) zdb_dump_block_raw(buf, BPE_GET_LSIZE(&bp), 0); } -static boolean_t -pool_match(nvlist_t *cfg, char *tgt) -{ - uint64_t v, guid = strtoull(tgt, NULL, 0); - char *s; - - if (guid != 0) { - if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_GUID, &v) == 0) - return (v == guid); - } else { - if (nvlist_lookup_string(cfg, ZPOOL_CONFIG_POOL_NAME, &s) == 0) - return (strcmp(s, tgt) == 0); - } - return (B_FALSE); -} - -static char * -find_zpool(char **target, nvlist_t **configp, int dirc, char **dirv) -{ - nvlist_t *pools; - nvlist_t *match = NULL; - char *name = NULL; - char *sepp = NULL; - char sep = '\0'; - int count = 0; - importargs_t args = { 0 }; - - args.paths = dirc; - args.path = dirv; - args.can_be_active = B_TRUE; - - if ((sepp = strpbrk(*target, "/@")) != NULL) { - sep = *sepp; - *sepp = '\0'; - } - - pools = zpool_search_import(g_zfs, &args); - - if (pools != NULL) { - nvpair_t *elem = NULL; - while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) { - verify(nvpair_value_nvlist(elem, configp) == 0); - if (pool_match(*configp, *target)) { - count++; - if (match != NULL) { - /* print previously found config */ - if (name != NULL) { - (void) printf("%s\n", name); - dump_nvlist(match, 8); - name = NULL; - } - (void) printf("%s\n", - nvpair_name(elem)); - dump_nvlist(*configp, 8); - } else { - match = *configp; - name = nvpair_name(elem); - } - } - } - } - if (count > 1) - (void) fatal("\tMatched %d pools - use pool GUID " - "instead of pool name or \n" - "\tpool name part of a dataset name to select pool", count); - - if (sepp) - *sepp = sep; - /* - * If pool GUID was specified for pool id, replace it with pool name - */ - if (name && (strstr(*target, name) != *target)) { - int sz = 1 + strlen(name) + ((sepp) ? strlen(sepp) : 0); - - *target = umem_alloc(sz, UMEM_NOFAIL); - (void) snprintf(*target, sz, "%s%s", name, sepp ? sepp : ""); - } - - *configp = name ? match : NULL; - - return (name); -} - int main(int argc, char **argv) { @@ -4424,21 +4353,31 @@ main(int argc, char **argv) target = argv[0]; if (dump_opt['e']) { + importargs_t args = { 0 }; nvlist_t *cfg = NULL; - char *name = find_zpool(&target, &cfg, nsearch, searchdirs); - error = ENOENT; - if (name) { - if (dump_opt['C'] > 1) { - (void) printf("\nConfiguration for import:\n"); - dump_nvlist(cfg, 8); - } + args.paths = nsearch; + args.path = searchdirs; + args.can_be_active = B_TRUE; + + error = zpool_tryimport(g_zfs, target, &cfg, &args); + if (error == 0) { if (nvlist_add_nvlist(cfg, ZPOOL_REWIND_POLICY, policy) != 0) { fatal("can't open '%s': %s", target, strerror(ENOMEM)); } - error = spa_import(name, cfg, NULL, flags); + + /* + * Disable the activity check to allow examination of + * active pools. + */ + if (dump_opt['C'] > 1) { + (void) printf("\nConfiguration for import:\n"); + dump_nvlist(cfg, 8); + } + error = spa_import(target, cfg, NULL, + flags | ZFS_IMPORT_SKIP_MMP); } } @@ -4453,6 +4392,16 @@ main(int argc, char **argv) if (error == 0) { if (target_is_spa || dump_opt['R']) { + /* + * Disable the activity check to allow examination of + * active pools. + */ + mutex_enter(&spa_namespace_lock); + if ((spa = spa_lookup(target)) != NULL) { + spa->spa_import_flags |= ZFS_IMPORT_SKIP_MMP; + } + mutex_exit(&spa_namespace_lock); + error = spa_open_rewind(target, &spa, FTAG, policy, NULL); if (error) { diff --git a/cmd/zhack/zhack.c b/cmd/zhack/zhack.c index e76945141..cc414dae1 100644 --- a/cmd/zhack/zhack.c +++ b/cmd/zhack/zhack.c @@ -121,16 +121,11 @@ space_delta_cb(dmu_object_type_t bonustype, void *data, * Target is the dataset whose pool we want to open. */ static void -import_pool(const char *target, boolean_t readonly) +zhack_import(char *target, boolean_t readonly) { nvlist_t *config; - nvlist_t *pools; - int error; - char *sepp; - spa_t *spa; - nvpair_t *elem; nvlist_t *props; - char *name; + int error; kernel_init(readonly ? FREAD : (FREAD | FWRITE)); g_zfs = libzfs_init(); @@ -139,43 +134,14 @@ import_pool(const char *target, boolean_t readonly) dmu_objset_register_type(DMU_OST_ZFS, space_delta_cb); g_readonly = readonly; - - /* - * If we only want readonly access, it's OK if we find - * a potentially-active (ie, imported into the kernel) pool from the - * default cachefile. - */ - if (readonly && spa_open(target, &spa, FTAG) == 0) { - spa_close(spa, FTAG); - return; - } - g_importargs.unique = B_TRUE; g_importargs.can_be_active = readonly; g_pool = strdup(target); - if ((sepp = strpbrk(g_pool, "/@")) != NULL) - *sepp = '\0'; - g_importargs.poolname = g_pool; - pools = zpool_search_import(g_zfs, &g_importargs); - - if (nvlist_empty(pools)) { - if (!g_importargs.can_be_active) { - g_importargs.can_be_active = B_TRUE; - if (zpool_search_import(g_zfs, &g_importargs) != NULL || - spa_open(target, &spa, FTAG) == 0) { - fatal(spa, FTAG, "cannot import '%s': pool is " - "active; run " "\"zpool export %s\" " - "first\n", g_pool, g_pool); - } - } - fatal(NULL, FTAG, "cannot import '%s': no such pool " - "available\n", g_pool); - } - - elem = nvlist_next_nvpair(pools, NULL); - name = nvpair_name(elem); - VERIFY(nvpair_value_nvlist(elem, &config) == 0); + error = zpool_tryimport(g_zfs, target, &config, &g_importargs); + if (error) + fatal(NULL, FTAG, "cannot import '%s': %s", target, + libzfs_error_description(g_zfs)); props = NULL; if (readonly) { @@ -185,22 +151,22 @@ import_pool(const char *target, boolean_t readonly) } zfeature_checks_disable = B_TRUE; - error = spa_import(name, config, props, ZFS_IMPORT_NORMAL); + error = spa_import(target, config, props, ZFS_IMPORT_NORMAL); zfeature_checks_disable = B_FALSE; if (error == EEXIST) error = 0; if (error) - fatal(NULL, FTAG, "can't import '%s': %s", name, + fatal(NULL, FTAG, "can't import '%s': %s", target, strerror(error)); } static void -zhack_spa_open(const char *target, boolean_t readonly, void *tag, spa_t **spa) +zhack_spa_open(char *target, boolean_t readonly, void *tag, spa_t **spa) { int err; - import_pool(target, readonly); + zhack_import(target, readonly); zfeature_checks_disable = B_TRUE; err = spa_open(target, spa, tag); diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 14181b083..052b429cc 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -1743,6 +1743,9 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name, (void) printf(gettext("split into new pool")); break; + case VDEV_AUX_ACTIVE: + (void) printf(gettext("currently in use")); + default: (void) printf(gettext("corrupted data")); break; @@ -1838,6 +1841,10 @@ print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv, (void) printf(gettext("too many errors")); break; + case VDEV_AUX_ACTIVE: + (void) printf(gettext("currently in use")); + break; + default: (void) printf(gettext("corrupted data")); break; @@ -1935,8 +1942,10 @@ show_import(nvlist_t *config) vdev_stat_t *vs; char *name; uint64_t guid; + uint64_t hostid = 0; char *msgid; - nvlist_t *nvroot; + char *hostname = "unknown"; + nvlist_t *nvroot, *nvinfo; zpool_status_t reason; zpool_errata_t errata; const char *health; @@ -2024,6 +2033,17 @@ show_import(nvlist_t *config) zpool_print_unsup_feat(config); break; + case ZPOOL_STATUS_HOSTID_ACTIVE: + (void) printf(gettext(" status: The pool is currently " + "imported by another system.\n")); + break; + + case ZPOOL_STATUS_HOSTID_REQUIRED: + (void) printf(gettext(" status: The pool has the " + "multihost property on. It cannot\n\tbe safely imported " + "when the system hostid is not set.\n")); + break; + case ZPOOL_STATUS_HOSTID_MISMATCH: (void) printf(gettext(" status: The pool was last accessed by " "another system.\n")); @@ -2137,6 +2157,27 @@ show_import(nvlist_t *config) "imported. Attach the missing\n\tdevices and try " "again.\n")); break; + case ZPOOL_STATUS_HOSTID_ACTIVE: + VERIFY0(nvlist_lookup_nvlist(config, + ZPOOL_CONFIG_LOAD_INFO, &nvinfo)); + + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME)) + hostname = fnvlist_lookup_string(nvinfo, + ZPOOL_CONFIG_MMP_HOSTNAME); + + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID)) + hostid = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_HOSTID); + + (void) printf(gettext(" action: The pool must be " + "exported from %s (hostid=%lx)\n\tbefore it " + "can be safely imported.\n"), hostname, + (unsigned long) hostid); + break; + case ZPOOL_STATUS_HOSTID_REQUIRED: + (void) printf(gettext(" action: Set a unique system " + "hostid with the genhostid(1) command.\n")); + break; default: (void) printf(gettext(" action: The pool cannot be " "imported due to damaged devices or data.\n")); @@ -2184,6 +2225,31 @@ show_import(nvlist_t *config) } } +static boolean_t +zfs_force_import_required(nvlist_t *config) +{ + uint64_t state; + uint64_t hostid = 0; + nvlist_t *nvinfo; + + state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE); + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid); + + if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid()) + return (B_TRUE); + + nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) { + mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_STATE); + + if (mmp_state != MMP_STATE_INACTIVE) + return (B_TRUE); + } + + return (B_FALSE); +} + /* * Perform the import for the given configuration. This passes the heavy * lifting off to zpool_import_props(), and then mounts the datasets contained @@ -2198,45 +2264,72 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts, uint64_t state; uint64_t version; - verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, - &name) == 0); + name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME); + state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE); + version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION); - verify(nvlist_lookup_uint64(config, - ZPOOL_CONFIG_POOL_STATE, &state) == 0); - verify(nvlist_lookup_uint64(config, - ZPOOL_CONFIG_VERSION, &version) == 0); if (!SPA_VERSION_IS_SUPPORTED(version)) { (void) fprintf(stderr, gettext("cannot import '%s': pool " "is formatted using an unsupported ZFS version\n"), name); return (1); - } else if (state != POOL_STATE_EXPORTED && + } else if (zfs_force_import_required(config) && !(flags & ZFS_IMPORT_ANY_HOST)) { - uint64_t hostid = 0; - unsigned long system_hostid = get_system_hostid(); + mmp_state_t mmp_state = MMP_STATE_INACTIVE; + nvlist_t *nvinfo; - (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, - &hostid); + nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) + mmp_state = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_STATE); - if (hostid != 0 && (unsigned long)hostid != system_hostid) { - char *hostname; - uint64_t timestamp; - time_t t; + if (mmp_state == MMP_STATE_ACTIVE) { + char *hostname = "<unknown>"; + uint64_t hostid = 0; - verify(nvlist_lookup_string(config, - ZPOOL_CONFIG_HOSTNAME, &hostname) == 0); - verify(nvlist_lookup_uint64(config, - ZPOOL_CONFIG_TIMESTAMP, ×tamp) == 0); - t = timestamp; - (void) fprintf(stderr, gettext("cannot import " - "'%s': pool may be in use from other " - "system, it was last accessed by %s " - "(hostid: 0x%lx) on %s"), name, hostname, - (unsigned long)hostid, - asctime(localtime(&t))); - (void) fprintf(stderr, gettext("use '-f' to " - "import anyway\n")); - return (1); + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME)) + hostname = fnvlist_lookup_string(nvinfo, + ZPOOL_CONFIG_MMP_HOSTNAME); + + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID)) + hostid = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_HOSTID); + + (void) fprintf(stderr, gettext("cannot import '%s': " + "pool is imported on %s (hostid: " + "0x%lx)\nExport the pool on the other system, " + "then run 'zpool import'.\n"), + name, hostname, (unsigned long) hostid); + } else if (mmp_state == MMP_STATE_NO_HOSTID) { + (void) fprintf(stderr, gettext("Cannot import '%s': " + "pool has the multihost property on and the\n" + "system's hostid is not set. Set a unique hostid " + "with the genhostid(1) command.\n"), name); + } else { + char *hostname = "<unknown>"; + uint64_t timestamp = 0; + uint64_t hostid = 0; + + if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME)) + hostname = fnvlist_lookup_string(config, + ZPOOL_CONFIG_HOSTNAME); + + if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP)) + timestamp = fnvlist_lookup_uint64(config, + ZPOOL_CONFIG_TIMESTAMP); + + if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID)) + hostid = fnvlist_lookup_uint64(config, + ZPOOL_CONFIG_HOSTID); + + (void) fprintf(stderr, gettext("cannot import '%s': " + "pool was previously in use from another system.\n" + "Last accessed by %s (hostid=%lx) at %s" + "The pool can be imported, use 'zpool import -f' " + "to import the pool.\n"), name, hostname, + (unsigned long)hostid, ctime((time_t *)×tamp)); } + + return (1); } if (zpool_import_props(g_zfs, config, newname, props, flags) != 0) diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c index d698628de..b4cedbdba 100644 --- a/cmd/ztest/ztest.c +++ b/cmd/ztest/ztest.c @@ -126,6 +126,7 @@ #include <sys/fs/zfs.h> #include <zfs_fletcher.h> #include <libnvpair.h> +#include <libzfs.h> #ifdef __GLIBC__ #include <execinfo.h> /* for backtrace() */ #endif @@ -166,6 +167,7 @@ typedef struct ztest_shared_opts { uint64_t zo_time; uint64_t zo_maxloops; uint64_t zo_metaslab_gang_bang; + int zo_mmp_test; } ztest_shared_opts_t; static const ztest_shared_opts_t ztest_opts_defaults = { @@ -184,6 +186,7 @@ static const ztest_shared_opts_t ztest_opts_defaults = { .zo_passtime = 60, /* 60 seconds */ .zo_killrate = 70, /* 70% kill rate */ .zo_verbose = 0, + .zo_mmp_test = 0, .zo_init = 1, .zo_time = 300, /* 5 minutes */ .zo_maxloops = 50, /* max loops during spa_freeze() */ @@ -623,6 +626,7 @@ usage(boolean_t requested) "\t[-k kill_percentage (default: %llu%%)]\n" "\t[-p pool_name (default: %s)]\n" "\t[-f dir (default: %s)] file directory for vdev files\n" + "\t[-M] Multi-host simulate pool imported on remote host\n" "\t[-V] verbose (use multiple times for ever more blather)\n" "\t[-E] use existing pool instead of creating new one\n" "\t[-T time (default: %llu sec)] total run time\n" @@ -666,7 +670,7 @@ process_options(int argc, char **argv) bcopy(&ztest_opts_defaults, zo, sizeof (*zo)); while ((opt = getopt(argc, argv, - "v:s:a:m:r:R:d:t:g:i:k:p:f:VET:P:hF:B:o:")) != EOF) { + "v:s:a:m:r:R:d:t:g:i:k:p:f:MVET:P:hF:B:o:")) != EOF) { value = 0; switch (opt) { case 'v': @@ -736,6 +740,9 @@ process_options(int argc, char **argv) free(path); } break; + case 'M': + zo->zo_mmp_test = 1; + break; case 'V': zo->zo_verbose++; break; @@ -2619,6 +2626,9 @@ ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id) spa_t *spa; nvlist_t *nvroot; + if (zo->zo_mmp_test) + return; + /* * Attempt to create using a bad file. */ @@ -2660,6 +2670,9 @@ ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id) nvlist_t *nvroot, *props; char *name; + if (ztest_opts.zo_mmp_test) + return; + mutex_enter(&ztest_vdev_lock); name = kmem_asprintf("%s_upgrade", ztest_opts.zo_pool); @@ -2773,6 +2786,9 @@ ztest_vdev_add_remove(ztest_ds_t *zd, uint64_t id) nvlist_t *nvroot; int error; + if (ztest_opts.zo_mmp_test) + return; + mutex_enter(&ztest_vdev_lock); leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) * ztest_opts.zo_raidz; @@ -2844,6 +2860,9 @@ ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id) uint64_t guid = 0; int error; + if (ztest_opts.zo_mmp_test) + return; + path = umem_alloc(MAXPATHLEN, UMEM_NOFAIL); if (ztest_random(2) == 0) { @@ -2929,6 +2948,9 @@ ztest_split_pool(ztest_ds_t *zd, uint64_t id) uint_t c, children, schildren = 0, lastlogid = 0; int error = 0; + if (ztest_opts.zo_mmp_test) + return; + mutex_enter(&ztest_vdev_lock); /* ensure we have a useable config; mirrors of raidz aren't supported */ @@ -3036,6 +3058,9 @@ ztest_vdev_attach_detach(ztest_ds_t *zd, uint64_t id) int oldvd_is_log; int error, expected_error; + if (ztest_opts.zo_mmp_test) + return; + oldpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL); newpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL); @@ -5624,6 +5649,9 @@ ztest_spa_rename(ztest_ds_t *zd, uint64_t id) char *oldname, *newname; spa_t *spa; + if (ztest_opts.zo_mmp_test) + return; + (void) rw_wrlock(&ztest_name_lock); oldname = ztest_opts.zo_pool; @@ -6414,7 +6442,7 @@ ztest_run(ztest_shared_t *zs) * Verify that we can export the pool and reimport it under a * different name. */ - if (ztest_random(2) == 0) { + if ((ztest_random(2) == 0) && !ztest_opts.zo_mmp_test) { char name[ZFS_MAX_DATASET_NAME_LEN]; (void) snprintf(name, sizeof (name), "%s_import", ztest_opts.zo_pool); @@ -6562,6 +6590,56 @@ make_random_props(void) } /* + * Import a storage pool with the given name. + */ +static void +ztest_import(ztest_shared_t *zs) +{ + libzfs_handle_t *hdl; + importargs_t args = { 0 }; + spa_t *spa; + nvlist_t *cfg = NULL; + int nsearch = 1; + char *searchdirs[nsearch]; + char *name = ztest_opts.zo_pool; + int flags = ZFS_IMPORT_MISSING_LOG; + int error; + + mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL); + VERIFY(rwlock_init(&ztest_name_lock, USYNC_THREAD, NULL) == 0); + + kernel_init(FREAD | FWRITE); + hdl = libzfs_init(); + + searchdirs[0] = ztest_opts.zo_dir; + args.paths = nsearch; + args.path = searchdirs; + args.can_be_active = B_FALSE; + + error = zpool_tryimport(hdl, name, &cfg, &args); + if (error) + (void) fatal(0, "No pools found\n"); + + VERIFY0(spa_import(name, cfg, NULL, flags)); + VERIFY0(spa_open(name, &spa, FTAG)); + zs->zs_metaslab_sz = + 1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift; + spa_close(spa, FTAG); + + libzfs_fini(hdl); + kernel_fini(); + + if (!ztest_opts.zo_mmp_test) { + ztest_run_zdb(ztest_opts.zo_pool); + ztest_freeze(); + ztest_run_zdb(ztest_opts.zo_pool); + } + + (void) rwlock_destroy(&ztest_name_lock); + mutex_destroy(&ztest_vdev_lock); +} + +/* * Create a storage pool with the given name and initial vdev size. * Then test spa_freeze() functionality. */ @@ -6605,11 +6683,11 @@ ztest_init(ztest_shared_t *zs) kernel_fini(); - ztest_run_zdb(ztest_opts.zo_pool); - - ztest_freeze(); - - ztest_run_zdb(ztest_opts.zo_pool); + if (!ztest_opts.zo_mmp_test) { + ztest_run_zdb(ztest_opts.zo_pool); + ztest_freeze(); + ztest_run_zdb(ztest_opts.zo_pool); + } (void) rwlock_destroy(&ztest_name_lock); mutex_destroy(&ztest_vdev_lock); @@ -6769,13 +6847,19 @@ ztest_run_init(void) ztest_shared_t *zs = ztest_shared; - ASSERT(ztest_opts.zo_init != 0); - /* * Blow away any existing copy of zpool.cache */ (void) remove(spa_config_path); + if (ztest_opts.zo_init == 0) { + if (ztest_opts.zo_verbose >= 1) + (void) printf("Importing pool %s\n", + ztest_opts.zo_pool); + ztest_import(zs); + return; + } + /* * Create and initialize our storage pool. */ @@ -7002,7 +7086,8 @@ main(int argc, char **argv) } kernel_fini(); - ztest_run_zdb(ztest_opts.zo_pool); + if (!ztest_opts.zo_mmp_test) + ztest_run_zdb(ztest_opts.zo_pool); } if (ztest_opts.zo_verbose >= 1) { diff --git a/configure.ac b/configure.ac index c52d72e3d..514fcd98e 100644 --- a/configure.ac +++ b/configure.ac @@ -253,6 +253,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/libzfs/Makefile tests/zfs-tests/tests/functional/migration/Makefile tests/zfs-tests/tests/functional/mmap/Makefile + tests/zfs-tests/tests/functional/mmp/Makefile tests/zfs-tests/tests/functional/mount/Makefile tests/zfs-tests/tests/functional/mv_files/Makefile tests/zfs-tests/tests/functional/nestedfs/Makefile diff --git a/include/libzfs.h b/include/libzfs.h index c03ff5710..147589bbf 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -148,6 +148,7 @@ typedef enum zfs_error { EZFS_DIFFDATA, /* bad zfs diff data */ EZFS_POOLREADONLY, /* pool is in read-only mode */ EZFS_SCRUB_PAUSED, /* scrub currently paused */ + EZFS_ACTIVE_POOL, /* pool is imported on a different system */ EZFS_UNKNOWN } zfs_error_t; @@ -326,6 +327,8 @@ typedef enum { ZPOOL_STATUS_FAILING_DEV, /* device experiencing errors */ ZPOOL_STATUS_VERSION_NEWER, /* newer on-disk version */ ZPOOL_STATUS_HOSTID_MISMATCH, /* last accessed by another system */ + ZPOOL_STATUS_HOSTID_ACTIVE, /* currently active on another system */ + ZPOOL_STATUS_HOSTID_REQUIRED, /* multihost=on and hostid=0 */ ZPOOL_STATUS_IO_FAILURE_WAIT, /* failed I/O, failmode 'wait' */ ZPOOL_STATUS_IO_FAILURE_CONTINUE, /* failed I/O, failmode 'continue' */ ZPOOL_STATUS_BAD_LOG, /* cannot read log chain(s) */ @@ -408,6 +411,8 @@ typedef struct importargs { } importargs_t; extern nvlist_t *zpool_search_import(libzfs_handle_t *, importargs_t *); +extern int zpool_tryimport(libzfs_handle_t *hdl, char *target, + nvlist_t **configp, importargs_t *args); /* legacy pool search routines */ extern nvlist_t *zpool_find_import(libzfs_handle_t *, int, char **); diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index 75a7bc249..2efd85e31 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -131,7 +131,7 @@ typedef enum { SHARED_SMB = 0x4 } zfs_share_type_t; -#define CONFIG_BUF_MINSIZE 65536 +#define CONFIG_BUF_MINSIZE 262144 int zfs_error(libzfs_handle_t *, int, const char *); int zfs_error_fmt(libzfs_handle_t *, int, const char *, ...); diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am index 956643801..be606b8c6 100644 --- a/include/sys/Makefile.am +++ b/include/sys/Makefile.am @@ -36,6 +36,7 @@ COMMON_H = \ $(top_srcdir)/include/sys/efi_partition.h \ $(top_srcdir)/include/sys/metaslab.h \ $(top_srcdir)/include/sys/metaslab_impl.h \ + $(top_srcdir)/include/sys/mmp.h \ $(top_srcdir)/include/sys/mntent.h \ $(top_srcdir)/include/sys/multilist.h \ $(top_srcdir)/include/sys/nvpair.h \ diff --git a/include/sys/dsl_pool.h b/include/sys/dsl_pool.h index 674dd25b5..d2dabda6d 100644 --- a/include/sys/dsl_pool.h +++ b/include/sys/dsl_pool.h @@ -38,6 +38,7 @@ #include <sys/bpobj.h> #include <sys/bptree.h> #include <sys/rrwlock.h> +#include <sys/mmp.h> #ifdef __cplusplus extern "C" { diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 9fbcfbef3..13b25a695 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -223,6 +223,7 @@ typedef enum { ZPOOL_PROP_MAXBLOCKSIZE, ZPOOL_PROP_TNAME, ZPOOL_PROP_MAXDNODESIZE, + ZPOOL_PROP_MULTIHOST, ZPOOL_NUM_PROPS } zpool_prop_t; @@ -651,6 +652,11 @@ typedef struct zpool_rewind_policy { #define ZPOOL_CONFIG_VDEV_TOP_ZAP "com.delphix:vdev_zap_top" #define ZPOOL_CONFIG_VDEV_LEAF_ZAP "com.delphix:vdev_zap_leaf" #define ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS "com.delphix:has_per_vdev_zaps" +#define ZPOOL_CONFIG_MMP_STATE "mmp_state" /* not stored on disk */ +#define ZPOOL_CONFIG_MMP_TXG "mmp_txg" /* not stored on disk */ +#define ZPOOL_CONFIG_MMP_HOSTNAME "mmp_hostname" /* not stored on disk */ +#define ZPOOL_CONFIG_MMP_HOSTID "mmp_hostid" /* not stored on disk */ + /* * The persistent vdev state is stored as separate values rather than a single * 'vdev_state' entry. This is because a device can be in multiple states, such @@ -744,7 +750,8 @@ typedef enum vdev_aux { VDEV_AUX_EXTERNAL, /* external diagnosis or forced fault */ VDEV_AUX_SPLIT_POOL, /* vdev was split off into another pool */ VDEV_AUX_BAD_ASHIFT, /* vdev ashift is invalid */ - VDEV_AUX_EXTERNAL_PERSIST /* persistent forced fault */ + VDEV_AUX_EXTERNAL_PERSIST, /* persistent forced fault */ + VDEV_AUX_ACTIVE, /* vdev active on a different host */ } vdev_aux_t; /* @@ -765,6 +772,16 @@ typedef enum pool_state { } pool_state_t; /* + * mmp state. The following states provide additional detail describing + * why a pool couldn't be safely imported. + */ +typedef enum mmp_state { + MMP_STATE_ACTIVE = 0, /* In active use */ + MMP_STATE_INACTIVE, /* Inactive and safe to import */ + MMP_STATE_NO_HOSTID /* System hostid is not set */ +} mmp_state_t; + +/* * Scan Functions. */ typedef enum pool_scan_func { @@ -1126,6 +1143,7 @@ typedef enum { #define ZFS_IMPORT_MISSING_LOG 0x4 #define ZFS_IMPORT_ONLY 0x8 #define ZFS_IMPORT_TEMP_NAME 0x10 +#define ZFS_IMPORT_SKIP_MMP 0x20 /* * Sysevent payload members. ZFS will generate the following sysevents with the diff --git a/include/sys/mmp.h b/include/sys/mmp.h new file mode 100644 index 000000000..4da612d6a --- /dev/null +++ b/include/sys/mmp.h @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2017 by Lawrence Livermore National Security, LLC. + */ + +#ifndef _SYS_MMP_H +#define _SYS_MMP_H + +#include <sys/spa.h> +#include <sys/zfs_context.h> +#include <sys/uberblock_impl.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define MMP_MIN_INTERVAL 100 /* ms */ +#define MMP_DEFAULT_INTERVAL 1000 /* ms */ +#define MMP_DEFAULT_IMPORT_INTERVALS 10 +#define MMP_DEFAULT_FAIL_INTERVALS 5 + +typedef struct mmp_thread { + kmutex_t mmp_thread_lock; /* protect thread mgmt fields */ + kcondvar_t mmp_thread_cv; + kthread_t *mmp_thread; + uint8_t mmp_thread_exiting; + kmutex_t mmp_io_lock; /* protect below */ + hrtime_t mmp_last_write; /* last successful MMP write */ + uint64_t mmp_delay; /* decaying avg ns between MMP writes */ + uberblock_t mmp_ub; /* last ub written by sync */ + zio_t *mmp_zio_root; /* root of mmp write zios */ +} mmp_thread_t; + + +extern void mmp_init(struct spa *spa); +extern void mmp_fini(struct spa *spa); +extern void mmp_thread_start(struct spa *spa); +extern void mmp_thread_stop(struct spa *spa); +extern void mmp_update_uberblock(struct spa *spa, struct uberblock *ub); + +/* Global tuning */ +extern ulong_t zfs_multihost_interval; +extern uint_t zfs_multihost_fail_intervals; +extern uint_t zfs_multihost_import_intervals; + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MMP_H */ diff --git a/include/sys/spa.h b/include/sys/spa.h index dd86aad40..de942ad2b 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -729,6 +729,7 @@ typedef struct spa_stats { spa_stats_history_t txg_history; spa_stats_history_t tx_assign_histogram; spa_stats_history_t io_history; + spa_stats_history_t mmp_history; } spa_stats_t; typedef enum txg_state { @@ -758,6 +759,8 @@ extern txg_stat_t *spa_txg_history_init_io(spa_t *, uint64_t, struct dsl_pool *); extern void spa_txg_history_fini_io(spa_t *, txg_stat_t *); extern void spa_tx_assign_add_nsecs(spa_t *spa, uint64_t nsecs); +extern void spa_mmp_history_add(uint64_t txg, uint64_t timestamp, + uint64_t mmp_delay, vdev_t *vd, int label); /* Pool configuration locks */ extern int spa_config_tryenter(spa_t *spa, int locks, void *tag, krw_t rw); @@ -860,6 +863,8 @@ extern boolean_t spa_has_pending_synctask(spa_t *spa); extern int spa_maxblocksize(spa_t *spa); extern int spa_maxdnodesize(spa_t *spa); extern void zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp); +extern boolean_t spa_multihost(spa_t *spa); +extern unsigned long spa_get_hostid(void); extern int spa_mode(spa_t *spa); extern uint64_t zfs_strtonum(const char *str, char **nptr); diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index 9142242cb..06de24421 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -275,6 +275,8 @@ struct spa { spa_stats_t spa_stats; /* assorted spa statistics */ hrtime_t spa_ccw_fail_time; /* Conf cache write fail time */ taskq_t *spa_zvol_taskq; /* Taskq for minor management */ + uint64_t spa_multihost; /* multihost aware (mmp) */ + mmp_thread_t spa_mmp; /* multihost mmp thread */ /* * spa_refcount & spa_config_lock must be the last elements diff --git a/include/sys/uberblock.h b/include/sys/uberblock.h index 21e7ae0de..044e43838 100644 --- a/include/sys/uberblock.h +++ b/include/sys/uberblock.h @@ -40,7 +40,8 @@ extern "C" { typedef struct uberblock uberblock_t; extern int uberblock_verify(uberblock_t *); -extern boolean_t uberblock_update(uberblock_t *, vdev_t *, uint64_t); +extern boolean_t uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg, + uint64_t mmp_delay); #ifdef __cplusplus } diff --git a/include/sys/uberblock_impl.h b/include/sys/uberblock_impl.h index 6ab6aa313..9fdc70b91 100644 --- a/include/sys/uberblock_impl.h +++ b/include/sys/uberblock_impl.h @@ -43,6 +43,7 @@ extern "C" { */ #define UBERBLOCK_MAGIC 0x00bab10c /* oo-ba-bloc! */ #define UBERBLOCK_SHIFT 10 /* up to 1K */ +#define MMP_MAGIC 0xa11cea11 /* all-see-all */ struct uberblock { uint64_t ub_magic; /* UBERBLOCK_MAGIC */ @@ -54,6 +55,11 @@ struct uberblock { /* highest SPA_VERSION supported by software that wrote this txg */ uint64_t ub_software_version; + + /* Maybe missing in uberblocks we read, but always written */ + uint64_t ub_mmp_magic; /* MMP_MAGIC */ + uint64_t ub_mmp_delay; /* nanosec since last MMP write */ + uint64_t ub_mmp_seq; /* reserved for sequence number */ }; #ifdef __cplusplus diff --git a/include/sys/vdev.h b/include/sys/vdev.h index 63b4904c5..7157ef43f 100644 --- a/include/sys/vdev.h +++ b/include/sys/vdev.h @@ -155,6 +155,8 @@ extern int vdev_label_number(uint64_t psise, uint64_t offset); extern nvlist_t *vdev_label_read_config(vdev_t *vd, uint64_t txg); extern void vdev_uberblock_load(vdev_t *, struct uberblock *, nvlist_t **); extern void vdev_config_generate_stats(vdev_t *vd, nvlist_t *nv); +extern void vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t + offset, uint64_t size, zio_done_func_t *done, void *private, int flags); typedef enum { VDEV_LABEL_CREATE, /* create/add a new device */ diff --git a/include/sys/vdev_impl.h b/include/sys/vdev_impl.h index 835d2dbbf..7c5e54b08 100644 --- a/include/sys/vdev_impl.h +++ b/include/sys/vdev_impl.h @@ -238,6 +238,7 @@ struct vdev { zio_t *vdev_probe_zio; /* root of current probe */ vdev_aux_t vdev_label_aux; /* on-disk aux state */ uint64_t vdev_leaf_zap; + hrtime_t vdev_mmp_pending; /* 0 if write finished */ /* * For DTrace to work in userland (libzpool) context, these fields must @@ -268,6 +269,12 @@ struct vdev { #define VDEV_PHYS_SIZE (112 << 10) #define VDEV_UBERBLOCK_RING (128 << 10) +/* + * MMP blocks occupy the last MMP_BLOCKS_PER_LABEL slots in the uberblock + * ring when MMP is enabled. + */ +#define MMP_BLOCKS_PER_LABEL 1 + /* The largest uberblock we support is 8k. */ #define MAX_UBERBLOCK_SHIFT (13) #define VDEV_UBERBLOCK_SHIFT(vd) \ diff --git a/lib/libzfs/libzfs_import.c b/lib/libzfs/libzfs_import.c index 02264433e..f371d925b 100644 --- a/lib/libzfs/libzfs_import.c +++ b/lib/libzfs/libzfs_import.c @@ -2166,6 +2166,80 @@ zpool_search_import(libzfs_handle_t *hdl, importargs_t *import) return (zpool_find_import_impl(hdl, import)); } +static boolean_t +pool_match(nvlist_t *cfg, char *tgt) +{ + uint64_t v, guid = strtoull(tgt, NULL, 0); + char *s; + + if (guid != 0) { + if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_GUID, &v) == 0) + return (v == guid); + } else { + if (nvlist_lookup_string(cfg, ZPOOL_CONFIG_POOL_NAME, &s) == 0) + return (strcmp(s, tgt) == 0); + } + return (B_FALSE); +} + +int +zpool_tryimport(libzfs_handle_t *hdl, char *target, nvlist_t **configp, + importargs_t *args) +{ + nvlist_t *pools; + nvlist_t *match = NULL; + nvlist_t *config = NULL; + char *name = NULL, *sepp = NULL; + char sep = '\0'; + int count = 0; + char *targetdup = strdup(target); + + *configp = NULL; + + if ((sepp = strpbrk(targetdup, "/@")) != NULL) { + sep = *sepp; + *sepp = '\0'; + } + + pools = zpool_search_import(hdl, args); + + if (pools != NULL) { + nvpair_t *elem = NULL; + while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) { + VERIFY0(nvpair_value_nvlist(elem, &config)); + if (pool_match(config, targetdup)) { + count++; + if (match != NULL) { + /* multiple matches found */ + continue; + } else { + match = config; + name = nvpair_name(elem); + } + } + } + } + + if (count == 0) { + (void) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "no pools found")); + free(targetdup); + return (ENOENT); + } + + if (count > 1) { + (void) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "%d pools found, use pool GUID\n"), count); + free(targetdup); + return (EINVAL); + } + + *configp = match; + free(targetdup); + + return (0); +} + boolean_t find_guid(nvlist_t *nv, uint64_t guid) { diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index 746e6a8cc..58d91c8f4 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -676,7 +676,14 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname, goto error; } break; - + case ZPOOL_PROP_MULTIHOST: + if (get_system_hostid() == 0) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "requires a non-zero system hostid")); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + goto error; + } + break; default: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "property '%s'(%d) not defined"), propname, prop); @@ -1779,6 +1786,7 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname, if (error) { char desc[1024]; + char aux[256]; /* * Dry-run failed, but we print out what success @@ -1824,6 +1832,47 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname, (void) zfs_error(hdl, EZFS_BADVERSION, desc); break; + case EREMOTEIO: + if (nv != NULL && nvlist_lookup_nvlist(nv, + ZPOOL_CONFIG_LOAD_INFO, &nvinfo) == 0) { + char *hostname = "<unknown>"; + uint64_t hostid = 0; + mmp_state_t mmp_state; + + mmp_state = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_STATE); + + if (nvlist_exists(nvinfo, + ZPOOL_CONFIG_MMP_HOSTNAME)) + hostname = fnvlist_lookup_string(nvinfo, + ZPOOL_CONFIG_MMP_HOSTNAME); + + if (nvlist_exists(nvinfo, + ZPOOL_CONFIG_MMP_HOSTID)) + hostid = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_HOSTID); + + if (mmp_state == MMP_STATE_ACTIVE) { + (void) snprintf(aux, sizeof (aux), + dgettext(TEXT_DOMAIN, "pool is imp" + "orted on host '%s' (hostid=%lx).\n" + "Export the pool on the other " + "system, then run 'zpool import'."), + hostname, (unsigned long) hostid); + } else if (mmp_state == MMP_STATE_NO_HOSTID) { + (void) snprintf(aux, sizeof (aux), + dgettext(TEXT_DOMAIN, "pool has " + "the multihost property on and " + "the\nsystem's hostid is not set. " + "Set a unique system hostid with " + "the genhostid(1) command.\n")); + } + + (void) zfs_error_aux(hdl, aux); + } + (void) zfs_error(hdl, EZFS_ACTIVE_POOL, desc); + break; + case EINVAL: (void) zfs_error(hdl, EZFS_INVALCONFIG, desc); break; @@ -2206,7 +2255,7 @@ zpool_find_vdev(zpool_handle_t *zhp, const char *path, boolean_t *avail_spare, } static int -vdev_online(nvlist_t *nv) +vdev_is_online(nvlist_t *nv) { uint64_t ival; @@ -2274,7 +2323,7 @@ vdev_get_physpaths(nvlist_t *nv, char *physpath, size_t phypath_size, return (EZFS_INVALCONFIG); } - if (vdev_online(nv)) { + if (vdev_is_online(nv)) { if ((ret = vdev_get_one_physpath(nv, physpath, phypath_size, rsz)) != 0) return (ret); diff --git a/lib/libzfs/libzfs_status.c b/lib/libzfs/libzfs_status.c index d5470f412..05a9afce8 100644 --- a/lib/libzfs/libzfs_status.c +++ b/lib/libzfs/libzfs_status.c @@ -64,6 +64,8 @@ static char *zfs_msgid_table[] = { "ZFS-8000-9P", "ZFS-8000-A5", "ZFS-8000-EY", + "ZFS-8000-EY", + "ZFS-8000-EY", "ZFS-8000-HC", "ZFS-8000-JQ", "ZFS-8000-K4", @@ -216,6 +218,26 @@ check_status(nvlist_t *config, boolean_t isimport, zpool_errata_t *erratap) return (ZPOOL_STATUS_RESILVERING); /* + * The multihost property is set and the pool may be active. + */ + if (vs->vs_state == VDEV_STATE_CANT_OPEN && + vs->vs_aux == VDEV_AUX_ACTIVE) { + mmp_state_t mmp_state; + nvlist_t *nvinfo; + + nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); + mmp_state = fnvlist_lookup_uint64(nvinfo, + ZPOOL_CONFIG_MMP_STATE); + + if (mmp_state == MMP_STATE_ACTIVE) + return (ZPOOL_STATUS_HOSTID_ACTIVE); + else if (mmp_state == MMP_STATE_NO_HOSTID) + return (ZPOOL_STATUS_HOSTID_REQUIRED); + else + return (ZPOOL_STATUS_HOSTID_MISMATCH); + } + + /* * Pool last accessed by another system. */ (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid); @@ -344,8 +366,9 @@ check_status(nvlist_t *config, boolean_t isimport, zpool_errata_t *erratap) if (isimport) { feat = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); - feat = fnvlist_lookup_nvlist(feat, - ZPOOL_CONFIG_ENABLED_FEAT); + if (nvlist_exists(feat, ZPOOL_CONFIG_ENABLED_FEAT)) + feat = fnvlist_lookup_nvlist(feat, + ZPOOL_CONFIG_ENABLED_FEAT); } else { feat = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS); diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index 30255d643..bc51a76a8 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -261,6 +261,9 @@ libzfs_error_description(libzfs_handle_t *hdl) return (dgettext(TEXT_DOMAIN, "invalid diff data")); case EZFS_POOLREADONLY: return (dgettext(TEXT_DOMAIN, "pool is read-only")); + case EZFS_ACTIVE_POOL: + return (dgettext(TEXT_DOMAIN, "pool is imported on a " + "different host")); case EZFS_UNKNOWN: return (dgettext(TEXT_DOMAIN, "unknown error")); default: @@ -423,6 +426,9 @@ zfs_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...) "pool I/O is currently suspended")); zfs_verror(hdl, EZFS_POOLUNAVAIL, fmt, ap); break; + case EREMOTEIO: + zfs_verror(hdl, EZFS_ACTIVE_POOL, fmt, ap); + break; default: zfs_error_aux(hdl, strerror(error)); zfs_verror(hdl, EZFS_UNKNOWN, fmt, ap); @@ -511,6 +517,9 @@ zpool_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...) "block size out of range or does not match")); zfs_verror(hdl, EZFS_BADPROP, fmt, ap); break; + case EREMOTEIO: + zfs_verror(hdl, EZFS_ACTIVE_POOL, fmt, ap); + break; default: zfs_error_aux(hdl, strerror(error)); diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index 1e95f8064..a8c72c98d 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -71,6 +71,7 @@ KERNEL_C = \ lzjb.c \ lz4.c \ metaslab.c \ + mmp.c \ multilist.c \ pathname.c \ range_tree.c \ diff --git a/man/man1/ztest.1 b/man/man1/ztest.1 index ebfacd46d..8cb07fa99 100644 --- a/man/man1/ztest.1 +++ b/man/man1/ztest.1 @@ -147,6 +147,12 @@ ztest -f / -V -T 120 .SH "ENVIRONMENT VARIABLES" .TP +.B "ZFS_HOSTID=id" +Use \fBid\fR instead of the SPL hostid to identify this host. Intended for use +with ztest, but this environment variable will affect any utility which uses +libzpool, including \fBzpool(8)\fR. Since the kernel is unaware of this setting +results with utilites other than ztest are undefined. +.TP .B "ZFS_STACK_SIZE=stacksize" Limit the default stack size to \fBstacksize\fR bytes for the purpose of detecting and debugging kernel stack overflows. This value defaults to @@ -160,6 +166,7 @@ required for a NULL procedure in user space. By default the stack size is limited to 256K. .SH "SEE ALSO" +.BR "spl-module-parameters (5)" "," .BR "zpool (1)" "," .BR "zfs (1)" "," .BR "zdb (1)" "," diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index ab1c15841..0447debf7 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -1413,6 +1413,79 @@ Default value: \fB0\fR. .sp .ne 2 .na +\fBzfs_multihost_history\fR (int) +.ad +.RS 12n +Historical statistics for the last N multihost updates will be available in +\fB/proc/spl/kstat/zfs/<pool>/multihost\fR +.sp +Default value: \fB0\fR. +.RE + +.sp +.ne 2 +.na +\fBzfs_multihost_interval\fR (ulong) +.ad +.RS 12n +Used to control the frequency of multihost writes which are performed when the +\fBmultihost\fR pool property is on. This is one factor used to determine +the length of the activity check during import. +.sp +The multihost write period is \fBzfs_multihost_interval / leaf-vdevs\fR milliseconds. +This means that on average a multihost write will be issued for each leaf vdev every +\fBzfs_multihost_interval\fR milliseconds. In practice, the observed period can +vary with the I/O load and this observed value is the delay which is stored in +the uberblock. +.sp +On import the activity check waits a minimum amount of time determined by +\fBzfs_multihost_interval * zfs_multihost_import_intervals\fR. The activity +check time may be further extended if the value of mmp delay found in the best +uberblock indicates actual multihost updates happened at longer intervals than +\fBzfs_multihost_interval\fR. A minimum value of \fB100ms\fR is enforced. +.sp +Default value: \fB1000\fR. +.RE + +.sp +.ne 2 +.na +\fBzfs_multihost_import_intervals\fR (uint) +.ad +.RS 12n +Used to control the duration of the activity test on import. Smaller values of +\fBzfs_multihost_import_intervals\fR will reduce the import time but increase +the risk of failing to detect an active pool. The total activity check time is +never allowed to drop below one second. A value of 0 is ignored and treated as +if it was set to 1 +.sp +Default value: \fB10\fR. +.RE + +.sp +.ne 2 +.na +\fBzfs_multihost_fail_intervals\fR (uint) +.ad +.RS 12n +Controls the behavior of the pool when multihost write failures are detected. +.sp +When \fBzfs_multihost_fail_intervals = 0\fR then multihost write failures are ignored. +The failures will still be reported to the ZED which depending on its +configuration may take action such as suspending the pool or offlining a device. +.sp +When \fBzfs_multihost_fail_intervals > 0\fR then sequential multihost write failures +will cause the pool to be suspended. This occurs when +\fBzfs_multihost_fail_intervals * zfs_multihost_interval\fR milliseconds have +passed since the last successful multihost write. This guarantees the activity test +will see multihost writes if the pool is imported. +.sp +Default value: \fB5\fR. +.RE + +.sp +.ne 2 +.na \fBzfs_no_scrub_io\fR (int) .ad .RS 12n @@ -1529,8 +1602,8 @@ Default value: \fB1,048,576\fR. \fBzfs_read_history\fR (int) .ad .RS 12n -Historic statistics for the last N reads will be available in -\fR/proc/spl/kstat/zfs/POOLNAME/reads\fB +Historical statistics for the last N reads will be available in +\fB/proc/spl/kstat/zfs/<pool>/reads\fR .sp Default value: \fB0\fR (no data is kept). .RE @@ -1684,8 +1757,8 @@ Default value: \fB32\fR. \fBzfs_txg_history\fR (int) .ad .RS 12n -Historic statistics for the last N txgs will be available in -\fR/proc/spl/kstat/zfs/POOLNAME/txgs\fB +Historical statistics for the last N txgs will be available in +\fB/proc/spl/kstat/zfs/<pool>/txgs\fR .sp Default value: \fB0\fR. .RE diff --git a/man/man8/zpool.8 b/man/man8/zpool.8 index ea25da300..69d77e961 100644 --- a/man/man8/zpool.8 +++ b/man/man8/zpool.8 @@ -717,6 +717,25 @@ The default value is .Sy off . This property can also be referred to by its shortened name, .Sy listsnaps . +.It Sy multihost Ns = Ns Sy on Ns | Ns Sy off +Controls whether a pool activity check should be performed during +.Nm zpool Cm import . +When a pool is determined to be active it cannot be imported, even with the +.Fl f +option. This property is intended to be used in failover configurations +where multiple hosts have access to a pool on shared storage. When this +property is on, periodic writes to storage occur to show the pool is in use. +See +.Sy zfs_multihost_interval +in the +.Xr zfs-module-parameters 5 +man page. In order to enable this property each host must set a unique hostid. +See +.Xr genhostid 1 +and +.Xr spl-module-paramters 5 +for additional details. The default value is +.Sy off . .It Sy version Ns = Ns Ar version The current on-disk version of the pool. This can be increased, but never decreased. @@ -2335,7 +2354,7 @@ is not set, it is assumed that the user is allowed to run .Sh INTERFACE STABILITY .Sy Evolving .Sh SEE ALSO -.Xr zed 8 +.Xr zed 8 , .Xr zfs 8 , .Xr zfs-events 5 , .Xr zfs-module-parameters 5 , diff --git a/module/zcommon/zpool_prop.c b/module/zcommon/zpool_prop.c index 77b4d62ff..fd21f3117 100644 --- a/module/zcommon/zpool_prop.c +++ b/module/zcommon/zpool_prop.c @@ -120,6 +120,9 @@ zpool_prop_init(void) PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "EXPAND", boolean_table); zprop_register_index(ZPOOL_PROP_READONLY, "readonly", 0, PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "RDONLY", boolean_table); + zprop_register_index(ZPOOL_PROP_MULTIHOST, "multihost", 0, + PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "MULTIHOST", + boolean_table); /* default index properties */ zprop_register_index(ZPOOL_PROP_FAILUREMODE, "failmode", diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in index f8d54f4dd..d6336f314 100644 --- a/module/zfs/Makefile.in +++ b/module/zfs/Makefile.in @@ -43,6 +43,7 @@ $(MODULE)-objs += gzip.o $(MODULE)-objs += lzjb.o $(MODULE)-objs += lz4.o $(MODULE)-objs += metaslab.o +$(MODULE)-objs += mmp.o $(MODULE)-objs += multilist.o $(MODULE)-objs += pathname.o $(MODULE)-objs += policy.o diff --git a/module/zfs/dsl_pool.c b/module/zfs/dsl_pool.c index 97eb0cced..c16708048 100644 --- a/module/zfs/dsl_pool.c +++ b/module/zfs/dsl_pool.c @@ -48,6 +48,7 @@ #include <sys/zil_impl.h> #include <sys/dsl_userhold.h> #include <sys/trace_txg.h> +#include <sys/mmp.h> /* * ZFS Write Throttle @@ -160,6 +161,7 @@ dsl_pool_open_impl(spa_t *spa, uint64_t txg) dp->dp_meta_rootbp = *bp; rrw_init(&dp->dp_config_rwlock, B_TRUE); txg_init(dp, txg); + mmp_init(spa); txg_list_create(&dp->dp_dirty_datasets, spa, offsetof(dsl_dataset_t, ds_dirty_link)); @@ -342,6 +344,7 @@ dsl_pool_close(dsl_pool_t *dp) */ arc_flush(dp->dp_spa, FALSE); + mmp_fini(dp->dp_spa); txg_fini(dp); dsl_scan_fini(dp); dmu_buf_user_evict_wait(); diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c new file mode 100644 index 000000000..35348f8b4 --- /dev/null +++ b/module/zfs/mmp.c @@ -0,0 +1,475 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2017 by Lawrence Livermore National Security, LLC. + */ + +#include <sys/abd.h> +#include <sys/mmp.h> +#include <sys/spa.h> +#include <sys/spa_impl.h> +#include <sys/vdev.h> +#include <sys/vdev_impl.h> +#include <sys/zfs_context.h> +#include <sys/callb.h> + +/* + * Multi-Modifier Protection (MMP) attempts to prevent a user from importing + * or opening a pool on more than one host at a time. In particular, it + * prevents "zpool import -f" on a host from succeeding while the pool is + * already imported on another host. There are many other ways in which a + * device could be used by two hosts for different purposes at the same time + * resulting in pool damage. This implementation does not attempt to detect + * those cases. + * + * MMP operates by ensuring there are frequent visible changes on disk (a + * "heartbeat") at all times. And by altering the import process to check + * for these changes and failing the import when they are detected. This + * functionality is enabled by setting the 'multihost' pool property to on. + * + * Uberblocks written by the txg_sync thread always go into the first + * (N-MMP_BLOCKS_PER_LABEL) slots, the remaining slots are reserved for MMP. + * They are used to hold uberblocks which are exactly the same as the last + * synced uberblock except that the ub_timestamp is frequently updated. + * Like all other uberblocks, the slot is written with an embedded checksum, + * and slots with invalid checksums are ignored. This provides the + * "heartbeat", with no risk of overwriting good uberblocks that must be + * preserved, e.g. previous txgs and associated block pointers. + * + * Two optional fields are added to uberblock structure: ub_mmp_magic and + * ub_mmp_delay. The magic field allows zfs to tell whether ub_mmp_delay is + * valid. The delay field is a decaying average of the amount of time between + * completion of successive MMP writes, in nanoseconds. It is used to predict + * how long the import must wait to detect activity in the pool, before + * concluding it is not in use. + * + * During import an activity test may now be performed to determine if + * the pool is in use. The activity test is typically required if the + * ZPOOL_CONFIG_HOSTID does not match the system hostid, the pool state is + * POOL_STATE_ACTIVE, and the pool is not a root pool. + * + * The activity test finds the "best" uberblock (highest txg & timestamp), + * waits some time, and then finds the "best" uberblock again. If the txg + * and timestamp in both "best" uberblocks do not match, the pool is in use + * by another host and the import fails. Since the granularity of the + * timestamp is in seconds this activity test must take a bare minimum of one + * second. In order to assure the accuracy of the activity test, the default + * values result in an activity test duration of 10x the mmp write interval. + * + * The "zpool import" activity test can be expected to take a minimum time of + * zfs_multihost_import_intervals * zfs_multihost_interval milliseconds. If the + * "best" uberblock has a valid ub_mmp_delay field, then the duration of the + * test may take longer if MMP writes were occurring less frequently than + * expected. Additionally, the duration is then extended by a random 25% to + * attempt to to detect simultaneous imports. For example, if both partner + * hosts are rebooted at the same time and automatically attempt to import the + * pool. + */ + +/* + * Used to control the frequency of mmp writes which are performed when the + * 'multihost' pool property is on. This is one factor used to determine the + * length of the activity check during import. + * + * The mmp write period is zfs_multihost_interval / leaf-vdevs milliseconds. + * This means that on average an mmp write will be issued for each leaf vdev + * every zfs_multihost_interval milliseconds. In practice, the observed period + * can vary with the I/O load and this observed value is the delay which is + * stored in the uberblock. The minimum allowed value is 100 ms. + */ +ulong_t zfs_multihost_interval = MMP_DEFAULT_INTERVAL; + +/* + * Used to control the duration of the activity test on import. Smaller values + * of zfs_multihost_import_intervals will reduce the import time but increase + * the risk of failing to detect an active pool. The total activity check time + * is never allowed to drop below one second. A value of 0 is ignored and + * treated as if it was set to 1. + */ +uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS; + +/* + * Controls the behavior of the pool when mmp write failures are detected. + * + * When zfs_multihost_fail_intervals = 0 then mmp write failures are ignored. + * The failures will still be reported to the ZED which depending on its + * configuration may take action such as suspending the pool or taking a + * device offline. + * + * When zfs_multihost_fail_intervals > 0 then sequential mmp write failures will + * cause the pool to be suspended. This occurs when + * zfs_multihost_fail_intervals * zfs_multihost_interval milliseconds have + * passed since the last successful mmp write. This guarantees the activity + * test will see mmp writes if the + * pool is imported. + */ +uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS; + +static void mmp_thread(spa_t *spa); + +void +mmp_init(spa_t *spa) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + + mutex_init(&mmp->mmp_thread_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&mmp->mmp_thread_cv, NULL, CV_DEFAULT, NULL); + mutex_init(&mmp->mmp_io_lock, NULL, MUTEX_DEFAULT, NULL); +} + +void +mmp_fini(spa_t *spa) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + + mutex_destroy(&mmp->mmp_thread_lock); + cv_destroy(&mmp->mmp_thread_cv); + mutex_destroy(&mmp->mmp_io_lock); +} + +static void +mmp_thread_enter(mmp_thread_t *mmp, callb_cpr_t *cpr) +{ + CALLB_CPR_INIT(cpr, &mmp->mmp_thread_lock, callb_generic_cpr, FTAG); + mutex_enter(&mmp->mmp_thread_lock); +} + +static void +mmp_thread_exit(mmp_thread_t *mmp, kthread_t **mpp, callb_cpr_t *cpr) +{ + ASSERT(*mpp != NULL); + *mpp = NULL; + cv_broadcast(&mmp->mmp_thread_cv); + CALLB_CPR_EXIT(cpr); /* drops &mmp->mmp_thread_lock */ + thread_exit(); +} + +void +mmp_thread_start(spa_t *spa) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + + if (spa_writeable(spa)) { + mutex_enter(&mmp->mmp_thread_lock); + if (!mmp->mmp_thread) { + dprintf("mmp_thread_start pool %s\n", + spa->spa_name); + mmp->mmp_thread = thread_create(NULL, 0, mmp_thread, + spa, 0, &p0, TS_RUN, defclsyspri); + } + mutex_exit(&mmp->mmp_thread_lock); + } +} + +void +mmp_thread_stop(spa_t *spa) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + + mutex_enter(&mmp->mmp_thread_lock); + mmp->mmp_thread_exiting = 1; + cv_broadcast(&mmp->mmp_thread_cv); + + while (mmp->mmp_thread) { + cv_wait(&mmp->mmp_thread_cv, &mmp->mmp_thread_lock); + } + mutex_exit(&mmp->mmp_thread_lock); + + ASSERT(mmp->mmp_thread == NULL); + mmp->mmp_thread_exiting = 0; +} + +/* + * Randomly choose a leaf vdev, to write an MMP block to. It must be + * writable. It must not have an outstanding mmp write (if so then + * there is a problem, and a new write will also block). + * + * We try 10 times to pick a random leaf without an outstanding write. + * If 90% of the leaves have pending writes, this gives us a >65% + * chance of finding one we can write to. There will be at least + * (zfs_multihost_fail_intervals) tries before the inability to write an MMP + * block causes serious problems. + */ +static vdev_t * +vdev_random_leaf(spa_t *spa) +{ + vdev_t *vd, *child; + int pending_writes = 10; + + ASSERT(spa); + ASSERT(spa_config_held(spa, SCL_STATE, RW_READER) == SCL_STATE); + + /* + * Since we hold SCL_STATE, neither pool nor vdev state can + * change. Therefore, if the root is not dead, there is a + * child that is not dead, and so on down to a leaf. + */ + if (!vdev_writeable(spa->spa_root_vdev)) + return (NULL); + + vd = spa->spa_root_vdev; + while (!vd->vdev_ops->vdev_op_leaf) { + child = vd->vdev_child[spa_get_random(vd->vdev_children)]; + + if (!vdev_writeable(child)) + continue; + + if (child->vdev_ops->vdev_op_leaf && child->vdev_mmp_pending) { + if (pending_writes-- > 0) + continue; + else + return (NULL); + } + + vd = child; + } + return (vd); +} + +static void +mmp_write_done(zio_t *zio) +{ + spa_t *spa = zio->io_spa; + vdev_t *vd = zio->io_vd; + mmp_thread_t *mts = zio->io_private; + + mutex_enter(&mts->mmp_io_lock); + vd->vdev_mmp_pending = 0; + + if (zio->io_error) + goto unlock; + + /* + * Mmp writes are queued on a fixed schedule, but under many + * circumstances, such as a busy device or faulty hardware, + * the writes will complete at variable, much longer, + * intervals. In these cases, another node checking for + * activity must wait longer to account for these delays. + * + * The mmp_delay is calculated as a decaying average of the interval + * between completed mmp writes. This is used to predict how long + * the import must wait to detect activity in the pool, before + * concluding it is not in use. + * + * Do not set mmp_delay if the multihost property is not on, + * so as not to trigger an activity check on import. + */ + if (spa_multihost(spa)) { + hrtime_t delay = gethrtime() - mts->mmp_last_write; + + if (delay > mts->mmp_delay) + mts->mmp_delay = delay; + else + mts->mmp_delay = (delay + mts->mmp_delay * 127) / + 128; + } else { + mts->mmp_delay = 0; + } + mts->mmp_last_write = gethrtime(); + +unlock: + mutex_exit(&mts->mmp_io_lock); + + abd_free(zio->io_abd); +} + +/* + * When the uberblock on-disk is updated by a spa_sync, + * creating a new "best" uberblock, update the one stored + * in the mmp thread state, used for mmp writes. + */ +void +mmp_update_uberblock(spa_t *spa, uberblock_t *ub) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + + mutex_enter(&mmp->mmp_io_lock); + mmp->mmp_ub = *ub; + mmp->mmp_ub.ub_timestamp = gethrestime_sec(); + mutex_exit(&mmp->mmp_io_lock); +} + +/* + * Choose a random vdev, label, and MMP block, and write over it + * with a copy of the last-synced uberblock, whose timestamp + * has been updated to reflect that the pool is in use. + */ +static void +mmp_write_uberblock(spa_t *spa) +{ + int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL; + mmp_thread_t *mmp = &spa->spa_mmp; + uberblock_t *ub; + vdev_t *vd; + int label; + uint64_t offset; + + vd = vdev_random_leaf(spa); + if (vd == NULL || !vdev_writeable(vd)) + return; + + mutex_enter(&mmp->mmp_io_lock); + + if (mmp->mmp_zio_root == NULL) + mmp->mmp_zio_root = zio_root(spa, NULL, NULL, + flags | ZIO_FLAG_GODFATHER); + + ub = &mmp->mmp_ub; + ub->ub_timestamp = gethrestime_sec(); + ub->ub_mmp_magic = MMP_MAGIC; + ub->ub_mmp_delay = mmp->mmp_delay; + vd->vdev_mmp_pending = gethrtime(); + + zio_t *zio = zio_null(mmp->mmp_zio_root, spa, NULL, NULL, NULL, flags); + abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE); + abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd)); + abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t)); + + mutex_exit(&mmp->mmp_io_lock); + + offset = VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) - + MMP_BLOCKS_PER_LABEL + spa_get_random(MMP_BLOCKS_PER_LABEL)); + + label = spa_get_random(VDEV_LABELS); + vdev_label_write(zio, vd, label, ub_abd, offset, + VDEV_UBERBLOCK_SIZE(vd), mmp_write_done, mmp, + flags | ZIO_FLAG_DONT_PROPAGATE); + + spa_mmp_history_add(ub->ub_txg, ub->ub_timestamp, ub->ub_mmp_delay, vd, + label); + + zio_nowait(zio); +} + +static void +mmp_thread(spa_t *spa) +{ + mmp_thread_t *mmp = &spa->spa_mmp; + boolean_t last_spa_suspended = spa_suspended(spa); + boolean_t last_spa_multihost = spa_multihost(spa); + callb_cpr_t cpr; + hrtime_t max_fail_ns = zfs_multihost_fail_intervals * + MSEC2NSEC(MAX(zfs_multihost_interval, MMP_MIN_INTERVAL)); + + mmp_thread_enter(mmp, &cpr); + + /* + * The mmp_write_done() function calculates mmp_delay based on the + * prior value of mmp_delay and the elapsed time since the last write. + * For the first mmp write, there is no "last write", so we start + * with fake, but reasonable, default non-zero values. + */ + mmp->mmp_delay = MSEC2NSEC(MAX(zfs_multihost_interval, + MMP_MIN_INTERVAL)) / vdev_count_leaves(spa); + mmp->mmp_last_write = gethrtime() - mmp->mmp_delay; + + while (!mmp->mmp_thread_exiting) { + uint64_t mmp_fail_intervals = zfs_multihost_fail_intervals; + uint64_t mmp_interval = MSEC2NSEC( + MAX(zfs_multihost_interval, MMP_MIN_INTERVAL)); + boolean_t suspended = spa_suspended(spa); + boolean_t multihost = spa_multihost(spa); + hrtime_t start, next_time; + + start = gethrtime(); + if (multihost) { + next_time = start + mmp_interval / + vdev_count_leaves(spa); + } else { + next_time = start + MSEC2NSEC(MMP_DEFAULT_INTERVAL); + } + + /* + * When MMP goes off => on, or spa goes suspended => + * !suspended, we know no writes occurred recently. We + * update mmp_last_write to give us some time to try. + */ + if ((!last_spa_multihost && multihost) || + (last_spa_suspended && !suspended)) { + mutex_enter(&mmp->mmp_io_lock); + mmp->mmp_last_write = gethrtime(); + mutex_exit(&mmp->mmp_io_lock); + } else if (last_spa_multihost && !multihost) { + mutex_enter(&mmp->mmp_io_lock); + mmp->mmp_delay = 0; + mutex_exit(&mmp->mmp_io_lock); + } + last_spa_multihost = multihost; + last_spa_suspended = suspended; + + /* + * Smooth max_fail_ns when its factors are decreased, because + * making (max_fail_ns < mmp_interval) results in the pool being + * immediately suspended before writes can occur at the new + * higher frequency. + */ + if ((mmp_interval * mmp_fail_intervals) < max_fail_ns) { + max_fail_ns = ((31 * max_fail_ns) + (mmp_interval * + mmp_fail_intervals)) / 32; + } else { + max_fail_ns = mmp_interval * mmp_fail_intervals; + } + + /* + * Suspend the pool if no MMP write has succeeded in over + * mmp_interval * mmp_fail_intervals nanoseconds. + */ + if (!suspended && mmp_fail_intervals && multihost && + (start - mmp->mmp_last_write) > max_fail_ns) { + zio_suspend(spa, NULL); + } + + if (multihost) { + spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); + mmp_write_uberblock(spa); + spa_config_exit(spa, SCL_STATE, FTAG); + } + + CALLB_CPR_SAFE_BEGIN(&cpr); + (void) cv_timedwait_sig(&mmp->mmp_thread_cv, + &mmp->mmp_thread_lock, ddi_get_lbolt() + + ((next_time - gethrtime()) / (NANOSEC / HZ))); + CALLB_CPR_SAFE_END(&cpr, &mmp->mmp_thread_lock); + } + + /* Outstanding writes are allowed to complete. */ + if (mmp->mmp_zio_root) + zio_wait(mmp->mmp_zio_root); + + mmp->mmp_zio_root = NULL; + mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +/* BEGIN CSTYLED */ +module_param(zfs_multihost_fail_intervals, uint, 0644); +MODULE_PARM_DESC(zfs_multihost_fail_intervals, + "Max allowed period without a successful mmp write"); + +module_param(zfs_multihost_interval, ulong, 0644); +MODULE_PARM_DESC(zfs_multihost_interval, + "Milliseconds between mmp writes to each leaf"); + +module_param(zfs_multihost_import_intervals, uint, 0644); +MODULE_PARM_DESC(zfs_multihost_import_intervals, + "Number of zfs_multihost_interval periods to wait for activity"); +/* END CSTYLED */ +#endif diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 52420194c..7edf0459c 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -55,6 +55,7 @@ #include <sys/vdev_disk.h> #include <sys/metaslab.h> #include <sys/metaslab_impl.h> +#include <sys/mmp.h> #include <sys/uberblock_impl.h> #include <sys/txg.h> #include <sys/avl.h> @@ -491,6 +492,16 @@ spa_prop_validate(spa_t *spa, nvlist_t *props) error = SET_ERROR(EINVAL); break; + case ZPOOL_PROP_MULTIHOST: + error = nvpair_value_uint64(elem, &intval); + if (!error && intval > 1) + error = SET_ERROR(EINVAL); + + if (!error && !spa_get_hostid()) + error = SET_ERROR(ENOTSUP); + + break; + case ZPOOL_PROP_BOOTFS: /* * If the pool version is less than SPA_VERSION_BOOTFS, @@ -1346,6 +1357,9 @@ spa_unload(spa_t *spa) spa_config_exit(spa, SCL_ALL, FTAG); } + if (spa->spa_mmp.mmp_thread) + mmp_thread_stop(spa); + /* * Wait for any outstanding async I/O to complete. */ @@ -2324,6 +2338,197 @@ vdev_count_verify_zaps(vdev_t *vd) #endif /* + * Determine whether the activity check is required. + */ +static boolean_t +spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *config) +{ + uint64_t state = 0; + uint64_t hostid = 0; + uint64_t tryconfig_txg = 0; + uint64_t tryconfig_timestamp = 0; + nvlist_t *nvinfo; + + if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { + nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); + (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, + &tryconfig_txg); + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, + &tryconfig_timestamp); + } + + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid); + + /* + * Disable the MMP activity check - This is used by zdb which + * is intended to be used on potentially active pools. + */ + if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) + return (B_FALSE); + + /* + * Skip the activity check when the MMP feature is disabled. + */ + if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) + return (B_FALSE); + /* + * If the tryconfig_* values are nonzero, they are the results of an + * earlier tryimport. If they match the uberblock we just found, then + * the pool has not changed and we return false so we do not test a + * second time. + */ + if (tryconfig_txg && tryconfig_txg == ub->ub_txg && + tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp) + return (B_FALSE); + + /* + * Allow the activity check to be skipped when importing the pool + * on the same host which last imported it. + */ + if (hostid == spa_get_hostid()) + return (B_FALSE); + + /* + * Skip the activity test when the pool was cleanly exported. + */ + if (state != POOL_STATE_ACTIVE) + return (B_FALSE); + + return (B_TRUE); +} + +/* + * Perform the import activity check. If the user canceled the import or + * we detected activity then fail. + */ +static int +spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) +{ + uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); + uint64_t txg = ub->ub_txg; + uint64_t timestamp = ub->ub_timestamp; + uint64_t import_delay = NANOSEC; + hrtime_t import_expire; + nvlist_t *mmp_label = NULL; + vdev_t *rvd = spa->spa_root_vdev; + kcondvar_t cv; + kmutex_t mtx; + int error = 0; + + cv_init(&cv, NULL, CV_DEFAULT, NULL); + mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); + mutex_enter(&mtx); + + /* + * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed + * during the earlier tryimport. If the txg recorded there is 0 then + * the pool is known to be active on another host. + * + * Otherwise, the pool might be in use on another node. Check for + * changes in the uberblocks on disk if necessary. + */ + if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { + nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, + ZPOOL_CONFIG_LOAD_INFO); + + if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && + fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { + vdev_uberblock_load(rvd, ub, &mmp_label); + error = SET_ERROR(EREMOTEIO); + goto out; + } + } + + /* + * Preferentially use the zfs_multihost_interval from the node which + * last imported the pool. This value is stored in an MMP uberblock as. + * + * ub_mmp_delay * vdev_count_leaves() == zfs_multihost_interval + */ + if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay) + import_delay = MAX(import_delay, import_intervals * + ub->ub_mmp_delay * vdev_count_leaves(spa)); + + /* Apply a floor using the local default values. */ + import_delay = MAX(import_delay, import_intervals * + MSEC2NSEC(MAX(zfs_multihost_interval, MMP_MIN_INTERVAL))); + + /* Add a small random factor in case of simultaneous imports (0-25%) */ + import_expire = gethrtime() + import_delay + + (import_delay * spa_get_random(250) / 1000); + + while (gethrtime() < import_expire) { + vdev_uberblock_load(rvd, ub, &mmp_label); + + if (txg != ub->ub_txg || timestamp != ub->ub_timestamp) { + error = SET_ERROR(EREMOTEIO); + break; + } + + if (mmp_label) { + nvlist_free(mmp_label); + mmp_label = NULL; + } + + error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); + if (error != -1) { + error = SET_ERROR(EINTR); + break; + } + error = 0; + } + +out: + mutex_exit(&mtx); + mutex_destroy(&mtx); + cv_destroy(&cv); + + /* + * If the pool is determined to be active store the status in the + * spa->spa_load_info nvlist. If the remote hostname or hostid are + * available from configuration read from disk store them as well. + * This allows 'zpool import' to generate a more useful message. + * + * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) + * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool + * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool + */ + if (error == EREMOTEIO) { + char *hostname = "<unknown>"; + uint64_t hostid = 0; + + if (mmp_label) { + if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { + hostname = fnvlist_lookup_string(mmp_label, + ZPOOL_CONFIG_HOSTNAME); + fnvlist_add_string(spa->spa_load_info, + ZPOOL_CONFIG_MMP_HOSTNAME, hostname); + } + + if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { + hostid = fnvlist_lookup_uint64(mmp_label, + ZPOOL_CONFIG_HOSTID); + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_HOSTID, hostid); + } + } + + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_TXG, 0); + + error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); + } + + if (mmp_label) + nvlist_free(mmp_label); + + return (error); +} + +/* * Load an existing storage pool, using the pool's builtin spa_config as a * source of configuration information. */ @@ -2343,6 +2548,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, int parse, i; uint64_t obj; boolean_t missing_feat_write = B_FALSE; + boolean_t activity_check = B_FALSE; nvlist_t *mos_config; /* @@ -2441,6 +2647,33 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, } /* + * For pools which have the multihost property on determine if the + * pool is truly inactive and can be safely imported. Prevent + * hosts which don't have a hostid set from importing the pool. + */ + activity_check = spa_activity_check_required(spa, ub, config); + if (activity_check) { + error = spa_activity_check(spa, ub, config); + if (error) { + nvlist_free(label); + return (error); + } + + if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && + spa_get_hostid() == 0) { + nvlist_free(label); + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); + return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); + } + + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); + } + + /* * If the pool has an unsupported version we can't open it. */ if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { @@ -2667,24 +2900,9 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, VERIFY(nvlist_lookup_string(nvconfig, ZPOOL_CONFIG_HOSTNAME, &hostname) == 0); -#ifdef _KERNEL - myhostid = zone_get_hostid(NULL); -#else /* _KERNEL */ - /* - * We're emulating the system's hostid in userland, so - * we can't use zone_get_hostid(). - */ - (void) ddi_strtoul(hw_serial, NULL, 10, &myhostid); -#endif /* _KERNEL */ - if (hostid != 0 && myhostid != 0 && - hostid != myhostid) { + myhostid = spa_get_hostid(); + if (hostid && myhostid && hostid != myhostid) { nvlist_free(nvconfig); - cmn_err(CE_WARN, "pool '%s' could not be " - "loaded as it was last accessed by another " - "system (host: %s hostid: 0x%lx). See: " - "http://zfsonlinux.org/msg/ZFS-8000-EY", - spa_name(spa), hostname, - (unsigned long)hostid); return (SET_ERROR(EBADF)); } } @@ -2850,6 +3068,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); + spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); spa_prop_find(spa, ZPOOL_PROP_DEDUPDITTO, &spa->spa_dedup_ditto); @@ -2857,6 +3076,18 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, } /* + * If the 'multihost' property is set, then never allow a pool to + * be imported when the system hostid is zero. The exception to + * this rule is zdb which is always allowed to access pools. + */ + if (spa_multihost(spa) && spa_get_hostid() == 0 && + (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { + fnvlist_add_uint64(spa->spa_load_info, + ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); + return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); + } + + /* * If the 'autoreplace' property is set, then post a resource notifying * the ZFS DE that it should not issue any faults for unopenable * devices. We also iterate over the vdevs, and post a sysevent for any @@ -2980,6 +3211,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, spa_set_log_state(spa, SPA_LOG_GOOD); spa->spa_sync_on = B_TRUE; txg_sync_start(spa->spa_dsl_pool); + mmp_thread_start(spa); /* * Wait for all claims to sync. We sync up to the highest @@ -3632,18 +3864,6 @@ spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, goto out; } - /* - * The L2ARC currently only supports disk devices in - * kernel context. For user-level testing, we allow it. - */ -#ifdef _KERNEL - if ((strcmp(config, ZPOOL_CONFIG_L2CACHE) == 0) && - strcmp(vd->vdev_ops->vdev_op_type, VDEV_TYPE_DISK) != 0) { - error = SET_ERROR(ENOTBLK); - vdev_free(vd); - goto out; - } -#endif vd->vdev_top = vd; if ((error = vdev_open(vd)) == 0 && @@ -3986,6 +4206,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); + spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); if (props != NULL) { spa_configfile_set(spa, props, B_FALSE); @@ -3996,6 +4217,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, spa->spa_sync_on = B_TRUE; txg_sync_start(spa->spa_dsl_pool); + mmp_thread_start(spa); /* * We explicitly wait for the first transaction to complete so that our @@ -6405,6 +6627,9 @@ spa_sync_props(void *arg, dmu_tx_t *tx) spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); break; + case ZPOOL_PROP_MULTIHOST: + spa->spa_multihost = intval; + break; case ZPOOL_PROP_DEDUPDITTO: spa->spa_dedup_ditto = intval; break; diff --git a/module/zfs/spa_config.c b/module/zfs/spa_config.c index 2715898cc..5b792b868 100644 --- a/module/zfs/spa_config.c +++ b/module/zfs/spa_config.c @@ -436,15 +436,7 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) fnvlist_add_string(config, ZPOOL_CONFIG_COMMENT, spa->spa_comment); -#ifdef _KERNEL - hostid = zone_get_hostid(NULL); -#else /* _KERNEL */ - /* - * We're emulating the system's hostid in userland, so we can't use - * zone_get_hostid(). - */ - (void) ddi_strtoul(hw_serial, NULL, 10, &hostid); -#endif /* _KERNEL */ + hostid = spa_get_hostid(); if (hostid != 0) fnvlist_add_uint64(config, ZPOOL_CONFIG_HOSTID, hostid); fnvlist_add_string(config, ZPOOL_CONFIG_HOSTNAME, utsname()->nodename); diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 09e5067a5..3787e010f 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -1384,6 +1384,9 @@ spa_get_random(uint64_t range) ASSERT(range != 0); + if (range == 1) + return (0); + (void) random_get_pseudo_bytes((void *)&r, sizeof (uint64_t)); return (r % range); @@ -2073,6 +2076,30 @@ spa_maxdnodesize(spa_t *spa) return (DNODE_MIN_SIZE); } +boolean_t +spa_multihost(spa_t *spa) +{ + return (spa->spa_multihost ? B_TRUE : B_FALSE); +} + +unsigned long +spa_get_hostid(void) +{ + unsigned long myhostid; + +#ifdef _KERNEL + myhostid = zone_get_hostid(NULL); +#else /* _KERNEL */ + /* + * We're emulating the system's hostid in userland, so + * we can't use zone_get_hostid(). + */ + (void) ddi_strtoul(hw_serial, NULL, 10, &myhostid); +#endif /* _KERNEL */ + + return (myhostid); +} + #if defined(_KERNEL) && defined(HAVE_SPL) /* Namespace manipulation */ EXPORT_SYMBOL(spa_lookup); diff --git a/module/zfs/spa_stats.c b/module/zfs/spa_stats.c index ac1fb8c6f..7ca359806 100644 --- a/module/zfs/spa_stats.c +++ b/module/zfs/spa_stats.c @@ -21,6 +21,7 @@ #include <sys/zfs_context.h> #include <sys/spa_impl.h> +#include <sys/vdev_impl.h> /* * Keeps stats on last N reads per spa_t, disabled by default. @@ -38,6 +39,11 @@ int zfs_read_history_hits = 0; int zfs_txg_history = 0; /* + * Keeps stats on the last N MMP updates, disabled by default. + */ +int zfs_multihost_history = 0; + +/* * ========================================================================== * SPA Read History Routines * ========================================================================== @@ -701,6 +707,198 @@ spa_io_history_destroy(spa_t *spa) mutex_destroy(&ssh->lock); } +/* + * ========================================================================== + * SPA MMP History Routines + * ========================================================================== + */ + +/* + * MMP statistics - Information exported regarding each MMP update + */ + +typedef struct spa_mmp_history { + uint64_t txg; /* txg of last sync */ + uint64_t timestamp; /* UTC time of of last sync */ + uint64_t mmp_delay; /* nanosec since last MMP write */ + uint64_t vdev_guid; /* unique ID of leaf vdev */ + char *vdev_path; + uint64_t vdev_label; /* vdev label */ + list_node_t smh_link; +} spa_mmp_history_t; + +static int +spa_mmp_history_headers(char *buf, size_t size) +{ + (void) snprintf(buf, size, "%-10s %-10s %-12s %-24s %-10s %s\n", + "txg", "timestamp", "mmp_delay", "vdev_guid", "vdev_label", + "vdev_path"); + return (0); +} + +static int +spa_mmp_history_data(char *buf, size_t size, void *data) +{ + spa_mmp_history_t *smh = (spa_mmp_history_t *)data; + + (void) snprintf(buf, size, "%-10llu %-10llu %-12llu %-24llu %-10llu " + "%s\n", + (u_longlong_t)smh->txg, (u_longlong_t)smh->timestamp, + (u_longlong_t)smh->mmp_delay, (u_longlong_t)smh->vdev_guid, + (u_longlong_t)smh->vdev_label, + (smh->vdev_path ? smh->vdev_path : "-")); + + return (0); +} + +/* + * Calculate the address for the next spa_stats_history_t entry. The + * ssh->lock will be held until ksp->ks_ndata entries are processed. + */ +static void * +spa_mmp_history_addr(kstat_t *ksp, loff_t n) +{ + spa_t *spa = ksp->ks_private; + spa_stats_history_t *ssh = &spa->spa_stats.mmp_history; + + ASSERT(MUTEX_HELD(&ssh->lock)); + + if (n == 0) + ssh->private = list_tail(&ssh->list); + else if (ssh->private) + ssh->private = list_prev(&ssh->list, ssh->private); + + return (ssh->private); +} + +/* + * When the kstat is written discard all spa_mmp_history_t entries. The + * ssh->lock will be held until ksp->ks_ndata entries are processed. + */ +static int +spa_mmp_history_update(kstat_t *ksp, int rw) +{ + spa_t *spa = ksp->ks_private; + spa_stats_history_t *ssh = &spa->spa_stats.mmp_history; + + ASSERT(MUTEX_HELD(&ssh->lock)); + + if (rw == KSTAT_WRITE) { + spa_mmp_history_t *smh; + + while ((smh = list_remove_head(&ssh->list))) { + ssh->size--; + if (smh->vdev_path) + strfree(smh->vdev_path); + kmem_free(smh, sizeof (spa_mmp_history_t)); + } + + ASSERT3U(ssh->size, ==, 0); + } + + ksp->ks_ndata = ssh->size; + ksp->ks_data_size = ssh->size * sizeof (spa_mmp_history_t); + + return (0); +} + +static void +spa_mmp_history_init(spa_t *spa) +{ + spa_stats_history_t *ssh = &spa->spa_stats.mmp_history; + char name[KSTAT_STRLEN]; + kstat_t *ksp; + + mutex_init(&ssh->lock, NULL, MUTEX_DEFAULT, NULL); + list_create(&ssh->list, sizeof (spa_mmp_history_t), + offsetof(spa_mmp_history_t, smh_link)); + + ssh->count = 0; + ssh->size = 0; + ssh->private = NULL; + + (void) snprintf(name, KSTAT_STRLEN, "zfs/%s", spa_name(spa)); + + ksp = kstat_create(name, 0, "multihost", "misc", + KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + ssh->kstat = ksp; + + if (ksp) { + ksp->ks_lock = &ssh->lock; + ksp->ks_data = NULL; + ksp->ks_private = spa; + ksp->ks_update = spa_mmp_history_update; + kstat_set_raw_ops(ksp, spa_mmp_history_headers, + spa_mmp_history_data, spa_mmp_history_addr); + kstat_install(ksp); + } +} + +static void +spa_mmp_history_destroy(spa_t *spa) +{ + spa_stats_history_t *ssh = &spa->spa_stats.mmp_history; + spa_mmp_history_t *smh; + kstat_t *ksp; + + ksp = ssh->kstat; + if (ksp) + kstat_delete(ksp); + + mutex_enter(&ssh->lock); + while ((smh = list_remove_head(&ssh->list))) { + ssh->size--; + if (smh->vdev_path) + strfree(smh->vdev_path); + kmem_free(smh, sizeof (spa_mmp_history_t)); + } + + ASSERT3U(ssh->size, ==, 0); + list_destroy(&ssh->list); + mutex_exit(&ssh->lock); + + mutex_destroy(&ssh->lock); +} + +/* + * Add a new MMP update to historical record. + */ +void +spa_mmp_history_add(uint64_t txg, uint64_t timestamp, uint64_t mmp_delay, + vdev_t *vd, int label) +{ + spa_t *spa = vd->vdev_spa; + spa_stats_history_t *ssh = &spa->spa_stats.mmp_history; + spa_mmp_history_t *smh, *rm; + + if (zfs_multihost_history == 0 && ssh->size == 0) + return; + + smh = kmem_zalloc(sizeof (spa_mmp_history_t), KM_SLEEP); + smh->txg = txg; + smh->timestamp = timestamp; + smh->mmp_delay = mmp_delay; + smh->vdev_guid = vd->vdev_guid; + if (vd->vdev_path) + smh->vdev_path = strdup(vd->vdev_path); + smh->vdev_label = label; + + mutex_enter(&ssh->lock); + + list_insert_head(&ssh->list, smh); + ssh->size++; + + while (ssh->size > zfs_multihost_history) { + ssh->size--; + rm = list_remove_tail(&ssh->list); + if (rm->vdev_path) + strfree(rm->vdev_path); + kmem_free(rm, sizeof (spa_mmp_history_t)); + } + + mutex_exit(&ssh->lock); +} + void spa_stats_init(spa_t *spa) { @@ -708,6 +906,7 @@ spa_stats_init(spa_t *spa) spa_txg_history_init(spa); spa_tx_assign_init(spa); spa_io_history_init(spa); + spa_mmp_history_init(spa); } void @@ -717,15 +916,25 @@ spa_stats_destroy(spa_t *spa) spa_txg_history_destroy(spa); spa_read_history_destroy(spa); spa_io_history_destroy(spa); + spa_mmp_history_destroy(spa); } #if defined(_KERNEL) && defined(HAVE_SPL) +/* CSTYLED */ module_param(zfs_read_history, int, 0644); -MODULE_PARM_DESC(zfs_read_history, "Historic statistics for the last N reads"); +MODULE_PARM_DESC(zfs_read_history, + "Historical statistics for the last N reads"); module_param(zfs_read_history_hits, int, 0644); -MODULE_PARM_DESC(zfs_read_history_hits, "Include cache hits in read history"); +MODULE_PARM_DESC(zfs_read_history_hits, + "Include cache hits in read history"); module_param(zfs_txg_history, int, 0644); -MODULE_PARM_DESC(zfs_txg_history, "Historic statistics for the last N txgs"); +MODULE_PARM_DESC(zfs_txg_history, + "Historical statistics for the last N txgs"); + +module_param(zfs_multihost_history, int, 0644); +MODULE_PARM_DESC(zfs_multihost_history, + "Historical statistics for last N multihost writes"); +/* END CSTYLED */ #endif diff --git a/module/zfs/uberblock.c b/module/zfs/uberblock.c index f8bdecdf5..c1e85bdce 100644 --- a/module/zfs/uberblock.c +++ b/module/zfs/uberblock.c @@ -44,7 +44,7 @@ uberblock_verify(uberblock_t *ub) * transaction group. */ boolean_t -uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg) +uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg, uint64_t mmp_delay) { ASSERT(ub->ub_txg < txg); @@ -57,6 +57,9 @@ uberblock_update(uberblock_t *ub, vdev_t *rvd, uint64_t txg) ub->ub_guid_sum = rvd->vdev_guid_sum; ub->ub_timestamp = gethrestime_sec(); ub->ub_software_version = SPA_VERSION; + ub->ub_mmp_magic = MMP_MAGIC; + ub->ub_mmp_delay = spa_multihost(rvd->vdev_spa) ? mmp_delay : 0; + ub->ub_mmp_seq = 0; return (ub->ub_rootbp.blk_birth == txg); } diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index 021f4774b..a0a02366e 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -193,7 +193,7 @@ vdev_label_read(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset, ZIO_PRIORITY_SYNC_READ, flags, B_TRUE)); } -static void +void vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t offset, uint64_t size, zio_done_func_t *done, void *private, int flags) { @@ -1082,14 +1082,12 @@ static void vdev_uberblock_load_impl(zio_t *zio, vdev_t *vd, int flags, struct ubl_cbdata *cbp) { - int c, l, n; - - for (c = 0; c < vd->vdev_children; c++) + for (int c = 0; c < vd->vdev_children; c++) vdev_uberblock_load_impl(zio, vd->vdev_child[c], flags, cbp); if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) { - for (l = 0; l < VDEV_LABELS; l++) { - for (n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) { + for (int l = 0; l < VDEV_LABELS; l++) { + for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) { vdev_label_read(zio, vd, l, abd_alloc_linear(VDEV_UBERBLOCK_SIZE(vd), B_TRUE), VDEV_UBERBLOCK_OFFSET(vd, n), @@ -1213,10 +1211,7 @@ vdev_uberblock_sync_done(zio_t *zio) static void vdev_uberblock_sync(zio_t *zio, uberblock_t *ub, vdev_t *vd, int flags) { - abd_t *ub_abd; - int c, l, n; - - for (c = 0; c < vd->vdev_children; c++) + for (int c = 0; c < vd->vdev_children; c++) vdev_uberblock_sync(zio, ub, vd->vdev_child[c], flags); if (!vd->vdev_ops->vdev_op_leaf) @@ -1232,14 +1227,15 @@ vdev_uberblock_sync(zio_t *zio, uberblock_t *ub, vdev_t *vd, int flags) vd->vdev_copy_uberblocks = B_FALSE; } - n = ub->ub_txg & (VDEV_UBERBLOCK_COUNT(vd) - 1); + int m = spa_multihost(vd->vdev_spa) ? MMP_BLOCKS_PER_LABEL : 0; + int n = ub->ub_txg % (VDEV_UBERBLOCK_COUNT(vd) - m); /* Copy the uberblock_t into the ABD */ - ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE); + abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE); abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd)); abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t)); - for (l = 0; l < VDEV_LABELS; l++) + for (int l = 0; l < VDEV_LABELS; l++) vdev_label_write(zio, vd, l, ub_abd, VDEV_UBERBLOCK_OFFSET(vd, n), VDEV_UBERBLOCK_SIZE(vd), vdev_uberblock_sync_done, zio->io_private, @@ -1448,10 +1444,13 @@ retry: * and the vdev configuration hasn't changed, * then there's nothing to do. */ - if (ub->ub_txg < txg && - uberblock_update(ub, spa->spa_root_vdev, txg) == B_FALSE && - list_is_empty(&spa->spa_config_dirty_list)) - return (0); + if (ub->ub_txg < txg) { + boolean_t changed = uberblock_update(ub, spa->spa_root_vdev, + txg, spa->spa_mmp.mmp_delay); + + if (!changed && list_is_empty(&spa->spa_config_dirty_list)) + return (0); + } if (txg > spa_freeze_txg(spa)) return (0); @@ -1502,6 +1501,10 @@ retry: if ((error = vdev_uberblock_sync_list(svd, svdcount, ub, flags)) != 0) goto retry; + + if (spa_multihost(spa)) + mmp_update_uberblock(spa, ub); + /* * Sync out odd labels for every dirty vdev. If the system dies * in the middle of this process, the even labels and the new diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index 728b02377..b2f5db584 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -1650,7 +1650,7 @@ zfs_ioc_pool_stats(zfs_cmd_t *zc) static int zfs_ioc_pool_tryimport(zfs_cmd_t *zc) { - nvlist_t *tryconfig, *config; + nvlist_t *tryconfig, *config = NULL; int error; if ((error = get_nvlist(zc->zc_nvlist_conf, zc->zc_nvlist_conf_size, diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index c4db08d78..eb2ce3828 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -43,7 +43,8 @@ SINGLETEST=() SINGLETESTUSER="root" ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh" ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh" -TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG"} +ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh" +TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"} # # Attempt to remove loopback devices and files which where created earlier diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 9f195628c..e556f1e8c 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -410,6 +410,10 @@ tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos', [tests/functional/mmap] tests = ['mmap_write_001_pos', 'mmap_read_001_pos'] +[tests/functional/mmp] +tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval', + 'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import'] + [tests/functional/mount] tests = ['umount_001', 'umountall_001'] diff --git a/tests/zfs-tests/callbacks/Makefile.am b/tests/zfs-tests/callbacks/Makefile.am index 71947f308..30e847241 100644 --- a/tests/zfs-tests/callbacks/Makefile.am +++ b/tests/zfs-tests/callbacks/Makefile.am @@ -1,4 +1,5 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/callbacks dist_pkgdata_SCRIPTS = \ zfs_dbgmsg.ksh \ - zfs_dmesg.ksh + zfs_dmesg.ksh \ + zfs_mmp.ksh diff --git a/tests/zfs-tests/callbacks/zfs_mmp.ksh b/tests/zfs-tests/callbacks/zfs_mmp.ksh new file mode 100755 index 000000000..df2cd132d --- /dev/null +++ b/tests/zfs-tests/callbacks/zfs_mmp.ksh @@ -0,0 +1,37 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security. +# All rights reserved. +# + +# $1: number of lines to output (default: 40) +typeset lines=${1:-40} +typeset history=$(cat /sys/module/zfs/parameters/zfs_multihost_history) + +if [ $history -eq 0 ]; then + exit +fi + +for f in /proc/spl/kstat/zfs/*/multihost; do + echo "=================================================================" + echo " Last $lines lines of $f" + echo "=================================================================" + + sudo tail -n $lines $f +done + +echo "=================================================================" +echo " End of zfs multihost log" +echo "=================================================================" diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 968ab3cd1..57b1bd315 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -52,6 +52,7 @@ export SYSTEM_FILES='arp gunzip gzip head + hostid hostname id iostat diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index ddfe550bf..1d1c57e7a 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -3428,3 +3428,83 @@ function swap_cleanup return 0 } + +# +# Set a global system tunable (64-bit value) +# +# $1 tunable name +# $2 tunable values +# +function set_tunable64 +{ + set_tunable_impl "$1" "$2" Z +} + +# +# Set a global system tunable (32-bit value) +# +# $1 tunable name +# $2 tunable values +# +function set_tunable32 +{ + set_tunable_impl "$1" "$2" W +} + +function set_tunable_impl +{ + typeset tunable="$1" + typeset value="$2" + typeset mdb_cmd="$3" + typeset module="${4:-zfs}" + + [[ -z "$tunable" ]] && return 1 + [[ -z "$value" ]] && return 1 + [[ -z "$mdb_cmd" ]] && return 1 + + case "$(uname)" in + Linux) + typeset zfs_tunables="/sys/module/$module/parameters" + [[ -w "$zfs_tunables/$tunable" ]] || return 1 + echo -n "$value" > "$zfs_tunables/$tunable" + return "$?" + ;; + SunOS) + [[ "$module" -eq "zfs" ]] || return 1 + echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw + return "$?" + ;; + esac +} + +# +# Get a global system tunable +# +# $1 tunable name +# +function get_tunable +{ + get_tunable_impl "$1" +} + +function get_tunable_impl +{ + typeset tunable="$1" + typeset module="${2:-zfs}" + + [[ -z "$tunable" ]] && return 1 + + case "$(uname)" in + Linux) + typeset zfs_tunables="/sys/module/$module/parameters" + [[ -f "$zfs_tunables/$tunable" ]] || return 1 + cat $zfs_tunables/$tunable + return "$?" + ;; + SunOS) + [[ "$module" -eq "zfs" ]] || return 1 + ;; + esac + + return 1 +} diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index 1c3f4b1a8..d68f254ef 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -29,6 +29,7 @@ SUBDIRS = \ link_count \ migration \ mmap \ + mmp \ mount \ mv_files \ nestedfs \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index 0ffd6f510..e1537806f 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -54,6 +54,7 @@ typeset -a properties=( "freeing" "fragmentation" "leaked" + "multihost" "feature@async_destroy" "feature@empty_bpobj" "feature@lz4_compress" diff --git a/tests/zfs-tests/tests/functional/mmp/Makefile.am b/tests/zfs-tests/tests/functional/mmp/Makefile.am new file mode 100644 index 000000000..2399a6c37 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/Makefile.am @@ -0,0 +1,13 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/mmp +dist_pkgdata_SCRIPTS = \ + mmp_on_thread.ksh \ + mmp_on_uberblocks.ksh \ + mmp_on_off.ksh \ + mmp_interval.ksh \ + mmp_active_import.ksh \ + mmp_inactive_import.ksh \ + mmp_exported_import.ksh \ + setup.ksh \ + cleanup.ksh \ + mmp.kshlib \ + mmp.cfg diff --git a/tests/zfs-tests/tests/functional/mmp/cleanup.ksh b/tests/zfs-tests/tests/functional/mmp/cleanup.ksh new file mode 100755 index 000000000..6e438d88d --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/cleanup.ksh @@ -0,0 +1,28 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg + +verify_runnable "global" + +log_must set_tunable64 zfs_multihost_history 0 + +log_pass "mmp cleanup passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp.cfg b/tests/zfs-tests/tests/functional/mmp/mmp.cfg new file mode 100644 index 000000000..f17108a87 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp.cfg @@ -0,0 +1,38 @@ +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +export PREV_UBER="$TEST_BASE_DIR/mmp-uber-prev.txt" +export CURR_UBER="$TEST_BASE_DIR/mmp-uber-curr.txt" +export DISK=${DISKS%% *} + +export HOSTID_FILE="/etc/hostid" +export HOSTID1=01234567 +export HOSTID2=89abcdef + +export TXG_TIMEOUT_LONG=5000 +export TXG_TIMEOUT_DEFAULT=5 + +export MMP_POOL=mmppool +export MMP_DIR=$TEST_BASE_DIR/mmp +export MMP_HISTORY=100 + +export MMP_INTERVAL_DEFAULT=1000 +export MMP_INTERVAL_MIN=100 + +export ZPOOL_IMPORT_DURATION=9 diff --git a/tests/zfs-tests/tests/functional/mmp/mmp.kshlib b/tests/zfs-tests/tests/functional/mmp/mmp.kshlib new file mode 100644 index 000000000..a81779b0c --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp.kshlib @@ -0,0 +1,186 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Use is subject to license terms. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg + + +function check_pool_import # pool opts token keyword +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + typeset token=$3 + typeset keyword=$4 + + zpool import $opts 2>&1 | \ + nawk -v token="$token:" '($1==token) {print $0}' | \ + grep -i "$keyword" > /dev/null 2>&1 + + return $? +} + +function is_pool_imported # pool opts +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + + check_pool_import "$pool" "$opts" "status" \ + "The pool is currently imported" + return $? +} + +function wait_pool_imported # pool opts +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + + while is_pool_imported "$pool" "$opts"; do + log_must sleep 5 + done + + return 0 +} + +function try_pool_import # pool opts message +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + typeset msg=$3 + + zpool import $opts $pool 2>&1 | grep -i "$msg" + + return $? +} + +function mmp_set_hostid +{ + typeset hostid=$1 + + a=${hostid:6:2} + b=${hostid:4:2} + c=${hostid:2:2} + d=${hostid:0:2} + + printf "\\x$a\\x$b\\x$c\\x$d" >$HOSTID_FILE + + if [ $(hostid) != "$hostid" ]; then + return 1 + fi + + return 0 +} + +function mmp_clear_hostid +{ + rm -f $HOSTID_FILE +} + +function mmp_pool_create # pool dir +{ + typeset pool=${1:-$MMP_POOL} + typeset dir=${2:-$MMP_DIR} + typeset opts="-T120 -M -k0 -f $dir -E -p $pool" + + log_must mkdir -p $dir + log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 + + log_must mmp_set_hostid $HOSTID1 + log_must zpool create -f $pool mirror $dir/vdev1 $dir/vdev2 + log_must zpool set multihost=on $pool + log_must zpool export $pool + log_must mmp_set_hostid $HOSTID2 + + log_note "Starting ztest in the background as hostid $HOSTID1" + log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >/dev/null 2>&1 &" + + while ! is_pool_imported "$pool" "-d $dir"; do + log_must pgrep ztest + log_must sleep 5 + done +} + +function mmp_pool_destroy # pool dir +{ + typeset pool=${1:-$MMP_POOL} + typeset dir=${2:-$MMP_DIR} + + ZTESTPID=$(pgrep ztest) + if [ -n "$ZTESTPID" ]; then + log_must kill $ZTESTPID + wait $ZTESTPID + fi + + if poolexists $pool; then + destroy_pool $pool + fi + + rm -Rf $dir + mmp_clear_hostid +} + +function mmp_pool_set_hostid # pool hostid +{ + typeset pool=$1 + typeset hostid=$2 + + log_must mmp_set_hostid $hostid + log_must zpool export $pool + log_must zpool import $pool + + return 0 +} + +function import_no_activity_check # pool opts +{ + typeset pool=$1 + typeset opts=$2 + + SECONDS=0 + zpool import $opts $pool + typeset rc=$? + + if [[ $SECONDS -gt $ZPOOL_IMPORT_DURATION ]]; then + log_fail "unexpected activity check (${SECONDS}s)" + fi + + return $rc +} + +function import_activity_check # pool opts +{ + typeset pool=$1 + typeset opts=$2 + + SECONDS=0 + zpool import $opts $pool + typeset rc=$? + + if [[ $SECONDS -le $ZPOOL_IMPORT_DURATION ]]; then + log_fail "expected activity check (${SECONDS}s)" + fi + + return $rc +} diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh new file mode 100755 index 000000000..92eb9ce2d --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh @@ -0,0 +1,104 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Under no circumstances when multihost is active, should an active pool +# with one hostid be importable by a host with a different hostid. +# +# STRATEGY: +# 1. Simulate an active pool on another host with ztest. +# 2. Verify 'zpool import' reports an active pool. +# 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. +# 4. Kill ztest to make pool eligible for import. +# 5. Verify 'zpool import' fails with the expected error message. +# 6. Verify 'zpool import $MMP_POOL' fails with the expected message. +# 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. +# 8. Verify pool may be exported/imported without -f argument. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + mmp_pool_destroy $MMP_DIR $MMP_POOL + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off active pool activity checks" +log_onexit cleanup + +# 1. Simulate an active pool on another host with ztest. +mmp_pool_destroy $MMP_POOL $MMP_DIR +mmp_pool_create $MMP_POOL $MMP_DIR + +# 2. Verify 'zpool import' reports an active pool. +log_must mmp_set_hostid $HOSTID2 +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must is_pool_imported $MMP_POOL "-d $MMP_DIR" + +# 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. +MMP_IMPORTED_MSG="Cannot import '$MMP_POOL': pool is imported" +log_must try_pool_import $MMP_POOL "-d $MMP_DIR" $MMP_IMPORTED_MSG +for i in {1..10}; do + log_must pgrep ztest >/dev/null + log_must try_pool_import $MMP_POOL "-f -d $MMP_DIR" $MMP_IMPORTED_MSG +done + +# 4. Kill ztest to make pool eligible for import. Poll with 'zpool status'. +ZTESTPID=$(pgrep ztest) +if [ -n "$ZTESTPID" ]; then + log_must kill -9 $ZTESTPID +fi +log_must wait_pool_imported $MMP_POOL "-d $MMP_DIR" + +# 5. Verify 'zpool import' fails with the expected error message, when +# - hostid=0: - configuration error +# - hostid=matches - safe to import the pool +# - hostid=different - previously imported on a different system +# +log_must mmp_clear_hostid +MMP_IMPORTED_MSG="Set the system hostid" +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" $MMP_IMPORTED_MSG + +log_must mmp_set_hostid $HOSTID1 +MMP_IMPORTED_MSG="The pool can be imported" +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" $MMP_IMPORTED_MSG + +log_must mmp_set_hostid $HOSTID2 +MMP_IMPORTED_MSG="The pool was last accessed by another system." +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "status" $MMP_IMPORTED_MSG + +# 6. Verify 'zpool import $MMP_POOL' fails with the expected message. +MMP_IMPORTED_MSG="pool was previously in use from another system." +log_must try_pool_import $MMP_POOL "-d $MMP_DIR" $MMP_IMPORTED_MSG + +# 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. +log_must import_activity_check $MMP_POOL "-f -d $MMP_DIR" + +# 8 Verify pool may be exported/imported without -f argument. +log_must zpool export $MMP_POOL +log_must import_no_activity_check $MMP_POOL "-d $MMP_DIR" + +log_pass "multihost=on|off active pool activity checks passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh new file mode 100755 index 000000000..d65ca5b3d --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh @@ -0,0 +1,104 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Verify import behavior for exported pool (no activity check) +# +# STRATEGY: +# 1. Create a zpool +# 2. Verify multihost=off and hostids match (no activity check) +# 3. Verify multihost=off and hostids differ (no activity check) +# 4. Verify multihost=off and hostid zero allowed (no activity check) +# 5. Verify multihost=on and hostids match (no activity check) +# 6. Verify multihost=on and hostids differ (no activity check) +# 7. Verify multihost=on and hostid zero fails (no activity check) +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off activity checks exported pool" +log_onexit cleanup + +# 1. Create a zpool +log_must mmp_set_hostid $HOSTID1 +default_setup_noexit $DISK + +# 2. Verify multihost=off and hostids match (no activity check) +log_must zpool set multihost=off $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 3. Verify multihost=off and hostids differ (no activity check) +for opt in "" "-f"; do + log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 + log_must zpool export $TESTPOOL + log_must mmp_set_hostid $HOSTID2 + log_must import_no_activity_check $TESTPOOL $opt +done + +# 4. Verify multihost=off and hostid zero allowed (no activity check) +log_must mmp_clear_hostid + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 5. Verify multihost=on and hostids match (no activity check) +log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 +log_must zpool set multihost=on $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 6. Verify multihost=on and hostids differ (no activity check) +for opt in "" "-f"; do + log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 + log_must zpool export $TESTPOOL + log_must mmp_set_hostid $HOSTID2 + log_must import_no_activity_check $TESTPOOL $opt +done + +# 7. Verify multihost=on and hostid zero fails (no activity check) +log_must zpool export $TESTPOOL +log_must mmp_clear_hostid + +for opt in "" "-f"; do + MMP_IMPORTED_MSG="Set the system hostid" + log_must check_pool_import $TESTPOOL "" "action" $MMP_IMPORTED_MSG + log_mustnot import_no_activity_check $TESTPOOL $opt +done + +log_pass "multihost=on|off exported pool activity checks passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh new file mode 100755 index 000000000..c944b6b28 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh @@ -0,0 +1,95 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Verify import behavior for inactive, but not exported, pools +# +# STRATEGY: +# 1. Create a zpool +# 2. Verify multihost=off and hostids match (no activity check) +# 3. Verify multihost=off and hostids differ (no activity check) +# 4. Verify multihost=off and hostid allowed (no activity check) +# 5. Verify multihost=on and hostids match (no activity check) +# 6. Verify multihost=on and hostids differ (activity check) +# 7. Verify multihost=on and hostid zero fails (activity check) +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off inactive pool activity checks" +log_onexit cleanup + +# 1. Create a zpool +log_must mmp_set_hostid $HOSTID1 +default_setup_noexit $DISK + +# 2. Verify multihost=off and hostids match (no activity check) +log_must zpool set multihost=off $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export -F $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 3. Verify multihost=off and hostids differ (no activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_set_hostid $HOSTID2 +log_mustnot import_no_activity_check $TESTPOOL "" +log_must import_no_activity_check $TESTPOOL "-f" + +# 4. Verify multihost=off and hostid zero allowed (no activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +log_mustnot import_no_activity_check $TESTPOOL "" +log_must import_no_activity_check $TESTPOOL "-f" + +# 5. Verify multihost=on and hostids match (no activity check) +log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 +log_must zpool set multihost=on $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export -F $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 6. Verify multihost=on and hostids differ (activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_set_hostid $HOSTID2 +log_mustnot import_activity_check $TESTPOOL "" +log_must import_activity_check $TESTPOOL "-f" + +# 7. Verify multihost=on and hostid zero fails (activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +MMP_IMPORTED_MSG="Set the system hostid" +log_must check_pool_import $TESTPOOL "-f" "action" $MMP_IMPORTED_MSG +log_mustnot import_activity_check $TESTPOOL "-f" + +log_pass "multihost=on|off inactive pool activity checks passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_interval.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_interval.ksh new file mode 100755 index 000000000..fb44d6191 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_interval.ksh @@ -0,0 +1,47 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# zfs_multihost_interval should only accept valid values. +# +# STRATEGY: +# 1. Set zfs_multihost_interval to invalid values (negative). +# 2. Set zfs_multihost_interval to valid values. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT +} + +log_assert "zfs_multihost_interval cannot be set to an invalid value" +log_onexit cleanup + +log_mustnot set_tunable64 zfs_multihost_interval -1 +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + +log_pass "zfs_multihost_interval cannot be set to an invalid value" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh new file mode 100755 index 000000000..8bef86a0f --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh @@ -0,0 +1,79 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# When multihost=off ensure that leaf vdev uberblocks are not updated. +# +# STRATEGY: +# 1. Set multihost=off (disables mmp) +# 2. Set zfs_txg_timeout to large value +# 3. Create a zpool +# 4. Find the current "best" uberblock +# 5. Sleep for enough time for uberblocks to change +# 6. Find the current "best" uberblock +# 7. If the uberblock changed, fail +# 8. Set multihost=on +# 9. Sleep for enough time for uberblocks to change +# 10. Find the current "best" uberblock +# 11. If uberblocks didn't change, fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + log_must rm -f $PREV_UBER $CURR_UBER + log_must mmp_clear_hostid +} + +log_assert "mmp thread won't write uberblocks with multihost=off" +log_onexit cleanup + +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=off $TESTPOOL + +log_must zdb -u $TESTPOOL > $PREV_UBER +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if ! diff "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp thread has updated an uberblock" +fi + +log_must zpool set multihost=on $TESTPOOL +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if diff "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp failed to update uberblocks" +fi + +log_pass "mmp thread won't write uberblocks with multihost=off passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh new file mode 100755 index 000000000..07384c623 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Ensure that the MMP thread is writing uberblocks. +# +# STRATEGY: +# 1. Set zfs_txg_timeout to large value +# 2. Create a zpool +# 3. Find the current "best" uberblock +# 4. Sleep for enough time for a potential uberblock update +# 5. Find the current "best" uberblock +# 6. If the uberblock never changed, fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must rm -f $PREV_UBER $CURR_UBER + log_must mmp_clear_hostid +} + +log_assert "mmp thread writes uberblocks (MMP)" +log_onexit cleanup + +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=on $TESTPOOL + +log_must zdb -u $TESTPOOL > $PREV_UBER +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if diff -u "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp failed to update uberblocks" +fi + +log_pass "mmp thread writes uberblocks (MMP) passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh new file mode 100755 index 000000000..2e21e2fef --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh @@ -0,0 +1,79 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Ensure that MMP updates uberblocks at the expected intervals. +# +# STRATEGY: +# 1. Set zfs_txg_timeout to large value +# 2. Create a zpool +# 3. Find the current "best" uberblock +# 4. Loop for 10 seconds, increment counter for each change in UB +# 5. If number of changes seen is less than min threshold, then fail +# 6. If number of changes seen is more than max threshold, then fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +UBER_CHANGES=0 + +function cleanup +{ + default_cleanup_noexit + set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must rm -f $PREV_UBER $CURR_UBER + log_must mmp_clear_hostid +} + +log_assert "Ensure MMP uberblocks update at the correct interval" +log_onexit cleanup + +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=on $TESTPOOL + +log_must zdb -u $TESTPOOL > $PREV_UBER + +SECONDS=0 +while [[ $SECONDS -le 10 ]]; do + log_must zdb -u $TESTPOOL > $CURR_UBER + if ! diff -u "$CURR_UBER" "$PREV_UBER"; then + (( UBER_CHANGES = UBER_CHANGES + 1 )) + log_must mv "$CURR_UBER" "$PREV_UBER" + fi +done + +log_note "Uberblock changed $UBER_CHANGES times" + +if [[ $UBER_CHANGES -lt 8 ]]; then + log_fail "Fewer uberblock writes occured than expected (10)" +fi + +if [[ $UBER_CHANGES -gt 12 ]]; then + log_fail "More uberblock writes occured than expected (10)" +fi + +log_pass "Ensure MMP uberblocks update at the correct interval passed" diff --git a/tests/zfs-tests/tests/functional/mmp/setup.ksh b/tests/zfs-tests/tests/functional/mmp/setup.ksh new file mode 100755 index 000000000..fde5e3bb7 --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/setup.ksh @@ -0,0 +1,32 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg + +verify_runnable "global" + +if [ -e $HOSTID_FILE ]; then + log_unsupported "System has existing $HOSTID_FILE file" +fi + +log_must set_tunable64 zfs_multihost_history $MMP_HISTORY + +log_pass "mmp setup pass" |